s3db.js 12.2.4 → 12.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +117 -0
- package/dist/s3db.cjs.js +1596 -167
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.es.js +1499 -73
- package/dist/s3db.es.js.map +1 -1
- package/package.json +2 -2
- package/src/behaviors/body-only.js +15 -5
- package/src/behaviors/body-overflow.js +9 -0
- package/src/behaviors/user-managed.js +8 -1
- package/src/clients/index.js +14 -0
- package/src/clients/memory-client.class.js +883 -0
- package/src/clients/memory-client.md +917 -0
- package/src/clients/memory-storage.class.js +504 -0
- package/src/{client.class.js → clients/s3-client.class.js} +11 -10
- package/src/concerns/typescript-generator.js +12 -2
- package/src/database.class.js +2 -2
- package/src/index.js +2 -1
- package/src/plugins/api/utils/openapi-generator.js +21 -2
- package/src/plugins/replicators/bigquery-replicator.class.js +109 -21
- package/src/plugins/replicators/mysql-replicator.class.js +9 -1
- package/src/plugins/replicators/planetscale-replicator.class.js +9 -1
- package/src/plugins/replicators/postgres-replicator.class.js +9 -1
- package/src/plugins/replicators/schema-sync.helper.js +53 -2
- package/src/plugins/replicators/turso-replicator.class.js +9 -1
- package/src/plugins/tfstate/s3-driver.js +3 -3
- package/src/plugins/vector.plugin.js +3 -3
- package/src/resource.class.js +203 -4
- package/src/schema.class.js +223 -33
package/dist/s3db.cjs.js
CHANGED
|
@@ -13447,7 +13447,7 @@ function generateMySQLAlterTable(tableName, attributes, existingSchema) {
|
|
|
13447
13447
|
}
|
|
13448
13448
|
return alterStatements;
|
|
13449
13449
|
}
|
|
13450
|
-
function generateBigQuerySchema(attributes) {
|
|
13450
|
+
function generateBigQuerySchema(attributes, mutability = "append-only") {
|
|
13451
13451
|
const fields = [];
|
|
13452
13452
|
fields.push({
|
|
13453
13453
|
name: "id",
|
|
@@ -13471,6 +13471,14 @@ function generateBigQuerySchema(attributes) {
|
|
|
13471
13471
|
if (!attributes.updatedAt) {
|
|
13472
13472
|
fields.push({ name: "updated_at", type: "TIMESTAMP", mode: "NULLABLE" });
|
|
13473
13473
|
}
|
|
13474
|
+
if (mutability === "append-only" || mutability === "immutable") {
|
|
13475
|
+
fields.push({ name: "_operation_type", type: "STRING", mode: "NULLABLE" });
|
|
13476
|
+
fields.push({ name: "_operation_timestamp", type: "TIMESTAMP", mode: "NULLABLE" });
|
|
13477
|
+
}
|
|
13478
|
+
if (mutability === "immutable") {
|
|
13479
|
+
fields.push({ name: "_is_deleted", type: "BOOL", mode: "NULLABLE" });
|
|
13480
|
+
fields.push({ name: "_version", type: "INT64", mode: "NULLABLE" });
|
|
13481
|
+
}
|
|
13474
13482
|
return fields;
|
|
13475
13483
|
}
|
|
13476
13484
|
async function getBigQueryTableSchema(bigqueryClient, datasetId, tableId) {
|
|
@@ -13492,7 +13500,7 @@ async function getBigQueryTableSchema(bigqueryClient, datasetId, tableId) {
|
|
|
13492
13500
|
}
|
|
13493
13501
|
return schema;
|
|
13494
13502
|
}
|
|
13495
|
-
function generateBigQuerySchemaUpdate(attributes, existingSchema) {
|
|
13503
|
+
function generateBigQuerySchemaUpdate(attributes, existingSchema, mutability = "append-only") {
|
|
13496
13504
|
const newFields = [];
|
|
13497
13505
|
for (const [fieldName, fieldConfig] of Object.entries(attributes)) {
|
|
13498
13506
|
if (fieldName === "id") continue;
|
|
@@ -13506,6 +13514,22 @@ function generateBigQuerySchemaUpdate(attributes, existingSchema) {
|
|
|
13506
13514
|
mode: required ? "REQUIRED" : "NULLABLE"
|
|
13507
13515
|
});
|
|
13508
13516
|
}
|
|
13517
|
+
if (mutability === "append-only" || mutability === "immutable") {
|
|
13518
|
+
if (!existingSchema["_operation_type"]) {
|
|
13519
|
+
newFields.push({ name: "_operation_type", type: "STRING", mode: "NULLABLE" });
|
|
13520
|
+
}
|
|
13521
|
+
if (!existingSchema["_operation_timestamp"]) {
|
|
13522
|
+
newFields.push({ name: "_operation_timestamp", type: "TIMESTAMP", mode: "NULLABLE" });
|
|
13523
|
+
}
|
|
13524
|
+
}
|
|
13525
|
+
if (mutability === "immutable") {
|
|
13526
|
+
if (!existingSchema["_is_deleted"]) {
|
|
13527
|
+
newFields.push({ name: "_is_deleted", type: "BOOL", mode: "NULLABLE" });
|
|
13528
|
+
}
|
|
13529
|
+
if (!existingSchema["_version"]) {
|
|
13530
|
+
newFields.push({ name: "_version", type: "INT64", mode: "NULLABLE" });
|
|
13531
|
+
}
|
|
13532
|
+
}
|
|
13509
13533
|
return newFields;
|
|
13510
13534
|
}
|
|
13511
13535
|
function s3dbTypeToSQLite(fieldType, fieldOptions = {}) {
|
|
@@ -13588,6 +13612,8 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
13588
13612
|
this.credentials = config.credentials;
|
|
13589
13613
|
this.location = config.location || "US";
|
|
13590
13614
|
this.logTable = config.logTable;
|
|
13615
|
+
this.mutability = config.mutability || "append-only";
|
|
13616
|
+
this._validateMutability(this.mutability);
|
|
13591
13617
|
this.schemaSync = {
|
|
13592
13618
|
enabled: config.schemaSync?.enabled || false,
|
|
13593
13619
|
strategy: config.schemaSync?.strategy || "alter",
|
|
@@ -13596,6 +13622,13 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
13596
13622
|
autoCreateColumns: config.schemaSync?.autoCreateColumns !== false
|
|
13597
13623
|
};
|
|
13598
13624
|
this.resources = this.parseResourcesConfig(resources);
|
|
13625
|
+
this.versionCounters = /* @__PURE__ */ new Map();
|
|
13626
|
+
}
|
|
13627
|
+
_validateMutability(mutability) {
|
|
13628
|
+
const validModes = ["append-only", "mutable", "immutable"];
|
|
13629
|
+
if (!validModes.includes(mutability)) {
|
|
13630
|
+
throw new Error(`Invalid mutability mode: ${mutability}. Must be one of: ${validModes.join(", ")}`);
|
|
13631
|
+
}
|
|
13599
13632
|
}
|
|
13600
13633
|
parseResourcesConfig(resources) {
|
|
13601
13634
|
const parsed = {};
|
|
@@ -13604,24 +13637,31 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
13604
13637
|
parsed[resourceName] = [{
|
|
13605
13638
|
table: config,
|
|
13606
13639
|
actions: ["insert"],
|
|
13607
|
-
transform: null
|
|
13640
|
+
transform: null,
|
|
13641
|
+
mutability: this.mutability
|
|
13608
13642
|
}];
|
|
13609
13643
|
} else if (Array.isArray(config)) {
|
|
13610
13644
|
parsed[resourceName] = config.map((item) => {
|
|
13611
13645
|
if (typeof item === "string") {
|
|
13612
|
-
return { table: item, actions: ["insert"], transform: null };
|
|
13646
|
+
return { table: item, actions: ["insert"], transform: null, mutability: this.mutability };
|
|
13613
13647
|
}
|
|
13648
|
+
const itemMutability = item.mutability || this.mutability;
|
|
13649
|
+
this._validateMutability(itemMutability);
|
|
13614
13650
|
return {
|
|
13615
13651
|
table: item.table,
|
|
13616
13652
|
actions: item.actions || ["insert"],
|
|
13617
|
-
transform: item.transform || null
|
|
13653
|
+
transform: item.transform || null,
|
|
13654
|
+
mutability: itemMutability
|
|
13618
13655
|
};
|
|
13619
13656
|
});
|
|
13620
13657
|
} else if (typeof config === "object") {
|
|
13658
|
+
const configMutability = config.mutability || this.mutability;
|
|
13659
|
+
this._validateMutability(configMutability);
|
|
13621
13660
|
parsed[resourceName] = [{
|
|
13622
13661
|
table: config.table,
|
|
13623
13662
|
actions: config.actions || ["insert"],
|
|
13624
|
-
transform: config.transform || null
|
|
13663
|
+
transform: config.transform || null,
|
|
13664
|
+
mutability: configMutability
|
|
13625
13665
|
}];
|
|
13626
13666
|
}
|
|
13627
13667
|
}
|
|
@@ -13693,11 +13733,16 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
13693
13733
|
}
|
|
13694
13734
|
continue;
|
|
13695
13735
|
}
|
|
13696
|
-
const
|
|
13736
|
+
const allAttributes = resource.config.versions[resource.config.currentVersion]?.attributes || {};
|
|
13737
|
+
const pluginAttrNames = resource.schema?._pluginAttributes ? Object.values(resource.schema._pluginAttributes).flat() : [];
|
|
13738
|
+
const attributes = Object.fromEntries(
|
|
13739
|
+
Object.entries(allAttributes).filter(([name]) => !pluginAttrNames.includes(name))
|
|
13740
|
+
);
|
|
13697
13741
|
for (const tableConfig of tableConfigs) {
|
|
13698
13742
|
const tableName = tableConfig.table;
|
|
13743
|
+
const mutability = tableConfig.mutability;
|
|
13699
13744
|
const [okSync, errSync] = await tryFn(async () => {
|
|
13700
|
-
await this.syncTableSchema(tableName, attributes);
|
|
13745
|
+
await this.syncTableSchema(tableName, attributes, mutability);
|
|
13701
13746
|
});
|
|
13702
13747
|
if (!okSync) {
|
|
13703
13748
|
const message = `Schema sync failed for table ${tableName}: ${errSync.message}`;
|
|
@@ -13717,7 +13762,7 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
13717
13762
|
/**
|
|
13718
13763
|
* Sync a single table schema in BigQuery
|
|
13719
13764
|
*/
|
|
13720
|
-
async syncTableSchema(tableName, attributes) {
|
|
13765
|
+
async syncTableSchema(tableName, attributes, mutability = "append-only") {
|
|
13721
13766
|
const dataset = this.bigqueryClient.dataset(this.datasetId);
|
|
13722
13767
|
const table = dataset.table(tableName);
|
|
13723
13768
|
const [exists] = await table.exists();
|
|
@@ -13728,15 +13773,16 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
13728
13773
|
if (this.schemaSync.strategy === "validate-only") {
|
|
13729
13774
|
throw new Error(`Table ${tableName} does not exist (validate-only mode)`);
|
|
13730
13775
|
}
|
|
13731
|
-
const schema = generateBigQuerySchema(attributes);
|
|
13776
|
+
const schema = generateBigQuerySchema(attributes, mutability);
|
|
13732
13777
|
if (this.config.verbose) {
|
|
13733
|
-
console.log(`[BigQueryReplicator] Creating table ${tableName} with schema:`, schema);
|
|
13778
|
+
console.log(`[BigQueryReplicator] Creating table ${tableName} with schema (mutability: ${mutability}):`, schema);
|
|
13734
13779
|
}
|
|
13735
13780
|
await dataset.createTable(tableName, { schema });
|
|
13736
13781
|
this.emit("table_created", {
|
|
13737
13782
|
replicator: this.name,
|
|
13738
13783
|
tableName,
|
|
13739
|
-
attributes: Object.keys(attributes)
|
|
13784
|
+
attributes: Object.keys(attributes),
|
|
13785
|
+
mutability
|
|
13740
13786
|
});
|
|
13741
13787
|
return;
|
|
13742
13788
|
}
|
|
@@ -13745,18 +13791,19 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
13745
13791
|
console.warn(`[BigQueryReplicator] Dropping and recreating table ${tableName}`);
|
|
13746
13792
|
}
|
|
13747
13793
|
await table.delete();
|
|
13748
|
-
const schema = generateBigQuerySchema(attributes);
|
|
13794
|
+
const schema = generateBigQuerySchema(attributes, mutability);
|
|
13749
13795
|
await dataset.createTable(tableName, { schema });
|
|
13750
13796
|
this.emit("table_recreated", {
|
|
13751
13797
|
replicator: this.name,
|
|
13752
13798
|
tableName,
|
|
13753
|
-
attributes: Object.keys(attributes)
|
|
13799
|
+
attributes: Object.keys(attributes),
|
|
13800
|
+
mutability
|
|
13754
13801
|
});
|
|
13755
13802
|
return;
|
|
13756
13803
|
}
|
|
13757
13804
|
if (this.schemaSync.strategy === "alter" && this.schemaSync.autoCreateColumns) {
|
|
13758
13805
|
const existingSchema = await getBigQueryTableSchema(this.bigqueryClient, this.datasetId, tableName);
|
|
13759
|
-
const newFields = generateBigQuerySchemaUpdate(attributes, existingSchema);
|
|
13806
|
+
const newFields = generateBigQuerySchemaUpdate(attributes, existingSchema, mutability);
|
|
13760
13807
|
if (newFields.length > 0) {
|
|
13761
13808
|
if (this.config.verbose) {
|
|
13762
13809
|
console.log(`[BigQueryReplicator] Adding ${newFields.length} field(s) to table ${tableName}:`, newFields);
|
|
@@ -13774,7 +13821,7 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
13774
13821
|
}
|
|
13775
13822
|
if (this.schemaSync.strategy === "validate-only") {
|
|
13776
13823
|
const existingSchema = await getBigQueryTableSchema(this.bigqueryClient, this.datasetId, tableName);
|
|
13777
|
-
const newFields = generateBigQuerySchemaUpdate(attributes, existingSchema);
|
|
13824
|
+
const newFields = generateBigQuerySchemaUpdate(attributes, existingSchema, mutability);
|
|
13778
13825
|
if (newFields.length > 0) {
|
|
13779
13826
|
throw new Error(`Table ${tableName} schema mismatch. Missing columns: ${newFields.length}`);
|
|
13780
13827
|
}
|
|
@@ -13793,7 +13840,8 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
13793
13840
|
if (!this.resources[resourceName]) return [];
|
|
13794
13841
|
return this.resources[resourceName].filter((tableConfig) => tableConfig.actions.includes(operation)).map((tableConfig) => ({
|
|
13795
13842
|
table: tableConfig.table,
|
|
13796
|
-
transform: tableConfig.transform
|
|
13843
|
+
transform: tableConfig.transform,
|
|
13844
|
+
mutability: tableConfig.mutability
|
|
13797
13845
|
}));
|
|
13798
13846
|
}
|
|
13799
13847
|
applyTransform(data, transformFn) {
|
|
@@ -13812,6 +13860,32 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
13812
13860
|
});
|
|
13813
13861
|
return cleanData;
|
|
13814
13862
|
}
|
|
13863
|
+
/**
|
|
13864
|
+
* Add tracking fields for append-only and immutable modes
|
|
13865
|
+
* @private
|
|
13866
|
+
*/
|
|
13867
|
+
_addTrackingFields(data, operation, mutability, id) {
|
|
13868
|
+
const tracked = { ...data };
|
|
13869
|
+
if (mutability === "append-only" || mutability === "immutable") {
|
|
13870
|
+
tracked._operation_type = operation;
|
|
13871
|
+
tracked._operation_timestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
13872
|
+
}
|
|
13873
|
+
if (mutability === "immutable") {
|
|
13874
|
+
tracked._is_deleted = operation === "delete";
|
|
13875
|
+
tracked._version = this._getNextVersion(id);
|
|
13876
|
+
}
|
|
13877
|
+
return tracked;
|
|
13878
|
+
}
|
|
13879
|
+
/**
|
|
13880
|
+
* Get next version number for immutable mode
|
|
13881
|
+
* @private
|
|
13882
|
+
*/
|
|
13883
|
+
_getNextVersion(id) {
|
|
13884
|
+
const current = this.versionCounters.get(id) || 0;
|
|
13885
|
+
const next = current + 1;
|
|
13886
|
+
this.versionCounters.set(id, next);
|
|
13887
|
+
return next;
|
|
13888
|
+
}
|
|
13815
13889
|
async replicate(resourceName, operation, data, id, beforeData = null) {
|
|
13816
13890
|
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
13817
13891
|
return { skipped: true, reason: "resource_not_included" };
|
|
@@ -13830,9 +13904,14 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
13830
13904
|
for (const tableConfig of tableConfigs) {
|
|
13831
13905
|
const [okTable, errTable] = await tryFn(async () => {
|
|
13832
13906
|
const table = dataset.table(tableConfig.table);
|
|
13907
|
+
const mutability = tableConfig.mutability;
|
|
13833
13908
|
let job;
|
|
13834
|
-
|
|
13835
|
-
|
|
13909
|
+
const shouldConvertToInsert = (mutability === "append-only" || mutability === "immutable") && (operation === "update" || operation === "delete");
|
|
13910
|
+
if (operation === "insert" || shouldConvertToInsert) {
|
|
13911
|
+
let transformedData = this.applyTransform(data, tableConfig.transform);
|
|
13912
|
+
if (shouldConvertToInsert) {
|
|
13913
|
+
transformedData = this._addTrackingFields(transformedData, operation, mutability, id);
|
|
13914
|
+
}
|
|
13836
13915
|
try {
|
|
13837
13916
|
job = await table.insert([transformedData]);
|
|
13838
13917
|
} catch (error) {
|
|
@@ -13844,7 +13923,7 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
13844
13923
|
}
|
|
13845
13924
|
throw error;
|
|
13846
13925
|
}
|
|
13847
|
-
} else if (operation === "update") {
|
|
13926
|
+
} else if (operation === "update" && mutability === "mutable") {
|
|
13848
13927
|
const transformedData = this.applyTransform(data, tableConfig.transform);
|
|
13849
13928
|
const keys = Object.keys(transformedData).filter((k) => k !== "id");
|
|
13850
13929
|
const setClause = keys.map((k) => `${k} = @${k}`).join(", ");
|
|
@@ -13886,7 +13965,7 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
13886
13965
|
}
|
|
13887
13966
|
}
|
|
13888
13967
|
if (!job) throw lastError;
|
|
13889
|
-
} else if (operation === "delete") {
|
|
13968
|
+
} else if (operation === "delete" && mutability === "mutable") {
|
|
13890
13969
|
const query = `DELETE FROM \`${this.projectId}.${this.datasetId}.${tableConfig.table}\` WHERE id = @id`;
|
|
13891
13970
|
try {
|
|
13892
13971
|
const [deleteJob] = await this.bigqueryClient.createQueryJob({
|
|
@@ -14022,7 +14101,8 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
14022
14101
|
datasetId: this.datasetId,
|
|
14023
14102
|
resources: this.resources,
|
|
14024
14103
|
logTable: this.logTable,
|
|
14025
|
-
schemaSync: this.schemaSync
|
|
14104
|
+
schemaSync: this.schemaSync,
|
|
14105
|
+
mutability: this.mutability
|
|
14026
14106
|
};
|
|
14027
14107
|
}
|
|
14028
14108
|
}
|
|
@@ -14711,7 +14791,11 @@ class MySQLReplicator extends BaseReplicator {
|
|
|
14711
14791
|
}
|
|
14712
14792
|
continue;
|
|
14713
14793
|
}
|
|
14714
|
-
const
|
|
14794
|
+
const allAttributes = resource.config.versions[resource.config.currentVersion]?.attributes || {};
|
|
14795
|
+
const pluginAttrNames = resource.schema?._pluginAttributes ? Object.values(resource.schema._pluginAttributes).flat() : [];
|
|
14796
|
+
const attributes = Object.fromEntries(
|
|
14797
|
+
Object.entries(allAttributes).filter(([name]) => !pluginAttrNames.includes(name))
|
|
14798
|
+
);
|
|
14715
14799
|
for (const tableConfig of tableConfigs) {
|
|
14716
14800
|
const tableName = tableConfig.table;
|
|
14717
14801
|
const [okSync, errSync] = await tryFn(async () => {
|
|
@@ -15090,7 +15174,11 @@ class PlanetScaleReplicator extends BaseReplicator {
|
|
|
15090
15174
|
}
|
|
15091
15175
|
continue;
|
|
15092
15176
|
}
|
|
15093
|
-
const
|
|
15177
|
+
const allAttributes = resource.config.versions[resource.config.currentVersion]?.attributes || {};
|
|
15178
|
+
const pluginAttrNames = resource.schema?._pluginAttributes ? Object.values(resource.schema._pluginAttributes).flat() : [];
|
|
15179
|
+
const attributes = Object.fromEntries(
|
|
15180
|
+
Object.entries(allAttributes).filter(([name]) => !pluginAttrNames.includes(name))
|
|
15181
|
+
);
|
|
15094
15182
|
for (const tableConfig of tableConfigs) {
|
|
15095
15183
|
const tableName = tableConfig.table;
|
|
15096
15184
|
const [okSync, errSync] = await tryFn(async () => {
|
|
@@ -15411,7 +15499,11 @@ class PostgresReplicator extends BaseReplicator {
|
|
|
15411
15499
|
}
|
|
15412
15500
|
continue;
|
|
15413
15501
|
}
|
|
15414
|
-
const
|
|
15502
|
+
const allAttributes = resource.config.versions[resource.config.currentVersion]?.attributes || {};
|
|
15503
|
+
const pluginAttrNames = resource.schema?._pluginAttributes ? Object.values(resource.schema._pluginAttributes).flat() : [];
|
|
15504
|
+
const attributes = Object.fromEntries(
|
|
15505
|
+
Object.entries(allAttributes).filter(([name]) => !pluginAttrNames.includes(name))
|
|
15506
|
+
);
|
|
15415
15507
|
for (const tableConfig of tableConfigs) {
|
|
15416
15508
|
const tableName = tableConfig.table;
|
|
15417
15509
|
const [okSync, errSync] = await tryFn(async () => {
|
|
@@ -15731,11 +15823,11 @@ class ConnectionString {
|
|
|
15731
15823
|
}
|
|
15732
15824
|
}
|
|
15733
15825
|
|
|
15734
|
-
class
|
|
15826
|
+
class S3Client extends EventEmitter {
|
|
15735
15827
|
constructor({
|
|
15736
15828
|
verbose = false,
|
|
15737
15829
|
id = null,
|
|
15738
|
-
AwsS3Client,
|
|
15830
|
+
AwsS3Client: AwsS3Client2,
|
|
15739
15831
|
connectionString,
|
|
15740
15832
|
parallelism = 10,
|
|
15741
15833
|
httpClientOptions = {}
|
|
@@ -15758,7 +15850,7 @@ class Client extends EventEmitter {
|
|
|
15758
15850
|
// 60 second timeout
|
|
15759
15851
|
...httpClientOptions
|
|
15760
15852
|
};
|
|
15761
|
-
this.client =
|
|
15853
|
+
this.client = AwsS3Client2 || this.createClient();
|
|
15762
15854
|
}
|
|
15763
15855
|
createClient() {
|
|
15764
15856
|
const httpAgent = new http.Agent(this.httpClientOptions);
|
|
@@ -16533,6 +16625,32 @@ function generateBase62Mapping(keys) {
|
|
|
16533
16625
|
});
|
|
16534
16626
|
return { mapping, reversedMapping };
|
|
16535
16627
|
}
|
|
16628
|
+
function generatePluginAttributeHash(pluginName, attributeName) {
|
|
16629
|
+
const input = `${pluginName}:${attributeName}`;
|
|
16630
|
+
const hash = crypto$1.createHash("sha256").update(input).digest();
|
|
16631
|
+
const num = hash.readUInt32BE(0);
|
|
16632
|
+
const base62Hash = encode(num);
|
|
16633
|
+
const paddedHash = base62Hash.padStart(3, "0").substring(0, 3);
|
|
16634
|
+
return "p" + paddedHash.toLowerCase();
|
|
16635
|
+
}
|
|
16636
|
+
function generatePluginMapping(attributes) {
|
|
16637
|
+
const mapping = {};
|
|
16638
|
+
const reversedMapping = {};
|
|
16639
|
+
const usedHashes = /* @__PURE__ */ new Set();
|
|
16640
|
+
for (const { key, pluginName } of attributes) {
|
|
16641
|
+
let hash = generatePluginAttributeHash(pluginName, key);
|
|
16642
|
+
let counter = 1;
|
|
16643
|
+
let finalHash = hash;
|
|
16644
|
+
while (usedHashes.has(finalHash)) {
|
|
16645
|
+
finalHash = `${hash}${counter}`;
|
|
16646
|
+
counter++;
|
|
16647
|
+
}
|
|
16648
|
+
usedHashes.add(finalHash);
|
|
16649
|
+
mapping[key] = finalHash;
|
|
16650
|
+
reversedMapping[finalHash] = key;
|
|
16651
|
+
}
|
|
16652
|
+
return { mapping, reversedMapping };
|
|
16653
|
+
}
|
|
16536
16654
|
const SchemaActions = {
|
|
16537
16655
|
trim: (value) => value == null ? value : value.trim(),
|
|
16538
16656
|
encrypt: async (value, { passphrase }) => {
|
|
@@ -16923,11 +17041,14 @@ class Schema {
|
|
|
16923
17041
|
constructor(args) {
|
|
16924
17042
|
const {
|
|
16925
17043
|
map,
|
|
17044
|
+
pluginMap,
|
|
16926
17045
|
name,
|
|
16927
17046
|
attributes,
|
|
16928
17047
|
passphrase,
|
|
16929
17048
|
version = 1,
|
|
16930
|
-
options = {}
|
|
17049
|
+
options = {},
|
|
17050
|
+
_pluginAttributeMetadata,
|
|
17051
|
+
_pluginAttributes
|
|
16931
17052
|
} = args;
|
|
16932
17053
|
this.name = name;
|
|
16933
17054
|
this.version = version;
|
|
@@ -16935,6 +17056,8 @@ class Schema {
|
|
|
16935
17056
|
this.passphrase = passphrase ?? "secret";
|
|
16936
17057
|
this.options = lodashEs.merge({}, this.defaultOptions(), options);
|
|
16937
17058
|
this.allNestedObjectsOptional = this.options.allNestedObjectsOptional ?? false;
|
|
17059
|
+
this._pluginAttributeMetadata = _pluginAttributeMetadata || {};
|
|
17060
|
+
this._pluginAttributes = _pluginAttributes || {};
|
|
16938
17061
|
const processedAttributes = this.preprocessAttributesForValidation(this.attributes);
|
|
16939
17062
|
this.validator = new ValidatorManager({ autoEncrypt: false }).compile(lodashEs.merge(
|
|
16940
17063
|
{ $$async: true, $$strict: false },
|
|
@@ -16949,9 +17072,43 @@ class Schema {
|
|
|
16949
17072
|
const leafKeys = Object.keys(flatAttrs).filter((k) => !k.includes("$$"));
|
|
16950
17073
|
const objectKeys = this.extractObjectKeys(this.attributes);
|
|
16951
17074
|
const allKeys = [.../* @__PURE__ */ new Set([...leafKeys, ...objectKeys])];
|
|
16952
|
-
const
|
|
17075
|
+
const userKeys = [];
|
|
17076
|
+
const pluginAttributes = [];
|
|
17077
|
+
for (const key of allKeys) {
|
|
17078
|
+
const attrDef = this.getAttributeDefinition(key);
|
|
17079
|
+
if (typeof attrDef === "object" && attrDef !== null && attrDef.__plugin__) {
|
|
17080
|
+
pluginAttributes.push({ key, pluginName: attrDef.__plugin__ });
|
|
17081
|
+
} else if (typeof attrDef === "string" && this._pluginAttributeMetadata && this._pluginAttributeMetadata[key]) {
|
|
17082
|
+
const pluginName = this._pluginAttributeMetadata[key].__plugin__;
|
|
17083
|
+
pluginAttributes.push({ key, pluginName });
|
|
17084
|
+
} else {
|
|
17085
|
+
userKeys.push(key);
|
|
17086
|
+
}
|
|
17087
|
+
}
|
|
17088
|
+
const { mapping, reversedMapping } = generateBase62Mapping(userKeys);
|
|
16953
17089
|
this.map = mapping;
|
|
16954
17090
|
this.reversedMap = reversedMapping;
|
|
17091
|
+
const { mapping: pMapping, reversedMapping: pReversedMapping } = generatePluginMapping(pluginAttributes);
|
|
17092
|
+
this.pluginMap = pMapping;
|
|
17093
|
+
this.reversedPluginMap = pReversedMapping;
|
|
17094
|
+
this._pluginAttributes = {};
|
|
17095
|
+
for (const { key, pluginName } of pluginAttributes) {
|
|
17096
|
+
if (!this._pluginAttributes[pluginName]) {
|
|
17097
|
+
this._pluginAttributes[pluginName] = [];
|
|
17098
|
+
}
|
|
17099
|
+
this._pluginAttributes[pluginName].push(key);
|
|
17100
|
+
}
|
|
17101
|
+
}
|
|
17102
|
+
if (!lodashEs.isEmpty(pluginMap)) {
|
|
17103
|
+
this.pluginMap = pluginMap;
|
|
17104
|
+
this.reversedPluginMap = lodashEs.invert(pluginMap);
|
|
17105
|
+
}
|
|
17106
|
+
if (!this.pluginMap) {
|
|
17107
|
+
this.pluginMap = {};
|
|
17108
|
+
this.reversedPluginMap = {};
|
|
17109
|
+
}
|
|
17110
|
+
if (!this._pluginAttributes) {
|
|
17111
|
+
this._pluginAttributes = {};
|
|
16955
17112
|
}
|
|
16956
17113
|
}
|
|
16957
17114
|
defaultOptions() {
|
|
@@ -17180,6 +17337,8 @@ class Schema {
|
|
|
17180
17337
|
static import(data) {
|
|
17181
17338
|
let {
|
|
17182
17339
|
map,
|
|
17340
|
+
pluginMap,
|
|
17341
|
+
_pluginAttributeMetadata,
|
|
17183
17342
|
name,
|
|
17184
17343
|
options,
|
|
17185
17344
|
version,
|
|
@@ -17190,11 +17349,15 @@ class Schema {
|
|
|
17190
17349
|
attributes = attrs;
|
|
17191
17350
|
const schema = new Schema({
|
|
17192
17351
|
map,
|
|
17352
|
+
pluginMap: pluginMap || {},
|
|
17193
17353
|
name,
|
|
17194
17354
|
options,
|
|
17195
17355
|
version,
|
|
17196
17356
|
attributes
|
|
17197
17357
|
});
|
|
17358
|
+
if (_pluginAttributeMetadata) {
|
|
17359
|
+
schema._pluginAttributeMetadata = _pluginAttributeMetadata;
|
|
17360
|
+
}
|
|
17198
17361
|
return schema;
|
|
17199
17362
|
}
|
|
17200
17363
|
/**
|
|
@@ -17232,7 +17395,10 @@ class Schema {
|
|
|
17232
17395
|
name: this.name,
|
|
17233
17396
|
options: this.options,
|
|
17234
17397
|
attributes: this._exportAttributes(this.attributes),
|
|
17235
|
-
map: this.map
|
|
17398
|
+
map: this.map,
|
|
17399
|
+
pluginMap: this.pluginMap || {},
|
|
17400
|
+
_pluginAttributeMetadata: this._pluginAttributeMetadata || {},
|
|
17401
|
+
_pluginAttributes: this._pluginAttributes || {}
|
|
17236
17402
|
};
|
|
17237
17403
|
return data;
|
|
17238
17404
|
}
|
|
@@ -17285,7 +17451,7 @@ class Schema {
|
|
|
17285
17451
|
const flattenedObj = flat.flatten(obj, { safe: true });
|
|
17286
17452
|
const rest = { "_v": this.version + "" };
|
|
17287
17453
|
for (const [key, value] of Object.entries(flattenedObj)) {
|
|
17288
|
-
const mappedKey = this.map[key] || key;
|
|
17454
|
+
const mappedKey = this.pluginMap[key] || this.map[key] || key;
|
|
17289
17455
|
const attrDef = this.getAttributeDefinition(key);
|
|
17290
17456
|
if (typeof value === "number" && typeof attrDef === "string" && attrDef.includes("number")) {
|
|
17291
17457
|
rest[mappedKey] = encode(value);
|
|
@@ -17306,14 +17472,18 @@ class Schema {
|
|
|
17306
17472
|
await this.applyHooksActions(rest, "afterMap");
|
|
17307
17473
|
return rest;
|
|
17308
17474
|
}
|
|
17309
|
-
async unmapper(mappedResourceItem, mapOverride) {
|
|
17475
|
+
async unmapper(mappedResourceItem, mapOverride, pluginMapOverride) {
|
|
17310
17476
|
let obj = lodashEs.cloneDeep(mappedResourceItem);
|
|
17311
17477
|
delete obj._v;
|
|
17312
17478
|
obj = await this.applyHooksActions(obj, "beforeUnmap");
|
|
17313
17479
|
const reversedMap = mapOverride ? lodashEs.invert(mapOverride) : this.reversedMap;
|
|
17480
|
+
const reversedPluginMap = pluginMapOverride ? lodashEs.invert(pluginMapOverride) : this.reversedPluginMap;
|
|
17314
17481
|
const rest = {};
|
|
17315
17482
|
for (const [key, value] of Object.entries(obj)) {
|
|
17316
|
-
|
|
17483
|
+
let originalKey = reversedPluginMap[key] || reversedMap[key] || key;
|
|
17484
|
+
if (!originalKey) {
|
|
17485
|
+
originalKey = key;
|
|
17486
|
+
}
|
|
17317
17487
|
let parsedValue = value;
|
|
17318
17488
|
const attrDef = this.getAttributeDefinition(originalKey);
|
|
17319
17489
|
const hasAfterUnmapHook = this.options.hooks?.afterUnmap?.[originalKey];
|
|
@@ -17380,6 +17550,37 @@ class Schema {
|
|
|
17380
17550
|
}
|
|
17381
17551
|
return def;
|
|
17382
17552
|
}
|
|
17553
|
+
/**
|
|
17554
|
+
* Regenerate plugin attribute mapping
|
|
17555
|
+
* Called when plugin attributes are added or removed
|
|
17556
|
+
* @returns {void}
|
|
17557
|
+
*/
|
|
17558
|
+
regeneratePluginMapping() {
|
|
17559
|
+
const flatAttrs = flat.flatten(this.attributes, { safe: true });
|
|
17560
|
+
const leafKeys = Object.keys(flatAttrs).filter((k) => !k.includes("$$"));
|
|
17561
|
+
const objectKeys = this.extractObjectKeys(this.attributes);
|
|
17562
|
+
const allKeys = [.../* @__PURE__ */ new Set([...leafKeys, ...objectKeys])];
|
|
17563
|
+
const pluginAttributes = [];
|
|
17564
|
+
for (const key of allKeys) {
|
|
17565
|
+
const attrDef = this.getAttributeDefinition(key);
|
|
17566
|
+
if (typeof attrDef === "object" && attrDef !== null && attrDef.__plugin__) {
|
|
17567
|
+
pluginAttributes.push({ key, pluginName: attrDef.__plugin__ });
|
|
17568
|
+
} else if (typeof attrDef === "string" && this._pluginAttributeMetadata && this._pluginAttributeMetadata[key]) {
|
|
17569
|
+
const pluginName = this._pluginAttributeMetadata[key].__plugin__;
|
|
17570
|
+
pluginAttributes.push({ key, pluginName });
|
|
17571
|
+
}
|
|
17572
|
+
}
|
|
17573
|
+
const { mapping, reversedMapping } = generatePluginMapping(pluginAttributes);
|
|
17574
|
+
this.pluginMap = mapping;
|
|
17575
|
+
this.reversedPluginMap = reversedMapping;
|
|
17576
|
+
this._pluginAttributes = {};
|
|
17577
|
+
for (const { key, pluginName } of pluginAttributes) {
|
|
17578
|
+
if (!this._pluginAttributes[pluginName]) {
|
|
17579
|
+
this._pluginAttributes[pluginName] = [];
|
|
17580
|
+
}
|
|
17581
|
+
this._pluginAttributes[pluginName].push(key);
|
|
17582
|
+
}
|
|
17583
|
+
}
|
|
17383
17584
|
/**
|
|
17384
17585
|
* Preprocess attributes to convert nested objects into validator-compatible format
|
|
17385
17586
|
* @param {Object} attributes - Original attributes
|
|
@@ -17449,37 +17650,38 @@ class Schema {
|
|
|
17449
17650
|
} else if (typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
17450
17651
|
const hasValidatorType = value.type !== void 0 && key !== "$$type";
|
|
17451
17652
|
if (hasValidatorType) {
|
|
17452
|
-
|
|
17453
|
-
|
|
17454
|
-
|
|
17455
|
-
|
|
17456
|
-
|
|
17457
|
-
|
|
17458
|
-
|
|
17459
|
-
|
|
17460
|
-
|
|
17653
|
+
const { __plugin__, __pluginCreated__, ...cleanValue } = value;
|
|
17654
|
+
if (cleanValue.type === "ip4") {
|
|
17655
|
+
processed[key] = { ...cleanValue, type: "string" };
|
|
17656
|
+
} else if (cleanValue.type === "ip6") {
|
|
17657
|
+
processed[key] = { ...cleanValue, type: "string" };
|
|
17658
|
+
} else if (cleanValue.type === "money" || cleanValue.type === "crypto") {
|
|
17659
|
+
processed[key] = { ...cleanValue, type: "number", min: cleanValue.min !== void 0 ? cleanValue.min : 0 };
|
|
17660
|
+
} else if (cleanValue.type === "decimal") {
|
|
17661
|
+
processed[key] = { ...cleanValue, type: "number" };
|
|
17662
|
+
} else if (cleanValue.type === "geo:lat" || cleanValue.type === "geo-lat") {
|
|
17461
17663
|
processed[key] = {
|
|
17462
|
-
...
|
|
17664
|
+
...cleanValue,
|
|
17463
17665
|
type: "number",
|
|
17464
|
-
min:
|
|
17465
|
-
max:
|
|
17666
|
+
min: cleanValue.min !== void 0 ? cleanValue.min : -90,
|
|
17667
|
+
max: cleanValue.max !== void 0 ? cleanValue.max : 90
|
|
17466
17668
|
};
|
|
17467
|
-
} else if (
|
|
17669
|
+
} else if (cleanValue.type === "geo:lon" || cleanValue.type === "geo-lon") {
|
|
17468
17670
|
processed[key] = {
|
|
17469
|
-
...
|
|
17671
|
+
...cleanValue,
|
|
17470
17672
|
type: "number",
|
|
17471
|
-
min:
|
|
17472
|
-
max:
|
|
17673
|
+
min: cleanValue.min !== void 0 ? cleanValue.min : -180,
|
|
17674
|
+
max: cleanValue.max !== void 0 ? cleanValue.max : 180
|
|
17473
17675
|
};
|
|
17474
|
-
} else if (
|
|
17475
|
-
processed[key] = { ...
|
|
17476
|
-
} else if (
|
|
17676
|
+
} else if (cleanValue.type === "geo:point" || cleanValue.type === "geo-point") {
|
|
17677
|
+
processed[key] = { ...cleanValue, type: "any" };
|
|
17678
|
+
} else if (cleanValue.type === "object" && cleanValue.properties) {
|
|
17477
17679
|
processed[key] = {
|
|
17478
|
-
...
|
|
17479
|
-
properties: this.preprocessAttributesForValidation(
|
|
17680
|
+
...cleanValue,
|
|
17681
|
+
properties: this.preprocessAttributesForValidation(cleanValue.properties)
|
|
17480
17682
|
};
|
|
17481
17683
|
} else {
|
|
17482
|
-
processed[key] =
|
|
17684
|
+
processed[key] = cleanValue;
|
|
17483
17685
|
}
|
|
17484
17686
|
} else {
|
|
17485
17687
|
const isExplicitRequired = value.$$type && value.$$type.includes("required");
|
|
@@ -17797,7 +17999,11 @@ async function handleInsert$3({ resource, data, mappedData, originalData }) {
|
|
|
17797
17999
|
excess: totalSize - 2047,
|
|
17798
18000
|
data: originalData || data
|
|
17799
18001
|
});
|
|
17800
|
-
|
|
18002
|
+
const metadataOnly = { _v: mappedData._v };
|
|
18003
|
+
if (resource.schema?.pluginMap && Object.keys(resource.schema.pluginMap).length > 0) {
|
|
18004
|
+
metadataOnly._pluginMap = JSON.stringify(resource.schema.pluginMap);
|
|
18005
|
+
}
|
|
18006
|
+
return { mappedData: metadataOnly, body: JSON.stringify(mappedData) };
|
|
17801
18007
|
}
|
|
17802
18008
|
return { mappedData, body: "" };
|
|
17803
18009
|
}
|
|
@@ -18000,6 +18206,12 @@ async function handleInsert$1({ resource, data, mappedData, originalData }) {
|
|
|
18000
18206
|
metadataFields._v = mappedData._v;
|
|
18001
18207
|
currentSize += attributeSizes._v;
|
|
18002
18208
|
}
|
|
18209
|
+
if (resource.schema?.pluginMap && Object.keys(resource.schema.pluginMap).length > 0) {
|
|
18210
|
+
const pluginMapStr = JSON.stringify(resource.schema.pluginMap);
|
|
18211
|
+
const pluginMapSize = calculateUTF8Bytes("_pluginMap") + calculateUTF8Bytes(pluginMapStr);
|
|
18212
|
+
metadataFields._pluginMap = pluginMapStr;
|
|
18213
|
+
currentSize += pluginMapSize;
|
|
18214
|
+
}
|
|
18003
18215
|
let reservedLimit = effectiveLimit;
|
|
18004
18216
|
for (const [fieldName, size] of sortedFields) {
|
|
18005
18217
|
if (fieldName === "_v") continue;
|
|
@@ -18059,6 +18271,9 @@ async function handleInsert({ resource, data, mappedData }) {
|
|
|
18059
18271
|
"_v": mappedData._v || String(resource.version)
|
|
18060
18272
|
};
|
|
18061
18273
|
metadataOnly._map = JSON.stringify(resource.schema.map);
|
|
18274
|
+
if (resource.schema.pluginMap && Object.keys(resource.schema.pluginMap).length > 0) {
|
|
18275
|
+
metadataOnly._pluginMap = JSON.stringify(resource.schema.pluginMap);
|
|
18276
|
+
}
|
|
18062
18277
|
const body = JSON.stringify(mappedData);
|
|
18063
18278
|
return { mappedData: metadataOnly, body };
|
|
18064
18279
|
}
|
|
@@ -18067,6 +18282,9 @@ async function handleUpdate({ resource, id, data, mappedData }) {
|
|
|
18067
18282
|
"_v": mappedData._v || String(resource.version)
|
|
18068
18283
|
};
|
|
18069
18284
|
metadataOnly._map = JSON.stringify(resource.schema.map);
|
|
18285
|
+
if (resource.schema.pluginMap && Object.keys(resource.schema.pluginMap).length > 0) {
|
|
18286
|
+
metadataOnly._pluginMap = JSON.stringify(resource.schema.pluginMap);
|
|
18287
|
+
}
|
|
18070
18288
|
const body = JSON.stringify(mappedData);
|
|
18071
18289
|
return { mappedData: metadataOnly, body };
|
|
18072
18290
|
}
|
|
@@ -18447,6 +18665,118 @@ ${errorDetails}`,
|
|
|
18447
18665
|
this.applyConfiguration();
|
|
18448
18666
|
return { oldAttributes, newAttributes };
|
|
18449
18667
|
}
|
|
18668
|
+
/**
|
|
18669
|
+
* Add a plugin-created attribute to the resource schema
|
|
18670
|
+
* This ensures plugin attributes don't interfere with user-defined attributes
|
|
18671
|
+
* by using a separate mapping namespace (p0, p1, p2, ...)
|
|
18672
|
+
*
|
|
18673
|
+
* @param {string} name - Attribute name (e.g., '_hasEmbedding', 'clusterId')
|
|
18674
|
+
* @param {Object|string} definition - Attribute definition
|
|
18675
|
+
* @param {string} pluginName - Name of plugin adding the attribute
|
|
18676
|
+
* @returns {void}
|
|
18677
|
+
*
|
|
18678
|
+
* @example
|
|
18679
|
+
* // VectorPlugin adding tracking field
|
|
18680
|
+
* resource.addPluginAttribute('_hasEmbedding', {
|
|
18681
|
+
* type: 'boolean',
|
|
18682
|
+
* optional: true,
|
|
18683
|
+
* default: false
|
|
18684
|
+
* }, 'VectorPlugin');
|
|
18685
|
+
*
|
|
18686
|
+
* // Shorthand notation
|
|
18687
|
+
* resource.addPluginAttribute('clusterId', 'string|optional', 'VectorPlugin');
|
|
18688
|
+
*/
|
|
18689
|
+
addPluginAttribute(name, definition, pluginName) {
|
|
18690
|
+
if (!pluginName) {
|
|
18691
|
+
throw new ResourceError(
|
|
18692
|
+
"Plugin name is required when adding plugin attributes",
|
|
18693
|
+
{ resource: this.name, attribute: name }
|
|
18694
|
+
);
|
|
18695
|
+
}
|
|
18696
|
+
const existingDef = this.schema.getAttributeDefinition(name);
|
|
18697
|
+
if (existingDef && (!existingDef.__plugin__ || existingDef.__plugin__ !== pluginName)) {
|
|
18698
|
+
throw new ResourceError(
|
|
18699
|
+
`Attribute '${name}' already exists and is not from plugin '${pluginName}'`,
|
|
18700
|
+
{ resource: this.name, attribute: name, plugin: pluginName }
|
|
18701
|
+
);
|
|
18702
|
+
}
|
|
18703
|
+
let defObject = definition;
|
|
18704
|
+
if (typeof definition === "object" && definition !== null) {
|
|
18705
|
+
defObject = { ...definition };
|
|
18706
|
+
}
|
|
18707
|
+
if (typeof defObject === "object" && defObject !== null) {
|
|
18708
|
+
defObject.__plugin__ = pluginName;
|
|
18709
|
+
defObject.__pluginCreated__ = Date.now();
|
|
18710
|
+
}
|
|
18711
|
+
this.schema.attributes[name] = defObject;
|
|
18712
|
+
this.attributes[name] = defObject;
|
|
18713
|
+
if (typeof defObject === "string") {
|
|
18714
|
+
if (!this.schema._pluginAttributeMetadata) {
|
|
18715
|
+
this.schema._pluginAttributeMetadata = {};
|
|
18716
|
+
}
|
|
18717
|
+
this.schema._pluginAttributeMetadata[name] = {
|
|
18718
|
+
__plugin__: pluginName,
|
|
18719
|
+
__pluginCreated__: Date.now()
|
|
18720
|
+
};
|
|
18721
|
+
}
|
|
18722
|
+
this.schema.regeneratePluginMapping();
|
|
18723
|
+
if (this.schema.options.generateAutoHooks) {
|
|
18724
|
+
this.schema.generateAutoHooks();
|
|
18725
|
+
}
|
|
18726
|
+
const processedAttributes = this.schema.preprocessAttributesForValidation(this.schema.attributes);
|
|
18727
|
+
this.schema.validator = new ValidatorManager({ autoEncrypt: false }).compile(lodashEs.merge(
|
|
18728
|
+
{ $$async: true, $$strict: false },
|
|
18729
|
+
processedAttributes
|
|
18730
|
+
));
|
|
18731
|
+
if (this.database) {
|
|
18732
|
+
this.database.emit("plugin-attribute-added", {
|
|
18733
|
+
resource: this.name,
|
|
18734
|
+
attribute: name,
|
|
18735
|
+
plugin: pluginName,
|
|
18736
|
+
definition: defObject
|
|
18737
|
+
});
|
|
18738
|
+
}
|
|
18739
|
+
}
|
|
18740
|
+
/**
|
|
18741
|
+
* Remove a plugin-created attribute from the resource schema
|
|
18742
|
+
* Called when a plugin is uninstalled or no longer needs the attribute
|
|
18743
|
+
*
|
|
18744
|
+
* @param {string} name - Attribute name to remove
|
|
18745
|
+
* @param {string} [pluginName] - Optional plugin name for safety check
|
|
18746
|
+
* @returns {boolean} True if attribute was removed, false if not found
|
|
18747
|
+
*
|
|
18748
|
+
* @example
|
|
18749
|
+
* resource.removePluginAttribute('_hasEmbedding', 'VectorPlugin');
|
|
18750
|
+
*/
|
|
18751
|
+
removePluginAttribute(name, pluginName = null) {
|
|
18752
|
+
const attrDef = this.schema.getAttributeDefinition(name);
|
|
18753
|
+
const metadata = this.schema._pluginAttributeMetadata?.[name];
|
|
18754
|
+
const isPluginAttr = typeof attrDef === "object" && attrDef?.__plugin__ || metadata;
|
|
18755
|
+
if (!attrDef || !isPluginAttr) {
|
|
18756
|
+
return false;
|
|
18757
|
+
}
|
|
18758
|
+
const actualPlugin = attrDef?.__plugin__ || metadata?.__plugin__;
|
|
18759
|
+
if (pluginName && actualPlugin !== pluginName) {
|
|
18760
|
+
throw new ResourceError(
|
|
18761
|
+
`Attribute '${name}' belongs to plugin '${actualPlugin}', not '${pluginName}'`,
|
|
18762
|
+
{ resource: this.name, attribute: name, actualPlugin, requestedPlugin: pluginName }
|
|
18763
|
+
);
|
|
18764
|
+
}
|
|
18765
|
+
delete this.schema.attributes[name];
|
|
18766
|
+
delete this.attributes[name];
|
|
18767
|
+
if (this.schema._pluginAttributeMetadata?.[name]) {
|
|
18768
|
+
delete this.schema._pluginAttributeMetadata[name];
|
|
18769
|
+
}
|
|
18770
|
+
this.schema.regeneratePluginMapping();
|
|
18771
|
+
if (this.database) {
|
|
18772
|
+
this.database.emit("plugin-attribute-removed", {
|
|
18773
|
+
resource: this.name,
|
|
18774
|
+
attribute: name,
|
|
18775
|
+
plugin: actualPlugin
|
|
18776
|
+
});
|
|
18777
|
+
}
|
|
18778
|
+
return true;
|
|
18779
|
+
}
|
|
18450
18780
|
/**
|
|
18451
18781
|
* Add a hook function for a specific event
|
|
18452
18782
|
* @param {string} event - Hook event (beforeInsert, afterInsert, etc.)
|
|
@@ -20748,8 +21078,9 @@ ${errorDetails}`,
|
|
|
20748
21078
|
const filterInternalFields = (obj) => {
|
|
20749
21079
|
if (!obj || typeof obj !== "object") return obj;
|
|
20750
21080
|
const filtered2 = {};
|
|
21081
|
+
const pluginAttrNames = this.schema._pluginAttributes ? Object.values(this.schema._pluginAttributes).flat() : [];
|
|
20751
21082
|
for (const [key, value] of Object.entries(obj)) {
|
|
20752
|
-
if (!key.startsWith("_") || key === "_geohash" || key.startsWith("_geohash_zoom")) {
|
|
21083
|
+
if (!key.startsWith("_") || key === "_geohash" || key.startsWith("_geohash_zoom") || pluginAttrNames.includes(key)) {
|
|
20753
21084
|
filtered2[key] = value;
|
|
20754
21085
|
}
|
|
20755
21086
|
}
|
|
@@ -20775,7 +21106,16 @@ ${errorDetails}`,
|
|
|
20775
21106
|
if (hasOverflow && body) {
|
|
20776
21107
|
const [okBody, errBody, parsedBody] = await tryFn(() => Promise.resolve(JSON.parse(body)));
|
|
20777
21108
|
if (okBody) {
|
|
20778
|
-
|
|
21109
|
+
let pluginMapFromMeta = null;
|
|
21110
|
+
if (metadata && metadata._pluginmap) {
|
|
21111
|
+
const [okPluginMap, errPluginMap, parsedPluginMap] = await tryFn(
|
|
21112
|
+
() => Promise.resolve(typeof metadata._pluginmap === "string" ? JSON.parse(metadata._pluginmap) : metadata._pluginmap)
|
|
21113
|
+
);
|
|
21114
|
+
pluginMapFromMeta = okPluginMap ? parsedPluginMap : null;
|
|
21115
|
+
}
|
|
21116
|
+
const [okUnmap, errUnmap, unmappedBody] = await tryFn(
|
|
21117
|
+
() => this.schema.unmapper(parsedBody, void 0, pluginMapFromMeta)
|
|
21118
|
+
);
|
|
20779
21119
|
bodyData = okUnmap ? unmappedBody : {};
|
|
20780
21120
|
}
|
|
20781
21121
|
}
|
|
@@ -20792,11 +21132,16 @@ ${errorDetails}`,
|
|
|
20792
21132
|
if (behavior === "body-only") {
|
|
20793
21133
|
const [okBody, errBody, parsedBody] = await tryFn(() => Promise.resolve(body ? JSON.parse(body) : {}));
|
|
20794
21134
|
let mapFromMeta = this.schema.map;
|
|
21135
|
+
let pluginMapFromMeta = null;
|
|
20795
21136
|
if (metadata && metadata._map) {
|
|
20796
21137
|
const [okMap, errMap, parsedMap] = await tryFn(() => Promise.resolve(typeof metadata._map === "string" ? JSON.parse(metadata._map) : metadata._map));
|
|
20797
21138
|
mapFromMeta = okMap ? parsedMap : this.schema.map;
|
|
20798
21139
|
}
|
|
20799
|
-
|
|
21140
|
+
if (metadata && metadata._pluginmap) {
|
|
21141
|
+
const [okPluginMap, errPluginMap, parsedPluginMap] = await tryFn(() => Promise.resolve(typeof metadata._pluginmap === "string" ? JSON.parse(metadata._pluginmap) : metadata._pluginmap));
|
|
21142
|
+
pluginMapFromMeta = okPluginMap ? parsedPluginMap : null;
|
|
21143
|
+
}
|
|
21144
|
+
const [okUnmap, errUnmap, unmappedBody] = await tryFn(() => this.schema.unmapper(parsedBody, mapFromMeta, pluginMapFromMeta));
|
|
20800
21145
|
const result2 = okUnmap ? { ...unmappedBody, id } : { id };
|
|
20801
21146
|
Object.keys(result2).forEach((k) => {
|
|
20802
21147
|
result2[k] = fixValue(result2[k]);
|
|
@@ -20806,7 +21151,16 @@ ${errorDetails}`,
|
|
|
20806
21151
|
if (behavior === "user-managed" && body && body.trim() !== "") {
|
|
20807
21152
|
const [okBody, errBody, parsedBody] = await tryFn(() => Promise.resolve(JSON.parse(body)));
|
|
20808
21153
|
if (okBody) {
|
|
20809
|
-
|
|
21154
|
+
let pluginMapFromMeta = null;
|
|
21155
|
+
if (metadata && metadata._pluginmap) {
|
|
21156
|
+
const [okPluginMap, errPluginMap, parsedPluginMap] = await tryFn(
|
|
21157
|
+
() => Promise.resolve(typeof metadata._pluginmap === "string" ? JSON.parse(metadata._pluginmap) : metadata._pluginmap)
|
|
21158
|
+
);
|
|
21159
|
+
pluginMapFromMeta = okPluginMap ? parsedPluginMap : null;
|
|
21160
|
+
}
|
|
21161
|
+
const [okUnmap, errUnmap, unmappedBody] = await tryFn(
|
|
21162
|
+
() => this.schema.unmapper(parsedBody, void 0, pluginMapFromMeta)
|
|
21163
|
+
);
|
|
20810
21164
|
const bodyData = okUnmap ? unmappedBody : {};
|
|
20811
21165
|
const merged = { ...bodyData, ...unmappedMetadata, id };
|
|
20812
21166
|
Object.keys(merged).forEach((k) => {
|
|
@@ -21054,7 +21408,7 @@ class Database extends EventEmitter {
|
|
|
21054
21408
|
this.id = idGenerator(7);
|
|
21055
21409
|
this.version = "1";
|
|
21056
21410
|
this.s3dbVersion = (() => {
|
|
21057
|
-
const [ok, err, version] = tryFn(() => true ? "12.
|
|
21411
|
+
const [ok, err, version] = tryFn(() => true ? "12.4.0" : "latest");
|
|
21058
21412
|
return ok ? version : "latest";
|
|
21059
21413
|
})();
|
|
21060
21414
|
this._resourcesMap = {};
|
|
@@ -21110,7 +21464,7 @@ class Database extends EventEmitter {
|
|
|
21110
21464
|
connectionString = `s3://${encodeURIComponent(accessKeyId)}:${encodeURIComponent(secretAccessKey)}@${bucket || "s3db"}?${params.toString()}`;
|
|
21111
21465
|
}
|
|
21112
21466
|
}
|
|
21113
|
-
this.client = options.client || new
|
|
21467
|
+
this.client = options.client || new S3Client({
|
|
21114
21468
|
verbose: this.verbose,
|
|
21115
21469
|
parallelism: this.parallelism,
|
|
21116
21470
|
connectionString
|
|
@@ -23044,7 +23398,11 @@ class TursoReplicator extends BaseReplicator {
|
|
|
23044
23398
|
}
|
|
23045
23399
|
continue;
|
|
23046
23400
|
}
|
|
23047
|
-
const
|
|
23401
|
+
const allAttributes = resource.config.versions[resource.config.currentVersion]?.attributes || {};
|
|
23402
|
+
const pluginAttrNames = resource.schema?._pluginAttributes ? Object.values(resource.schema._pluginAttributes).flat() : [];
|
|
23403
|
+
const attributes = Object.fromEntries(
|
|
23404
|
+
Object.entries(allAttributes).filter(([name]) => !pluginAttrNames.includes(name))
|
|
23405
|
+
);
|
|
23048
23406
|
for (const tableConfig of tableConfigs) {
|
|
23049
23407
|
const tableName = tableConfig.table;
|
|
23050
23408
|
const [okSync, errSync] = await tryFn(async () => {
|
|
@@ -26012,7 +26370,7 @@ class S3TfStateDriver extends TfStateDriver {
|
|
|
26012
26370
|
*/
|
|
26013
26371
|
async initialize() {
|
|
26014
26372
|
const { bucket, credentials, region } = this.connectionConfig;
|
|
26015
|
-
this.client = new
|
|
26373
|
+
this.client = new S3Client({
|
|
26016
26374
|
bucketName: bucket,
|
|
26017
26375
|
credentials,
|
|
26018
26376
|
region
|
|
@@ -36953,11 +37311,11 @@ class VectorPlugin extends Plugin {
|
|
|
36953
37311
|
}
|
|
36954
37312
|
};
|
|
36955
37313
|
if (!resource.schema.attributes[trackingFieldName]) {
|
|
36956
|
-
resource.
|
|
37314
|
+
resource.addPluginAttribute(trackingFieldName, {
|
|
36957
37315
|
type: "boolean",
|
|
36958
37316
|
optional: true,
|
|
36959
37317
|
default: false
|
|
36960
|
-
};
|
|
37318
|
+
}, "VectorPlugin");
|
|
36961
37319
|
}
|
|
36962
37320
|
this.emit("vector:partition-created", {
|
|
36963
37321
|
resource: resource.name,
|
|
@@ -37638,114 +37996,1182 @@ class VectorPlugin extends Plugin {
|
|
|
37638
37996
|
}
|
|
37639
37997
|
}
|
|
37640
37998
|
|
|
37641
|
-
|
|
37642
|
-
|
|
37643
|
-
|
|
37644
|
-
|
|
37645
|
-
|
|
37646
|
-
|
|
37647
|
-
|
|
37648
|
-
|
|
37649
|
-
|
|
37650
|
-
|
|
37651
|
-
"secret": "string",
|
|
37652
|
-
"email": "string",
|
|
37653
|
-
"url": "string",
|
|
37654
|
-
"date": "string",
|
|
37655
|
-
// ISO date string
|
|
37656
|
-
"datetime": "string",
|
|
37657
|
-
// ISO datetime string
|
|
37658
|
-
"ip4": "string",
|
|
37659
|
-
"ip6": "string"
|
|
37660
|
-
};
|
|
37661
|
-
if (baseType.startsWith("embedding:")) {
|
|
37662
|
-
const dimensions = parseInt(baseType.split(":")[1]);
|
|
37663
|
-
return `number[] /* ${dimensions} dimensions */`;
|
|
37999
|
+
class MemoryStorage {
|
|
38000
|
+
constructor(config = {}) {
|
|
38001
|
+
this.objects = /* @__PURE__ */ new Map();
|
|
38002
|
+
this.bucket = config.bucket || "s3db";
|
|
38003
|
+
this.enforceLimits = config.enforceLimits || false;
|
|
38004
|
+
this.metadataLimit = config.metadataLimit || 2048;
|
|
38005
|
+
this.maxObjectSize = config.maxObjectSize || 5 * 1024 * 1024 * 1024;
|
|
38006
|
+
this.persistPath = config.persistPath;
|
|
38007
|
+
this.autoPersist = config.autoPersist || false;
|
|
38008
|
+
this.verbose = config.verbose || false;
|
|
37664
38009
|
}
|
|
37665
|
-
|
|
37666
|
-
|
|
37667
|
-
|
|
37668
|
-
|
|
37669
|
-
|
|
38010
|
+
/**
|
|
38011
|
+
* Generate ETag (MD5 hash) for object body
|
|
38012
|
+
*/
|
|
38013
|
+
_generateETag(body) {
|
|
38014
|
+
const buffer = Buffer.isBuffer(body) ? body : Buffer.from(body || "");
|
|
38015
|
+
return crypto$1.createHash("md5").update(buffer).digest("hex");
|
|
37670
38016
|
}
|
|
37671
|
-
|
|
37672
|
-
|
|
38017
|
+
/**
|
|
38018
|
+
* Calculate metadata size in bytes
|
|
38019
|
+
*/
|
|
38020
|
+
_calculateMetadataSize(metadata) {
|
|
38021
|
+
if (!metadata) return 0;
|
|
38022
|
+
let size = 0;
|
|
38023
|
+
for (const [key, value] of Object.entries(metadata)) {
|
|
38024
|
+
size += Buffer.byteLength(key, "utf8");
|
|
38025
|
+
size += Buffer.byteLength(String(value), "utf8");
|
|
38026
|
+
}
|
|
38027
|
+
return size;
|
|
37673
38028
|
}
|
|
37674
|
-
|
|
37675
|
-
|
|
37676
|
-
|
|
37677
|
-
|
|
37678
|
-
|
|
37679
|
-
|
|
37680
|
-
|
|
37681
|
-
|
|
37682
|
-
|
|
37683
|
-
|
|
37684
|
-
|
|
37685
|
-
const
|
|
37686
|
-
|
|
37687
|
-
|
|
37688
|
-
|
|
37689
|
-
|
|
37690
|
-
|
|
37691
|
-
|
|
37692
|
-
|
|
37693
|
-
|
|
37694
|
-
|
|
37695
|
-
|
|
37696
|
-
|
|
37697
|
-
|
|
37698
|
-
|
|
37699
|
-
|
|
37700
|
-
|
|
37701
|
-
if (fieldDef.type === "array" && fieldDef.items) {
|
|
37702
|
-
const itemType = mapFieldTypeToTypeScript(fieldDef.items);
|
|
37703
|
-
tsType = `Array<${itemType}>`;
|
|
38029
|
+
/**
|
|
38030
|
+
* Validate limits if enforceLimits is enabled
|
|
38031
|
+
*/
|
|
38032
|
+
_validateLimits(body, metadata) {
|
|
38033
|
+
if (!this.enforceLimits) return;
|
|
38034
|
+
const metadataSize = this._calculateMetadataSize(metadata);
|
|
38035
|
+
if (metadataSize > this.metadataLimit) {
|
|
38036
|
+
throw new Error(
|
|
38037
|
+
`Metadata size (${metadataSize} bytes) exceeds limit of ${this.metadataLimit} bytes`
|
|
38038
|
+
);
|
|
38039
|
+
}
|
|
38040
|
+
const bodySize = Buffer.isBuffer(body) ? body.length : Buffer.byteLength(body || "", "utf8");
|
|
38041
|
+
if (bodySize > this.maxObjectSize) {
|
|
38042
|
+
throw new Error(
|
|
38043
|
+
`Object size (${bodySize} bytes) exceeds limit of ${this.maxObjectSize} bytes`
|
|
38044
|
+
);
|
|
38045
|
+
}
|
|
38046
|
+
}
|
|
38047
|
+
/**
|
|
38048
|
+
* Store an object
|
|
38049
|
+
*/
|
|
38050
|
+
async put(key, { body, metadata, contentType, contentEncoding, contentLength, ifMatch }) {
|
|
38051
|
+
this._validateLimits(body, metadata);
|
|
38052
|
+
if (ifMatch !== void 0) {
|
|
38053
|
+
const existing = this.objects.get(key);
|
|
38054
|
+
if (existing && existing.etag !== ifMatch) {
|
|
38055
|
+
throw new Error(`Precondition failed: ETag mismatch for key "${key}"`);
|
|
37704
38056
|
}
|
|
37705
|
-
} else {
|
|
37706
|
-
tsType = "any";
|
|
37707
38057
|
}
|
|
37708
|
-
|
|
37709
|
-
|
|
38058
|
+
const buffer = Buffer.isBuffer(body) ? body : Buffer.from(body || "");
|
|
38059
|
+
const etag = this._generateETag(buffer);
|
|
38060
|
+
const lastModified = (/* @__PURE__ */ new Date()).toISOString();
|
|
38061
|
+
const size = buffer.length;
|
|
38062
|
+
const objectData = {
|
|
38063
|
+
body: buffer,
|
|
38064
|
+
metadata: metadata || {},
|
|
38065
|
+
contentType: contentType || "application/octet-stream",
|
|
38066
|
+
etag,
|
|
38067
|
+
lastModified,
|
|
38068
|
+
size,
|
|
38069
|
+
contentEncoding,
|
|
38070
|
+
contentLength: contentLength || size
|
|
38071
|
+
};
|
|
38072
|
+
this.objects.set(key, objectData);
|
|
38073
|
+
if (this.verbose) {
|
|
38074
|
+
console.log(`[MemoryStorage] PUT ${key} (${size} bytes, etag: ${etag})`);
|
|
37710
38075
|
}
|
|
37711
|
-
|
|
38076
|
+
if (this.autoPersist && this.persistPath) {
|
|
38077
|
+
await this.saveToDisk();
|
|
38078
|
+
}
|
|
38079
|
+
return {
|
|
38080
|
+
ETag: etag,
|
|
38081
|
+
VersionId: null,
|
|
38082
|
+
// Memory storage doesn't support versioning
|
|
38083
|
+
ServerSideEncryption: null,
|
|
38084
|
+
Location: `/${this.bucket}/${key}`
|
|
38085
|
+
};
|
|
37712
38086
|
}
|
|
37713
|
-
|
|
37714
|
-
|
|
37715
|
-
|
|
37716
|
-
|
|
37717
|
-
|
|
37718
|
-
|
|
38087
|
+
/**
|
|
38088
|
+
* Retrieve an object
|
|
38089
|
+
*/
|
|
38090
|
+
async get(key) {
|
|
38091
|
+
const obj = this.objects.get(key);
|
|
38092
|
+
if (!obj) {
|
|
38093
|
+
const error = new Error(`Object not found: ${key}`);
|
|
38094
|
+
error.name = "NoSuchKey";
|
|
38095
|
+
error.$metadata = {
|
|
38096
|
+
httpStatusCode: 404,
|
|
38097
|
+
requestId: "memory-" + Date.now(),
|
|
38098
|
+
attempts: 1,
|
|
38099
|
+
totalRetryDelay: 0
|
|
38100
|
+
};
|
|
38101
|
+
throw error;
|
|
38102
|
+
}
|
|
38103
|
+
if (this.verbose) {
|
|
38104
|
+
console.log(`[MemoryStorage] GET ${key} (${obj.size} bytes)`);
|
|
38105
|
+
}
|
|
38106
|
+
const bodyStream = stream$1.Readable.from(obj.body);
|
|
38107
|
+
return {
|
|
38108
|
+
Body: bodyStream,
|
|
38109
|
+
Metadata: { ...obj.metadata },
|
|
38110
|
+
ContentType: obj.contentType,
|
|
38111
|
+
ContentLength: obj.size,
|
|
38112
|
+
ETag: obj.etag,
|
|
38113
|
+
LastModified: new Date(obj.lastModified),
|
|
38114
|
+
ContentEncoding: obj.contentEncoding
|
|
38115
|
+
};
|
|
37719
38116
|
}
|
|
37720
|
-
|
|
37721
|
-
|
|
37722
|
-
|
|
37723
|
-
|
|
37724
|
-
|
|
37725
|
-
|
|
37726
|
-
}
|
|
37727
|
-
|
|
37728
|
-
|
|
37729
|
-
|
|
37730
|
-
|
|
37731
|
-
|
|
37732
|
-
|
|
37733
|
-
|
|
37734
|
-
|
|
37735
|
-
|
|
37736
|
-
|
|
37737
|
-
|
|
37738
|
-
|
|
37739
|
-
|
|
37740
|
-
|
|
37741
|
-
|
|
37742
|
-
|
|
38117
|
+
/**
|
|
38118
|
+
* Get object metadata only (like S3 HeadObject)
|
|
38119
|
+
*/
|
|
38120
|
+
async head(key) {
|
|
38121
|
+
const obj = this.objects.get(key);
|
|
38122
|
+
if (!obj) {
|
|
38123
|
+
const error = new Error(`Object not found: ${key}`);
|
|
38124
|
+
error.name = "NoSuchKey";
|
|
38125
|
+
error.$metadata = {
|
|
38126
|
+
httpStatusCode: 404,
|
|
38127
|
+
requestId: "memory-" + Date.now(),
|
|
38128
|
+
attempts: 1,
|
|
38129
|
+
totalRetryDelay: 0
|
|
38130
|
+
};
|
|
38131
|
+
throw error;
|
|
38132
|
+
}
|
|
38133
|
+
if (this.verbose) {
|
|
38134
|
+
console.log(`[MemoryStorage] HEAD ${key}`);
|
|
38135
|
+
}
|
|
38136
|
+
return {
|
|
38137
|
+
Metadata: { ...obj.metadata },
|
|
38138
|
+
ContentType: obj.contentType,
|
|
38139
|
+
ContentLength: obj.size,
|
|
38140
|
+
ETag: obj.etag,
|
|
38141
|
+
LastModified: new Date(obj.lastModified),
|
|
38142
|
+
ContentEncoding: obj.contentEncoding
|
|
38143
|
+
};
|
|
37743
38144
|
}
|
|
37744
|
-
|
|
37745
|
-
|
|
37746
|
-
|
|
37747
|
-
|
|
37748
|
-
const
|
|
38145
|
+
/**
|
|
38146
|
+
* Copy an object
|
|
38147
|
+
*/
|
|
38148
|
+
async copy(from, to, { metadata, metadataDirective, contentType }) {
|
|
38149
|
+
const source = this.objects.get(from);
|
|
38150
|
+
if (!source) {
|
|
38151
|
+
const error = new Error(`Source object not found: ${from}`);
|
|
38152
|
+
error.name = "NoSuchKey";
|
|
38153
|
+
throw error;
|
|
38154
|
+
}
|
|
38155
|
+
let finalMetadata = { ...source.metadata };
|
|
38156
|
+
if (metadataDirective === "REPLACE" && metadata) {
|
|
38157
|
+
finalMetadata = metadata;
|
|
38158
|
+
} else if (metadata) {
|
|
38159
|
+
finalMetadata = { ...finalMetadata, ...metadata };
|
|
38160
|
+
}
|
|
38161
|
+
const result = await this.put(to, {
|
|
38162
|
+
body: source.body,
|
|
38163
|
+
metadata: finalMetadata,
|
|
38164
|
+
contentType: contentType || source.contentType,
|
|
38165
|
+
contentEncoding: source.contentEncoding
|
|
38166
|
+
});
|
|
38167
|
+
if (this.verbose) {
|
|
38168
|
+
console.log(`[MemoryStorage] COPY ${from} \u2192 ${to}`);
|
|
38169
|
+
}
|
|
38170
|
+
return result;
|
|
38171
|
+
}
|
|
38172
|
+
/**
|
|
38173
|
+
* Check if object exists
|
|
38174
|
+
*/
|
|
38175
|
+
exists(key) {
|
|
38176
|
+
return this.objects.has(key);
|
|
38177
|
+
}
|
|
38178
|
+
/**
|
|
38179
|
+
* Delete an object
|
|
38180
|
+
*/
|
|
38181
|
+
async delete(key) {
|
|
38182
|
+
const existed = this.objects.has(key);
|
|
38183
|
+
this.objects.delete(key);
|
|
38184
|
+
if (this.verbose) {
|
|
38185
|
+
console.log(`[MemoryStorage] DELETE ${key} (existed: ${existed})`);
|
|
38186
|
+
}
|
|
38187
|
+
if (this.autoPersist && this.persistPath) {
|
|
38188
|
+
await this.saveToDisk();
|
|
38189
|
+
}
|
|
38190
|
+
return {
|
|
38191
|
+
DeleteMarker: false,
|
|
38192
|
+
VersionId: null
|
|
38193
|
+
};
|
|
38194
|
+
}
|
|
38195
|
+
/**
|
|
38196
|
+
* Delete multiple objects (batch)
|
|
38197
|
+
*/
|
|
38198
|
+
async deleteMultiple(keys) {
|
|
38199
|
+
const deleted = [];
|
|
38200
|
+
const errors = [];
|
|
38201
|
+
for (const key of keys) {
|
|
38202
|
+
try {
|
|
38203
|
+
await this.delete(key);
|
|
38204
|
+
deleted.push({ Key: key });
|
|
38205
|
+
} catch (error) {
|
|
38206
|
+
errors.push({
|
|
38207
|
+
Key: key,
|
|
38208
|
+
Code: error.name || "InternalError",
|
|
38209
|
+
Message: error.message
|
|
38210
|
+
});
|
|
38211
|
+
}
|
|
38212
|
+
}
|
|
38213
|
+
if (this.verbose) {
|
|
38214
|
+
console.log(`[MemoryStorage] DELETE BATCH (${deleted.length} deleted, ${errors.length} errors)`);
|
|
38215
|
+
}
|
|
38216
|
+
return { Deleted: deleted, Errors: errors };
|
|
38217
|
+
}
|
|
38218
|
+
/**
|
|
38219
|
+
* List objects with prefix/delimiter support
|
|
38220
|
+
*/
|
|
38221
|
+
async list({ prefix = "", delimiter = null, maxKeys = 1e3, continuationToken = null }) {
|
|
38222
|
+
const allKeys = Array.from(this.objects.keys());
|
|
38223
|
+
let filteredKeys = prefix ? allKeys.filter((key) => key.startsWith(prefix)) : allKeys;
|
|
38224
|
+
filteredKeys.sort();
|
|
38225
|
+
let startIndex = 0;
|
|
38226
|
+
if (continuationToken) {
|
|
38227
|
+
startIndex = parseInt(continuationToken) || 0;
|
|
38228
|
+
}
|
|
38229
|
+
const paginatedKeys = filteredKeys.slice(startIndex, startIndex + maxKeys);
|
|
38230
|
+
const isTruncated = startIndex + maxKeys < filteredKeys.length;
|
|
38231
|
+
const nextContinuationToken = isTruncated ? String(startIndex + maxKeys) : null;
|
|
38232
|
+
const commonPrefixes = /* @__PURE__ */ new Set();
|
|
38233
|
+
const contents = [];
|
|
38234
|
+
for (const key of paginatedKeys) {
|
|
38235
|
+
if (delimiter && prefix) {
|
|
38236
|
+
const suffix = key.substring(prefix.length);
|
|
38237
|
+
const delimiterIndex = suffix.indexOf(delimiter);
|
|
38238
|
+
if (delimiterIndex !== -1) {
|
|
38239
|
+
const commonPrefix = prefix + suffix.substring(0, delimiterIndex + 1);
|
|
38240
|
+
commonPrefixes.add(commonPrefix);
|
|
38241
|
+
continue;
|
|
38242
|
+
}
|
|
38243
|
+
}
|
|
38244
|
+
const obj = this.objects.get(key);
|
|
38245
|
+
contents.push({
|
|
38246
|
+
Key: key,
|
|
38247
|
+
Size: obj.size,
|
|
38248
|
+
LastModified: new Date(obj.lastModified),
|
|
38249
|
+
ETag: obj.etag,
|
|
38250
|
+
StorageClass: "STANDARD"
|
|
38251
|
+
});
|
|
38252
|
+
}
|
|
38253
|
+
if (this.verbose) {
|
|
38254
|
+
console.log(`[MemoryStorage] LIST prefix="${prefix}" (${contents.length} objects, ${commonPrefixes.size} prefixes)`);
|
|
38255
|
+
}
|
|
38256
|
+
return {
|
|
38257
|
+
Contents: contents,
|
|
38258
|
+
CommonPrefixes: Array.from(commonPrefixes).map((prefix2) => ({ Prefix: prefix2 })),
|
|
38259
|
+
IsTruncated: isTruncated,
|
|
38260
|
+
NextContinuationToken: nextContinuationToken,
|
|
38261
|
+
KeyCount: contents.length + commonPrefixes.size,
|
|
38262
|
+
MaxKeys: maxKeys,
|
|
38263
|
+
Prefix: prefix,
|
|
38264
|
+
Delimiter: delimiter
|
|
38265
|
+
};
|
|
38266
|
+
}
|
|
38267
|
+
/**
|
|
38268
|
+
* Create a snapshot of current state
|
|
38269
|
+
*/
|
|
38270
|
+
snapshot() {
|
|
38271
|
+
const snapshot = {
|
|
38272
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
38273
|
+
bucket: this.bucket,
|
|
38274
|
+
objectCount: this.objects.size,
|
|
38275
|
+
objects: {}
|
|
38276
|
+
};
|
|
38277
|
+
for (const [key, obj] of this.objects.entries()) {
|
|
38278
|
+
snapshot.objects[key] = {
|
|
38279
|
+
body: obj.body.toString("base64"),
|
|
38280
|
+
metadata: obj.metadata,
|
|
38281
|
+
contentType: obj.contentType,
|
|
38282
|
+
etag: obj.etag,
|
|
38283
|
+
lastModified: obj.lastModified,
|
|
38284
|
+
size: obj.size,
|
|
38285
|
+
contentEncoding: obj.contentEncoding,
|
|
38286
|
+
contentLength: obj.contentLength
|
|
38287
|
+
};
|
|
38288
|
+
}
|
|
38289
|
+
return snapshot;
|
|
38290
|
+
}
|
|
38291
|
+
/**
|
|
38292
|
+
* Restore from a snapshot
|
|
38293
|
+
*/
|
|
38294
|
+
restore(snapshot) {
|
|
38295
|
+
if (!snapshot || !snapshot.objects) {
|
|
38296
|
+
throw new Error("Invalid snapshot format");
|
|
38297
|
+
}
|
|
38298
|
+
this.objects.clear();
|
|
38299
|
+
for (const [key, obj] of Object.entries(snapshot.objects)) {
|
|
38300
|
+
this.objects.set(key, {
|
|
38301
|
+
body: Buffer.from(obj.body, "base64"),
|
|
38302
|
+
metadata: obj.metadata,
|
|
38303
|
+
contentType: obj.contentType,
|
|
38304
|
+
etag: obj.etag,
|
|
38305
|
+
lastModified: obj.lastModified,
|
|
38306
|
+
size: obj.size,
|
|
38307
|
+
contentEncoding: obj.contentEncoding,
|
|
38308
|
+
contentLength: obj.contentLength
|
|
38309
|
+
});
|
|
38310
|
+
}
|
|
38311
|
+
if (this.verbose) {
|
|
38312
|
+
console.log(`[MemoryStorage] Restored snapshot with ${this.objects.size} objects`);
|
|
38313
|
+
}
|
|
38314
|
+
}
|
|
38315
|
+
/**
|
|
38316
|
+
* Save current state to disk
|
|
38317
|
+
*/
|
|
38318
|
+
async saveToDisk(customPath) {
|
|
38319
|
+
const path = customPath || this.persistPath;
|
|
38320
|
+
if (!path) {
|
|
38321
|
+
throw new Error("No persist path configured");
|
|
38322
|
+
}
|
|
38323
|
+
const snapshot = this.snapshot();
|
|
38324
|
+
const json = JSON.stringify(snapshot, null, 2);
|
|
38325
|
+
const [ok, err] = await tryFn(() => promises.writeFile(path, json, "utf-8"));
|
|
38326
|
+
if (!ok) {
|
|
38327
|
+
throw new Error(`Failed to save to disk: ${err.message}`);
|
|
38328
|
+
}
|
|
38329
|
+
if (this.verbose) {
|
|
38330
|
+
console.log(`[MemoryStorage] Saved ${this.objects.size} objects to ${path}`);
|
|
38331
|
+
}
|
|
38332
|
+
return path;
|
|
38333
|
+
}
|
|
38334
|
+
/**
|
|
38335
|
+
* Load state from disk
|
|
38336
|
+
*/
|
|
38337
|
+
async loadFromDisk(customPath) {
|
|
38338
|
+
const path = customPath || this.persistPath;
|
|
38339
|
+
if (!path) {
|
|
38340
|
+
throw new Error("No persist path configured");
|
|
38341
|
+
}
|
|
38342
|
+
const [ok, err, json] = await tryFn(() => promises.readFile(path, "utf-8"));
|
|
38343
|
+
if (!ok) {
|
|
38344
|
+
throw new Error(`Failed to load from disk: ${err.message}`);
|
|
38345
|
+
}
|
|
38346
|
+
const snapshot = JSON.parse(json);
|
|
38347
|
+
this.restore(snapshot);
|
|
38348
|
+
if (this.verbose) {
|
|
38349
|
+
console.log(`[MemoryStorage] Loaded ${this.objects.size} objects from ${path}`);
|
|
38350
|
+
}
|
|
38351
|
+
return snapshot;
|
|
38352
|
+
}
|
|
38353
|
+
/**
|
|
38354
|
+
* Get storage statistics
|
|
38355
|
+
*/
|
|
38356
|
+
getStats() {
|
|
38357
|
+
let totalSize = 0;
|
|
38358
|
+
const keys = [];
|
|
38359
|
+
for (const [key, obj] of this.objects.entries()) {
|
|
38360
|
+
totalSize += obj.size;
|
|
38361
|
+
keys.push(key);
|
|
38362
|
+
}
|
|
38363
|
+
return {
|
|
38364
|
+
objectCount: this.objects.size,
|
|
38365
|
+
totalSize,
|
|
38366
|
+
totalSizeFormatted: this._formatBytes(totalSize),
|
|
38367
|
+
keys: keys.sort(),
|
|
38368
|
+
bucket: this.bucket
|
|
38369
|
+
};
|
|
38370
|
+
}
|
|
38371
|
+
/**
|
|
38372
|
+
* Format bytes for human reading
|
|
38373
|
+
*/
|
|
38374
|
+
_formatBytes(bytes) {
|
|
38375
|
+
if (bytes === 0) return "0 Bytes";
|
|
38376
|
+
const k = 1024;
|
|
38377
|
+
const sizes = ["Bytes", "KB", "MB", "GB"];
|
|
38378
|
+
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
|
38379
|
+
return Math.round(bytes / Math.pow(k, i) * 100) / 100 + " " + sizes[i];
|
|
38380
|
+
}
|
|
38381
|
+
/**
|
|
38382
|
+
* Clear all objects
|
|
38383
|
+
*/
|
|
38384
|
+
clear() {
|
|
38385
|
+
this.objects.clear();
|
|
38386
|
+
if (this.verbose) {
|
|
38387
|
+
console.log(`[MemoryStorage] Cleared all objects`);
|
|
38388
|
+
}
|
|
38389
|
+
}
|
|
38390
|
+
}
|
|
38391
|
+
|
|
38392
|
+
class MemoryClient extends EventEmitter {
|
|
38393
|
+
constructor(config = {}) {
|
|
38394
|
+
super();
|
|
38395
|
+
this.id = config.id || idGenerator(77);
|
|
38396
|
+
this.verbose = config.verbose || false;
|
|
38397
|
+
this.parallelism = config.parallelism || 10;
|
|
38398
|
+
this.bucket = config.bucket || "s3db";
|
|
38399
|
+
this.keyPrefix = config.keyPrefix || "";
|
|
38400
|
+
this.region = config.region || "us-east-1";
|
|
38401
|
+
this.storage = new MemoryStorage({
|
|
38402
|
+
bucket: this.bucket,
|
|
38403
|
+
enforceLimits: config.enforceLimits || false,
|
|
38404
|
+
metadataLimit: config.metadataLimit || 2048,
|
|
38405
|
+
maxObjectSize: config.maxObjectSize || 5 * 1024 * 1024 * 1024,
|
|
38406
|
+
persistPath: config.persistPath,
|
|
38407
|
+
autoPersist: config.autoPersist || false,
|
|
38408
|
+
verbose: this.verbose
|
|
38409
|
+
});
|
|
38410
|
+
this.config = {
|
|
38411
|
+
bucket: this.bucket,
|
|
38412
|
+
keyPrefix: this.keyPrefix,
|
|
38413
|
+
region: this.region,
|
|
38414
|
+
endpoint: "memory://localhost",
|
|
38415
|
+
forcePathStyle: true
|
|
38416
|
+
};
|
|
38417
|
+
if (this.verbose) {
|
|
38418
|
+
console.log(`[MemoryClient] Initialized (id: ${this.id}, bucket: ${this.bucket})`);
|
|
38419
|
+
}
|
|
38420
|
+
}
|
|
38421
|
+
/**
|
|
38422
|
+
* Simulate sendCommand from AWS SDK
|
|
38423
|
+
* Used by Database/Resource to send AWS SDK commands
|
|
38424
|
+
*/
|
|
38425
|
+
async sendCommand(command) {
|
|
38426
|
+
const commandName = command.constructor.name;
|
|
38427
|
+
const input = command.input || {};
|
|
38428
|
+
this.emit("command.request", commandName, input);
|
|
38429
|
+
let response;
|
|
38430
|
+
try {
|
|
38431
|
+
switch (commandName) {
|
|
38432
|
+
case "PutObjectCommand":
|
|
38433
|
+
response = await this._handlePutObject(input);
|
|
38434
|
+
break;
|
|
38435
|
+
case "GetObjectCommand":
|
|
38436
|
+
response = await this._handleGetObject(input);
|
|
38437
|
+
break;
|
|
38438
|
+
case "HeadObjectCommand":
|
|
38439
|
+
response = await this._handleHeadObject(input);
|
|
38440
|
+
break;
|
|
38441
|
+
case "CopyObjectCommand":
|
|
38442
|
+
response = await this._handleCopyObject(input);
|
|
38443
|
+
break;
|
|
38444
|
+
case "DeleteObjectCommand":
|
|
38445
|
+
response = await this._handleDeleteObject(input);
|
|
38446
|
+
break;
|
|
38447
|
+
case "DeleteObjectsCommand":
|
|
38448
|
+
response = await this._handleDeleteObjects(input);
|
|
38449
|
+
break;
|
|
38450
|
+
case "ListObjectsV2Command":
|
|
38451
|
+
response = await this._handleListObjects(input);
|
|
38452
|
+
break;
|
|
38453
|
+
default:
|
|
38454
|
+
throw new Error(`Unsupported command: ${commandName}`);
|
|
38455
|
+
}
|
|
38456
|
+
this.emit("command.response", commandName, response, input);
|
|
38457
|
+
return response;
|
|
38458
|
+
} catch (error) {
|
|
38459
|
+
const mappedError = mapAwsError(error, {
|
|
38460
|
+
bucket: this.bucket,
|
|
38461
|
+
key: input.Key,
|
|
38462
|
+
commandName,
|
|
38463
|
+
commandInput: input
|
|
38464
|
+
});
|
|
38465
|
+
throw mappedError;
|
|
38466
|
+
}
|
|
38467
|
+
}
|
|
38468
|
+
/**
|
|
38469
|
+
* PutObjectCommand handler
|
|
38470
|
+
*/
|
|
38471
|
+
async _handlePutObject(input) {
|
|
38472
|
+
const key = input.Key;
|
|
38473
|
+
const metadata = input.Metadata || {};
|
|
38474
|
+
const contentType = input.ContentType;
|
|
38475
|
+
const body = input.Body;
|
|
38476
|
+
const contentEncoding = input.ContentEncoding;
|
|
38477
|
+
const contentLength = input.ContentLength;
|
|
38478
|
+
const ifMatch = input.IfMatch;
|
|
38479
|
+
return await this.storage.put(key, {
|
|
38480
|
+
body,
|
|
38481
|
+
metadata,
|
|
38482
|
+
contentType,
|
|
38483
|
+
contentEncoding,
|
|
38484
|
+
contentLength,
|
|
38485
|
+
ifMatch
|
|
38486
|
+
});
|
|
38487
|
+
}
|
|
38488
|
+
/**
|
|
38489
|
+
* GetObjectCommand handler
|
|
38490
|
+
*/
|
|
38491
|
+
async _handleGetObject(input) {
|
|
38492
|
+
const key = input.Key;
|
|
38493
|
+
return await this.storage.get(key);
|
|
38494
|
+
}
|
|
38495
|
+
/**
|
|
38496
|
+
* HeadObjectCommand handler
|
|
38497
|
+
*/
|
|
38498
|
+
async _handleHeadObject(input) {
|
|
38499
|
+
const key = input.Key;
|
|
38500
|
+
return await this.storage.head(key);
|
|
38501
|
+
}
|
|
38502
|
+
/**
|
|
38503
|
+
* CopyObjectCommand handler
|
|
38504
|
+
*/
|
|
38505
|
+
async _handleCopyObject(input) {
|
|
38506
|
+
const copySource = input.CopySource;
|
|
38507
|
+
const parts = copySource.split("/");
|
|
38508
|
+
const sourceKey = parts.slice(1).join("/");
|
|
38509
|
+
const destinationKey = input.Key;
|
|
38510
|
+
const metadata = input.Metadata;
|
|
38511
|
+
const metadataDirective = input.MetadataDirective;
|
|
38512
|
+
const contentType = input.ContentType;
|
|
38513
|
+
return await this.storage.copy(sourceKey, destinationKey, {
|
|
38514
|
+
metadata,
|
|
38515
|
+
metadataDirective,
|
|
38516
|
+
contentType
|
|
38517
|
+
});
|
|
38518
|
+
}
|
|
38519
|
+
/**
|
|
38520
|
+
* DeleteObjectCommand handler
|
|
38521
|
+
*/
|
|
38522
|
+
async _handleDeleteObject(input) {
|
|
38523
|
+
const key = input.Key;
|
|
38524
|
+
return await this.storage.delete(key);
|
|
38525
|
+
}
|
|
38526
|
+
/**
|
|
38527
|
+
* DeleteObjectsCommand handler
|
|
38528
|
+
*/
|
|
38529
|
+
async _handleDeleteObjects(input) {
|
|
38530
|
+
const objects = input.Delete?.Objects || [];
|
|
38531
|
+
const keys = objects.map((obj) => obj.Key);
|
|
38532
|
+
return await this.storage.deleteMultiple(keys);
|
|
38533
|
+
}
|
|
38534
|
+
/**
|
|
38535
|
+
* ListObjectsV2Command handler
|
|
38536
|
+
*/
|
|
38537
|
+
async _handleListObjects(input) {
|
|
38538
|
+
const fullPrefix = this.keyPrefix && input.Prefix ? path$1.join(this.keyPrefix, input.Prefix) : this.keyPrefix || input.Prefix || "";
|
|
38539
|
+
return await this.storage.list({
|
|
38540
|
+
prefix: fullPrefix,
|
|
38541
|
+
delimiter: input.Delimiter,
|
|
38542
|
+
maxKeys: input.MaxKeys,
|
|
38543
|
+
continuationToken: input.ContinuationToken
|
|
38544
|
+
});
|
|
38545
|
+
}
|
|
38546
|
+
/**
|
|
38547
|
+
* Put an object (Client interface method)
|
|
38548
|
+
*/
|
|
38549
|
+
async putObject({ key, metadata, contentType, body, contentEncoding, contentLength, ifMatch }) {
|
|
38550
|
+
const fullKey = this.keyPrefix ? path$1.join(this.keyPrefix, key) : key;
|
|
38551
|
+
const stringMetadata = {};
|
|
38552
|
+
if (metadata) {
|
|
38553
|
+
for (const [k, v] of Object.entries(metadata)) {
|
|
38554
|
+
const validKey = String(k).replace(/[^a-zA-Z0-9\-_]/g, "_");
|
|
38555
|
+
const { encoded } = metadataEncode(v);
|
|
38556
|
+
stringMetadata[validKey] = encoded;
|
|
38557
|
+
}
|
|
38558
|
+
}
|
|
38559
|
+
const response = await this.storage.put(fullKey, {
|
|
38560
|
+
body,
|
|
38561
|
+
metadata: stringMetadata,
|
|
38562
|
+
contentType,
|
|
38563
|
+
contentEncoding,
|
|
38564
|
+
contentLength,
|
|
38565
|
+
ifMatch
|
|
38566
|
+
});
|
|
38567
|
+
this.emit("putObject", null, { key, metadata, contentType, body, contentEncoding, contentLength });
|
|
38568
|
+
return response;
|
|
38569
|
+
}
|
|
38570
|
+
/**
|
|
38571
|
+
* Get an object (Client interface method)
|
|
38572
|
+
*/
|
|
38573
|
+
async getObject(key) {
|
|
38574
|
+
const fullKey = this.keyPrefix ? path$1.join(this.keyPrefix, key) : key;
|
|
38575
|
+
const response = await this.storage.get(fullKey);
|
|
38576
|
+
const decodedMetadata = {};
|
|
38577
|
+
if (response.Metadata) {
|
|
38578
|
+
for (const [k, v] of Object.entries(response.Metadata)) {
|
|
38579
|
+
decodedMetadata[k] = metadataDecode(v);
|
|
38580
|
+
}
|
|
38581
|
+
}
|
|
38582
|
+
this.emit("getObject", null, { key });
|
|
38583
|
+
return {
|
|
38584
|
+
...response,
|
|
38585
|
+
Metadata: decodedMetadata
|
|
38586
|
+
};
|
|
38587
|
+
}
|
|
38588
|
+
/**
|
|
38589
|
+
* Head object (get metadata only)
|
|
38590
|
+
*/
|
|
38591
|
+
async headObject(key) {
|
|
38592
|
+
const fullKey = this.keyPrefix ? path$1.join(this.keyPrefix, key) : key;
|
|
38593
|
+
const response = await this.storage.head(fullKey);
|
|
38594
|
+
const decodedMetadata = {};
|
|
38595
|
+
if (response.Metadata) {
|
|
38596
|
+
for (const [k, v] of Object.entries(response.Metadata)) {
|
|
38597
|
+
decodedMetadata[k] = metadataDecode(v);
|
|
38598
|
+
}
|
|
38599
|
+
}
|
|
38600
|
+
this.emit("headObject", null, { key });
|
|
38601
|
+
return {
|
|
38602
|
+
...response,
|
|
38603
|
+
Metadata: decodedMetadata
|
|
38604
|
+
};
|
|
38605
|
+
}
|
|
38606
|
+
/**
|
|
38607
|
+
* Copy an object
|
|
38608
|
+
*/
|
|
38609
|
+
async copyObject({ from, to, metadata, metadataDirective, contentType }) {
|
|
38610
|
+
const fullFrom = this.keyPrefix ? path$1.join(this.keyPrefix, from) : from;
|
|
38611
|
+
const fullTo = this.keyPrefix ? path$1.join(this.keyPrefix, to) : to;
|
|
38612
|
+
const encodedMetadata = {};
|
|
38613
|
+
if (metadata) {
|
|
38614
|
+
for (const [k, v] of Object.entries(metadata)) {
|
|
38615
|
+
const validKey = String(k).replace(/[^a-zA-Z0-9\-_]/g, "_");
|
|
38616
|
+
const { encoded } = metadataEncode(v);
|
|
38617
|
+
encodedMetadata[validKey] = encoded;
|
|
38618
|
+
}
|
|
38619
|
+
}
|
|
38620
|
+
const response = await this.storage.copy(fullFrom, fullTo, {
|
|
38621
|
+
metadata: encodedMetadata,
|
|
38622
|
+
metadataDirective,
|
|
38623
|
+
contentType
|
|
38624
|
+
});
|
|
38625
|
+
this.emit("copyObject", null, { from, to, metadata, metadataDirective });
|
|
38626
|
+
return response;
|
|
38627
|
+
}
|
|
38628
|
+
/**
|
|
38629
|
+
* Check if object exists
|
|
38630
|
+
*/
|
|
38631
|
+
async exists(key) {
|
|
38632
|
+
const fullKey = this.keyPrefix ? path$1.join(this.keyPrefix, key) : key;
|
|
38633
|
+
return this.storage.exists(fullKey);
|
|
38634
|
+
}
|
|
38635
|
+
/**
|
|
38636
|
+
* Delete an object
|
|
38637
|
+
*/
|
|
38638
|
+
async deleteObject(key) {
|
|
38639
|
+
const fullKey = this.keyPrefix ? path$1.join(this.keyPrefix, key) : key;
|
|
38640
|
+
const response = await this.storage.delete(fullKey);
|
|
38641
|
+
this.emit("deleteObject", null, { key });
|
|
38642
|
+
return response;
|
|
38643
|
+
}
|
|
38644
|
+
/**
|
|
38645
|
+
* Delete multiple objects (batch)
|
|
38646
|
+
*/
|
|
38647
|
+
async deleteObjects(keys) {
|
|
38648
|
+
const fullKeys = keys.map(
|
|
38649
|
+
(key) => this.keyPrefix ? path$1.join(this.keyPrefix, key) : key
|
|
38650
|
+
);
|
|
38651
|
+
const batches = lodashEs.chunk(fullKeys, this.parallelism);
|
|
38652
|
+
const allResults = { Deleted: [], Errors: [] };
|
|
38653
|
+
const { results } = await promisePool.PromisePool.withConcurrency(this.parallelism).for(batches).process(async (batch) => {
|
|
38654
|
+
return await this.storage.deleteMultiple(batch);
|
|
38655
|
+
});
|
|
38656
|
+
for (const result of results) {
|
|
38657
|
+
allResults.Deleted.push(...result.Deleted);
|
|
38658
|
+
allResults.Errors.push(...result.Errors);
|
|
38659
|
+
}
|
|
38660
|
+
this.emit("deleteObjects", null, { keys, count: allResults.Deleted.length });
|
|
38661
|
+
return allResults;
|
|
38662
|
+
}
|
|
38663
|
+
/**
|
|
38664
|
+
* List objects with pagination support
|
|
38665
|
+
*/
|
|
38666
|
+
async listObjects({ prefix = "", delimiter = null, maxKeys = 1e3, continuationToken = null }) {
|
|
38667
|
+
const fullPrefix = this.keyPrefix ? path$1.join(this.keyPrefix, prefix) : prefix;
|
|
38668
|
+
const response = await this.storage.list({
|
|
38669
|
+
prefix: fullPrefix,
|
|
38670
|
+
delimiter,
|
|
38671
|
+
maxKeys,
|
|
38672
|
+
continuationToken
|
|
38673
|
+
});
|
|
38674
|
+
this.emit("listObjects", null, { prefix, count: response.Contents.length });
|
|
38675
|
+
return response;
|
|
38676
|
+
}
|
|
38677
|
+
/**
|
|
38678
|
+
* Get a page of keys with offset/limit pagination
|
|
38679
|
+
*/
|
|
38680
|
+
async getKeysPage(params = {}) {
|
|
38681
|
+
const { prefix = "", offset = 0, amount = 100 } = params;
|
|
38682
|
+
let keys = [];
|
|
38683
|
+
let truncated = true;
|
|
38684
|
+
let continuationToken;
|
|
38685
|
+
if (offset > 0) {
|
|
38686
|
+
const fullPrefix = this.keyPrefix ? path$1.join(this.keyPrefix, prefix) : prefix;
|
|
38687
|
+
const response = await this.storage.list({
|
|
38688
|
+
prefix: fullPrefix,
|
|
38689
|
+
maxKeys: offset + amount
|
|
38690
|
+
});
|
|
38691
|
+
keys = response.Contents.map((x) => x.Key).slice(offset, offset + amount);
|
|
38692
|
+
} else {
|
|
38693
|
+
while (truncated) {
|
|
38694
|
+
const options = {
|
|
38695
|
+
prefix,
|
|
38696
|
+
continuationToken,
|
|
38697
|
+
maxKeys: amount - keys.length
|
|
38698
|
+
};
|
|
38699
|
+
const res = await this.listObjects(options);
|
|
38700
|
+
if (res.Contents) {
|
|
38701
|
+
keys = keys.concat(res.Contents.map((x) => x.Key));
|
|
38702
|
+
}
|
|
38703
|
+
truncated = res.IsTruncated || false;
|
|
38704
|
+
continuationToken = res.NextContinuationToken;
|
|
38705
|
+
if (keys.length >= amount) {
|
|
38706
|
+
keys = keys.slice(0, amount);
|
|
38707
|
+
break;
|
|
38708
|
+
}
|
|
38709
|
+
}
|
|
38710
|
+
}
|
|
38711
|
+
if (this.keyPrefix) {
|
|
38712
|
+
keys = keys.map((x) => x.replace(this.keyPrefix, "")).map((x) => x.startsWith("/") ? x.replace("/", "") : x);
|
|
38713
|
+
}
|
|
38714
|
+
this.emit("getKeysPage", keys, params);
|
|
38715
|
+
return keys;
|
|
38716
|
+
}
|
|
38717
|
+
/**
|
|
38718
|
+
* Get all keys with a given prefix
|
|
38719
|
+
*/
|
|
38720
|
+
async getAllKeys({ prefix = "" }) {
|
|
38721
|
+
const fullPrefix = this.keyPrefix ? path$1.join(this.keyPrefix, prefix) : prefix;
|
|
38722
|
+
const response = await this.storage.list({
|
|
38723
|
+
prefix: fullPrefix,
|
|
38724
|
+
maxKeys: 1e5
|
|
38725
|
+
// Large number to get all
|
|
38726
|
+
});
|
|
38727
|
+
let keys = response.Contents.map((x) => x.Key);
|
|
38728
|
+
if (this.keyPrefix) {
|
|
38729
|
+
keys = keys.map((x) => x.replace(this.keyPrefix, "")).map((x) => x.startsWith("/") ? x.replace("/", "") : x);
|
|
38730
|
+
}
|
|
38731
|
+
this.emit("getAllKeys", keys, { prefix });
|
|
38732
|
+
return keys;
|
|
38733
|
+
}
|
|
38734
|
+
/**
|
|
38735
|
+
* Count total objects under a prefix
|
|
38736
|
+
*/
|
|
38737
|
+
async count({ prefix = "" } = {}) {
|
|
38738
|
+
const keys = await this.getAllKeys({ prefix });
|
|
38739
|
+
const count = keys.length;
|
|
38740
|
+
this.emit("count", count, { prefix });
|
|
38741
|
+
return count;
|
|
38742
|
+
}
|
|
38743
|
+
/**
|
|
38744
|
+
* Delete all objects under a prefix
|
|
38745
|
+
*/
|
|
38746
|
+
async deleteAll({ prefix = "" } = {}) {
|
|
38747
|
+
const keys = await this.getAllKeys({ prefix });
|
|
38748
|
+
let totalDeleted = 0;
|
|
38749
|
+
if (keys.length > 0) {
|
|
38750
|
+
const result = await this.deleteObjects(keys);
|
|
38751
|
+
totalDeleted = result.Deleted.length;
|
|
38752
|
+
this.emit("deleteAll", {
|
|
38753
|
+
prefix,
|
|
38754
|
+
batch: totalDeleted,
|
|
38755
|
+
total: totalDeleted
|
|
38756
|
+
});
|
|
38757
|
+
}
|
|
38758
|
+
this.emit("deleteAllComplete", {
|
|
38759
|
+
prefix,
|
|
38760
|
+
totalDeleted
|
|
38761
|
+
});
|
|
38762
|
+
return totalDeleted;
|
|
38763
|
+
}
|
|
38764
|
+
/**
|
|
38765
|
+
* Get continuation token after skipping offset items
|
|
38766
|
+
*/
|
|
38767
|
+
async getContinuationTokenAfterOffset({ prefix = "", offset = 1e3 } = {}) {
|
|
38768
|
+
if (offset === 0) return null;
|
|
38769
|
+
const keys = await this.getAllKeys({ prefix });
|
|
38770
|
+
if (offset >= keys.length) {
|
|
38771
|
+
this.emit("getContinuationTokenAfterOffset", null, { prefix, offset });
|
|
38772
|
+
return null;
|
|
38773
|
+
}
|
|
38774
|
+
const token = keys[offset];
|
|
38775
|
+
this.emit("getContinuationTokenAfterOffset", token, { prefix, offset });
|
|
38776
|
+
return token;
|
|
38777
|
+
}
|
|
38778
|
+
/**
|
|
38779
|
+
* Move an object from one key to another
|
|
38780
|
+
*/
|
|
38781
|
+
async moveObject({ from, to }) {
|
|
38782
|
+
await this.copyObject({ from, to, metadataDirective: "COPY" });
|
|
38783
|
+
await this.deleteObject(from);
|
|
38784
|
+
}
|
|
38785
|
+
/**
|
|
38786
|
+
* Move all objects from one prefix to another
|
|
38787
|
+
*/
|
|
38788
|
+
async moveAllObjects({ prefixFrom, prefixTo }) {
|
|
38789
|
+
const keys = await this.getAllKeys({ prefix: prefixFrom });
|
|
38790
|
+
const results = [];
|
|
38791
|
+
const errors = [];
|
|
38792
|
+
for (const key of keys) {
|
|
38793
|
+
try {
|
|
38794
|
+
const to = key.replace(prefixFrom, prefixTo);
|
|
38795
|
+
await this.moveObject({ from: key, to });
|
|
38796
|
+
results.push(to);
|
|
38797
|
+
} catch (error) {
|
|
38798
|
+
errors.push({
|
|
38799
|
+
message: error.message,
|
|
38800
|
+
raw: error,
|
|
38801
|
+
key
|
|
38802
|
+
});
|
|
38803
|
+
}
|
|
38804
|
+
}
|
|
38805
|
+
this.emit("moveAllObjects", { results, errors }, { prefixFrom, prefixTo });
|
|
38806
|
+
if (errors.length > 0) {
|
|
38807
|
+
const error = new Error("Some objects could not be moved");
|
|
38808
|
+
error.context = {
|
|
38809
|
+
bucket: this.bucket,
|
|
38810
|
+
operation: "moveAllObjects",
|
|
38811
|
+
prefixFrom,
|
|
38812
|
+
prefixTo,
|
|
38813
|
+
totalKeys: keys.length,
|
|
38814
|
+
failedCount: errors.length,
|
|
38815
|
+
successCount: results.length,
|
|
38816
|
+
errors
|
|
38817
|
+
};
|
|
38818
|
+
throw error;
|
|
38819
|
+
}
|
|
38820
|
+
return results;
|
|
38821
|
+
}
|
|
38822
|
+
/**
|
|
38823
|
+
* Create a snapshot of current storage state
|
|
38824
|
+
*/
|
|
38825
|
+
snapshot() {
|
|
38826
|
+
return this.storage.snapshot();
|
|
38827
|
+
}
|
|
38828
|
+
/**
|
|
38829
|
+
* Restore from a snapshot
|
|
38830
|
+
*/
|
|
38831
|
+
restore(snapshot) {
|
|
38832
|
+
return this.storage.restore(snapshot);
|
|
38833
|
+
}
|
|
38834
|
+
/**
|
|
38835
|
+
* Save current state to disk (persistence)
|
|
38836
|
+
*/
|
|
38837
|
+
async saveToDisk(path2) {
|
|
38838
|
+
return await this.storage.saveToDisk(path2);
|
|
38839
|
+
}
|
|
38840
|
+
/**
|
|
38841
|
+
* Load state from disk
|
|
38842
|
+
*/
|
|
38843
|
+
async loadFromDisk(path2) {
|
|
38844
|
+
return await this.storage.loadFromDisk(path2);
|
|
38845
|
+
}
|
|
38846
|
+
/**
|
|
38847
|
+
* Export to BackupPlugin-compatible format (s3db.json + JSONL files)
|
|
38848
|
+
* Compatible with BackupPlugin for easy migration
|
|
38849
|
+
*
|
|
38850
|
+
* @param {string} outputDir - Output directory path
|
|
38851
|
+
* @param {Object} options - Export options
|
|
38852
|
+
* @param {Array<string>} options.resources - Resource names to export (default: all)
|
|
38853
|
+
* @param {boolean} options.compress - Use gzip compression (default: true)
|
|
38854
|
+
* @param {Object} options.database - Database instance for schema metadata
|
|
38855
|
+
* @returns {Promise<Object>} Export manifest with file paths and stats
|
|
38856
|
+
*/
|
|
38857
|
+
async exportBackup(outputDir, options = {}) {
|
|
38858
|
+
const { mkdir, writeFile } = await import('fs/promises');
|
|
38859
|
+
const zlib = await import('zlib');
|
|
38860
|
+
const { promisify } = await import('util');
|
|
38861
|
+
const gzip = promisify(zlib.gzip);
|
|
38862
|
+
await mkdir(outputDir, { recursive: true });
|
|
38863
|
+
const compress = options.compress !== false;
|
|
38864
|
+
const database = options.database;
|
|
38865
|
+
const resourceFilter = options.resources;
|
|
38866
|
+
const allKeys = await this.getAllKeys({});
|
|
38867
|
+
const resourceMap = /* @__PURE__ */ new Map();
|
|
38868
|
+
for (const key of allKeys) {
|
|
38869
|
+
const match = key.match(/^resource=([^/]+)\//);
|
|
38870
|
+
if (match) {
|
|
38871
|
+
const resourceName = match[1];
|
|
38872
|
+
if (!resourceFilter || resourceFilter.includes(resourceName)) {
|
|
38873
|
+
if (!resourceMap.has(resourceName)) {
|
|
38874
|
+
resourceMap.set(resourceName, []);
|
|
38875
|
+
}
|
|
38876
|
+
resourceMap.get(resourceName).push(key);
|
|
38877
|
+
}
|
|
38878
|
+
}
|
|
38879
|
+
}
|
|
38880
|
+
const exportedFiles = {};
|
|
38881
|
+
const resourceStats = {};
|
|
38882
|
+
for (const [resourceName, keys] of resourceMap.entries()) {
|
|
38883
|
+
const records = [];
|
|
38884
|
+
for (const key of keys) {
|
|
38885
|
+
const obj = await this.getObject(key);
|
|
38886
|
+
const idMatch = key.match(/\/id=([^/]+)/);
|
|
38887
|
+
const recordId = idMatch ? idMatch[1] : null;
|
|
38888
|
+
const record = { ...obj.Metadata };
|
|
38889
|
+
if (recordId && !record.id) {
|
|
38890
|
+
record.id = recordId;
|
|
38891
|
+
}
|
|
38892
|
+
if (obj.Body) {
|
|
38893
|
+
const chunks = [];
|
|
38894
|
+
for await (const chunk2 of obj.Body) {
|
|
38895
|
+
chunks.push(chunk2);
|
|
38896
|
+
}
|
|
38897
|
+
const bodyBuffer = Buffer.concat(chunks);
|
|
38898
|
+
const bodyStr = bodyBuffer.toString("utf-8");
|
|
38899
|
+
if (bodyStr.startsWith("{") || bodyStr.startsWith("[")) {
|
|
38900
|
+
try {
|
|
38901
|
+
const bodyData = JSON.parse(bodyStr);
|
|
38902
|
+
Object.assign(record, bodyData);
|
|
38903
|
+
} catch {
|
|
38904
|
+
record._body = bodyStr;
|
|
38905
|
+
}
|
|
38906
|
+
} else if (bodyStr) {
|
|
38907
|
+
record._body = bodyStr;
|
|
38908
|
+
}
|
|
38909
|
+
}
|
|
38910
|
+
records.push(record);
|
|
38911
|
+
}
|
|
38912
|
+
const jsonl = records.map((r) => JSON.stringify(r)).join("\n");
|
|
38913
|
+
const filename = compress ? `${resourceName}.jsonl.gz` : `${resourceName}.jsonl`;
|
|
38914
|
+
const filePath = `${outputDir}/${filename}`;
|
|
38915
|
+
if (compress) {
|
|
38916
|
+
const compressed = await gzip(jsonl);
|
|
38917
|
+
await writeFile(filePath, compressed);
|
|
38918
|
+
} else {
|
|
38919
|
+
await writeFile(filePath, jsonl, "utf-8");
|
|
38920
|
+
}
|
|
38921
|
+
exportedFiles[resourceName] = filePath;
|
|
38922
|
+
resourceStats[resourceName] = {
|
|
38923
|
+
recordCount: records.length,
|
|
38924
|
+
fileSize: compress ? (await gzip(jsonl)).length : Buffer.byteLength(jsonl)
|
|
38925
|
+
};
|
|
38926
|
+
}
|
|
38927
|
+
const s3dbMetadata = {
|
|
38928
|
+
version: "1.0",
|
|
38929
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
38930
|
+
bucket: this.bucket,
|
|
38931
|
+
keyPrefix: this.keyPrefix || "",
|
|
38932
|
+
compressed: compress,
|
|
38933
|
+
resources: {},
|
|
38934
|
+
totalRecords: 0,
|
|
38935
|
+
totalSize: 0
|
|
38936
|
+
};
|
|
38937
|
+
if (database && database.resources) {
|
|
38938
|
+
for (const [resourceName, resource] of Object.entries(database.resources)) {
|
|
38939
|
+
if (resourceMap.has(resourceName)) {
|
|
38940
|
+
s3dbMetadata.resources[resourceName] = {
|
|
38941
|
+
schema: resource.schema ? {
|
|
38942
|
+
attributes: resource.schema.attributes,
|
|
38943
|
+
partitions: resource.schema.partitions,
|
|
38944
|
+
behavior: resource.schema.behavior,
|
|
38945
|
+
timestamps: resource.schema.timestamps
|
|
38946
|
+
} : null,
|
|
38947
|
+
stats: resourceStats[resourceName]
|
|
38948
|
+
};
|
|
38949
|
+
}
|
|
38950
|
+
}
|
|
38951
|
+
} else {
|
|
38952
|
+
for (const [resourceName, stats] of Object.entries(resourceStats)) {
|
|
38953
|
+
s3dbMetadata.resources[resourceName] = { stats };
|
|
38954
|
+
}
|
|
38955
|
+
}
|
|
38956
|
+
for (const stats of Object.values(resourceStats)) {
|
|
38957
|
+
s3dbMetadata.totalRecords += stats.recordCount;
|
|
38958
|
+
s3dbMetadata.totalSize += stats.fileSize;
|
|
38959
|
+
}
|
|
38960
|
+
const s3dbPath = `${outputDir}/s3db.json`;
|
|
38961
|
+
await writeFile(s3dbPath, JSON.stringify(s3dbMetadata, null, 2), "utf-8");
|
|
38962
|
+
return {
|
|
38963
|
+
manifest: s3dbPath,
|
|
38964
|
+
files: exportedFiles,
|
|
38965
|
+
stats: s3dbMetadata,
|
|
38966
|
+
resourceCount: resourceMap.size,
|
|
38967
|
+
totalRecords: s3dbMetadata.totalRecords,
|
|
38968
|
+
totalSize: s3dbMetadata.totalSize
|
|
38969
|
+
};
|
|
38970
|
+
}
|
|
38971
|
+
/**
|
|
38972
|
+
* Import from BackupPlugin-compatible format
|
|
38973
|
+
* Loads data from s3db.json + JSONL files created by BackupPlugin or exportBackup()
|
|
38974
|
+
*
|
|
38975
|
+
* @param {string} backupDir - Backup directory path containing s3db.json
|
|
38976
|
+
* @param {Object} options - Import options
|
|
38977
|
+
* @param {Array<string>} options.resources - Resource names to import (default: all)
|
|
38978
|
+
* @param {boolean} options.clear - Clear existing data first (default: false)
|
|
38979
|
+
* @param {Object} options.database - Database instance to recreate schemas
|
|
38980
|
+
* @returns {Promise<Object>} Import stats
|
|
38981
|
+
*/
|
|
38982
|
+
async importBackup(backupDir, options = {}) {
|
|
38983
|
+
const { readFile, readdir } = await import('fs/promises');
|
|
38984
|
+
const zlib = await import('zlib');
|
|
38985
|
+
const { promisify } = await import('util');
|
|
38986
|
+
const gunzip = promisify(zlib.gunzip);
|
|
38987
|
+
if (options.clear) {
|
|
38988
|
+
this.clear();
|
|
38989
|
+
}
|
|
38990
|
+
const s3dbPath = `${backupDir}/s3db.json`;
|
|
38991
|
+
const s3dbContent = await readFile(s3dbPath, "utf-8");
|
|
38992
|
+
const metadata = JSON.parse(s3dbContent);
|
|
38993
|
+
const database = options.database;
|
|
38994
|
+
const resourceFilter = options.resources;
|
|
38995
|
+
const importStats = {
|
|
38996
|
+
resourcesImported: 0,
|
|
38997
|
+
recordsImported: 0,
|
|
38998
|
+
errors: []
|
|
38999
|
+
};
|
|
39000
|
+
if (database && metadata.resources) {
|
|
39001
|
+
for (const [resourceName, resourceMeta] of Object.entries(metadata.resources)) {
|
|
39002
|
+
if (resourceFilter && !resourceFilter.includes(resourceName)) continue;
|
|
39003
|
+
if (resourceMeta.schema) {
|
|
39004
|
+
try {
|
|
39005
|
+
await database.createResource({
|
|
39006
|
+
name: resourceName,
|
|
39007
|
+
...resourceMeta.schema
|
|
39008
|
+
});
|
|
39009
|
+
} catch (error) {
|
|
39010
|
+
}
|
|
39011
|
+
}
|
|
39012
|
+
}
|
|
39013
|
+
}
|
|
39014
|
+
const files = await readdir(backupDir);
|
|
39015
|
+
for (const file of files) {
|
|
39016
|
+
if (!file.endsWith(".jsonl") && !file.endsWith(".jsonl.gz")) continue;
|
|
39017
|
+
const resourceName = file.replace(/\.jsonl(\.gz)?$/, "");
|
|
39018
|
+
if (resourceFilter && !resourceFilter.includes(resourceName)) continue;
|
|
39019
|
+
const filePath = `${backupDir}/${file}`;
|
|
39020
|
+
let content = await readFile(filePath);
|
|
39021
|
+
if (file.endsWith(".gz")) {
|
|
39022
|
+
content = await gunzip(content);
|
|
39023
|
+
}
|
|
39024
|
+
const jsonl = content.toString("utf-8");
|
|
39025
|
+
const lines = jsonl.split("\n").filter((line) => line.trim());
|
|
39026
|
+
for (const line of lines) {
|
|
39027
|
+
try {
|
|
39028
|
+
const record = JSON.parse(line);
|
|
39029
|
+
const id = record.id || record._id || `imported_${Date.now()}_${Math.random()}`;
|
|
39030
|
+
const { _body, id: _, _id: __, ...metadata2 } = record;
|
|
39031
|
+
await this.putObject({
|
|
39032
|
+
key: `resource=${resourceName}/id=${id}`,
|
|
39033
|
+
metadata: metadata2,
|
|
39034
|
+
body: _body ? Buffer.from(_body) : void 0
|
|
39035
|
+
});
|
|
39036
|
+
importStats.recordsImported++;
|
|
39037
|
+
} catch (error) {
|
|
39038
|
+
importStats.errors.push({
|
|
39039
|
+
resource: resourceName,
|
|
39040
|
+
error: error.message,
|
|
39041
|
+
line
|
|
39042
|
+
});
|
|
39043
|
+
}
|
|
39044
|
+
}
|
|
39045
|
+
importStats.resourcesImported++;
|
|
39046
|
+
}
|
|
39047
|
+
return importStats;
|
|
39048
|
+
}
|
|
39049
|
+
/**
|
|
39050
|
+
* Get storage statistics
|
|
39051
|
+
*/
|
|
39052
|
+
getStats() {
|
|
39053
|
+
return this.storage.getStats();
|
|
39054
|
+
}
|
|
39055
|
+
/**
|
|
39056
|
+
* Clear all objects
|
|
39057
|
+
*/
|
|
39058
|
+
clear() {
|
|
39059
|
+
this.storage.clear();
|
|
39060
|
+
}
|
|
39061
|
+
}
|
|
39062
|
+
|
|
39063
|
+
function mapFieldTypeToTypeScript(fieldType) {
|
|
39064
|
+
const baseType = fieldType.split("|")[0].trim();
|
|
39065
|
+
const typeMap = {
|
|
39066
|
+
"string": "string",
|
|
39067
|
+
"number": "number",
|
|
39068
|
+
"integer": "number",
|
|
39069
|
+
"boolean": "boolean",
|
|
39070
|
+
"array": "any[]",
|
|
39071
|
+
"object": "Record<string, any>",
|
|
39072
|
+
"json": "Record<string, any>",
|
|
39073
|
+
"secret": "string",
|
|
39074
|
+
"email": "string",
|
|
39075
|
+
"url": "string",
|
|
39076
|
+
"date": "string",
|
|
39077
|
+
// ISO date string
|
|
39078
|
+
"datetime": "string",
|
|
39079
|
+
// ISO datetime string
|
|
39080
|
+
"ip4": "string",
|
|
39081
|
+
"ip6": "string"
|
|
39082
|
+
};
|
|
39083
|
+
if (baseType.startsWith("embedding:")) {
|
|
39084
|
+
const dimensions = parseInt(baseType.split(":")[1]);
|
|
39085
|
+
return `number[] /* ${dimensions} dimensions */`;
|
|
39086
|
+
}
|
|
39087
|
+
return typeMap[baseType] || "any";
|
|
39088
|
+
}
|
|
39089
|
+
function isFieldRequired(fieldDef) {
|
|
39090
|
+
if (typeof fieldDef === "string") {
|
|
39091
|
+
return fieldDef.includes("|required");
|
|
39092
|
+
}
|
|
39093
|
+
if (typeof fieldDef === "object" && fieldDef.required) {
|
|
39094
|
+
return true;
|
|
39095
|
+
}
|
|
39096
|
+
return false;
|
|
39097
|
+
}
|
|
39098
|
+
function generateResourceInterface(resourceName, attributes, timestamps = false) {
|
|
39099
|
+
const interfaceName = toPascalCase(resourceName);
|
|
39100
|
+
const lines = [];
|
|
39101
|
+
lines.push(`export interface ${interfaceName} {`);
|
|
39102
|
+
lines.push(` /** Resource ID (auto-generated) */`);
|
|
39103
|
+
lines.push(` id: string;`);
|
|
39104
|
+
lines.push("");
|
|
39105
|
+
for (const [fieldName, fieldDef] of Object.entries(attributes)) {
|
|
39106
|
+
const required = isFieldRequired(fieldDef);
|
|
39107
|
+
const optional = required ? "" : "?";
|
|
39108
|
+
let tsType;
|
|
39109
|
+
if (typeof fieldDef === "string") {
|
|
39110
|
+
tsType = mapFieldTypeToTypeScript(fieldDef);
|
|
39111
|
+
} else if (typeof fieldDef === "object" && fieldDef.type) {
|
|
39112
|
+
tsType = mapFieldTypeToTypeScript(fieldDef.type);
|
|
39113
|
+
if (fieldDef.type === "object" && fieldDef.props) {
|
|
39114
|
+
tsType = "{\n";
|
|
39115
|
+
for (const [propName, propDef] of Object.entries(fieldDef.props)) {
|
|
39116
|
+
const propType = typeof propDef === "string" ? mapFieldTypeToTypeScript(propDef) : mapFieldTypeToTypeScript(propDef.type);
|
|
39117
|
+
const propRequired = isFieldRequired(propDef);
|
|
39118
|
+
tsType += ` ${propName}${propRequired ? "" : "?"}: ${propType};
|
|
39119
|
+
`;
|
|
39120
|
+
}
|
|
39121
|
+
tsType += " }";
|
|
39122
|
+
}
|
|
39123
|
+
if (fieldDef.type === "array" && fieldDef.items) {
|
|
39124
|
+
const itemType = mapFieldTypeToTypeScript(fieldDef.items);
|
|
39125
|
+
tsType = `Array<${itemType}>`;
|
|
39126
|
+
}
|
|
39127
|
+
} else {
|
|
39128
|
+
tsType = "any";
|
|
39129
|
+
}
|
|
39130
|
+
if (fieldDef.description) {
|
|
39131
|
+
lines.push(` /** ${fieldDef.description} */`);
|
|
39132
|
+
}
|
|
39133
|
+
lines.push(` ${fieldName}${optional}: ${tsType};`);
|
|
39134
|
+
}
|
|
39135
|
+
if (timestamps) {
|
|
39136
|
+
lines.push("");
|
|
39137
|
+
lines.push(` /** Creation timestamp (ISO 8601) */`);
|
|
39138
|
+
lines.push(` createdAt: string;`);
|
|
39139
|
+
lines.push(` /** Last update timestamp (ISO 8601) */`);
|
|
39140
|
+
lines.push(` updatedAt: string;`);
|
|
39141
|
+
}
|
|
39142
|
+
lines.push("}");
|
|
39143
|
+
lines.push("");
|
|
39144
|
+
return lines.join("\n");
|
|
39145
|
+
}
|
|
39146
|
+
function toPascalCase(str) {
|
|
39147
|
+
return str.split(/[_-]/).map((word) => word.charAt(0).toUpperCase() + word.slice(1)).join("");
|
|
39148
|
+
}
|
|
39149
|
+
async function generateTypes(database, options = {}) {
|
|
39150
|
+
const {
|
|
39151
|
+
outputPath = "./types/database.d.ts",
|
|
39152
|
+
moduleName = "s3db.js",
|
|
39153
|
+
includeResource = true
|
|
39154
|
+
} = options;
|
|
39155
|
+
const lines = [];
|
|
39156
|
+
lines.push("/**");
|
|
39157
|
+
lines.push(" * Auto-generated TypeScript definitions for s3db.js resources");
|
|
39158
|
+
lines.push(" * Generated at: " + (/* @__PURE__ */ new Date()).toISOString());
|
|
39159
|
+
lines.push(" * DO NOT EDIT - This file is auto-generated");
|
|
39160
|
+
lines.push(" */");
|
|
39161
|
+
lines.push("");
|
|
39162
|
+
if (includeResource) {
|
|
39163
|
+
lines.push(`import { Resource, Database } from '${moduleName}';`);
|
|
39164
|
+
lines.push("");
|
|
39165
|
+
}
|
|
39166
|
+
const resourceInterfaces = [];
|
|
39167
|
+
for (const [name, resource] of Object.entries(database.resources)) {
|
|
39168
|
+
const allAttributes = resource.config?.attributes || resource.attributes || {};
|
|
39169
|
+
const timestamps = resource.config?.timestamps || false;
|
|
39170
|
+
const pluginAttrNames = resource.schema?._pluginAttributes ? Object.values(resource.schema._pluginAttributes).flat() : [];
|
|
39171
|
+
const userAttributes = Object.fromEntries(
|
|
39172
|
+
Object.entries(allAttributes).filter(([name2]) => !pluginAttrNames.includes(name2))
|
|
39173
|
+
);
|
|
39174
|
+
const interfaceDef = generateResourceInterface(name, userAttributes, timestamps);
|
|
37749
39175
|
lines.push(interfaceDef);
|
|
37750
39176
|
resourceInterfaces.push({
|
|
37751
39177
|
name,
|
|
@@ -38554,7 +39980,7 @@ exports.BigqueryReplicator = BigqueryReplicator;
|
|
|
38554
39980
|
exports.CONSUMER_DRIVERS = CONSUMER_DRIVERS;
|
|
38555
39981
|
exports.Cache = Cache;
|
|
38556
39982
|
exports.CachePlugin = CachePlugin;
|
|
38557
|
-
exports.Client =
|
|
39983
|
+
exports.Client = S3Client;
|
|
38558
39984
|
exports.ConnectionString = ConnectionString;
|
|
38559
39985
|
exports.ConnectionStringError = ConnectionStringError;
|
|
38560
39986
|
exports.CostsPlugin = CostsPlugin;
|
|
@@ -38573,6 +39999,8 @@ exports.FullTextPlugin = FullTextPlugin;
|
|
|
38573
39999
|
exports.GeoPlugin = GeoPlugin;
|
|
38574
40000
|
exports.InvalidResourceItem = InvalidResourceItem;
|
|
38575
40001
|
exports.MemoryCache = MemoryCache;
|
|
40002
|
+
exports.MemoryClient = MemoryClient;
|
|
40003
|
+
exports.MemoryStorage = MemoryStorage;
|
|
38576
40004
|
exports.MetadataLimitError = MetadataLimitError;
|
|
38577
40005
|
exports.MetricsPlugin = MetricsPlugin;
|
|
38578
40006
|
exports.MissingMetadata = MissingMetadata;
|
|
@@ -38606,6 +40034,7 @@ exports.ResourceReader = ResourceReader;
|
|
|
38606
40034
|
exports.ResourceWriter = ResourceWriter;
|
|
38607
40035
|
exports.S3BackupDriver = S3BackupDriver;
|
|
38608
40036
|
exports.S3Cache = S3Cache;
|
|
40037
|
+
exports.S3Client = S3Client;
|
|
38609
40038
|
exports.S3QueuePlugin = S3QueuePlugin;
|
|
38610
40039
|
exports.S3db = Database;
|
|
38611
40040
|
exports.S3dbError = S3dbError;
|