s3db.js 12.3.0 → 12.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +117 -0
- package/dist/s3db.cjs.js +1171 -28
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.es.js +1171 -31
- package/dist/s3db.es.js.map +1 -1
- package/package.json +2 -2
- package/src/clients/index.js +14 -0
- package/src/clients/memory-client.class.js +883 -0
- package/src/clients/memory-client.md +917 -0
- package/src/clients/memory-storage.class.js +504 -0
- package/src/{client.class.js → clients/s3-client.class.js} +11 -10
- package/src/database.class.js +2 -2
- package/src/index.js +2 -1
- package/src/plugins/replicators/bigquery-replicator.class.js +100 -20
- package/src/plugins/replicators/schema-sync.helper.js +34 -2
- package/src/plugins/tfstate/s3-driver.js +3 -3
package/dist/s3db.cjs.js
CHANGED
|
@@ -13447,7 +13447,7 @@ function generateMySQLAlterTable(tableName, attributes, existingSchema) {
|
|
|
13447
13447
|
}
|
|
13448
13448
|
return alterStatements;
|
|
13449
13449
|
}
|
|
13450
|
-
function generateBigQuerySchema(attributes) {
|
|
13450
|
+
function generateBigQuerySchema(attributes, mutability = "append-only") {
|
|
13451
13451
|
const fields = [];
|
|
13452
13452
|
fields.push({
|
|
13453
13453
|
name: "id",
|
|
@@ -13471,6 +13471,14 @@ function generateBigQuerySchema(attributes) {
|
|
|
13471
13471
|
if (!attributes.updatedAt) {
|
|
13472
13472
|
fields.push({ name: "updated_at", type: "TIMESTAMP", mode: "NULLABLE" });
|
|
13473
13473
|
}
|
|
13474
|
+
if (mutability === "append-only" || mutability === "immutable") {
|
|
13475
|
+
fields.push({ name: "_operation_type", type: "STRING", mode: "NULLABLE" });
|
|
13476
|
+
fields.push({ name: "_operation_timestamp", type: "TIMESTAMP", mode: "NULLABLE" });
|
|
13477
|
+
}
|
|
13478
|
+
if (mutability === "immutable") {
|
|
13479
|
+
fields.push({ name: "_is_deleted", type: "BOOL", mode: "NULLABLE" });
|
|
13480
|
+
fields.push({ name: "_version", type: "INT64", mode: "NULLABLE" });
|
|
13481
|
+
}
|
|
13474
13482
|
return fields;
|
|
13475
13483
|
}
|
|
13476
13484
|
async function getBigQueryTableSchema(bigqueryClient, datasetId, tableId) {
|
|
@@ -13492,7 +13500,7 @@ async function getBigQueryTableSchema(bigqueryClient, datasetId, tableId) {
|
|
|
13492
13500
|
}
|
|
13493
13501
|
return schema;
|
|
13494
13502
|
}
|
|
13495
|
-
function generateBigQuerySchemaUpdate(attributes, existingSchema) {
|
|
13503
|
+
function generateBigQuerySchemaUpdate(attributes, existingSchema, mutability = "append-only") {
|
|
13496
13504
|
const newFields = [];
|
|
13497
13505
|
for (const [fieldName, fieldConfig] of Object.entries(attributes)) {
|
|
13498
13506
|
if (fieldName === "id") continue;
|
|
@@ -13506,6 +13514,22 @@ function generateBigQuerySchemaUpdate(attributes, existingSchema) {
|
|
|
13506
13514
|
mode: required ? "REQUIRED" : "NULLABLE"
|
|
13507
13515
|
});
|
|
13508
13516
|
}
|
|
13517
|
+
if (mutability === "append-only" || mutability === "immutable") {
|
|
13518
|
+
if (!existingSchema["_operation_type"]) {
|
|
13519
|
+
newFields.push({ name: "_operation_type", type: "STRING", mode: "NULLABLE" });
|
|
13520
|
+
}
|
|
13521
|
+
if (!existingSchema["_operation_timestamp"]) {
|
|
13522
|
+
newFields.push({ name: "_operation_timestamp", type: "TIMESTAMP", mode: "NULLABLE" });
|
|
13523
|
+
}
|
|
13524
|
+
}
|
|
13525
|
+
if (mutability === "immutable") {
|
|
13526
|
+
if (!existingSchema["_is_deleted"]) {
|
|
13527
|
+
newFields.push({ name: "_is_deleted", type: "BOOL", mode: "NULLABLE" });
|
|
13528
|
+
}
|
|
13529
|
+
if (!existingSchema["_version"]) {
|
|
13530
|
+
newFields.push({ name: "_version", type: "INT64", mode: "NULLABLE" });
|
|
13531
|
+
}
|
|
13532
|
+
}
|
|
13509
13533
|
return newFields;
|
|
13510
13534
|
}
|
|
13511
13535
|
function s3dbTypeToSQLite(fieldType, fieldOptions = {}) {
|
|
@@ -13588,6 +13612,8 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
13588
13612
|
this.credentials = config.credentials;
|
|
13589
13613
|
this.location = config.location || "US";
|
|
13590
13614
|
this.logTable = config.logTable;
|
|
13615
|
+
this.mutability = config.mutability || "append-only";
|
|
13616
|
+
this._validateMutability(this.mutability);
|
|
13591
13617
|
this.schemaSync = {
|
|
13592
13618
|
enabled: config.schemaSync?.enabled || false,
|
|
13593
13619
|
strategy: config.schemaSync?.strategy || "alter",
|
|
@@ -13596,6 +13622,13 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
13596
13622
|
autoCreateColumns: config.schemaSync?.autoCreateColumns !== false
|
|
13597
13623
|
};
|
|
13598
13624
|
this.resources = this.parseResourcesConfig(resources);
|
|
13625
|
+
this.versionCounters = /* @__PURE__ */ new Map();
|
|
13626
|
+
}
|
|
13627
|
+
_validateMutability(mutability) {
|
|
13628
|
+
const validModes = ["append-only", "mutable", "immutable"];
|
|
13629
|
+
if (!validModes.includes(mutability)) {
|
|
13630
|
+
throw new Error(`Invalid mutability mode: ${mutability}. Must be one of: ${validModes.join(", ")}`);
|
|
13631
|
+
}
|
|
13599
13632
|
}
|
|
13600
13633
|
parseResourcesConfig(resources) {
|
|
13601
13634
|
const parsed = {};
|
|
@@ -13604,24 +13637,31 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
13604
13637
|
parsed[resourceName] = [{
|
|
13605
13638
|
table: config,
|
|
13606
13639
|
actions: ["insert"],
|
|
13607
|
-
transform: null
|
|
13640
|
+
transform: null,
|
|
13641
|
+
mutability: this.mutability
|
|
13608
13642
|
}];
|
|
13609
13643
|
} else if (Array.isArray(config)) {
|
|
13610
13644
|
parsed[resourceName] = config.map((item) => {
|
|
13611
13645
|
if (typeof item === "string") {
|
|
13612
|
-
return { table: item, actions: ["insert"], transform: null };
|
|
13646
|
+
return { table: item, actions: ["insert"], transform: null, mutability: this.mutability };
|
|
13613
13647
|
}
|
|
13648
|
+
const itemMutability = item.mutability || this.mutability;
|
|
13649
|
+
this._validateMutability(itemMutability);
|
|
13614
13650
|
return {
|
|
13615
13651
|
table: item.table,
|
|
13616
13652
|
actions: item.actions || ["insert"],
|
|
13617
|
-
transform: item.transform || null
|
|
13653
|
+
transform: item.transform || null,
|
|
13654
|
+
mutability: itemMutability
|
|
13618
13655
|
};
|
|
13619
13656
|
});
|
|
13620
13657
|
} else if (typeof config === "object") {
|
|
13658
|
+
const configMutability = config.mutability || this.mutability;
|
|
13659
|
+
this._validateMutability(configMutability);
|
|
13621
13660
|
parsed[resourceName] = [{
|
|
13622
13661
|
table: config.table,
|
|
13623
13662
|
actions: config.actions || ["insert"],
|
|
13624
|
-
transform: config.transform || null
|
|
13663
|
+
transform: config.transform || null,
|
|
13664
|
+
mutability: configMutability
|
|
13625
13665
|
}];
|
|
13626
13666
|
}
|
|
13627
13667
|
}
|
|
@@ -13700,8 +13740,9 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
13700
13740
|
);
|
|
13701
13741
|
for (const tableConfig of tableConfigs) {
|
|
13702
13742
|
const tableName = tableConfig.table;
|
|
13743
|
+
const mutability = tableConfig.mutability;
|
|
13703
13744
|
const [okSync, errSync] = await tryFn(async () => {
|
|
13704
|
-
await this.syncTableSchema(tableName, attributes);
|
|
13745
|
+
await this.syncTableSchema(tableName, attributes, mutability);
|
|
13705
13746
|
});
|
|
13706
13747
|
if (!okSync) {
|
|
13707
13748
|
const message = `Schema sync failed for table ${tableName}: ${errSync.message}`;
|
|
@@ -13721,7 +13762,7 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
13721
13762
|
/**
|
|
13722
13763
|
* Sync a single table schema in BigQuery
|
|
13723
13764
|
*/
|
|
13724
|
-
async syncTableSchema(tableName, attributes) {
|
|
13765
|
+
async syncTableSchema(tableName, attributes, mutability = "append-only") {
|
|
13725
13766
|
const dataset = this.bigqueryClient.dataset(this.datasetId);
|
|
13726
13767
|
const table = dataset.table(tableName);
|
|
13727
13768
|
const [exists] = await table.exists();
|
|
@@ -13732,15 +13773,16 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
13732
13773
|
if (this.schemaSync.strategy === "validate-only") {
|
|
13733
13774
|
throw new Error(`Table ${tableName} does not exist (validate-only mode)`);
|
|
13734
13775
|
}
|
|
13735
|
-
const schema = generateBigQuerySchema(attributes);
|
|
13776
|
+
const schema = generateBigQuerySchema(attributes, mutability);
|
|
13736
13777
|
if (this.config.verbose) {
|
|
13737
|
-
console.log(`[BigQueryReplicator] Creating table ${tableName} with schema:`, schema);
|
|
13778
|
+
console.log(`[BigQueryReplicator] Creating table ${tableName} with schema (mutability: ${mutability}):`, schema);
|
|
13738
13779
|
}
|
|
13739
13780
|
await dataset.createTable(tableName, { schema });
|
|
13740
13781
|
this.emit("table_created", {
|
|
13741
13782
|
replicator: this.name,
|
|
13742
13783
|
tableName,
|
|
13743
|
-
attributes: Object.keys(attributes)
|
|
13784
|
+
attributes: Object.keys(attributes),
|
|
13785
|
+
mutability
|
|
13744
13786
|
});
|
|
13745
13787
|
return;
|
|
13746
13788
|
}
|
|
@@ -13749,18 +13791,19 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
13749
13791
|
console.warn(`[BigQueryReplicator] Dropping and recreating table ${tableName}`);
|
|
13750
13792
|
}
|
|
13751
13793
|
await table.delete();
|
|
13752
|
-
const schema = generateBigQuerySchema(attributes);
|
|
13794
|
+
const schema = generateBigQuerySchema(attributes, mutability);
|
|
13753
13795
|
await dataset.createTable(tableName, { schema });
|
|
13754
13796
|
this.emit("table_recreated", {
|
|
13755
13797
|
replicator: this.name,
|
|
13756
13798
|
tableName,
|
|
13757
|
-
attributes: Object.keys(attributes)
|
|
13799
|
+
attributes: Object.keys(attributes),
|
|
13800
|
+
mutability
|
|
13758
13801
|
});
|
|
13759
13802
|
return;
|
|
13760
13803
|
}
|
|
13761
13804
|
if (this.schemaSync.strategy === "alter" && this.schemaSync.autoCreateColumns) {
|
|
13762
13805
|
const existingSchema = await getBigQueryTableSchema(this.bigqueryClient, this.datasetId, tableName);
|
|
13763
|
-
const newFields = generateBigQuerySchemaUpdate(attributes, existingSchema);
|
|
13806
|
+
const newFields = generateBigQuerySchemaUpdate(attributes, existingSchema, mutability);
|
|
13764
13807
|
if (newFields.length > 0) {
|
|
13765
13808
|
if (this.config.verbose) {
|
|
13766
13809
|
console.log(`[BigQueryReplicator] Adding ${newFields.length} field(s) to table ${tableName}:`, newFields);
|
|
@@ -13778,7 +13821,7 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
13778
13821
|
}
|
|
13779
13822
|
if (this.schemaSync.strategy === "validate-only") {
|
|
13780
13823
|
const existingSchema = await getBigQueryTableSchema(this.bigqueryClient, this.datasetId, tableName);
|
|
13781
|
-
const newFields = generateBigQuerySchemaUpdate(attributes, existingSchema);
|
|
13824
|
+
const newFields = generateBigQuerySchemaUpdate(attributes, existingSchema, mutability);
|
|
13782
13825
|
if (newFields.length > 0) {
|
|
13783
13826
|
throw new Error(`Table ${tableName} schema mismatch. Missing columns: ${newFields.length}`);
|
|
13784
13827
|
}
|
|
@@ -13797,7 +13840,8 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
13797
13840
|
if (!this.resources[resourceName]) return [];
|
|
13798
13841
|
return this.resources[resourceName].filter((tableConfig) => tableConfig.actions.includes(operation)).map((tableConfig) => ({
|
|
13799
13842
|
table: tableConfig.table,
|
|
13800
|
-
transform: tableConfig.transform
|
|
13843
|
+
transform: tableConfig.transform,
|
|
13844
|
+
mutability: tableConfig.mutability
|
|
13801
13845
|
}));
|
|
13802
13846
|
}
|
|
13803
13847
|
applyTransform(data, transformFn) {
|
|
@@ -13816,6 +13860,32 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
13816
13860
|
});
|
|
13817
13861
|
return cleanData;
|
|
13818
13862
|
}
|
|
13863
|
+
/**
|
|
13864
|
+
* Add tracking fields for append-only and immutable modes
|
|
13865
|
+
* @private
|
|
13866
|
+
*/
|
|
13867
|
+
_addTrackingFields(data, operation, mutability, id) {
|
|
13868
|
+
const tracked = { ...data };
|
|
13869
|
+
if (mutability === "append-only" || mutability === "immutable") {
|
|
13870
|
+
tracked._operation_type = operation;
|
|
13871
|
+
tracked._operation_timestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
13872
|
+
}
|
|
13873
|
+
if (mutability === "immutable") {
|
|
13874
|
+
tracked._is_deleted = operation === "delete";
|
|
13875
|
+
tracked._version = this._getNextVersion(id);
|
|
13876
|
+
}
|
|
13877
|
+
return tracked;
|
|
13878
|
+
}
|
|
13879
|
+
/**
|
|
13880
|
+
* Get next version number for immutable mode
|
|
13881
|
+
* @private
|
|
13882
|
+
*/
|
|
13883
|
+
_getNextVersion(id) {
|
|
13884
|
+
const current = this.versionCounters.get(id) || 0;
|
|
13885
|
+
const next = current + 1;
|
|
13886
|
+
this.versionCounters.set(id, next);
|
|
13887
|
+
return next;
|
|
13888
|
+
}
|
|
13819
13889
|
async replicate(resourceName, operation, data, id, beforeData = null) {
|
|
13820
13890
|
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
13821
13891
|
return { skipped: true, reason: "resource_not_included" };
|
|
@@ -13834,9 +13904,14 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
13834
13904
|
for (const tableConfig of tableConfigs) {
|
|
13835
13905
|
const [okTable, errTable] = await tryFn(async () => {
|
|
13836
13906
|
const table = dataset.table(tableConfig.table);
|
|
13907
|
+
const mutability = tableConfig.mutability;
|
|
13837
13908
|
let job;
|
|
13838
|
-
|
|
13839
|
-
|
|
13909
|
+
const shouldConvertToInsert = (mutability === "append-only" || mutability === "immutable") && (operation === "update" || operation === "delete");
|
|
13910
|
+
if (operation === "insert" || shouldConvertToInsert) {
|
|
13911
|
+
let transformedData = this.applyTransform(data, tableConfig.transform);
|
|
13912
|
+
if (shouldConvertToInsert) {
|
|
13913
|
+
transformedData = this._addTrackingFields(transformedData, operation, mutability, id);
|
|
13914
|
+
}
|
|
13840
13915
|
try {
|
|
13841
13916
|
job = await table.insert([transformedData]);
|
|
13842
13917
|
} catch (error) {
|
|
@@ -13848,7 +13923,7 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
13848
13923
|
}
|
|
13849
13924
|
throw error;
|
|
13850
13925
|
}
|
|
13851
|
-
} else if (operation === "update") {
|
|
13926
|
+
} else if (operation === "update" && mutability === "mutable") {
|
|
13852
13927
|
const transformedData = this.applyTransform(data, tableConfig.transform);
|
|
13853
13928
|
const keys = Object.keys(transformedData).filter((k) => k !== "id");
|
|
13854
13929
|
const setClause = keys.map((k) => `${k} = @${k}`).join(", ");
|
|
@@ -13890,7 +13965,7 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
13890
13965
|
}
|
|
13891
13966
|
}
|
|
13892
13967
|
if (!job) throw lastError;
|
|
13893
|
-
} else if (operation === "delete") {
|
|
13968
|
+
} else if (operation === "delete" && mutability === "mutable") {
|
|
13894
13969
|
const query = `DELETE FROM \`${this.projectId}.${this.datasetId}.${tableConfig.table}\` WHERE id = @id`;
|
|
13895
13970
|
try {
|
|
13896
13971
|
const [deleteJob] = await this.bigqueryClient.createQueryJob({
|
|
@@ -14026,7 +14101,8 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
14026
14101
|
datasetId: this.datasetId,
|
|
14027
14102
|
resources: this.resources,
|
|
14028
14103
|
logTable: this.logTable,
|
|
14029
|
-
schemaSync: this.schemaSync
|
|
14104
|
+
schemaSync: this.schemaSync,
|
|
14105
|
+
mutability: this.mutability
|
|
14030
14106
|
};
|
|
14031
14107
|
}
|
|
14032
14108
|
}
|
|
@@ -15747,11 +15823,11 @@ class ConnectionString {
|
|
|
15747
15823
|
}
|
|
15748
15824
|
}
|
|
15749
15825
|
|
|
15750
|
-
class
|
|
15826
|
+
class S3Client extends EventEmitter {
|
|
15751
15827
|
constructor({
|
|
15752
15828
|
verbose = false,
|
|
15753
15829
|
id = null,
|
|
15754
|
-
AwsS3Client,
|
|
15830
|
+
AwsS3Client: AwsS3Client2,
|
|
15755
15831
|
connectionString,
|
|
15756
15832
|
parallelism = 10,
|
|
15757
15833
|
httpClientOptions = {}
|
|
@@ -15774,7 +15850,7 @@ class Client extends EventEmitter {
|
|
|
15774
15850
|
// 60 second timeout
|
|
15775
15851
|
...httpClientOptions
|
|
15776
15852
|
};
|
|
15777
|
-
this.client =
|
|
15853
|
+
this.client = AwsS3Client2 || this.createClient();
|
|
15778
15854
|
}
|
|
15779
15855
|
createClient() {
|
|
15780
15856
|
const httpAgent = new http.Agent(this.httpClientOptions);
|
|
@@ -21332,7 +21408,7 @@ class Database extends EventEmitter {
|
|
|
21332
21408
|
this.id = idGenerator(7);
|
|
21333
21409
|
this.version = "1";
|
|
21334
21410
|
this.s3dbVersion = (() => {
|
|
21335
|
-
const [ok, err, version] = tryFn(() => true ? "12.
|
|
21411
|
+
const [ok, err, version] = tryFn(() => true ? "12.4.0" : "latest");
|
|
21336
21412
|
return ok ? version : "latest";
|
|
21337
21413
|
})();
|
|
21338
21414
|
this._resourcesMap = {};
|
|
@@ -21388,7 +21464,7 @@ class Database extends EventEmitter {
|
|
|
21388
21464
|
connectionString = `s3://${encodeURIComponent(accessKeyId)}:${encodeURIComponent(secretAccessKey)}@${bucket || "s3db"}?${params.toString()}`;
|
|
21389
21465
|
}
|
|
21390
21466
|
}
|
|
21391
|
-
this.client = options.client || new
|
|
21467
|
+
this.client = options.client || new S3Client({
|
|
21392
21468
|
verbose: this.verbose,
|
|
21393
21469
|
parallelism: this.parallelism,
|
|
21394
21470
|
connectionString
|
|
@@ -26294,7 +26370,7 @@ class S3TfStateDriver extends TfStateDriver {
|
|
|
26294
26370
|
*/
|
|
26295
26371
|
async initialize() {
|
|
26296
26372
|
const { bucket, credentials, region } = this.connectionConfig;
|
|
26297
|
-
this.client = new
|
|
26373
|
+
this.client = new S3Client({
|
|
26298
26374
|
bucketName: bucket,
|
|
26299
26375
|
credentials,
|
|
26300
26376
|
region
|
|
@@ -37920,6 +37996,1070 @@ class VectorPlugin extends Plugin {
|
|
|
37920
37996
|
}
|
|
37921
37997
|
}
|
|
37922
37998
|
|
|
37999
|
+
class MemoryStorage {
|
|
38000
|
+
constructor(config = {}) {
|
|
38001
|
+
this.objects = /* @__PURE__ */ new Map();
|
|
38002
|
+
this.bucket = config.bucket || "s3db";
|
|
38003
|
+
this.enforceLimits = config.enforceLimits || false;
|
|
38004
|
+
this.metadataLimit = config.metadataLimit || 2048;
|
|
38005
|
+
this.maxObjectSize = config.maxObjectSize || 5 * 1024 * 1024 * 1024;
|
|
38006
|
+
this.persistPath = config.persistPath;
|
|
38007
|
+
this.autoPersist = config.autoPersist || false;
|
|
38008
|
+
this.verbose = config.verbose || false;
|
|
38009
|
+
}
|
|
38010
|
+
/**
|
|
38011
|
+
* Generate ETag (MD5 hash) for object body
|
|
38012
|
+
*/
|
|
38013
|
+
_generateETag(body) {
|
|
38014
|
+
const buffer = Buffer.isBuffer(body) ? body : Buffer.from(body || "");
|
|
38015
|
+
return crypto$1.createHash("md5").update(buffer).digest("hex");
|
|
38016
|
+
}
|
|
38017
|
+
/**
|
|
38018
|
+
* Calculate metadata size in bytes
|
|
38019
|
+
*/
|
|
38020
|
+
_calculateMetadataSize(metadata) {
|
|
38021
|
+
if (!metadata) return 0;
|
|
38022
|
+
let size = 0;
|
|
38023
|
+
for (const [key, value] of Object.entries(metadata)) {
|
|
38024
|
+
size += Buffer.byteLength(key, "utf8");
|
|
38025
|
+
size += Buffer.byteLength(String(value), "utf8");
|
|
38026
|
+
}
|
|
38027
|
+
return size;
|
|
38028
|
+
}
|
|
38029
|
+
/**
|
|
38030
|
+
* Validate limits if enforceLimits is enabled
|
|
38031
|
+
*/
|
|
38032
|
+
_validateLimits(body, metadata) {
|
|
38033
|
+
if (!this.enforceLimits) return;
|
|
38034
|
+
const metadataSize = this._calculateMetadataSize(metadata);
|
|
38035
|
+
if (metadataSize > this.metadataLimit) {
|
|
38036
|
+
throw new Error(
|
|
38037
|
+
`Metadata size (${metadataSize} bytes) exceeds limit of ${this.metadataLimit} bytes`
|
|
38038
|
+
);
|
|
38039
|
+
}
|
|
38040
|
+
const bodySize = Buffer.isBuffer(body) ? body.length : Buffer.byteLength(body || "", "utf8");
|
|
38041
|
+
if (bodySize > this.maxObjectSize) {
|
|
38042
|
+
throw new Error(
|
|
38043
|
+
`Object size (${bodySize} bytes) exceeds limit of ${this.maxObjectSize} bytes`
|
|
38044
|
+
);
|
|
38045
|
+
}
|
|
38046
|
+
}
|
|
38047
|
+
/**
|
|
38048
|
+
* Store an object
|
|
38049
|
+
*/
|
|
38050
|
+
async put(key, { body, metadata, contentType, contentEncoding, contentLength, ifMatch }) {
|
|
38051
|
+
this._validateLimits(body, metadata);
|
|
38052
|
+
if (ifMatch !== void 0) {
|
|
38053
|
+
const existing = this.objects.get(key);
|
|
38054
|
+
if (existing && existing.etag !== ifMatch) {
|
|
38055
|
+
throw new Error(`Precondition failed: ETag mismatch for key "${key}"`);
|
|
38056
|
+
}
|
|
38057
|
+
}
|
|
38058
|
+
const buffer = Buffer.isBuffer(body) ? body : Buffer.from(body || "");
|
|
38059
|
+
const etag = this._generateETag(buffer);
|
|
38060
|
+
const lastModified = (/* @__PURE__ */ new Date()).toISOString();
|
|
38061
|
+
const size = buffer.length;
|
|
38062
|
+
const objectData = {
|
|
38063
|
+
body: buffer,
|
|
38064
|
+
metadata: metadata || {},
|
|
38065
|
+
contentType: contentType || "application/octet-stream",
|
|
38066
|
+
etag,
|
|
38067
|
+
lastModified,
|
|
38068
|
+
size,
|
|
38069
|
+
contentEncoding,
|
|
38070
|
+
contentLength: contentLength || size
|
|
38071
|
+
};
|
|
38072
|
+
this.objects.set(key, objectData);
|
|
38073
|
+
if (this.verbose) {
|
|
38074
|
+
console.log(`[MemoryStorage] PUT ${key} (${size} bytes, etag: ${etag})`);
|
|
38075
|
+
}
|
|
38076
|
+
if (this.autoPersist && this.persistPath) {
|
|
38077
|
+
await this.saveToDisk();
|
|
38078
|
+
}
|
|
38079
|
+
return {
|
|
38080
|
+
ETag: etag,
|
|
38081
|
+
VersionId: null,
|
|
38082
|
+
// Memory storage doesn't support versioning
|
|
38083
|
+
ServerSideEncryption: null,
|
|
38084
|
+
Location: `/${this.bucket}/${key}`
|
|
38085
|
+
};
|
|
38086
|
+
}
|
|
38087
|
+
/**
|
|
38088
|
+
* Retrieve an object
|
|
38089
|
+
*/
|
|
38090
|
+
async get(key) {
|
|
38091
|
+
const obj = this.objects.get(key);
|
|
38092
|
+
if (!obj) {
|
|
38093
|
+
const error = new Error(`Object not found: ${key}`);
|
|
38094
|
+
error.name = "NoSuchKey";
|
|
38095
|
+
error.$metadata = {
|
|
38096
|
+
httpStatusCode: 404,
|
|
38097
|
+
requestId: "memory-" + Date.now(),
|
|
38098
|
+
attempts: 1,
|
|
38099
|
+
totalRetryDelay: 0
|
|
38100
|
+
};
|
|
38101
|
+
throw error;
|
|
38102
|
+
}
|
|
38103
|
+
if (this.verbose) {
|
|
38104
|
+
console.log(`[MemoryStorage] GET ${key} (${obj.size} bytes)`);
|
|
38105
|
+
}
|
|
38106
|
+
const bodyStream = stream$1.Readable.from(obj.body);
|
|
38107
|
+
return {
|
|
38108
|
+
Body: bodyStream,
|
|
38109
|
+
Metadata: { ...obj.metadata },
|
|
38110
|
+
ContentType: obj.contentType,
|
|
38111
|
+
ContentLength: obj.size,
|
|
38112
|
+
ETag: obj.etag,
|
|
38113
|
+
LastModified: new Date(obj.lastModified),
|
|
38114
|
+
ContentEncoding: obj.contentEncoding
|
|
38115
|
+
};
|
|
38116
|
+
}
|
|
38117
|
+
/**
|
|
38118
|
+
* Get object metadata only (like S3 HeadObject)
|
|
38119
|
+
*/
|
|
38120
|
+
async head(key) {
|
|
38121
|
+
const obj = this.objects.get(key);
|
|
38122
|
+
if (!obj) {
|
|
38123
|
+
const error = new Error(`Object not found: ${key}`);
|
|
38124
|
+
error.name = "NoSuchKey";
|
|
38125
|
+
error.$metadata = {
|
|
38126
|
+
httpStatusCode: 404,
|
|
38127
|
+
requestId: "memory-" + Date.now(),
|
|
38128
|
+
attempts: 1,
|
|
38129
|
+
totalRetryDelay: 0
|
|
38130
|
+
};
|
|
38131
|
+
throw error;
|
|
38132
|
+
}
|
|
38133
|
+
if (this.verbose) {
|
|
38134
|
+
console.log(`[MemoryStorage] HEAD ${key}`);
|
|
38135
|
+
}
|
|
38136
|
+
return {
|
|
38137
|
+
Metadata: { ...obj.metadata },
|
|
38138
|
+
ContentType: obj.contentType,
|
|
38139
|
+
ContentLength: obj.size,
|
|
38140
|
+
ETag: obj.etag,
|
|
38141
|
+
LastModified: new Date(obj.lastModified),
|
|
38142
|
+
ContentEncoding: obj.contentEncoding
|
|
38143
|
+
};
|
|
38144
|
+
}
|
|
38145
|
+
/**
|
|
38146
|
+
* Copy an object
|
|
38147
|
+
*/
|
|
38148
|
+
async copy(from, to, { metadata, metadataDirective, contentType }) {
|
|
38149
|
+
const source = this.objects.get(from);
|
|
38150
|
+
if (!source) {
|
|
38151
|
+
const error = new Error(`Source object not found: ${from}`);
|
|
38152
|
+
error.name = "NoSuchKey";
|
|
38153
|
+
throw error;
|
|
38154
|
+
}
|
|
38155
|
+
let finalMetadata = { ...source.metadata };
|
|
38156
|
+
if (metadataDirective === "REPLACE" && metadata) {
|
|
38157
|
+
finalMetadata = metadata;
|
|
38158
|
+
} else if (metadata) {
|
|
38159
|
+
finalMetadata = { ...finalMetadata, ...metadata };
|
|
38160
|
+
}
|
|
38161
|
+
const result = await this.put(to, {
|
|
38162
|
+
body: source.body,
|
|
38163
|
+
metadata: finalMetadata,
|
|
38164
|
+
contentType: contentType || source.contentType,
|
|
38165
|
+
contentEncoding: source.contentEncoding
|
|
38166
|
+
});
|
|
38167
|
+
if (this.verbose) {
|
|
38168
|
+
console.log(`[MemoryStorage] COPY ${from} \u2192 ${to}`);
|
|
38169
|
+
}
|
|
38170
|
+
return result;
|
|
38171
|
+
}
|
|
38172
|
+
/**
|
|
38173
|
+
* Check if object exists
|
|
38174
|
+
*/
|
|
38175
|
+
exists(key) {
|
|
38176
|
+
return this.objects.has(key);
|
|
38177
|
+
}
|
|
38178
|
+
/**
|
|
38179
|
+
* Delete an object
|
|
38180
|
+
*/
|
|
38181
|
+
async delete(key) {
|
|
38182
|
+
const existed = this.objects.has(key);
|
|
38183
|
+
this.objects.delete(key);
|
|
38184
|
+
if (this.verbose) {
|
|
38185
|
+
console.log(`[MemoryStorage] DELETE ${key} (existed: ${existed})`);
|
|
38186
|
+
}
|
|
38187
|
+
if (this.autoPersist && this.persistPath) {
|
|
38188
|
+
await this.saveToDisk();
|
|
38189
|
+
}
|
|
38190
|
+
return {
|
|
38191
|
+
DeleteMarker: false,
|
|
38192
|
+
VersionId: null
|
|
38193
|
+
};
|
|
38194
|
+
}
|
|
38195
|
+
/**
|
|
38196
|
+
* Delete multiple objects (batch)
|
|
38197
|
+
*/
|
|
38198
|
+
async deleteMultiple(keys) {
|
|
38199
|
+
const deleted = [];
|
|
38200
|
+
const errors = [];
|
|
38201
|
+
for (const key of keys) {
|
|
38202
|
+
try {
|
|
38203
|
+
await this.delete(key);
|
|
38204
|
+
deleted.push({ Key: key });
|
|
38205
|
+
} catch (error) {
|
|
38206
|
+
errors.push({
|
|
38207
|
+
Key: key,
|
|
38208
|
+
Code: error.name || "InternalError",
|
|
38209
|
+
Message: error.message
|
|
38210
|
+
});
|
|
38211
|
+
}
|
|
38212
|
+
}
|
|
38213
|
+
if (this.verbose) {
|
|
38214
|
+
console.log(`[MemoryStorage] DELETE BATCH (${deleted.length} deleted, ${errors.length} errors)`);
|
|
38215
|
+
}
|
|
38216
|
+
return { Deleted: deleted, Errors: errors };
|
|
38217
|
+
}
|
|
38218
|
+
/**
|
|
38219
|
+
* List objects with prefix/delimiter support
|
|
38220
|
+
*/
|
|
38221
|
+
async list({ prefix = "", delimiter = null, maxKeys = 1e3, continuationToken = null }) {
|
|
38222
|
+
const allKeys = Array.from(this.objects.keys());
|
|
38223
|
+
let filteredKeys = prefix ? allKeys.filter((key) => key.startsWith(prefix)) : allKeys;
|
|
38224
|
+
filteredKeys.sort();
|
|
38225
|
+
let startIndex = 0;
|
|
38226
|
+
if (continuationToken) {
|
|
38227
|
+
startIndex = parseInt(continuationToken) || 0;
|
|
38228
|
+
}
|
|
38229
|
+
const paginatedKeys = filteredKeys.slice(startIndex, startIndex + maxKeys);
|
|
38230
|
+
const isTruncated = startIndex + maxKeys < filteredKeys.length;
|
|
38231
|
+
const nextContinuationToken = isTruncated ? String(startIndex + maxKeys) : null;
|
|
38232
|
+
const commonPrefixes = /* @__PURE__ */ new Set();
|
|
38233
|
+
const contents = [];
|
|
38234
|
+
for (const key of paginatedKeys) {
|
|
38235
|
+
if (delimiter && prefix) {
|
|
38236
|
+
const suffix = key.substring(prefix.length);
|
|
38237
|
+
const delimiterIndex = suffix.indexOf(delimiter);
|
|
38238
|
+
if (delimiterIndex !== -1) {
|
|
38239
|
+
const commonPrefix = prefix + suffix.substring(0, delimiterIndex + 1);
|
|
38240
|
+
commonPrefixes.add(commonPrefix);
|
|
38241
|
+
continue;
|
|
38242
|
+
}
|
|
38243
|
+
}
|
|
38244
|
+
const obj = this.objects.get(key);
|
|
38245
|
+
contents.push({
|
|
38246
|
+
Key: key,
|
|
38247
|
+
Size: obj.size,
|
|
38248
|
+
LastModified: new Date(obj.lastModified),
|
|
38249
|
+
ETag: obj.etag,
|
|
38250
|
+
StorageClass: "STANDARD"
|
|
38251
|
+
});
|
|
38252
|
+
}
|
|
38253
|
+
if (this.verbose) {
|
|
38254
|
+
console.log(`[MemoryStorage] LIST prefix="${prefix}" (${contents.length} objects, ${commonPrefixes.size} prefixes)`);
|
|
38255
|
+
}
|
|
38256
|
+
return {
|
|
38257
|
+
Contents: contents,
|
|
38258
|
+
CommonPrefixes: Array.from(commonPrefixes).map((prefix2) => ({ Prefix: prefix2 })),
|
|
38259
|
+
IsTruncated: isTruncated,
|
|
38260
|
+
NextContinuationToken: nextContinuationToken,
|
|
38261
|
+
KeyCount: contents.length + commonPrefixes.size,
|
|
38262
|
+
MaxKeys: maxKeys,
|
|
38263
|
+
Prefix: prefix,
|
|
38264
|
+
Delimiter: delimiter
|
|
38265
|
+
};
|
|
38266
|
+
}
|
|
38267
|
+
/**
|
|
38268
|
+
* Create a snapshot of current state
|
|
38269
|
+
*/
|
|
38270
|
+
snapshot() {
|
|
38271
|
+
const snapshot = {
|
|
38272
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
38273
|
+
bucket: this.bucket,
|
|
38274
|
+
objectCount: this.objects.size,
|
|
38275
|
+
objects: {}
|
|
38276
|
+
};
|
|
38277
|
+
for (const [key, obj] of this.objects.entries()) {
|
|
38278
|
+
snapshot.objects[key] = {
|
|
38279
|
+
body: obj.body.toString("base64"),
|
|
38280
|
+
metadata: obj.metadata,
|
|
38281
|
+
contentType: obj.contentType,
|
|
38282
|
+
etag: obj.etag,
|
|
38283
|
+
lastModified: obj.lastModified,
|
|
38284
|
+
size: obj.size,
|
|
38285
|
+
contentEncoding: obj.contentEncoding,
|
|
38286
|
+
contentLength: obj.contentLength
|
|
38287
|
+
};
|
|
38288
|
+
}
|
|
38289
|
+
return snapshot;
|
|
38290
|
+
}
|
|
38291
|
+
/**
|
|
38292
|
+
* Restore from a snapshot
|
|
38293
|
+
*/
|
|
38294
|
+
restore(snapshot) {
|
|
38295
|
+
if (!snapshot || !snapshot.objects) {
|
|
38296
|
+
throw new Error("Invalid snapshot format");
|
|
38297
|
+
}
|
|
38298
|
+
this.objects.clear();
|
|
38299
|
+
for (const [key, obj] of Object.entries(snapshot.objects)) {
|
|
38300
|
+
this.objects.set(key, {
|
|
38301
|
+
body: Buffer.from(obj.body, "base64"),
|
|
38302
|
+
metadata: obj.metadata,
|
|
38303
|
+
contentType: obj.contentType,
|
|
38304
|
+
etag: obj.etag,
|
|
38305
|
+
lastModified: obj.lastModified,
|
|
38306
|
+
size: obj.size,
|
|
38307
|
+
contentEncoding: obj.contentEncoding,
|
|
38308
|
+
contentLength: obj.contentLength
|
|
38309
|
+
});
|
|
38310
|
+
}
|
|
38311
|
+
if (this.verbose) {
|
|
38312
|
+
console.log(`[MemoryStorage] Restored snapshot with ${this.objects.size} objects`);
|
|
38313
|
+
}
|
|
38314
|
+
}
|
|
38315
|
+
/**
|
|
38316
|
+
* Save current state to disk
|
|
38317
|
+
*/
|
|
38318
|
+
async saveToDisk(customPath) {
|
|
38319
|
+
const path = customPath || this.persistPath;
|
|
38320
|
+
if (!path) {
|
|
38321
|
+
throw new Error("No persist path configured");
|
|
38322
|
+
}
|
|
38323
|
+
const snapshot = this.snapshot();
|
|
38324
|
+
const json = JSON.stringify(snapshot, null, 2);
|
|
38325
|
+
const [ok, err] = await tryFn(() => promises.writeFile(path, json, "utf-8"));
|
|
38326
|
+
if (!ok) {
|
|
38327
|
+
throw new Error(`Failed to save to disk: ${err.message}`);
|
|
38328
|
+
}
|
|
38329
|
+
if (this.verbose) {
|
|
38330
|
+
console.log(`[MemoryStorage] Saved ${this.objects.size} objects to ${path}`);
|
|
38331
|
+
}
|
|
38332
|
+
return path;
|
|
38333
|
+
}
|
|
38334
|
+
/**
|
|
38335
|
+
* Load state from disk
|
|
38336
|
+
*/
|
|
38337
|
+
async loadFromDisk(customPath) {
|
|
38338
|
+
const path = customPath || this.persistPath;
|
|
38339
|
+
if (!path) {
|
|
38340
|
+
throw new Error("No persist path configured");
|
|
38341
|
+
}
|
|
38342
|
+
const [ok, err, json] = await tryFn(() => promises.readFile(path, "utf-8"));
|
|
38343
|
+
if (!ok) {
|
|
38344
|
+
throw new Error(`Failed to load from disk: ${err.message}`);
|
|
38345
|
+
}
|
|
38346
|
+
const snapshot = JSON.parse(json);
|
|
38347
|
+
this.restore(snapshot);
|
|
38348
|
+
if (this.verbose) {
|
|
38349
|
+
console.log(`[MemoryStorage] Loaded ${this.objects.size} objects from ${path}`);
|
|
38350
|
+
}
|
|
38351
|
+
return snapshot;
|
|
38352
|
+
}
|
|
38353
|
+
/**
|
|
38354
|
+
* Get storage statistics
|
|
38355
|
+
*/
|
|
38356
|
+
getStats() {
|
|
38357
|
+
let totalSize = 0;
|
|
38358
|
+
const keys = [];
|
|
38359
|
+
for (const [key, obj] of this.objects.entries()) {
|
|
38360
|
+
totalSize += obj.size;
|
|
38361
|
+
keys.push(key);
|
|
38362
|
+
}
|
|
38363
|
+
return {
|
|
38364
|
+
objectCount: this.objects.size,
|
|
38365
|
+
totalSize,
|
|
38366
|
+
totalSizeFormatted: this._formatBytes(totalSize),
|
|
38367
|
+
keys: keys.sort(),
|
|
38368
|
+
bucket: this.bucket
|
|
38369
|
+
};
|
|
38370
|
+
}
|
|
38371
|
+
/**
|
|
38372
|
+
* Format bytes for human reading
|
|
38373
|
+
*/
|
|
38374
|
+
_formatBytes(bytes) {
|
|
38375
|
+
if (bytes === 0) return "0 Bytes";
|
|
38376
|
+
const k = 1024;
|
|
38377
|
+
const sizes = ["Bytes", "KB", "MB", "GB"];
|
|
38378
|
+
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
|
38379
|
+
return Math.round(bytes / Math.pow(k, i) * 100) / 100 + " " + sizes[i];
|
|
38380
|
+
}
|
|
38381
|
+
/**
|
|
38382
|
+
* Clear all objects
|
|
38383
|
+
*/
|
|
38384
|
+
clear() {
|
|
38385
|
+
this.objects.clear();
|
|
38386
|
+
if (this.verbose) {
|
|
38387
|
+
console.log(`[MemoryStorage] Cleared all objects`);
|
|
38388
|
+
}
|
|
38389
|
+
}
|
|
38390
|
+
}
|
|
38391
|
+
|
|
38392
|
+
class MemoryClient extends EventEmitter {
|
|
38393
|
+
constructor(config = {}) {
|
|
38394
|
+
super();
|
|
38395
|
+
this.id = config.id || idGenerator(77);
|
|
38396
|
+
this.verbose = config.verbose || false;
|
|
38397
|
+
this.parallelism = config.parallelism || 10;
|
|
38398
|
+
this.bucket = config.bucket || "s3db";
|
|
38399
|
+
this.keyPrefix = config.keyPrefix || "";
|
|
38400
|
+
this.region = config.region || "us-east-1";
|
|
38401
|
+
this.storage = new MemoryStorage({
|
|
38402
|
+
bucket: this.bucket,
|
|
38403
|
+
enforceLimits: config.enforceLimits || false,
|
|
38404
|
+
metadataLimit: config.metadataLimit || 2048,
|
|
38405
|
+
maxObjectSize: config.maxObjectSize || 5 * 1024 * 1024 * 1024,
|
|
38406
|
+
persistPath: config.persistPath,
|
|
38407
|
+
autoPersist: config.autoPersist || false,
|
|
38408
|
+
verbose: this.verbose
|
|
38409
|
+
});
|
|
38410
|
+
this.config = {
|
|
38411
|
+
bucket: this.bucket,
|
|
38412
|
+
keyPrefix: this.keyPrefix,
|
|
38413
|
+
region: this.region,
|
|
38414
|
+
endpoint: "memory://localhost",
|
|
38415
|
+
forcePathStyle: true
|
|
38416
|
+
};
|
|
38417
|
+
if (this.verbose) {
|
|
38418
|
+
console.log(`[MemoryClient] Initialized (id: ${this.id}, bucket: ${this.bucket})`);
|
|
38419
|
+
}
|
|
38420
|
+
}
|
|
38421
|
+
/**
|
|
38422
|
+
* Simulate sendCommand from AWS SDK
|
|
38423
|
+
* Used by Database/Resource to send AWS SDK commands
|
|
38424
|
+
*/
|
|
38425
|
+
async sendCommand(command) {
|
|
38426
|
+
const commandName = command.constructor.name;
|
|
38427
|
+
const input = command.input || {};
|
|
38428
|
+
this.emit("command.request", commandName, input);
|
|
38429
|
+
let response;
|
|
38430
|
+
try {
|
|
38431
|
+
switch (commandName) {
|
|
38432
|
+
case "PutObjectCommand":
|
|
38433
|
+
response = await this._handlePutObject(input);
|
|
38434
|
+
break;
|
|
38435
|
+
case "GetObjectCommand":
|
|
38436
|
+
response = await this._handleGetObject(input);
|
|
38437
|
+
break;
|
|
38438
|
+
case "HeadObjectCommand":
|
|
38439
|
+
response = await this._handleHeadObject(input);
|
|
38440
|
+
break;
|
|
38441
|
+
case "CopyObjectCommand":
|
|
38442
|
+
response = await this._handleCopyObject(input);
|
|
38443
|
+
break;
|
|
38444
|
+
case "DeleteObjectCommand":
|
|
38445
|
+
response = await this._handleDeleteObject(input);
|
|
38446
|
+
break;
|
|
38447
|
+
case "DeleteObjectsCommand":
|
|
38448
|
+
response = await this._handleDeleteObjects(input);
|
|
38449
|
+
break;
|
|
38450
|
+
case "ListObjectsV2Command":
|
|
38451
|
+
response = await this._handleListObjects(input);
|
|
38452
|
+
break;
|
|
38453
|
+
default:
|
|
38454
|
+
throw new Error(`Unsupported command: ${commandName}`);
|
|
38455
|
+
}
|
|
38456
|
+
this.emit("command.response", commandName, response, input);
|
|
38457
|
+
return response;
|
|
38458
|
+
} catch (error) {
|
|
38459
|
+
const mappedError = mapAwsError(error, {
|
|
38460
|
+
bucket: this.bucket,
|
|
38461
|
+
key: input.Key,
|
|
38462
|
+
commandName,
|
|
38463
|
+
commandInput: input
|
|
38464
|
+
});
|
|
38465
|
+
throw mappedError;
|
|
38466
|
+
}
|
|
38467
|
+
}
|
|
38468
|
+
/**
|
|
38469
|
+
* PutObjectCommand handler
|
|
38470
|
+
*/
|
|
38471
|
+
async _handlePutObject(input) {
|
|
38472
|
+
const key = input.Key;
|
|
38473
|
+
const metadata = input.Metadata || {};
|
|
38474
|
+
const contentType = input.ContentType;
|
|
38475
|
+
const body = input.Body;
|
|
38476
|
+
const contentEncoding = input.ContentEncoding;
|
|
38477
|
+
const contentLength = input.ContentLength;
|
|
38478
|
+
const ifMatch = input.IfMatch;
|
|
38479
|
+
return await this.storage.put(key, {
|
|
38480
|
+
body,
|
|
38481
|
+
metadata,
|
|
38482
|
+
contentType,
|
|
38483
|
+
contentEncoding,
|
|
38484
|
+
contentLength,
|
|
38485
|
+
ifMatch
|
|
38486
|
+
});
|
|
38487
|
+
}
|
|
38488
|
+
/**
|
|
38489
|
+
* GetObjectCommand handler
|
|
38490
|
+
*/
|
|
38491
|
+
async _handleGetObject(input) {
|
|
38492
|
+
const key = input.Key;
|
|
38493
|
+
return await this.storage.get(key);
|
|
38494
|
+
}
|
|
38495
|
+
/**
|
|
38496
|
+
* HeadObjectCommand handler
|
|
38497
|
+
*/
|
|
38498
|
+
async _handleHeadObject(input) {
|
|
38499
|
+
const key = input.Key;
|
|
38500
|
+
return await this.storage.head(key);
|
|
38501
|
+
}
|
|
38502
|
+
/**
|
|
38503
|
+
* CopyObjectCommand handler
|
|
38504
|
+
*/
|
|
38505
|
+
async _handleCopyObject(input) {
|
|
38506
|
+
const copySource = input.CopySource;
|
|
38507
|
+
const parts = copySource.split("/");
|
|
38508
|
+
const sourceKey = parts.slice(1).join("/");
|
|
38509
|
+
const destinationKey = input.Key;
|
|
38510
|
+
const metadata = input.Metadata;
|
|
38511
|
+
const metadataDirective = input.MetadataDirective;
|
|
38512
|
+
const contentType = input.ContentType;
|
|
38513
|
+
return await this.storage.copy(sourceKey, destinationKey, {
|
|
38514
|
+
metadata,
|
|
38515
|
+
metadataDirective,
|
|
38516
|
+
contentType
|
|
38517
|
+
});
|
|
38518
|
+
}
|
|
38519
|
+
/**
|
|
38520
|
+
* DeleteObjectCommand handler
|
|
38521
|
+
*/
|
|
38522
|
+
async _handleDeleteObject(input) {
|
|
38523
|
+
const key = input.Key;
|
|
38524
|
+
return await this.storage.delete(key);
|
|
38525
|
+
}
|
|
38526
|
+
/**
|
|
38527
|
+
* DeleteObjectsCommand handler
|
|
38528
|
+
*/
|
|
38529
|
+
async _handleDeleteObjects(input) {
|
|
38530
|
+
const objects = input.Delete?.Objects || [];
|
|
38531
|
+
const keys = objects.map((obj) => obj.Key);
|
|
38532
|
+
return await this.storage.deleteMultiple(keys);
|
|
38533
|
+
}
|
|
38534
|
+
/**
|
|
38535
|
+
* ListObjectsV2Command handler
|
|
38536
|
+
*/
|
|
38537
|
+
async _handleListObjects(input) {
|
|
38538
|
+
const fullPrefix = this.keyPrefix && input.Prefix ? path$1.join(this.keyPrefix, input.Prefix) : this.keyPrefix || input.Prefix || "";
|
|
38539
|
+
return await this.storage.list({
|
|
38540
|
+
prefix: fullPrefix,
|
|
38541
|
+
delimiter: input.Delimiter,
|
|
38542
|
+
maxKeys: input.MaxKeys,
|
|
38543
|
+
continuationToken: input.ContinuationToken
|
|
38544
|
+
});
|
|
38545
|
+
}
|
|
38546
|
+
/**
|
|
38547
|
+
* Put an object (Client interface method)
|
|
38548
|
+
*/
|
|
38549
|
+
async putObject({ key, metadata, contentType, body, contentEncoding, contentLength, ifMatch }) {
|
|
38550
|
+
const fullKey = this.keyPrefix ? path$1.join(this.keyPrefix, key) : key;
|
|
38551
|
+
const stringMetadata = {};
|
|
38552
|
+
if (metadata) {
|
|
38553
|
+
for (const [k, v] of Object.entries(metadata)) {
|
|
38554
|
+
const validKey = String(k).replace(/[^a-zA-Z0-9\-_]/g, "_");
|
|
38555
|
+
const { encoded } = metadataEncode(v);
|
|
38556
|
+
stringMetadata[validKey] = encoded;
|
|
38557
|
+
}
|
|
38558
|
+
}
|
|
38559
|
+
const response = await this.storage.put(fullKey, {
|
|
38560
|
+
body,
|
|
38561
|
+
metadata: stringMetadata,
|
|
38562
|
+
contentType,
|
|
38563
|
+
contentEncoding,
|
|
38564
|
+
contentLength,
|
|
38565
|
+
ifMatch
|
|
38566
|
+
});
|
|
38567
|
+
this.emit("putObject", null, { key, metadata, contentType, body, contentEncoding, contentLength });
|
|
38568
|
+
return response;
|
|
38569
|
+
}
|
|
38570
|
+
/**
|
|
38571
|
+
* Get an object (Client interface method)
|
|
38572
|
+
*/
|
|
38573
|
+
async getObject(key) {
|
|
38574
|
+
const fullKey = this.keyPrefix ? path$1.join(this.keyPrefix, key) : key;
|
|
38575
|
+
const response = await this.storage.get(fullKey);
|
|
38576
|
+
const decodedMetadata = {};
|
|
38577
|
+
if (response.Metadata) {
|
|
38578
|
+
for (const [k, v] of Object.entries(response.Metadata)) {
|
|
38579
|
+
decodedMetadata[k] = metadataDecode(v);
|
|
38580
|
+
}
|
|
38581
|
+
}
|
|
38582
|
+
this.emit("getObject", null, { key });
|
|
38583
|
+
return {
|
|
38584
|
+
...response,
|
|
38585
|
+
Metadata: decodedMetadata
|
|
38586
|
+
};
|
|
38587
|
+
}
|
|
38588
|
+
/**
|
|
38589
|
+
* Head object (get metadata only)
|
|
38590
|
+
*/
|
|
38591
|
+
async headObject(key) {
|
|
38592
|
+
const fullKey = this.keyPrefix ? path$1.join(this.keyPrefix, key) : key;
|
|
38593
|
+
const response = await this.storage.head(fullKey);
|
|
38594
|
+
const decodedMetadata = {};
|
|
38595
|
+
if (response.Metadata) {
|
|
38596
|
+
for (const [k, v] of Object.entries(response.Metadata)) {
|
|
38597
|
+
decodedMetadata[k] = metadataDecode(v);
|
|
38598
|
+
}
|
|
38599
|
+
}
|
|
38600
|
+
this.emit("headObject", null, { key });
|
|
38601
|
+
return {
|
|
38602
|
+
...response,
|
|
38603
|
+
Metadata: decodedMetadata
|
|
38604
|
+
};
|
|
38605
|
+
}
|
|
38606
|
+
/**
|
|
38607
|
+
* Copy an object
|
|
38608
|
+
*/
|
|
38609
|
+
async copyObject({ from, to, metadata, metadataDirective, contentType }) {
|
|
38610
|
+
const fullFrom = this.keyPrefix ? path$1.join(this.keyPrefix, from) : from;
|
|
38611
|
+
const fullTo = this.keyPrefix ? path$1.join(this.keyPrefix, to) : to;
|
|
38612
|
+
const encodedMetadata = {};
|
|
38613
|
+
if (metadata) {
|
|
38614
|
+
for (const [k, v] of Object.entries(metadata)) {
|
|
38615
|
+
const validKey = String(k).replace(/[^a-zA-Z0-9\-_]/g, "_");
|
|
38616
|
+
const { encoded } = metadataEncode(v);
|
|
38617
|
+
encodedMetadata[validKey] = encoded;
|
|
38618
|
+
}
|
|
38619
|
+
}
|
|
38620
|
+
const response = await this.storage.copy(fullFrom, fullTo, {
|
|
38621
|
+
metadata: encodedMetadata,
|
|
38622
|
+
metadataDirective,
|
|
38623
|
+
contentType
|
|
38624
|
+
});
|
|
38625
|
+
this.emit("copyObject", null, { from, to, metadata, metadataDirective });
|
|
38626
|
+
return response;
|
|
38627
|
+
}
|
|
38628
|
+
/**
|
|
38629
|
+
* Check if object exists
|
|
38630
|
+
*/
|
|
38631
|
+
async exists(key) {
|
|
38632
|
+
const fullKey = this.keyPrefix ? path$1.join(this.keyPrefix, key) : key;
|
|
38633
|
+
return this.storage.exists(fullKey);
|
|
38634
|
+
}
|
|
38635
|
+
/**
|
|
38636
|
+
* Delete an object
|
|
38637
|
+
*/
|
|
38638
|
+
async deleteObject(key) {
|
|
38639
|
+
const fullKey = this.keyPrefix ? path$1.join(this.keyPrefix, key) : key;
|
|
38640
|
+
const response = await this.storage.delete(fullKey);
|
|
38641
|
+
this.emit("deleteObject", null, { key });
|
|
38642
|
+
return response;
|
|
38643
|
+
}
|
|
38644
|
+
/**
|
|
38645
|
+
* Delete multiple objects (batch)
|
|
38646
|
+
*/
|
|
38647
|
+
async deleteObjects(keys) {
|
|
38648
|
+
const fullKeys = keys.map(
|
|
38649
|
+
(key) => this.keyPrefix ? path$1.join(this.keyPrefix, key) : key
|
|
38650
|
+
);
|
|
38651
|
+
const batches = lodashEs.chunk(fullKeys, this.parallelism);
|
|
38652
|
+
const allResults = { Deleted: [], Errors: [] };
|
|
38653
|
+
const { results } = await promisePool.PromisePool.withConcurrency(this.parallelism).for(batches).process(async (batch) => {
|
|
38654
|
+
return await this.storage.deleteMultiple(batch);
|
|
38655
|
+
});
|
|
38656
|
+
for (const result of results) {
|
|
38657
|
+
allResults.Deleted.push(...result.Deleted);
|
|
38658
|
+
allResults.Errors.push(...result.Errors);
|
|
38659
|
+
}
|
|
38660
|
+
this.emit("deleteObjects", null, { keys, count: allResults.Deleted.length });
|
|
38661
|
+
return allResults;
|
|
38662
|
+
}
|
|
38663
|
+
/**
|
|
38664
|
+
* List objects with pagination support
|
|
38665
|
+
*/
|
|
38666
|
+
async listObjects({ prefix = "", delimiter = null, maxKeys = 1e3, continuationToken = null }) {
|
|
38667
|
+
const fullPrefix = this.keyPrefix ? path$1.join(this.keyPrefix, prefix) : prefix;
|
|
38668
|
+
const response = await this.storage.list({
|
|
38669
|
+
prefix: fullPrefix,
|
|
38670
|
+
delimiter,
|
|
38671
|
+
maxKeys,
|
|
38672
|
+
continuationToken
|
|
38673
|
+
});
|
|
38674
|
+
this.emit("listObjects", null, { prefix, count: response.Contents.length });
|
|
38675
|
+
return response;
|
|
38676
|
+
}
|
|
38677
|
+
/**
|
|
38678
|
+
* Get a page of keys with offset/limit pagination
|
|
38679
|
+
*/
|
|
38680
|
+
async getKeysPage(params = {}) {
|
|
38681
|
+
const { prefix = "", offset = 0, amount = 100 } = params;
|
|
38682
|
+
let keys = [];
|
|
38683
|
+
let truncated = true;
|
|
38684
|
+
let continuationToken;
|
|
38685
|
+
if (offset > 0) {
|
|
38686
|
+
const fullPrefix = this.keyPrefix ? path$1.join(this.keyPrefix, prefix) : prefix;
|
|
38687
|
+
const response = await this.storage.list({
|
|
38688
|
+
prefix: fullPrefix,
|
|
38689
|
+
maxKeys: offset + amount
|
|
38690
|
+
});
|
|
38691
|
+
keys = response.Contents.map((x) => x.Key).slice(offset, offset + amount);
|
|
38692
|
+
} else {
|
|
38693
|
+
while (truncated) {
|
|
38694
|
+
const options = {
|
|
38695
|
+
prefix,
|
|
38696
|
+
continuationToken,
|
|
38697
|
+
maxKeys: amount - keys.length
|
|
38698
|
+
};
|
|
38699
|
+
const res = await this.listObjects(options);
|
|
38700
|
+
if (res.Contents) {
|
|
38701
|
+
keys = keys.concat(res.Contents.map((x) => x.Key));
|
|
38702
|
+
}
|
|
38703
|
+
truncated = res.IsTruncated || false;
|
|
38704
|
+
continuationToken = res.NextContinuationToken;
|
|
38705
|
+
if (keys.length >= amount) {
|
|
38706
|
+
keys = keys.slice(0, amount);
|
|
38707
|
+
break;
|
|
38708
|
+
}
|
|
38709
|
+
}
|
|
38710
|
+
}
|
|
38711
|
+
if (this.keyPrefix) {
|
|
38712
|
+
keys = keys.map((x) => x.replace(this.keyPrefix, "")).map((x) => x.startsWith("/") ? x.replace("/", "") : x);
|
|
38713
|
+
}
|
|
38714
|
+
this.emit("getKeysPage", keys, params);
|
|
38715
|
+
return keys;
|
|
38716
|
+
}
|
|
38717
|
+
/**
|
|
38718
|
+
* Get all keys with a given prefix
|
|
38719
|
+
*/
|
|
38720
|
+
async getAllKeys({ prefix = "" }) {
|
|
38721
|
+
const fullPrefix = this.keyPrefix ? path$1.join(this.keyPrefix, prefix) : prefix;
|
|
38722
|
+
const response = await this.storage.list({
|
|
38723
|
+
prefix: fullPrefix,
|
|
38724
|
+
maxKeys: 1e5
|
|
38725
|
+
// Large number to get all
|
|
38726
|
+
});
|
|
38727
|
+
let keys = response.Contents.map((x) => x.Key);
|
|
38728
|
+
if (this.keyPrefix) {
|
|
38729
|
+
keys = keys.map((x) => x.replace(this.keyPrefix, "")).map((x) => x.startsWith("/") ? x.replace("/", "") : x);
|
|
38730
|
+
}
|
|
38731
|
+
this.emit("getAllKeys", keys, { prefix });
|
|
38732
|
+
return keys;
|
|
38733
|
+
}
|
|
38734
|
+
/**
|
|
38735
|
+
* Count total objects under a prefix
|
|
38736
|
+
*/
|
|
38737
|
+
async count({ prefix = "" } = {}) {
|
|
38738
|
+
const keys = await this.getAllKeys({ prefix });
|
|
38739
|
+
const count = keys.length;
|
|
38740
|
+
this.emit("count", count, { prefix });
|
|
38741
|
+
return count;
|
|
38742
|
+
}
|
|
38743
|
+
/**
|
|
38744
|
+
* Delete all objects under a prefix
|
|
38745
|
+
*/
|
|
38746
|
+
async deleteAll({ prefix = "" } = {}) {
|
|
38747
|
+
const keys = await this.getAllKeys({ prefix });
|
|
38748
|
+
let totalDeleted = 0;
|
|
38749
|
+
if (keys.length > 0) {
|
|
38750
|
+
const result = await this.deleteObjects(keys);
|
|
38751
|
+
totalDeleted = result.Deleted.length;
|
|
38752
|
+
this.emit("deleteAll", {
|
|
38753
|
+
prefix,
|
|
38754
|
+
batch: totalDeleted,
|
|
38755
|
+
total: totalDeleted
|
|
38756
|
+
});
|
|
38757
|
+
}
|
|
38758
|
+
this.emit("deleteAllComplete", {
|
|
38759
|
+
prefix,
|
|
38760
|
+
totalDeleted
|
|
38761
|
+
});
|
|
38762
|
+
return totalDeleted;
|
|
38763
|
+
}
|
|
38764
|
+
/**
|
|
38765
|
+
* Get continuation token after skipping offset items
|
|
38766
|
+
*/
|
|
38767
|
+
async getContinuationTokenAfterOffset({ prefix = "", offset = 1e3 } = {}) {
|
|
38768
|
+
if (offset === 0) return null;
|
|
38769
|
+
const keys = await this.getAllKeys({ prefix });
|
|
38770
|
+
if (offset >= keys.length) {
|
|
38771
|
+
this.emit("getContinuationTokenAfterOffset", null, { prefix, offset });
|
|
38772
|
+
return null;
|
|
38773
|
+
}
|
|
38774
|
+
const token = keys[offset];
|
|
38775
|
+
this.emit("getContinuationTokenAfterOffset", token, { prefix, offset });
|
|
38776
|
+
return token;
|
|
38777
|
+
}
|
|
38778
|
+
/**
|
|
38779
|
+
* Move an object from one key to another
|
|
38780
|
+
*/
|
|
38781
|
+
async moveObject({ from, to }) {
|
|
38782
|
+
await this.copyObject({ from, to, metadataDirective: "COPY" });
|
|
38783
|
+
await this.deleteObject(from);
|
|
38784
|
+
}
|
|
38785
|
+
/**
|
|
38786
|
+
* Move all objects from one prefix to another
|
|
38787
|
+
*/
|
|
38788
|
+
async moveAllObjects({ prefixFrom, prefixTo }) {
|
|
38789
|
+
const keys = await this.getAllKeys({ prefix: prefixFrom });
|
|
38790
|
+
const results = [];
|
|
38791
|
+
const errors = [];
|
|
38792
|
+
for (const key of keys) {
|
|
38793
|
+
try {
|
|
38794
|
+
const to = key.replace(prefixFrom, prefixTo);
|
|
38795
|
+
await this.moveObject({ from: key, to });
|
|
38796
|
+
results.push(to);
|
|
38797
|
+
} catch (error) {
|
|
38798
|
+
errors.push({
|
|
38799
|
+
message: error.message,
|
|
38800
|
+
raw: error,
|
|
38801
|
+
key
|
|
38802
|
+
});
|
|
38803
|
+
}
|
|
38804
|
+
}
|
|
38805
|
+
this.emit("moveAllObjects", { results, errors }, { prefixFrom, prefixTo });
|
|
38806
|
+
if (errors.length > 0) {
|
|
38807
|
+
const error = new Error("Some objects could not be moved");
|
|
38808
|
+
error.context = {
|
|
38809
|
+
bucket: this.bucket,
|
|
38810
|
+
operation: "moveAllObjects",
|
|
38811
|
+
prefixFrom,
|
|
38812
|
+
prefixTo,
|
|
38813
|
+
totalKeys: keys.length,
|
|
38814
|
+
failedCount: errors.length,
|
|
38815
|
+
successCount: results.length,
|
|
38816
|
+
errors
|
|
38817
|
+
};
|
|
38818
|
+
throw error;
|
|
38819
|
+
}
|
|
38820
|
+
return results;
|
|
38821
|
+
}
|
|
38822
|
+
/**
|
|
38823
|
+
* Create a snapshot of current storage state
|
|
38824
|
+
*/
|
|
38825
|
+
snapshot() {
|
|
38826
|
+
return this.storage.snapshot();
|
|
38827
|
+
}
|
|
38828
|
+
/**
|
|
38829
|
+
* Restore from a snapshot
|
|
38830
|
+
*/
|
|
38831
|
+
restore(snapshot) {
|
|
38832
|
+
return this.storage.restore(snapshot);
|
|
38833
|
+
}
|
|
38834
|
+
/**
|
|
38835
|
+
* Save current state to disk (persistence)
|
|
38836
|
+
*/
|
|
38837
|
+
async saveToDisk(path2) {
|
|
38838
|
+
return await this.storage.saveToDisk(path2);
|
|
38839
|
+
}
|
|
38840
|
+
/**
|
|
38841
|
+
* Load state from disk
|
|
38842
|
+
*/
|
|
38843
|
+
async loadFromDisk(path2) {
|
|
38844
|
+
return await this.storage.loadFromDisk(path2);
|
|
38845
|
+
}
|
|
38846
|
+
/**
|
|
38847
|
+
* Export to BackupPlugin-compatible format (s3db.json + JSONL files)
|
|
38848
|
+
* Compatible with BackupPlugin for easy migration
|
|
38849
|
+
*
|
|
38850
|
+
* @param {string} outputDir - Output directory path
|
|
38851
|
+
* @param {Object} options - Export options
|
|
38852
|
+
* @param {Array<string>} options.resources - Resource names to export (default: all)
|
|
38853
|
+
* @param {boolean} options.compress - Use gzip compression (default: true)
|
|
38854
|
+
* @param {Object} options.database - Database instance for schema metadata
|
|
38855
|
+
* @returns {Promise<Object>} Export manifest with file paths and stats
|
|
38856
|
+
*/
|
|
38857
|
+
async exportBackup(outputDir, options = {}) {
|
|
38858
|
+
const { mkdir, writeFile } = await import('fs/promises');
|
|
38859
|
+
const zlib = await import('zlib');
|
|
38860
|
+
const { promisify } = await import('util');
|
|
38861
|
+
const gzip = promisify(zlib.gzip);
|
|
38862
|
+
await mkdir(outputDir, { recursive: true });
|
|
38863
|
+
const compress = options.compress !== false;
|
|
38864
|
+
const database = options.database;
|
|
38865
|
+
const resourceFilter = options.resources;
|
|
38866
|
+
const allKeys = await this.getAllKeys({});
|
|
38867
|
+
const resourceMap = /* @__PURE__ */ new Map();
|
|
38868
|
+
for (const key of allKeys) {
|
|
38869
|
+
const match = key.match(/^resource=([^/]+)\//);
|
|
38870
|
+
if (match) {
|
|
38871
|
+
const resourceName = match[1];
|
|
38872
|
+
if (!resourceFilter || resourceFilter.includes(resourceName)) {
|
|
38873
|
+
if (!resourceMap.has(resourceName)) {
|
|
38874
|
+
resourceMap.set(resourceName, []);
|
|
38875
|
+
}
|
|
38876
|
+
resourceMap.get(resourceName).push(key);
|
|
38877
|
+
}
|
|
38878
|
+
}
|
|
38879
|
+
}
|
|
38880
|
+
const exportedFiles = {};
|
|
38881
|
+
const resourceStats = {};
|
|
38882
|
+
for (const [resourceName, keys] of resourceMap.entries()) {
|
|
38883
|
+
const records = [];
|
|
38884
|
+
for (const key of keys) {
|
|
38885
|
+
const obj = await this.getObject(key);
|
|
38886
|
+
const idMatch = key.match(/\/id=([^/]+)/);
|
|
38887
|
+
const recordId = idMatch ? idMatch[1] : null;
|
|
38888
|
+
const record = { ...obj.Metadata };
|
|
38889
|
+
if (recordId && !record.id) {
|
|
38890
|
+
record.id = recordId;
|
|
38891
|
+
}
|
|
38892
|
+
if (obj.Body) {
|
|
38893
|
+
const chunks = [];
|
|
38894
|
+
for await (const chunk2 of obj.Body) {
|
|
38895
|
+
chunks.push(chunk2);
|
|
38896
|
+
}
|
|
38897
|
+
const bodyBuffer = Buffer.concat(chunks);
|
|
38898
|
+
const bodyStr = bodyBuffer.toString("utf-8");
|
|
38899
|
+
if (bodyStr.startsWith("{") || bodyStr.startsWith("[")) {
|
|
38900
|
+
try {
|
|
38901
|
+
const bodyData = JSON.parse(bodyStr);
|
|
38902
|
+
Object.assign(record, bodyData);
|
|
38903
|
+
} catch {
|
|
38904
|
+
record._body = bodyStr;
|
|
38905
|
+
}
|
|
38906
|
+
} else if (bodyStr) {
|
|
38907
|
+
record._body = bodyStr;
|
|
38908
|
+
}
|
|
38909
|
+
}
|
|
38910
|
+
records.push(record);
|
|
38911
|
+
}
|
|
38912
|
+
const jsonl = records.map((r) => JSON.stringify(r)).join("\n");
|
|
38913
|
+
const filename = compress ? `${resourceName}.jsonl.gz` : `${resourceName}.jsonl`;
|
|
38914
|
+
const filePath = `${outputDir}/${filename}`;
|
|
38915
|
+
if (compress) {
|
|
38916
|
+
const compressed = await gzip(jsonl);
|
|
38917
|
+
await writeFile(filePath, compressed);
|
|
38918
|
+
} else {
|
|
38919
|
+
await writeFile(filePath, jsonl, "utf-8");
|
|
38920
|
+
}
|
|
38921
|
+
exportedFiles[resourceName] = filePath;
|
|
38922
|
+
resourceStats[resourceName] = {
|
|
38923
|
+
recordCount: records.length,
|
|
38924
|
+
fileSize: compress ? (await gzip(jsonl)).length : Buffer.byteLength(jsonl)
|
|
38925
|
+
};
|
|
38926
|
+
}
|
|
38927
|
+
const s3dbMetadata = {
|
|
38928
|
+
version: "1.0",
|
|
38929
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
38930
|
+
bucket: this.bucket,
|
|
38931
|
+
keyPrefix: this.keyPrefix || "",
|
|
38932
|
+
compressed: compress,
|
|
38933
|
+
resources: {},
|
|
38934
|
+
totalRecords: 0,
|
|
38935
|
+
totalSize: 0
|
|
38936
|
+
};
|
|
38937
|
+
if (database && database.resources) {
|
|
38938
|
+
for (const [resourceName, resource] of Object.entries(database.resources)) {
|
|
38939
|
+
if (resourceMap.has(resourceName)) {
|
|
38940
|
+
s3dbMetadata.resources[resourceName] = {
|
|
38941
|
+
schema: resource.schema ? {
|
|
38942
|
+
attributes: resource.schema.attributes,
|
|
38943
|
+
partitions: resource.schema.partitions,
|
|
38944
|
+
behavior: resource.schema.behavior,
|
|
38945
|
+
timestamps: resource.schema.timestamps
|
|
38946
|
+
} : null,
|
|
38947
|
+
stats: resourceStats[resourceName]
|
|
38948
|
+
};
|
|
38949
|
+
}
|
|
38950
|
+
}
|
|
38951
|
+
} else {
|
|
38952
|
+
for (const [resourceName, stats] of Object.entries(resourceStats)) {
|
|
38953
|
+
s3dbMetadata.resources[resourceName] = { stats };
|
|
38954
|
+
}
|
|
38955
|
+
}
|
|
38956
|
+
for (const stats of Object.values(resourceStats)) {
|
|
38957
|
+
s3dbMetadata.totalRecords += stats.recordCount;
|
|
38958
|
+
s3dbMetadata.totalSize += stats.fileSize;
|
|
38959
|
+
}
|
|
38960
|
+
const s3dbPath = `${outputDir}/s3db.json`;
|
|
38961
|
+
await writeFile(s3dbPath, JSON.stringify(s3dbMetadata, null, 2), "utf-8");
|
|
38962
|
+
return {
|
|
38963
|
+
manifest: s3dbPath,
|
|
38964
|
+
files: exportedFiles,
|
|
38965
|
+
stats: s3dbMetadata,
|
|
38966
|
+
resourceCount: resourceMap.size,
|
|
38967
|
+
totalRecords: s3dbMetadata.totalRecords,
|
|
38968
|
+
totalSize: s3dbMetadata.totalSize
|
|
38969
|
+
};
|
|
38970
|
+
}
|
|
38971
|
+
/**
|
|
38972
|
+
* Import from BackupPlugin-compatible format
|
|
38973
|
+
* Loads data from s3db.json + JSONL files created by BackupPlugin or exportBackup()
|
|
38974
|
+
*
|
|
38975
|
+
* @param {string} backupDir - Backup directory path containing s3db.json
|
|
38976
|
+
* @param {Object} options - Import options
|
|
38977
|
+
* @param {Array<string>} options.resources - Resource names to import (default: all)
|
|
38978
|
+
* @param {boolean} options.clear - Clear existing data first (default: false)
|
|
38979
|
+
* @param {Object} options.database - Database instance to recreate schemas
|
|
38980
|
+
* @returns {Promise<Object>} Import stats
|
|
38981
|
+
*/
|
|
38982
|
+
async importBackup(backupDir, options = {}) {
|
|
38983
|
+
const { readFile, readdir } = await import('fs/promises');
|
|
38984
|
+
const zlib = await import('zlib');
|
|
38985
|
+
const { promisify } = await import('util');
|
|
38986
|
+
const gunzip = promisify(zlib.gunzip);
|
|
38987
|
+
if (options.clear) {
|
|
38988
|
+
this.clear();
|
|
38989
|
+
}
|
|
38990
|
+
const s3dbPath = `${backupDir}/s3db.json`;
|
|
38991
|
+
const s3dbContent = await readFile(s3dbPath, "utf-8");
|
|
38992
|
+
const metadata = JSON.parse(s3dbContent);
|
|
38993
|
+
const database = options.database;
|
|
38994
|
+
const resourceFilter = options.resources;
|
|
38995
|
+
const importStats = {
|
|
38996
|
+
resourcesImported: 0,
|
|
38997
|
+
recordsImported: 0,
|
|
38998
|
+
errors: []
|
|
38999
|
+
};
|
|
39000
|
+
if (database && metadata.resources) {
|
|
39001
|
+
for (const [resourceName, resourceMeta] of Object.entries(metadata.resources)) {
|
|
39002
|
+
if (resourceFilter && !resourceFilter.includes(resourceName)) continue;
|
|
39003
|
+
if (resourceMeta.schema) {
|
|
39004
|
+
try {
|
|
39005
|
+
await database.createResource({
|
|
39006
|
+
name: resourceName,
|
|
39007
|
+
...resourceMeta.schema
|
|
39008
|
+
});
|
|
39009
|
+
} catch (error) {
|
|
39010
|
+
}
|
|
39011
|
+
}
|
|
39012
|
+
}
|
|
39013
|
+
}
|
|
39014
|
+
const files = await readdir(backupDir);
|
|
39015
|
+
for (const file of files) {
|
|
39016
|
+
if (!file.endsWith(".jsonl") && !file.endsWith(".jsonl.gz")) continue;
|
|
39017
|
+
const resourceName = file.replace(/\.jsonl(\.gz)?$/, "");
|
|
39018
|
+
if (resourceFilter && !resourceFilter.includes(resourceName)) continue;
|
|
39019
|
+
const filePath = `${backupDir}/${file}`;
|
|
39020
|
+
let content = await readFile(filePath);
|
|
39021
|
+
if (file.endsWith(".gz")) {
|
|
39022
|
+
content = await gunzip(content);
|
|
39023
|
+
}
|
|
39024
|
+
const jsonl = content.toString("utf-8");
|
|
39025
|
+
const lines = jsonl.split("\n").filter((line) => line.trim());
|
|
39026
|
+
for (const line of lines) {
|
|
39027
|
+
try {
|
|
39028
|
+
const record = JSON.parse(line);
|
|
39029
|
+
const id = record.id || record._id || `imported_${Date.now()}_${Math.random()}`;
|
|
39030
|
+
const { _body, id: _, _id: __, ...metadata2 } = record;
|
|
39031
|
+
await this.putObject({
|
|
39032
|
+
key: `resource=${resourceName}/id=${id}`,
|
|
39033
|
+
metadata: metadata2,
|
|
39034
|
+
body: _body ? Buffer.from(_body) : void 0
|
|
39035
|
+
});
|
|
39036
|
+
importStats.recordsImported++;
|
|
39037
|
+
} catch (error) {
|
|
39038
|
+
importStats.errors.push({
|
|
39039
|
+
resource: resourceName,
|
|
39040
|
+
error: error.message,
|
|
39041
|
+
line
|
|
39042
|
+
});
|
|
39043
|
+
}
|
|
39044
|
+
}
|
|
39045
|
+
importStats.resourcesImported++;
|
|
39046
|
+
}
|
|
39047
|
+
return importStats;
|
|
39048
|
+
}
|
|
39049
|
+
/**
|
|
39050
|
+
* Get storage statistics
|
|
39051
|
+
*/
|
|
39052
|
+
getStats() {
|
|
39053
|
+
return this.storage.getStats();
|
|
39054
|
+
}
|
|
39055
|
+
/**
|
|
39056
|
+
* Clear all objects
|
|
39057
|
+
*/
|
|
39058
|
+
clear() {
|
|
39059
|
+
this.storage.clear();
|
|
39060
|
+
}
|
|
39061
|
+
}
|
|
39062
|
+
|
|
37923
39063
|
function mapFieldTypeToTypeScript(fieldType) {
|
|
37924
39064
|
const baseType = fieldType.split("|")[0].trim();
|
|
37925
39065
|
const typeMap = {
|
|
@@ -38840,7 +39980,7 @@ exports.BigqueryReplicator = BigqueryReplicator;
|
|
|
38840
39980
|
exports.CONSUMER_DRIVERS = CONSUMER_DRIVERS;
|
|
38841
39981
|
exports.Cache = Cache;
|
|
38842
39982
|
exports.CachePlugin = CachePlugin;
|
|
38843
|
-
exports.Client =
|
|
39983
|
+
exports.Client = S3Client;
|
|
38844
39984
|
exports.ConnectionString = ConnectionString;
|
|
38845
39985
|
exports.ConnectionStringError = ConnectionStringError;
|
|
38846
39986
|
exports.CostsPlugin = CostsPlugin;
|
|
@@ -38859,6 +39999,8 @@ exports.FullTextPlugin = FullTextPlugin;
|
|
|
38859
39999
|
exports.GeoPlugin = GeoPlugin;
|
|
38860
40000
|
exports.InvalidResourceItem = InvalidResourceItem;
|
|
38861
40001
|
exports.MemoryCache = MemoryCache;
|
|
40002
|
+
exports.MemoryClient = MemoryClient;
|
|
40003
|
+
exports.MemoryStorage = MemoryStorage;
|
|
38862
40004
|
exports.MetadataLimitError = MetadataLimitError;
|
|
38863
40005
|
exports.MetricsPlugin = MetricsPlugin;
|
|
38864
40006
|
exports.MissingMetadata = MissingMetadata;
|
|
@@ -38892,6 +40034,7 @@ exports.ResourceReader = ResourceReader;
|
|
|
38892
40034
|
exports.ResourceWriter = ResourceWriter;
|
|
38893
40035
|
exports.S3BackupDriver = S3BackupDriver;
|
|
38894
40036
|
exports.S3Cache = S3Cache;
|
|
40037
|
+
exports.S3Client = S3Client;
|
|
38895
40038
|
exports.S3QueuePlugin = S3QueuePlugin;
|
|
38896
40039
|
exports.S3db = Database;
|
|
38897
40040
|
exports.S3dbError = S3dbError;
|