rac-delta 1.0.11 → 1.0.13

package/README.md

@@ -34,7 +34,7 @@ In order to use the rac-delta SDK, you will need to create a RacDeltaClient, the
 ```ts
 import { RacDeltaClient } from 'rac-delta';
 
-const racDeltaClient = new RacDeltaClient({
+const racDeltaClient = await RacDeltaClient.create({
   chunkSize: 1024 * 1024,
   maxConcurrency: 6,
   storage: {

package/dist/core/services/reconstruction-service.d.ts

@@ -93,7 +93,8 @@ export interface ReconstructionService {
      *
      * @param entry The FileEntry containing the list of chunks of the file
      * @param chunkSource the source implementations of the chunks
+     * @param maxConcurrency the max concurrent entries that will be processed
      */
-    reconstructToStream(entry: FileEntry, chunkSource: ChunkSource): Promise<Readable>;
+    reconstructToStream(entry: FileEntry, chunkSource: ChunkSource, maxConcurrency: number): Promise<Readable>;
 }
 //# sourceMappingURL=reconstruction-service.d.ts.map

package/dist/core/services/reconstruction-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"reconstruction-service.d.ts","sourceRoot":"","sources":["../../../src/core/services/reconstruction-service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAElC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AAEnC,eAAO,MAAM,yCAAyC,QAAoB,CAAC;AAE3E,MAAM,WAAW,qBAAqB;IACpC;;OAEG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAEhC;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAEtC;;;;;;;;;;;;;;OAcG;IACH,8BAA8B,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAEjD;;OAEG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAElC;;;;;;;OAOG;IACH,UAAU,CAAC,EAAE,CACX,mBAAmB,EAAE,MAAM,EAC3B,SAAS,EAAE,MAAM,EACjB,eAAe,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,EACjC,YAAY,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,KAC3B,IAAI,CAAC;CACX;AAED,MAAM,WAAW,WAAW;IAC1B;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAExC;;OAEG;IACH,SAAS,CAAC,CACR,MAAM,EAAE,MAAM,EAAE,EAChB,OAAO,CAAC,EAAE,OAAO,CAAC;QAAE,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,GAC1C,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAEhC;;;;;;OAMG;IACH,YAAY,CAAC,CACX,MAAM,EAAE,MAAM,EAAE,EAChB,OAAO,CAAC,EAAE,OAAO,CAAC;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC,GACnE,cAAc,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,QAAQ,CAAA;KAAE,CAAC,CAAC;CACrD;AAED,MAAM,WAAW,qBAAqB;IACpC;;;;;;;;OAQG;IACH,eAAe,CACb,KAAK,EAAE,SAAS,EAChB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,GACvC,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;;;;;;OAOG;IACH,cAAc,CACZ,IAAI,EAAE,SAAS,EACf,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,GACvC,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;;;;OAKG;IACH,mBAAmB,CAAC,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;CACpF"}
+{"version":3,"file":"reconstruction-service.d.ts","sourceRoot":"","sources":["../../../src/core/services/reconstruction-service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAElC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AAEnC,eAAO,MAAM,yCAAyC,QAAoB,CAAC;AAE3E,MAAM,WAAW,qBAAqB;IACpC;;OAEG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAEhC;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAEtC;;;;;;;;;;;;;;OAcG;IACH,8BAA8B,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAEjD;;OAEG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAElC;;;;;;;OAOG;IACH,UAAU,CAAC,EAAE,CACX,mBAAmB,EAAE,MAAM,EAC3B,SAAS,EAAE,MAAM,EACjB,eAAe,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,EACjC,YAAY,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,KAC3B,IAAI,CAAC;CACX;AAED,MAAM,WAAW,WAAW;IAC1B;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAExC;;OAEG;IACH,SAAS,CAAC,CACR,MAAM,EAAE,MAAM,EAAE,EAChB,OAAO,CAAC,EAAE,OAAO,CAAC;QAAE,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,GAC1C,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAEhC;;;;;;OAMG;IACH,YAAY,CAAC,CACX,MAAM,EAAE,MAAM,EAAE,EAChB,OAAO,CAAC,EAAE,OAAO,CAAC;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC,GACnE,cAAc,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,QAAQ,CAAA;KAAE,CAAC,CAAC;CACrD;AAED,MAAM,WAAW,qBAAqB;IACpC;;;;;;;;OAQG;IACH,eAAe,CACb,KAAK,EAAE,SAAS,EAChB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,GACvC,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;;;;;;OAOG;IACH,cAAc,CACZ,IAAI,EAAE,SAAS,EACf,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,GACvC,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;;;;;OAMG;IACH,mBAAmB,CACjB,KAAK,EAAE,SAAS,EAChB,WAAW,EAAE,WAAW,EACxB,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,QAAQ,CAAC,CAAC;CACtB"}

package/dist/infrastructure/services/hash-wasm-hasher-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"hash-wasm-hasher-service.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/services/hash-wasm-hasher-service.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAErE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAGrD,qBAAa,qBAAsB,YAAW,aAAa;IACnD,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAiDlF,UAAU,CACd,MAAM,EAAE,gBAAgB,EACxB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,KAAK,IAAI,GAC5C,OAAO,CAAC,IAAI,CAAC;IA+EV,WAAW,CAAC,IAAI,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrE,UAAU,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC;IAM7C,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAgBhE,qBAAqB,IAAI,OAAO,CAAC,eAAe,CAAC;CAYxD"}
+{"version":3,"file":"hash-wasm-hasher-service.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/services/hash-wasm-hasher-service.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAErE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAGrD,qBAAa,qBAAsB,YAAW,aAAa;IACnD,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAiDlF,UAAU,CACd,MAAM,EAAE,gBAAgB,EACxB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,KAAK,IAAI,GAC5C,OAAO,CAAC,IAAI,CAAC;IAkEV,WAAW,CAAC,IAAI,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrE,UAAU,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC;IAM7C,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAgBhE,qBAAqB,IAAI,OAAO,CAAC,eAAe,CAAC;CAYxD"}

package/dist/infrastructure/services/hash-wasm-hasher-service.js

@@ -55,7 +55,6 @@ class HashWasmHasherService {
         let bufferLen = 0;
         try {
             for await (const data of stream) {
-                console.log('[HASHSTREAM] Received data length:', data.length, 'bufferLen', bufferLen, 'rss:', process.memoryUsage().rss);
                 let input = Buffer.isBuffer(data) ? data : Buffer.from(data);
                 while (input.length > 0) {
                     const take = Math.min(chunkSize - bufferLen, input.length);
@@ -69,9 +68,7 @@ class HashWasmHasherService {
                         chunkHasher.update(chunk);
                         const chunkHash = chunkHasher.digest('hex');
                         // onChunk could be a promise
-                        console.log('[HASHSTREAM] Emitting chunk', { offset, size: bufferLen });
                         await Promise.resolve(onChunk(chunk, { hash: chunkHash, offset, size: bufferLen }));
-                        console.log('[HASHSTREAM] onChunk done', { offset });
                         offset += bufferLen;
                         bufferLen = 0;
                     }
@@ -83,18 +80,15 @@ class HashWasmHasherService {
                 const chunkHasher = await (0, hash_wasm_1.createBLAKE3)();
                 chunkHasher.update(chunk);
                 const chunkHash = chunkHasher.digest('hex');
-                console.log('[HASHSTREAM] Emitting chunk on if', { offset, size: bufferLen });
                 await Promise.resolve(onChunk(chunk, {
                     hash: chunkHash,
                     offset,
                     size: bufferLen,
                 }));
-                console.log('[HASHSTREAM] onChunk if done', { offset });
             }
             if (stream.close) {
                 await stream.close();
             }
-            console.log('[HASHSTREAM] Finished.');
         }
         catch (err) {
             if (err instanceof Error) {

package/dist/infrastructure/services/memory-reconstruction-service.d.ts

@@ -7,7 +7,7 @@ export declare class MemoryReconstructionService implements ReconstructionServic
     constructor(hasher: HasherService);
     reconstructAll(plan: DeltaPlan, outputDir: string, chunkSource: ChunkSource, options?: Nullish<ReconstructionOptions>): Promise<void>;
     reconstructFile(entry: FileEntry, outputPath: string, chunkSource: ChunkSource, options?: ReconstructionOptions): Promise<void>;
-    reconstructToStream(entry: FileEntry, chunkSource: ChunkSource): Promise<Readable>;
+    reconstructToStream(entry: FileEntry, chunkSource: ChunkSource, maxConcurrency?: number): Promise<Readable>;
     private fileExists;
     /** Performs partial reconstruction directly in the existing file */
     private reconstructInPlace;

package/dist/infrastructure/services/memory-reconstruction-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"memory-reconstruction-service.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/services/memory-reconstruction-service.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAyB,MAAM,QAAQ,CAAC;AAIzD,OAAO,EACL,WAAW,EAEX,aAAa,EACb,qBAAqB,EACrB,qBAAqB,EACtB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,SAAS,EAAE,SAAS,EAAS,MAAM,mBAAmB,CAAC;AAEhE,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE3C,qBAAa,2BAA4B,YAAW,qBAAqB;IAC3D,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAAN,MAAM,EAAE,aAAa;IAE5C,cAAc,CAClB,IAAI,EAAE,SAAS,EACf,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,GACvC,OAAO,CAAC,IAAI,CAAC;IAmFV,eAAe,CACnB,KAAK,EAAE,SAAS,EAChB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,WAAW,EACxB,OAAO,GAAE,qBAIR,GACA,OAAO,CAAC,IAAI,CAAC;IAgDV,mBAAmB,CAAC,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,QAAQ,CAAC;YAsB1E,UAAU;IASxB,oEAAoE;YACtD,kBAAkB;IAiChC,oFAAoF;YACtE,iBAAiB;IAmF/B;;;OAGG;YACW,qBAAqB;YA2BrB,YAAY;YAcZ,aAAa;YAuDZ,gBAAgB;CAmChC"}
+{"version":3,"file":"memory-reconstruction-service.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/services/memory-reconstruction-service.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAyB,MAAM,QAAQ,CAAC;AAIzD,OAAO,EACL,WAAW,EAEX,aAAa,EACb,qBAAqB,EACrB,qBAAqB,EACtB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,SAAS,EAAE,SAAS,EAAS,MAAM,mBAAmB,CAAC;AAEhE,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE3C,qBAAa,2BAA4B,YAAW,qBAAqB;IAC3D,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAAN,MAAM,EAAE,aAAa;IAE5C,cAAc,CAClB,IAAI,EAAE,SAAS,EACf,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,GACvC,OAAO,CAAC,IAAI,CAAC;IAmFV,eAAe,CACnB,KAAK,EAAE,SAAS,EAChB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,WAAW,EACxB,OAAO,GAAE,qBAIR,GACA,OAAO,CAAC,IAAI,CAAC;IAgDV,mBAAmB,CACvB,KAAK,EAAE,SAAS,EAChB,WAAW,EAAE,WAAW,EACxB,cAAc,GAAE,MAAU,GACzB,OAAO,CAAC,QAAQ,CAAC;YAmCN,UAAU;IASxB,oEAAoE;YACtD,kBAAkB;IAiChC,oFAAoF;YACtE,iBAAiB;IAmF/B;;;OAGG;YACW,qBAAqB;YA2BrB,YAAY;YAcZ,aAAa;YAuDZ,gBAAgB;CAmChC"}

package/dist/infrastructure/services/memory-reconstruction-service.js

@@ -111,15 +111,24 @@ class MemoryReconstructionService {
         }
     }
     // This will reconstruct to stream, not to disk
-    async reconstructToStream(entry, chunkSource) {
+    async reconstructToStream(entry, chunkSource, maxConcurrency = 5) {
         const output = new stream_1.PassThrough({ highWaterMark: 1024 * 1024 });
         const chunks = entry.chunks ?? [];
         // Starts async reconstruction without blocking stream return
         const processChunks = async () => {
             try {
+                const active = new Set();
                 for await (const { data } of this.fetchChunksSmart(chunks, chunkSource)) {
-                    await this.writeToStream(data, output);
+                    const task = (async () => {
+                        await this.writeToStream(data, output);
+                    })();
+                    task.finally(() => active.delete(task));
+                    active.add(task);
+                    if (active.size >= maxConcurrency) {
+                        await Promise.race(active);
+                    }
                 }
+                await Promise.all(active);
                 output.end();
             }
             catch (err) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "rac-delta",
   "description": "Storage agnostic delta patching implementation of rac-delta protocol for NodeJs. With streaming support and file reconstruction.",
-  "version": "1.0.11",
+  "version": "1.0.13",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "repository": {
@@ -29,7 +29,7 @@
     "hash-wasm": "^4.12.0"
   },
   "peerDependencies": {
-    "@aws-sdk/client-s3": "^3.962.0",
+    "@aws-sdk/client-s3": ">=3.525.0 <4.0.0",
     "@azure/identity": "^4.13.0",
     "@azure/storage-blob": "^12.29.1",
     "@google-cloud/storage": "^7.18.0",