rac-delta 1.0.17 → 1.0.19

package/dist/core/services/reconstruction-service.d.ts

@@ -59,10 +59,12 @@ export interface ChunkSource {
      *
      * @param options.concurrency Number of parallel fetches (default 8)
      * @param options.preserveOrder Whether to yield in input order (default true)
+     * @param options.maxPrefetch Number of max chunk prefetchs to storage (default 12)
      */
     streamChunks?(hashes: string[], options?: Nullish<{
         concurrency?: number;
         preserveOrder?: boolean;
+        maxPrefetch?: number;
     }>): AsyncGenerator<{
         hash: string;
         data: Readable;

package/dist/core/services/reconstruction-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"reconstruction-service.d.ts","sourceRoot":"","sources":["../../../src/core/services/reconstruction-service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAElC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AAEnC,eAAO,MAAM,yCAAyC,QAAoB,CAAC;AAE3E,MAAM,WAAW,qBAAqB;IACpC;;OAEG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAEhC;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAEtC;;;;;;;;;;;;;;OAcG;IACH,8BAA8B,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAEjD;;OAEG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAElC;;;;;;;OAOG;IACH,UAAU,CAAC,EAAE,CACX,mBAAmB,EAAE,MAAM,EAC3B,SAAS,EAAE,MAAM,EACjB,eAAe,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,EACjC,YAAY,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,KAC3B,IAAI,CAAC;CACX;AAED,MAAM,WAAW,WAAW;IAC1B;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAExC;;OAEG;IACH,SAAS,CAAC,CACR,MAAM,EAAE,MAAM,EAAE,EAChB,OAAO,CAAC,EAAE,OAAO,CAAC;QAAE,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,GAC1C,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAEhC;;;;;;OAMG;IACH,YAAY,CAAC,CACX,MAAM,EAAE,MAAM,EAAE,EAChB,OAAO,CAAC,EAAE,OAAO,CAAC;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC,GACnE,cAAc,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,QAAQ,CAAA;KAAE,CAAC,CAAC;CACrD;AAED,MAAM,WAAW,qBAAqB;IACpC;;;;;;;;OAQG;IACH,eAAe,CACb,KAAK,EAAE,SAAS,EAChB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,GACvC,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;;;;;;OAOG;IACH,cAAc,CACZ,IAAI,EAAE,SAAS,EACf,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,GACvC,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;;;;;OAMG;IACH,mBAAmB,CACjB,KAAK,EAAE,SAAS,EAChB,WAAW,EAAE,WAAW,EACxB,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,QAAQ,CAAC,CAAC;CACtB"}
+{"version":3,"file":"reconstruction-service.d.ts","sourceRoot":"","sources":["../../../src/core/services/reconstruction-service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAElC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AAEnC,eAAO,MAAM,yCAAyC,QAAoB,CAAC;AAE3E,MAAM,WAAW,qBAAqB;IACpC;;OAEG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAEhC;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAEtC;;;;;;;;;;;;;;OAcG;IACH,8BAA8B,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAEjD;;OAEG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAElC;;;;;;;OAOG;IACH,UAAU,CAAC,EAAE,CACX,mBAAmB,EAAE,MAAM,EAC3B,SAAS,EAAE,MAAM,EACjB,eAAe,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,EACjC,YAAY,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,KAC3B,IAAI,CAAC;CACX;AAED,MAAM,WAAW,WAAW;IAC1B;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAExC;;OAEG;IACH,SAAS,CAAC,CACR,MAAM,EAAE,MAAM,EAAE,EAChB,OAAO,CAAC,EAAE,OAAO,CAAC;QAAE,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,GAC1C,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAEhC;;;;;;;OAOG;IACH,YAAY,CAAC,CACX,MAAM,EAAE,MAAM,EAAE,EAChB,OAAO,CAAC,EAAE,OAAO,CAAC;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,OAAO,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,GACzF,cAAc,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,QAAQ,CAAA;KAAE,CAAC,CAAC;CACrD;AAED,MAAM,WAAW,qBAAqB;IACpC;;;;;;;;OAQG;IACH,eAAe,CACb,KAAK,EAAE,SAAS,EAChB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,GACvC,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;;;;;;OAOG;IACH,cAAc,CACZ,IAAI,EAAE,SAAS,EACf,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,GACvC,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;;;;;OAMG;IACH,mBAAmB,CACjB,KAAK,EAAE,SAAS,EAChB,WAAW,EAAE,WAAW,EACxB,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,QAAQ,CAAC,CAAC;CACtB"}

package/dist/infrastructure/adapters/s3-storage-adapter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"s3-storage-adapter.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/adapters/s3-storage-adapter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAe,MAAM,QAAQ,CAAC;AAW/C,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE3C,qBAAa,gBAAiB,SAAQ,kBAAkB;IAG1C,OAAO,CAAC,QAAQ,CAAC,MAAM;IAFnC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAW;gBAED,MAAM,EAAE,eAAe;IAU9C,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAI9B,OAAO,CAAC,UAAU;IAIZ,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAqBhD,QAAQ,CACZ,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,QAAQ,GAAG,MAAM,EACvB,IAAI,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QAAC,IAAI,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;KAAE,GAC9D,OAAO,CAAC,IAAI,CAAC;IAoBV,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAY3C,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKxC,UAAU,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IA8B/B,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAqBpD,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAe7C,cAAc,IAAI,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;CA6BhD"}
+{"version":3,"file":"s3-storage-adapter.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/adapters/s3-storage-adapter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAe,MAAM,QAAQ,CAAC;AAW/C,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE3C,qBAAa,gBAAiB,SAAQ,kBAAkB;IAG1C,OAAO,CAAC,QAAQ,CAAC,MAAM;IAFnC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAW;gBAED,MAAM,EAAE,eAAe;IAU9C,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAI9B,OAAO,CAAC,UAAU;IAIZ,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAiBhD,QAAQ,CACZ,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,QAAQ,GAAG,MAAM,EACvB,IAAI,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QAAC,IAAI,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;KAAE,GAC9D,OAAO,CAAC,IAAI,CAAC;IAoBV,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAY3C,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKxC,UAAU,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IA8B/B,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAqBpD,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAe7C,cAAc,IAAI,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;CA6BhD"}

package/dist/infrastructure/adapters/s3-storage-adapter.js

@@ -1,7 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.S3StorageAdapter = void 0;
-const stream_1 = require("stream");
 const client_s3_1 = require("@aws-sdk/client-s3");
 const adapters_1 = require("../../core/adapters");
 class S3StorageAdapter extends adapters_1.HashStorageAdapter {
@@ -26,10 +25,7 @@ class S3StorageAdapter extends adapters_1.HashStorageAdapter {
         try {
             const key = this.resolveKey(hash);
             const res = await this.s3.send(new client_s3_1.GetObjectCommand({ Bucket: this.config.bucket, Key: key }));
-            const s3Stream = res.Body;
-            const pass = new stream_1.PassThrough({ highWaterMark: 1024 * 1024 });
-            s3Stream.pipe(pass);
-            return pass;
+            return res.Body;
         }
         catch (error) {
             if (error instanceof client_s3_1.NoSuchKey) {

package/dist/infrastructure/chunk-sources/storage-chunk-source.d.ts

@@ -10,9 +10,10 @@ export declare class StorageChunkSource implements ChunkSource {
     getChunks(hashes: string[], { concurrency }?: {
         concurrency?: number;
     }): Promise<Map<string, Buffer>>;
-    streamChunks(hashes: string[], { concurrency, preserveOrder, }?: {
+    streamChunks(hashes: string[], { concurrency, preserveOrder, maxPrefetch, }?: {
         concurrency?: number;
         preserveOrder?: boolean;
+        maxPrefetch?: number;
     }): AsyncGenerator<{
         hash: string;
         data: Readable;

package/dist/infrastructure/chunk-sources/storage-chunk-source.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"storage-chunk-source.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/chunk-sources/storage-chunk-source.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,cAAc,EAAqB,MAAM,qBAAqB,CAAC;AAG5F,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE3C,OAAO,EAAE,QAAQ,EAAe,MAAM,QAAQ,CAAC;AAE/C,qBAAa,kBAAmB,YAAW,WAAW;IAElD,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;gBADR,OAAO,EAAE,cAAc,EACvB,OAAO,CAAC,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAGnD,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAmBvC,SAAS,CACb,MAAM,EAAE,MAAM,EAAE,EAChB,EAAE,WAAe,EAAE,GAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAA;KAAO,GACjD,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAoBxB,YAAY,CACjB,MAAM,EAAE,MAAM,EAAE,EAChB,EACE,WAAe,EACf,aAAoB,GACrB,GAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,OAAO,CAAA;KAAO,GACxD,cAAc,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,QAAQ,CAAA;KAAE,CAAC;CA+GpD"}
+{"version":3,"file":"storage-chunk-source.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/chunk-sources/storage-chunk-source.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,cAAc,EAAqB,MAAM,qBAAqB,CAAC;AAG5F,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE3C,OAAO,EAAE,QAAQ,EAAe,MAAM,QAAQ,CAAC;AAE/C,qBAAa,kBAAmB,YAAW,WAAW;IAElD,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;gBADR,OAAO,EAAE,cAAc,EACvB,OAAO,CAAC,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAGnD,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAmBvC,SAAS,CACb,MAAM,EAAE,MAAM,EAAE,EAChB,EAAE,WAAe,EAAE,GAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAA;KAAO,GACjD,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAoBxB,YAAY,CACjB,MAAM,EAAE,MAAM,EAAE,EAChB,EACE,WAAe,EACf,aAAoB,EACpB,WAAgB,GACjB,GAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,OAAO,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAO,GAC9E,cAAc,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,QAAQ,CAAA;KAAE,CAAC;CAuIpD"}

package/dist/infrastructure/chunk-sources/storage-chunk-source.js

@@ -39,7 +39,7 @@ class StorageChunkSource {
         await Promise.all(workers);
         return results;
     }
-    async *streamChunks(hashes, { concurrency = 4, preserveOrder = true, } = {}) {
+    async *streamChunks(hashes, { concurrency = 4, preserveOrder = true, maxPrefetch = 12, } = {}) {
         if (hashes.length === 0) {
             return;
         }
@@ -49,6 +49,7 @@ class StorageChunkSource {
         let activeWorkers = 0;
         let workersDone = false;
         const pendingResolvers = [];
+        const pendingSlotResolvers = [];
         let workerError = null;
         const signalNext = () => {
             const resolver = pendingResolvers.shift();
@@ -56,6 +57,12 @@ class StorageChunkSource {
                 resolver();
             }
         };
+        const signalSlot = () => {
+            const resolver = pendingSlotResolvers.shift();
+            if (resolver) {
+                resolver();
+            }
+        };
         const waitForData = async () => {
             while ((preserveOrder && !results.has(nextIndexToEmit) && !workerError) ||
                 (!preserveOrder && results.size === 0 && !workerError)) {
@@ -66,7 +73,17 @@ class StorageChunkSource {
             activeWorkers++;
             try {
                 while (queue.length > 0 && !workerError) {
-                    const { hash, index } = queue.shift();
+                    while (results.size >= maxPrefetch && !workerError) {
+                        await new Promise((resolve) => pendingSlotResolvers.push(resolve));
+                    }
+                    if (workerError) {
+                        return;
+                    }
+                    const item = queue.shift();
+                    if (!item) {
+                        continue;
+                    }
+                    const { hash, index } = item;
                     try {
                         const stream = this.storage.type === 'hash'
                             ? await this.storage.getChunk(hash)
@@ -94,6 +111,7 @@ class StorageChunkSource {
                     workersDone = true;
                 }
                 signalNext();
+                signalSlot();
             }
         };
         const workers = Array.from({ length: Math.min(concurrency, queue.length) }, worker);
@@ -108,6 +126,7 @@ class StorageChunkSource {
                     while (results.has(nextIndexToEmit)) {
                         yield results.get(nextIndexToEmit);
                         results.delete(nextIndexToEmit);
+                        signalSlot();
                         nextIndexToEmit++;
                     }
                 }
@@ -116,6 +135,7 @@ class StorageChunkSource {
                     if (value !== undefined && index !== undefined) {
                         yield value;
                         results.delete(index);
+                        signalSlot();
                     }
                 }
                 if (workersDone && results.size === 0) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "rac-delta",
   "description": "Storage agnostic delta patching implementation of rac-delta protocol for NodeJs. With streaming support and file reconstruction.",
-  "version": "1.0.17",
+  "version": "1.0.19",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "repository": {