rac-delta 1.0.18 → 1.0.20

package/dist/core/services/reconstruction-service.d.ts

@@ -95,8 +95,7 @@ export interface ReconstructionService {
      *
      * @param entry The FileEntry containing the list of chunks of the file
      * @param chunkSource the source implementations of the chunks
-     * @param maxConcurrency the max concurrent entries that will be processed
      */
-    reconstructToStream(entry: FileEntry, chunkSource: ChunkSource, maxConcurrency: number): Promise<Readable>;
+    reconstructToStream(entry: FileEntry, chunkSource: ChunkSource): Promise<Readable>;
 }
 //# sourceMappingURL=reconstruction-service.d.ts.map

package/dist/core/services/reconstruction-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"reconstruction-service.d.ts","sourceRoot":"","sources":["../../../src/core/services/reconstruction-service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAElC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AAEnC,eAAO,MAAM,yCAAyC,QAAoB,CAAC;AAE3E,MAAM,WAAW,qBAAqB;IACpC;;OAEG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAEhC;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAEtC;;;;;;;;;;;;;;OAcG;IACH,8BAA8B,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAEjD;;OAEG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAElC;;;;;;;OAOG;IACH,UAAU,CAAC,EAAE,CACX,mBAAmB,EAAE,MAAM,EAC3B,SAAS,EAAE,MAAM,EACjB,eAAe,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,EACjC,YAAY,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,KAC3B,IAAI,CAAC;CACX;AAED,MAAM,WAAW,WAAW;IAC1B;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAExC;;OAEG;IACH,SAAS,CAAC,CACR,MAAM,EAAE,MAAM,EAAE,EAChB,OAAO,CAAC,EAAE,OAAO,CAAC;QAAE,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,GAC1C,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAEhC;;;;;;;OAOG;IACH,YAAY,CAAC,CACX,MAAM,EAAE,MAAM,EAAE,EAChB,OAAO,CAAC,EAAE,OAAO,CAAC;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,OAAO,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,GACzF,cAAc,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,QAAQ,CAAA;KAAE,CAAC,CAAC;CACrD;AAED,MAAM,WAAW,qBAAqB;IACpC;;;;;;;;OAQG;IACH,eAAe,CACb,KAAK,EAAE,SAAS,EAChB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,GACvC,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;;;;;;OAOG;IACH,cAAc,CACZ,IAAI,EAAE,SAAS,EACf,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,GACvC,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;;;;;OAMG;IACH,mBAAmB,CACjB,KAAK,EAAE,SAAS,EAChB,WAAW,EAAE,WAAW,EACxB,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,QAAQ,CAAC,CAAC;CACtB"}
+{"version":3,"file":"reconstruction-service.d.ts","sourceRoot":"","sources":["../../../src/core/services/reconstruction-service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAElC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AAEnC,eAAO,MAAM,yCAAyC,QAAoB,CAAC;AAE3E,MAAM,WAAW,qBAAqB;IACpC;;OAEG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAEhC;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAEtC;;;;;;;;;;;;;;OAcG;IACH,8BAA8B,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAEjD;;OAEG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAElC;;;;;;;OAOG;IACH,UAAU,CAAC,EAAE,CACX,mBAAmB,EAAE,MAAM,EAC3B,SAAS,EAAE,MAAM,EACjB,eAAe,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,EACjC,YAAY,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,KAC3B,IAAI,CAAC;CACX;AAED,MAAM,WAAW,WAAW;IAC1B;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAExC;;OAEG;IACH,SAAS,CAAC,CACR,MAAM,EAAE,MAAM,EAAE,EAChB,OAAO,CAAC,EAAE,OAAO,CAAC;QAAE,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,GAC1C,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAEhC;;;;;;;OAOG;IACH,YAAY,CAAC,CACX,MAAM,EAAE,MAAM,EAAE,EAChB,OAAO,CAAC,EAAE,OAAO,CAAC;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,OAAO,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,GACzF,cAAc,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,QAAQ,CAAA;KAAE,CAAC,CAAC;CACrD;AAED,MAAM,WAAW,qBAAqB;IACpC;;;;;;;;OAQG;IACH,eAAe,CACb,KAAK,EAAE,SAAS,EAChB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,GACvC,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;;;;;;OAOG;IACH,cAAc,CACZ,IAAI,EAAE,SAAS,EACf,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,GACvC,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;;;;OAKG;IACH,mBAAmB,CAAC,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;CACpF"}

package/dist/infrastructure/chunk-sources/storage-chunk-source.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"storage-chunk-source.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/chunk-sources/storage-chunk-source.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,cAAc,EAAqB,MAAM,qBAAqB,CAAC;AAG5F,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE3C,OAAO,EAAE,QAAQ,EAAe,MAAM,QAAQ,CAAC;AAE/C,qBAAa,kBAAmB,YAAW,WAAW;IAElD,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;gBADR,OAAO,EAAE,cAAc,EACvB,OAAO,CAAC,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAGnD,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAmBvC,SAAS,CACb,MAAM,EAAE,MAAM,EAAE,EAChB,EAAE,WAAe,EAAE,GAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAA;KAAO,GACjD,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAoBxB,YAAY,CACjB,MAAM,EAAE,MAAM,EAAE,EAChB,EACE,WAAe,EACf,aAAoB,EACpB,WAAgB,GACjB,GAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,OAAO,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAO,GAC9E,cAAc,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,QAAQ,CAAA;KAAE,CAAC;CAiIpD"}
+{"version":3,"file":"storage-chunk-source.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/chunk-sources/storage-chunk-source.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,cAAc,EAAqB,MAAM,qBAAqB,CAAC;AAG5F,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE3C,OAAO,EAAE,QAAQ,EAAe,MAAM,QAAQ,CAAC;AAE/C,qBAAa,kBAAmB,YAAW,WAAW;IAElD,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;gBADR,OAAO,EAAE,cAAc,EACvB,OAAO,CAAC,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAGnD,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAmBvC,SAAS,CACb,MAAM,EAAE,MAAM,EAAE,EAChB,EAAE,WAAe,EAAE,GAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAA;KAAO,GACjD,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAoBxB,YAAY,CACjB,MAAM,EAAE,MAAM,EAAE,EAChB,EACE,WAAe,EACf,aAAoB,EACpB,WAAgB,GACjB,GAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,OAAO,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAO,GAC9E,cAAc,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,QAAQ,CAAA;KAAE,CAAC;CAuIpD"}

package/dist/infrastructure/chunk-sources/storage-chunk-source.js

@@ -79,7 +79,11 @@ class StorageChunkSource {
                     if (workerError) {
                         return;
                     }
-                    const { hash, index } = queue.shift();
+                    const item = queue.shift();
+                    if (!item) {
+                        continue;
+                    }
+                    const { hash, index } = item;
                     try {
                         const stream = this.storage.type === 'hash'
                             ? await this.storage.getChunk(hash)
@@ -107,6 +111,7 @@ class StorageChunkSource {
                     workersDone = true;
                 }
                 signalNext();
+                signalSlot();
             }
         };
         const workers = Array.from({ length: Math.min(concurrency, queue.length) }, worker);

package/dist/infrastructure/services/memory-reconstruction-service.d.ts

@@ -7,7 +7,7 @@ export declare class MemoryReconstructionService implements ReconstructionServic
     constructor(hasher: HasherService);
     reconstructAll(plan: DeltaPlan, outputDir: string, chunkSource: ChunkSource, options?: Nullish<ReconstructionOptions>): Promise<void>;
     reconstructFile(entry: FileEntry, outputPath: string, chunkSource: ChunkSource, options?: ReconstructionOptions): Promise<void>;
-    reconstructToStream(entry: FileEntry, chunkSource: ChunkSource, maxConcurrency?: number): Promise<Readable>;
+    reconstructToStream(entry: FileEntry, chunkSource: ChunkSource): Promise<Readable>;
     private fileExists;
     /** Performs partial reconstruction directly in the existing file */
     private reconstructInPlace;

package/dist/infrastructure/services/memory-reconstruction-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"memory-reconstruction-service.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/services/memory-reconstruction-service.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAyB,MAAM,QAAQ,CAAC;AAIzD,OAAO,EACL,WAAW,EAEX,aAAa,EACb,qBAAqB,EACrB,qBAAqB,EACtB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,SAAS,EAAE,SAAS,EAAS,MAAM,mBAAmB,CAAC;AAEhE,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE3C,qBAAa,2BAA4B,YAAW,qBAAqB;IAC3D,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAAN,MAAM,EAAE,aAAa;IAE5C,cAAc,CAClB,IAAI,EAAE,SAAS,EACf,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,GACvC,OAAO,CAAC,IAAI,CAAC;IAmFV,eAAe,CACnB,KAAK,EAAE,SAAS,EAChB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,WAAW,EACxB,OAAO,GAAE,qBAIR,GACA,OAAO,CAAC,IAAI,CAAC;IAgDV,mBAAmB,CACvB,KAAK,EAAE,SAAS,EAChB,WAAW,EAAE,WAAW,EACxB,cAAc,SAAI,GACjB,OAAO,CAAC,QAAQ,CAAC;YA+CN,UAAU;IASxB,oEAAoE;YACtD,kBAAkB;IAiChC,oFAAoF;YACtE,iBAAiB;IAmF/B;;;OAGG;YACW,qBAAqB;YA2BrB,YAAY;YAcZ,aAAa;YAyCZ,gBAAgB;CAmChC"}
+{"version":3,"file":"memory-reconstruction-service.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/services/memory-reconstruction-service.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAyB,MAAM,QAAQ,CAAC;AAIzD,OAAO,EACL,WAAW,EAEX,aAAa,EACb,qBAAqB,EACrB,qBAAqB,EACtB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,SAAS,EAAE,SAAS,EAAS,MAAM,mBAAmB,CAAC;AAEhE,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE3C,qBAAa,2BAA4B,YAAW,qBAAqB;IAC3D,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAAN,MAAM,EAAE,aAAa;IAE5C,cAAc,CAClB,IAAI,EAAE,SAAS,EACf,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,GACvC,OAAO,CAAC,IAAI,CAAC;IAmFV,eAAe,CACnB,KAAK,EAAE,SAAS,EAChB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,WAAW,EACxB,OAAO,GAAE,qBAIR,GACA,OAAO,CAAC,IAAI,CAAC;IAgDV,mBAAmB,CAAC,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,QAAQ,CAAC;YA8C1E,UAAU;IASxB,oEAAoE;YACtD,kBAAkB;IAiChC,oFAAoF;YACtE,iBAAiB;IAmF/B;;;OAGG;YACW,qBAAqB;YA2BrB,YAAY;YAcZ,aAAa;YAyCZ,gBAAgB;CAmChC"}

package/dist/infrastructure/services/memory-reconstruction-service.js

@@ -111,42 +111,41 @@ class MemoryReconstructionService {
         }
     }
     // This will reconstruct to stream, not to disk
-    async reconstructToStream(entry, chunkSource, maxConcurrency = 5) {
+    async reconstructToStream(entry, chunkSource) {
         const chunks = entry.chunks ?? [];
-        const pass = new stream_1.PassThrough({ highWaterMark: 1024 * 1024 });
+        const pass = new stream_1.PassThrough({ highWaterMark: 2 * 1024 * 1024 });
         const iterator = this.fetchChunksSmart(chunks, chunkSource, true);
         pass.once('error', (err) => {
             pass.destroy(err);
         });
         (async () => {
             const active = new Set();
-            const pipeChunk = async (data) => {
-                if (Buffer.isBuffer(data)) {
-                    if (!pass.write(data)) {
-                        await new Promise((resolve) => pass.once('drain', resolve));
-                    }
-                }
-                else {
-                    await new Promise((resolve, reject) => {
-                        data.pipe(pass, { end: false });
-                        data.once('error', reject);
-                        data.once('end', resolve);
-                    });
-                }
-            };
             try {
                 for await (const { data } of iterator) {
-                    const task = pipeChunk(data);
-                    active.add(task);
-                    task.finally(() => active.delete(task));
-                    if (active.size >= maxConcurrency) {
-                        await Promise.race(active);
+                    if (Buffer.isBuffer(data)) {
+                        if (!pass.write(data)) {
+                            await new Promise((resolve) => pass.once('drain', resolve));
+                        }
+                    }
+                    else {
+                        await new Promise((resolve, reject) => {
+                            data.pipe(pass, { end: false });
+                            data.once('error', (err) => {
+                                (pass.destroy(err), reject(err));
+                            });
+                            data.once('end', resolve);
+                            data.once('close', () => {
+                                if (!data.destroyed) {
+                                    console.warn('Chunk stream closed unexpectedly');
+                                }
+                            });
+                        });
                     }
                 }
-                await Promise.all(active);
                 pass.end();
             }
             catch (err) {
+                console.error('Error during reconstruction: ', err);
                 pass.destroy(err instanceof Error ? err : new Error(String(err)));
             }
         })();

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "rac-delta",
   "description": "Storage agnostic delta patching implementation of rac-delta protocol for NodeJs. With streaming support and file reconstruction.",
-  "version": "1.0.18",
+  "version": "1.0.20",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "repository": {