rac-delta 1.0.15 → 1.0.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. package/dist/infrastructure/adapters/s3-storage-adapter.d.ts.map +1 -1
  2. package/dist/infrastructure/adapters/s3-storage-adapter.js +5 -1
  3. package/dist/infrastructure/chunk-sources/storage-chunk-source.d.ts.map +1 -1
  4. package/dist/infrastructure/chunk-sources/storage-chunk-source.js +10 -29
  5. package/dist/infrastructure/services/memory-reconstruction-service.d.ts.map +1 -1
  6. package/dist/infrastructure/services/memory-reconstruction-service.js +21 -18
  7. package/package.json +1 -1
package/dist/infrastructure/adapters/s3-storage-adapter.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"s3-storage-adapter.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/adapters/s3-storage-adapter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAWlC,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE3C,qBAAa,gBAAiB,SAAQ,kBAAkB;IAG1C,OAAO,CAAC,QAAQ,CAAC,MAAM;IAFnC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAW;gBAED,MAAM,EAAE,eAAe;IAU9C,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAI9B,OAAO,CAAC,UAAU;IAIZ,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAiBhD,QAAQ,CACZ,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,QAAQ,GAAG,MAAM,EACvB,IAAI,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QAAC,IAAI,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;KAAE,GAC9D,OAAO,CAAC,IAAI,CAAC;IAoBV,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAY3C,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKxC,UAAU,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IA8B/B,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAqBpD,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAe7C,cAAc,IAAI,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;CA6BhD"}
1
+ {"version":3,"file":"s3-storage-adapter.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/adapters/s3-storage-adapter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAe,MAAM,QAAQ,CAAC;AAW/C,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE3C,qBAAa,gBAAiB,SAAQ,kBAAkB;IAG1C,OAAO,CAAC,QAAQ,CAAC,MAAM;IAFnC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAW;gBAED,MAAM,EAAE,eAAe;IAU9C,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAI9B,OAAO,CAAC,UAAU;IAIZ,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAqBhD,QAAQ,CACZ,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,QAAQ,GAAG,MAAM,EACvB,IAAI,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QAAC,IAAI,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;KAAE,GAC9D,OAAO,CAAC,IAAI,CAAC;IAoBV,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAY3C,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKxC,UAAU,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IA8B/B,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAqBpD,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAe7C,cAAc,IAAI,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;CA6BhD"}
package/dist/infrastructure/adapters/s3-storage-adapter.js CHANGED
@@ -1,6 +1,7 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.S3StorageAdapter = void 0;
4
+ const stream_1 = require("stream");
4
5
  const client_s3_1 = require("@aws-sdk/client-s3");
5
6
  const adapters_1 = require("../../core/adapters");
6
7
  class S3StorageAdapter extends adapters_1.HashStorageAdapter {
@@ -25,7 +26,10 @@ class S3StorageAdapter extends adapters_1.HashStorageAdapter {
25
26
  try {
26
27
  const key = this.resolveKey(hash);
27
28
  const res = await this.s3.send(new client_s3_1.GetObjectCommand({ Bucket: this.config.bucket, Key: key }));
28
- return res.Body;
29
+ const s3Stream = res.Body;
30
+ const pass = new stream_1.PassThrough({ highWaterMark: 1024 * 1024 });
31
+ s3Stream.pipe(pass);
32
+ return pass;
29
33
  }
30
34
  catch (error) {
31
35
  if (error instanceof client_s3_1.NoSuchKey) {
package/dist/infrastructure/chunk-sources/storage-chunk-source.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"storage-chunk-source.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/chunk-sources/storage-chunk-source.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,cAAc,EAAqB,MAAM,qBAAqB,CAAC;AAG5F,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE3C,OAAO,EAAE,QAAQ,EAAe,MAAM,QAAQ,CAAC;AAE/C,qBAAa,kBAAmB,YAAW,WAAW;IAElD,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;gBADR,OAAO,EAAE,cAAc,EACvB,OAAO,CAAC,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAGnD,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAmBvC,SAAS,CACb,MAAM,EAAE,MAAM,EAAE,EAChB,EAAE,WAAe,EAAE,GAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAA;KAAO,GACjD,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAoBxB,YAAY,CACjB,MAAM,EAAE,MAAM,EAAE,EAChB,EACE,WAAe,EACf,aAAoB,GACrB,GAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,OAAO,CAAA;KAAO,GACxD,cAAc,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,QAAQ,CAAA;KAAE,CAAC;CAkIpD"}
1
+ {"version":3,"file":"storage-chunk-source.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/chunk-sources/storage-chunk-source.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,cAAc,EAAqB,MAAM,qBAAqB,CAAC;AAG5F,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE3C,OAAO,EAAE,QAAQ,EAAe,MAAM,QAAQ,CAAC;AAE/C,qBAAa,kBAAmB,YAAW,WAAW;IAElD,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;gBADR,OAAO,EAAE,cAAc,EACvB,OAAO,CAAC,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAGnD,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAmBvC,SAAS,CACb,MAAM,EAAE,MAAM,EAAE,EAChB,EAAE,WAAe,EAAE,GAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAA;KAAO,GACjD,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAoBxB,YAAY,CACjB,MAAM,EAAE,MAAM,EAAE,EAChB,EACE,WAAe,EACf,aAAoB,GACrB,GAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,OAAO,CAAA;KAAO,GACxD,cAAc,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,QAAQ,CAAA;KAAE,CAAC;CA+GpD"}
package/dist/infrastructure/chunk-sources/storage-chunk-source.js CHANGED
@@ -3,7 +3,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.StorageChunkSource = void 0;
4
4
  const stream_to_buffer_1 = require("../../core/utils/stream-to-buffer");
5
5
  const exceptions_1 = require("../../core/exceptions");
6
- const stream_1 = require("stream");
7
6
  class StorageChunkSource {
8
7
  storage;
9
8
  urlsMap;
@@ -44,7 +43,6 @@ class StorageChunkSource {
44
43
  if (hashes.length === 0) {
45
44
  return;
46
45
  }
47
- const controller = new AbortController();
48
46
  const queue = hashes.map((hash, index) => ({ hash, index }));
49
47
  const results = new Map();
50
48
  let nextIndexToEmit = 0;
@@ -59,45 +57,32 @@ class StorageChunkSource {
59
57
  }
60
58
  };
61
59
  const waitForData = async () => {
62
- while ((preserveOrder && !results.has(nextIndexToEmit) && workerError === null) ||
63
- (!preserveOrder && results.size === 0 && workerError === null)) {
60
+ while ((preserveOrder && !results.has(nextIndexToEmit) && !workerError) ||
61
+ (!preserveOrder && results.size === 0 && !workerError)) {
64
62
  await new Promise((resolve) => pendingResolvers.push(resolve));
65
63
  }
66
64
  };
67
65
  const worker = async () => {
68
66
  activeWorkers++;
69
67
  try {
70
- while (queue.length > 0 && !controller.signal.aborted) {
68
+ while (queue.length > 0 && !workerError) {
71
69
  const { hash, index } = queue.shift();
72
70
  try {
73
- const url = this.urlsMap?.get(hash);
74
- if (!url && this.storage.type === 'url') {
75
- throw new Error(`No URL found for hash: ${hash}`);
76
- }
77
71
  const stream = this.storage.type === 'hash'
78
72
  ? await this.storage.getChunk(hash)
79
- : await this.storage.getChunkByUrl(url);
73
+ : await this.storage.getChunkByUrl(this.urlsMap?.get(hash));
80
74
  if (!stream) {
81
- throw new exceptions_1.ChunkNotFoundException(`${hash} not found in storage`);
75
+ throw new exceptions_1.ChunkNotFoundException(`Chunk ${hash} not found in storage`);
82
76
  }
83
- const pass = new stream_1.PassThrough({ highWaterMark: 1024 * 1024 });
84
- stream.on('error', (err) => {
85
- workerError = err instanceof Error ? err : new Error(String(err));
86
- controller.abort();
87
- signalNext();
88
- });
89
- pass.on('error', (err) => {
77
+ stream.once('error', (err) => {
90
78
  workerError = err instanceof Error ? err : new Error(String(err));
91
- controller.abort();
92
79
  signalNext();
93
80
  });
94
- stream.pipe(pass);
95
- results.set(index, { hash, data: pass });
81
+ results.set(index, { hash, data: stream });
96
82
  signalNext();
97
83
  }
98
- catch (error) {
99
- workerError = error instanceof Error ? error : new Error(String(error));
100
- controller.abort();
84
+ catch (err) {
85
+ workerError = err instanceof Error ? err : new Error(String(err));
101
86
  signalNext();
102
87
  return;
103
88
  }
@@ -107,8 +92,8 @@ class StorageChunkSource {
107
92
  activeWorkers--;
108
93
  if (activeWorkers === 0) {
109
94
  workersDone = true;
110
- signalNext();
111
95
  }
96
+ signalNext();
112
97
  }
113
98
  };
114
99
  const workers = Array.from({ length: Math.min(concurrency, queue.length) }, worker);
@@ -116,11 +101,9 @@ class StorageChunkSource {
116
101
  while (true) {
117
102
  await waitForData();
118
103
  if (workerError) {
119
- // ensure workers settle so their promise rejections don't become unhandled
120
104
  await Promise.allSettled(workers);
121
105
  throw workerError;
122
106
  }
123
- // Emit strictly in order
124
107
  if (preserveOrder) {
125
108
  while (results.has(nextIndexToEmit)) {
126
109
  yield results.get(nextIndexToEmit);
@@ -128,7 +111,6 @@ class StorageChunkSource {
128
111
  nextIndexToEmit++;
129
112
  }
130
113
  }
131
- // Emit as soon as any result is ready
132
114
  if (!preserveOrder) {
133
115
  const [index, value] = results.entries().next().value ?? [];
134
116
  if (value !== undefined && index !== undefined) {
@@ -142,7 +124,6 @@ class StorageChunkSource {
142
124
  }
143
125
  }
144
126
  finally {
145
- controller.abort();
146
127
  await Promise.allSettled(workers);
147
128
  }
148
129
  }
package/dist/infrastructure/services/memory-reconstruction-service.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"memory-reconstruction-service.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/services/memory-reconstruction-service.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAyB,MAAM,QAAQ,CAAC;AAIzD,OAAO,EACL,WAAW,EAEX,aAAa,EACb,qBAAqB,EACrB,qBAAqB,EACtB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,SAAS,EAAE,SAAS,EAAS,MAAM,mBAAmB,CAAC;AAEhE,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE3C,qBAAa,2BAA4B,YAAW,qBAAqB;IAC3D,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAAN,MAAM,EAAE,aAAa;IAE5C,cAAc,CAClB,IAAI,EAAE,SAAS,EACf,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,GACvC,OAAO,CAAC,IAAI,CAAC;IAmFV,eAAe,CACnB,KAAK,EAAE,SAAS,EAChB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,WAAW,EACxB,OAAO,GAAE,qBAIR,GACA,OAAO,CAAC,IAAI,CAAC;IAgDV,mBAAmB,CACvB,KAAK,EAAE,SAAS,EAChB,WAAW,EAAE,WAAW,EACxB,cAAc,GAAE,MAAU,GACzB,OAAO,CAAC,QAAQ,CAAC;YA6CN,UAAU;IASxB,oEAAoE;YACtD,kBAAkB;IAiChC,oFAAoF;YACtE,iBAAiB;IAmF/B;;;OAGG;YACW,qBAAqB;YA2BrB,YAAY;YAcZ,aAAa;YAyCZ,gBAAgB;CAmChC"}
1
+ {"version":3,"file":"memory-reconstruction-service.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/services/memory-reconstruction-service.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAyB,MAAM,QAAQ,CAAC;AAIzD,OAAO,EACL,WAAW,EAEX,aAAa,EACb,qBAAqB,EACrB,qBAAqB,EACtB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,SAAS,EAAE,SAAS,EAAS,MAAM,mBAAmB,CAAC;AAEhE,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE3C,qBAAa,2BAA4B,YAAW,qBAAqB;IAC3D,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAAN,MAAM,EAAE,aAAa;IAE5C,cAAc,CAClB,IAAI,EAAE,SAAS,EACf,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,GACvC,OAAO,CAAC,IAAI,CAAC;IAmFV,eAAe,CACnB,KAAK,EAAE,SAAS,EAChB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,WAAW,EACxB,OAAO,GAAE,qBAIR,GACA,OAAO,CAAC,IAAI,CAAC;IAgDV,mBAAmB,CACvB,KAAK,EAAE,SAAS,EAChB,WAAW,EAAE,WAAW,EACxB,cAAc,SAAI,GACjB,OAAO,CAAC,QAAQ,CAAC;YA+CN,UAAU;IASxB,oEAAoE;YACtD,kBAAkB;IAiChC,oFAAoF;YACtE,iBAAiB;IAmF/B;;;OAGG;YACW,qBAAqB;YA2BrB,YAAY;YAcZ,aAAa;YAyCZ,gBAAgB;CAmChC"}
package/dist/infrastructure/services/memory-reconstruction-service.js CHANGED
@@ -113,31 +113,34 @@ class MemoryReconstructionService {
113
113
  // This will reconstruct to stream, not to disk
114
114
  async reconstructToStream(entry, chunkSource, maxConcurrency = 5) {
115
115
  const chunks = entry.chunks ?? [];
116
+ const pass = new stream_1.PassThrough({ highWaterMark: 1024 * 1024 });
116
117
  const iterator = this.fetchChunksSmart(chunks, chunkSource, true);
117
- const pass = new stream_1.PassThrough({ highWaterMark: 2 * 1024 * 1024 });
118
+ pass.once('error', (err) => {
119
+ pass.destroy(err);
120
+ });
118
121
  (async () => {
122
+ const active = new Set();
123
+ const pipeChunk = async (data) => {
124
+ if (Buffer.isBuffer(data)) {
125
+ if (!pass.write(data)) {
126
+ await new Promise((resolve) => pass.once('drain', resolve));
127
+ }
128
+ }
129
+ else {
130
+ await new Promise((resolve, reject) => {
131
+ data.pipe(pass, { end: false });
132
+ data.once('error', reject);
133
+ data.once('end', resolve);
134
+ });
135
+ }
136
+ };
119
137
  try {
120
- const active = new Set();
121
138
  for await (const { data } of iterator) {
122
- const task = (async () => {
123
- if (Buffer.isBuffer(data)) {
124
- if (!pass.write(data)) {
125
- await new Promise((resolve) => pass.once('drain', resolve));
126
- }
127
- }
128
- else {
129
- await new Promise((resolve, reject) => {
130
- data.on('error', reject);
131
- pass.on('error', reject);
132
- data.pipe(pass, { end: false });
133
- data.on('end', resolve);
134
- });
135
- }
136
- })();
139
+ const task = pipeChunk(data);
137
140
  active.add(task);
138
141
  task.finally(() => active.delete(task));
139
142
  if (active.size >= maxConcurrency) {
140
- const finished = await Promise.race(active);
143
+ await Promise.race(active);
141
144
  }
142
145
  }
143
146
  await Promise.all(active);
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "rac-delta",
3
3
  "description": "Storage agnostic delta patching implementation of rac-delta protocol for NodeJs. With streaming support and file reconstruction.",
4
- "version": "1.0.15",
4
+ "version": "1.0.17",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/index.d.ts",
7
7
  "repository": {