spliterator 3.0.1 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/index.d.ts +1 -0
- package/out/index.d.ts.map +1 -1
- package/out/index.js +1 -0
- package/out/index.js.map +1 -1
- package/out/lib/AsyncSpliterator.d.ts +18 -2
- package/out/lib/AsyncSpliterator.d.ts.map +1 -1
- package/out/lib/AsyncSpliterator.js +93 -25
- package/out/lib/AsyncSpliterator.js.map +1 -1
- package/out/lib/CSVSpliterator.d.ts +15 -0
- package/out/lib/CSVSpliterator.d.ts.map +1 -1
- package/out/lib/CSVSpliterator.js +63 -15
- package/out/lib/CSVSpliterator.js.map +1 -1
- package/out/lib/JSONSpliterator.d.ts +13 -0
- package/out/lib/JSONSpliterator.d.ts.map +1 -1
- package/out/lib/JSONSpliterator.js +13 -0
- package/out/lib/JSONSpliterator.js.map +1 -1
- package/out/lib/Spliterator.d.ts.map +1 -1
- package/out/lib/Spliterator.js +37 -43
- package/out/lib/Spliterator.js.map +1 -1
- package/out/lib/merge-async-iterators.d.ts +12 -0
- package/out/lib/merge-async-iterators.d.ts.map +1 -0
- package/out/lib/merge-async-iterators.js +38 -0
- package/out/lib/merge-async-iterators.js.map +1 -0
- package/out/lib/parallel-map-runtime.d.ts +20 -0
- package/out/lib/parallel-map-runtime.d.ts.map +1 -0
- package/out/lib/parallel-map-runtime.js +60 -0
- package/out/lib/parallel-map-runtime.js.map +1 -0
- package/out/lib/parallel-map-worker-entry.d.ts +11 -0
- package/out/lib/parallel-map-worker-entry.d.ts.map +1 -0
- package/out/lib/parallel-map-worker-entry.js +40 -0
- package/out/lib/parallel-map-worker-entry.js.map +1 -0
- package/out/lib/parallel-map.d.ts +45 -0
- package/out/lib/parallel-map.d.ts.map +1 -0
- package/out/lib/parallel-map.js +74 -0
- package/out/lib/parallel-map.js.map +1 -0
- package/out/lib/segment-workers.d.ts +2 -8
- package/out/lib/segment-workers.d.ts.map +1 -1
- package/out/lib/segment-workers.js +2 -34
- package/out/lib/segment-workers.js.map +1 -1
- package/out/lib/shared.d.ts +8 -3
- package/out/lib/shared.d.ts.map +1 -1
- package/out/lib/shared.js.map +1 -1
- package/package.json +3 -1
package/out/lib/Spliterator.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
import { AsyncSpliterator } from "./AsyncSpliterator.js";
|
|
7
|
-
import { CharacterSequence, debugAsVisibleCharacters, normalizeCharacterInput, } from "./CharacterSequence.js";
|
|
7
|
+
import { CharacterSequence, debugAsVisibleCharacters, Delimiters, normalizeCharacterInput, } from "./CharacterSequence.js";
|
|
8
8
|
import { IndexQueue } from "./IndexQueue.js";
|
|
9
9
|
import { isFileHandleLike } from "./shared.js";
|
|
10
10
|
export { AsyncSpliterator };
|
|
@@ -84,6 +84,7 @@ export class Spliterator {
|
|
|
84
84
|
this.#yieldByteEstimate = this.#source.byteLength - this.#readPosition;
|
|
85
85
|
this.#skipEmpty = init.skipEmpty ?? true;
|
|
86
86
|
this.#enableQuoteHandling = init.enableQuoteHandling ?? false;
|
|
87
|
+
this.#crlf = init.crlf ?? false;
|
|
87
88
|
this.#debug = init.debug ?? false;
|
|
88
89
|
this.#log = this.#debug ? console.debug.bind(console) : () => void 0;
|
|
89
90
|
}
|
|
@@ -110,6 +111,10 @@ export class Spliterator {
|
|
|
110
111
|
*/
|
|
111
112
|
#doubleQuoteSequence = new CharacterSequence('"');
|
|
112
113
|
#enableQuoteHandling;
|
|
114
|
+
/**
|
|
115
|
+
* Whether to trim a carriage return immediately preceding each delimiter match.
|
|
116
|
+
*/
|
|
117
|
+
#crlf;
|
|
113
118
|
/**
|
|
114
119
|
* How many yields to skip.
|
|
115
120
|
*/
|
|
@@ -199,9 +204,10 @@ export class Spliterator {
|
|
|
199
204
|
// resurfacing the entire delimiter run as one (non-empty) row.
|
|
200
205
|
this.#indices.enqueue([0, sourceByteLength]);
|
|
201
206
|
}
|
|
202
|
-
else
|
|
203
|
-
//
|
|
204
|
-
//
|
|
207
|
+
else {
|
|
208
|
+
// Emit an empty field for a trailing delimiter (match String.split). `#fill` always
|
|
209
|
+
// leaves the tail after the last consumed delimiter to this method, in both the plain
|
|
210
|
+
// and quote-aware paths — a source ending exactly on a delimiter has an empty tail.
|
|
205
211
|
const trailingDelimPos = sourceByteLength - this.#needle.length;
|
|
206
212
|
if (trailingDelimPos >= 0 && this.#needle.search(this.#source, trailingDelimPos) !== -1) {
|
|
207
213
|
this.#indices.enqueue([sourceByteLength, sourceByteLength]);
|
|
@@ -209,6 +215,16 @@ export class Spliterator {
|
|
|
209
215
|
}
|
|
210
216
|
this.#done = true;
|
|
211
217
|
}
|
|
218
|
+
/**
|
|
219
|
+
* Trim a carriage return immediately preceding a delimiter match, when {@linkcode SpliteratorInit.crlf} is set.
|
|
220
|
+
* Affects only the emitted range's end — `#readPosition` advancement always uses the raw match offset.
|
|
221
|
+
*/
|
|
222
|
+
#trimEnd(start, end) {
|
|
223
|
+
if (this.#crlf && end > start && this.#source[end - 1] === Delimiters.CarriageReturn) {
|
|
224
|
+
return end - 1;
|
|
225
|
+
}
|
|
226
|
+
return end;
|
|
227
|
+
}
|
|
212
228
|
/**
|
|
213
229
|
* Fill the buffer with data and search for delimiters.
|
|
214
230
|
*/
|
|
@@ -219,59 +235,37 @@ export class Spliterator {
|
|
|
219
235
|
const delimiterIndex = this.#needle.search(this.#source, this.#readPosition);
|
|
220
236
|
if (delimiterIndex === -1)
|
|
221
237
|
return;
|
|
222
|
-
this.#indices.enqueue([this.#readPosition, delimiterIndex]);
|
|
238
|
+
this.#indices.enqueue([this.#readPosition, this.#trimEnd(this.#readPosition, delimiterIndex)]);
|
|
223
239
|
this.#readPosition = delimiterIndex + this.#needle.length;
|
|
224
240
|
}
|
|
225
241
|
return;
|
|
226
242
|
}
|
|
243
|
+
// Quote-aware path: a delimiter inside a double-quoted region does not split, and the
|
|
244
|
+
// emitted slices keep their quotes verbatim — stripping and `""` unescaping belong to the
|
|
245
|
+
// consumer (CSVSpliterator does both). The tail after the last consumed delimiter is left
|
|
246
|
+
// to `#drain`, same as the plain path.
|
|
227
247
|
while (this.#readPosition < sourceByteLength && this.#indices.byteLength < this.#highWaterMark) {
|
|
228
248
|
const matches = this.#needle.searchMatches(this.#source, this.#doubleQuoteSequence, this.#readPosition, sourceByteLength);
|
|
229
249
|
if (matches.length === 0)
|
|
230
250
|
return;
|
|
231
251
|
let sliceStart = this.#readPosition;
|
|
232
252
|
let insideQuotes = false;
|
|
233
|
-
let quoteStart = -1;
|
|
234
253
|
for (const match of matches) {
|
|
235
254
|
if (match.patternId === 1) {
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
// Emit normal field before the opening quote
|
|
239
|
-
if (match.offset > sliceStart) {
|
|
240
|
-
this.#indices.enqueue([sliceStart, match.offset]);
|
|
241
|
-
}
|
|
242
|
-
insideQuotes = true;
|
|
243
|
-
quoteStart = match.offset;
|
|
244
|
-
}
|
|
245
|
-
else {
|
|
246
|
-
// Closing quote: emit quoted field, advance past quote
|
|
247
|
-
this.#indices.enqueue([quoteStart + 1, match.offset]);
|
|
248
|
-
insideQuotes = false;
|
|
249
|
-
sliceStart = match.offset + 1;
|
|
250
|
-
}
|
|
251
|
-
}
|
|
252
|
-
else {
|
|
253
|
-
// Delimiter found; one inside quotes is part of the field, so skip it.
|
|
254
|
-
if (insideQuotes)
|
|
255
|
-
continue;
|
|
256
|
-
// Emit content before this delimiter (empty if delimiter follows closing quote)
|
|
257
|
-
if (sliceStart < match.offset) {
|
|
258
|
-
this.#indices.enqueue([sliceStart, match.offset]);
|
|
259
|
-
}
|
|
260
|
-
sliceStart = match.offset + this.#needle.length;
|
|
255
|
+
insideQuotes = !insideQuotes;
|
|
256
|
+
continue;
|
|
261
257
|
}
|
|
258
|
+
// A delimiter inside quotes is part of the slice, so skip it.
|
|
259
|
+
if (insideQuotes)
|
|
260
|
+
continue;
|
|
261
|
+
this.#indices.enqueue([sliceStart, this.#trimEnd(sliceStart, match.offset)]);
|
|
262
|
+
sliceStart = match.offset + this.#needle.length;
|
|
262
263
|
}
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
this.#indices.enqueue([sliceStart, sourceByteLength]);
|
|
269
|
-
this.#readPosition = sourceByteLength;
|
|
270
|
-
// Case 2: Trailing delimiter (emit empty field to match String.split)
|
|
271
|
-
}
|
|
272
|
-
else if (sliceStart === sourceByteLength && this.#readPosition === sourceByteLength) {
|
|
273
|
-
this.#indices.enqueue([sourceByteLength, sourceByteLength]);
|
|
274
|
-
}
|
|
264
|
+
// No delimiter was consumed (e.g. an unclosed quote swallows the rest of the source) —
|
|
265
|
+
// leave the tail to `#drain` rather than rescanning the same matches forever.
|
|
266
|
+
if (sliceStart === this.#readPosition)
|
|
267
|
+
return;
|
|
268
|
+
this.#readPosition = sliceStart;
|
|
275
269
|
}
|
|
276
270
|
}
|
|
277
271
|
//#endregion
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Spliterator.js","sourceRoot":"","sources":["../../lib/Spliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,gBAAgB,EAAmD,MAAM,uBAAuB,CAAA;AACzG,OAAO,EACN,iBAAiB,EACjB,wBAAwB,EACxB,uBAAuB,GAEvB,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EAAE,gBAAgB,EAAmE,MAAM,aAAa,CAAA;AAE/G,OAAO,EAAE,gBAAgB,EAAE,CAAA;AAI3B;;GAEG;AACH,MAAM,OAAO,WAAW;IAGvB,mBAAmB;IAEnB;;;;;;;;OAQG;IACH,MAAM,CAAC,SAAS,GAAG,gBAAgB,CAAC,IAAI,CAAA;IAExC;;;;;OAKG;IACH,MAAM,CAAC,QAAQ,CAAmC,MAAS,EAAE,OAAwB,EAAE;QACtF,OAAO,IAAI,WAAW,CAAC,uBAAuB,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,CAAA;IAC9D,CAAC;IA0CD,MAAM,CAAC,IAAI,CACV,MAAuE,EACvE,OAA+C,EAAE;QAEjD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,aAAa,IAAI,MAAM,EAAE,CAAC;YAClE,OAAO,IAAI,gBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QAC1C,CAAC;QAED,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,YAAY,GAAG,IAAI,gBAAgB,CAAC,MAAM,CAAC,EAAE,CAAC;YACrF,OAAO,MAAM,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,mBAAmB,EAAE,EAAE,EAAE;gBAC3E,IAAI,aAAiC,CAAA;gBAErC,IAAI,CAAC;oBACJ,aAAa,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE;wBACjD,aAAa,EAAE,IAAI,CAAC,aAAa;qBACjC,CAAC,CAAA;gBACH,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,YAAY,GAAG,EAAE,CAAC;wBACzD,MAAM,OAAO,GAAG,IAAI,KAAK,CACxB,gHAAgH,CAChH,CAAA;wBAED,OAAO,CAAC,KAAK,GAAG,KAAK,CAAA;wBAErB,MAAM,OAAO,CAAA;oBACd,CAAC;oBAED,MAAM,KAAK,CAAA;gBACZ,CAAC;gBAED,OAAO,IAAI,gBAAgB,CAAC,aAAa,EAAE,IAAI,CAAC,CAAA;YACjD,CAAC,CAAC,CAAA;QACH,CAAC;QAED,OAAO,IAAI,WAAW,CAAC,uBAAuB,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,CAAA;IAC9D,CAAC;IAED;;OAEG;IACI,MAAM,CAAC,iBAAiB,CAAC,IAAqB;QACpD,OAAO,IAAI,eAAe,CAA2B;YACpD,SAAS,CAAC,KAAK,EAAE,UAAU;gBAC1B,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;gBAEhD,UAAU,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAA;YAC1C,CAAC;SACD,CAAC,CAAA;IACH,CAAC;IAED;;OAEG;IACI,CAAC,MAAM,CAAC,OAAO,CAAC;QACtB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;IACtB,CAAC;IAED,YAAY,MAAkB,EAAE,OAAwB,EAAE;QACzD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QAErB,IAAI,CAAC,OAAO,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QAEpD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAA;QAEvC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,CAAA;QAE7D,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAA;QAClD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,eAAe,CAAA;QAChF,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,aAAa,CAAA;QAEtE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAA;QACxC,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,mBAAmB,IAAI,KAAK,CAAA;
|
|
1
|
+
{"version":3,"file":"Spliterator.js","sourceRoot":"","sources":["../../lib/Spliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,gBAAgB,EAAmD,MAAM,uBAAuB,CAAA;AACzG,OAAO,EACN,iBAAiB,EACjB,wBAAwB,EACxB,UAAU,EACV,uBAAuB,GAEvB,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EAAE,gBAAgB,EAAmE,MAAM,aAAa,CAAA;AAE/G,OAAO,EAAE,gBAAgB,EAAE,CAAA;AAI3B;;GAEG;AACH,MAAM,OAAO,WAAW;IAGvB,mBAAmB;IAEnB;;;;;;;;OAQG;IACH,MAAM,CAAC,SAAS,GAAG,gBAAgB,CAAC,IAAI,CAAA;IAExC;;;;;OAKG;IACH,MAAM,CAAC,QAAQ,CAAmC,MAAS,EAAE,OAAwB,EAAE;QACtF,OAAO,IAAI,WAAW,CAAC,uBAAuB,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,CAAA;IAC9D,CAAC;IA0CD,MAAM,CAAC,IAAI,CACV,MAAuE,EACvE,OAA+C,EAAE;QAEjD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,aAAa,IAAI,MAAM,EAAE,CAAC;YAClE,OAAO,IAAI,gBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QAC1C,CAAC;QAED,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,YAAY,GAAG,IAAI,gBAAgB,CAAC,MAAM,CAAC,EAAE,CAAC;YACrF,OAAO,MAAM,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,mBAAmB,EAAE,EAAE,EAAE;gBAC3E,IAAI,aAAiC,CAAA;gBAErC,IAAI,CAAC;oBACJ,aAAa,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE;wBACjD,aAAa,EAAE,IAAI,CAAC,aAAa;qBACjC,CAAC,CAAA;gBACH,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,YAAY,GAAG,EAAE,CAAC;wBACzD,MAAM,OAAO,GAAG,IAAI,KAAK,CACxB,gHAAgH,CAChH,CAAA;wBAED,OAAO,CAAC,KAAK,GAAG,KAAK,CAAA;wBAErB,MAAM,OAAO,CAAA;oBACd,CAAC;oBAED,MAAM,KAAK,CAAA;gBACZ,CAAC;gBAED,OAAO,IAAI,gBAAgB,CAAC,aAAa,EAAE,IAAI,CAAC,CAAA;YACjD,CAAC,CAAC,CAAA;QACH,CAAC;QAED,OAAO,IAAI,WAAW,CAAC,uBAAuB,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,CAAA;IAC9D,CAAC;IAED;;OAEG;IACI,MAAM,CAAC,iBAAiB,CAAC,IAAqB;QACpD,OAAO,IAAI,eAAe,CAA2B;YACpD,SAAS,CAAC,KAAK,EAAE,UAAU;gBAC1B,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;gBAEhD,UAAU,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAA;YAC1C,CAAC;SACD,CAAC,CAAA;IACH,CAAC;IAED;;OAEG;IACI,CAAC,MAAM,CAAC,OAAO,CAAC;QACtB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;IACtB,CAAC;IAED,YAAY,MAAkB,EAAE,OAAwB,EAAE;QACzD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QAErB,IAAI,CAAC,OAAO,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QAEpD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAA;QAEvC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,CAAA;QAE7D,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAA;QAClD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,eAAe,CAAA;QAChF,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,aAAa,CAAA;QAEtE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAA;QACxC,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,mBAAmB,IAAI,KAAK,CAAA;QAC7D,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,IAAI,KAAK,CAAA;QAE/B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,IAAI,KAAK,CAAA;QACjC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAA;IACrE,CAAC;IAED,YAAY;IAEZ,4BAA4B;IAE5B;;OAEG;IACM,OAAO,CAAY;IAE5B;;;;;;OAMG;IACM,QAAQ,GAAG,IAAI,UAAU,EAAE,CAAA;IAEpC;;OAEG;IACM,OAAO,CAAmB;IAEnC;;OAEG;IACM,oBAAoB,GAAsB,IAAI,iBAAiB,CAAC,GAAG,CAAC,CAAA;IACpE,oBAAoB,CAAS;IAEtC;;OAEG;IACM,KAAK,CAAS;IAEvB;;OAEG;IACM,eAAe,CAAQ;IAEhC;;OAEG;IACM,eAAe,CAAQ;IAEhC;;;;OAIG;IACM,kBAAkB,CAAQ;IAEnC;;OAEG;IACM,UAAU,CAAS;IAE5B;;;;OAIG;IACM,cAAc,CAAQ;IAE/B;;OAEG;IACH,kBAAkB,GAAG,CAAC,CAAA;IAEtB;;OAEG;IACH,WAAW,GAAG,CAAC,CAAA;IAEf;;OAEG;IACH,aAAa,CAAQ;IAErB;;OAEG;IACH,kBAAkB,CAAuB;IAEzC;;OAEG;IACH,MAAM,CAAS;IAEf;;OAEG;IACH,KAAK,GAAG,KAAK,CAAA;IAEb,YAAY;IAEZ,yBAAyB;IAEzB,IAAI,CAA0B;IAE9B,SAAS;QACR,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB;;;eAGG;YACH,MAAM,6BAA6B,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;YAClF,MAAM,YAAY,GAAG,IAAI,CAAC,kBAAkB,GAAG,CAAC,IAAI,CAAC,kBAAkB,GAAG,6BAA6B,CAAC,CAAA;YAExG,IAAI,CAAC,IAAI,CAAC;gBACT,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU;gBACtC,YAAY,EAAE,IAAI,CAAC,aAAa;gBAChC,iBAAiB,EAAE,IAAI,CAAC,kBAAkB;gBAC1C,iBAAiB,EAAE,IAAI,CAAC,kBAAkB;gBAC1C,UAAU,EAAE,IAAI,CAAC,WAAW;gBAC5B,6BAA6B;gBAC7B,YAAY;aACZ,CAAC,CAAA;QACH,CAAC;QAED,OAAO;YACN,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,SAAS;SAChB,CAAA;IACF,CAAC;IAED,MAAM;QACL,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAA;QAEhD,IAAI,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAA;QAExD,qFAAqF;QACrF,MAAM,aAAa,GAAG,IAAI,CAAC,kBAAkB,CAAA;QAE7C,IAAI,IAAI,CAAC,aAAa,GAAG,gBAAgB,EAAE,CAAC;YAC3C,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,aAAa,EAAE,gBAAgB,CAAC,CAAC,CAAA;QAC9D,CAAC;aAAM,IAAI,IAAI,CAAC,WAAW,KAAK,CAAC,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;YAClE,sFAAsF;YACtF,mFAAmF;YACnF,iFAAiF;YACjF,+DAA+D;YAC/D,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC,CAAA;QAC7C,CAAC;aAAM,CAAC;YACP,oFAAoF;YACpF,sFAAsF;YACtF,oFAAoF;YACpF,MAAM,gBAAgB,GAAG,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;YAE/D,IAAI,gBAAgB,IAAI,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gBACzF,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,gBAAgB,EAAE,gBAAgB,CAAC,CAAC,CAAA;YAC5D,CAAC;QACF,CAAC;QAED,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;IAClB,CAAC;IAED;;;OAGG;IACH,QAAQ,CAAC,KAAa,EAAE,GAAW;QAClC,IAAI,IAAI,CAAC,KAAK,IAAI,GAAG,GAAG,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,UAAU,CAAC,cAAc,EAAE,CAAC;YACtF,OAAO,GAAG,GAAG,CAAC,CAAA;QACf,CAAC;QAED,OAAO,GAAG,CAAA;IACX,CAAC;IAED;;OAEG;IACH,KAAK;QACJ,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAA;QAEhD,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAChC,OAAO,IAAI,CAAC,aAAa,GAAG,gBAAgB,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;gBAChG,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,aAAa,CAAC,CAAA;gBAE5E,IAAI,cAAc,KAAK,CAAC,CAAC;oBAAE,OAAM;gBAEjC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC,CAAC,CAAA;gBAC9F,IAAI,CAAC,aAAa,GAAG,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;YAC1D,CAAC;YAED,OAAM;QACP,CAAC;QAED,sFAAsF;QACtF,0FAA0F;QAC1F,0FAA0F;QAC1F,uCAAuC;QACvC,OAAO,IAAI,CAAC,aAAa,GAAG,gBAAgB,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;YAChG,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CACzC,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,oBAAoB,EACzB,IAAI,CAAC,aAAa,EAClB,gBAAgB,CAChB,CAAA;YAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAM;YAEhC,IAAI,UAAU,GAAG,IAAI,CAAC,aAAa,CAAA;YACnC,IAAI,YAAY,GAAG,KAAK,CAAA;YAExB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC7B,IAAI,KAAK,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;oBAC3B,YAAY,GAAG,CAAC,YAAY,CAAA;oBAC5B,SAAQ;gBACT,CAAC;gBAED,8DAA8D;gBAC9D,IAAI,YAAY;oBAAE,SAAQ;gBAE1B,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;gBAC5E,UAAU,GAAG,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;YAChD,CAAC;YAED,uFAAuF;YACvF,8EAA8E;YAC9E,IAAI,UAAU,KAAK,IAAI,CAAC,aAAa;gBAAE,OAAM;YAE7C,IAAI,CAAC,aAAa,GAAG,UAAU,CAAA;QAChC,CAAC;IACF,CAAC;IAED,YAAY;IAEZ,0BAA0B;IAE1B;;OAEG;IACI,IAAI;QACV,kFAAkF;QAClF,iEAAiE;QACjE,OAAO,IAAI,EAAE,CAAC;YACb,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,eAAe;gBAAE,OAAO,IAAI,CAAC,SAAS,EAAE,CAAA;YAEnF,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI;gBAAE,IAAI,CAAC,KAAK,EAAE,CAAA;YAErC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACzB,IAAI,CAAC,MAAM,EAAE,CAAA;YACd,CAAC;YAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAA;YAChD,IAAI,CAAC,kBAAkB,GAAG,gBAAgB,CAAA;YAE1C,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACvB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;gBAEjB,OAAO,IAAI,CAAC,SAAS,EAAE,CAAA;YACxB,CAAC;YAED,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,gBAAgB,CAAA;YAErC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;YAE/C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC3C,SAAQ;YACT,CAAC;YAED,IAAI,CAAC,WAAW,EAAE,CAAA;YAElB,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;gBAC9C,SAAQ;YACT,CAAC;YAED,IAAI,CAAC,kBAAkB,IAAI,GAAG,GAAG,KAAK,CAAA;YAEtC,OAAO;gBACN,KAAK,EAAE,KAAqB;gBAC5B,IAAI,EAAE,KAAK;aACX,CAAA;QACF,CAAC;IACF,CAAC;IAED;;;;;OAKG;IACI,OAAO;QACb,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACxB,CAAC;IAED;;OAEG;IACI,cAAc,CAAC,OAAO,GAAG,IAAI,WAAW,EAAE;QAChD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;IAC1D,CAAC;IAEM,CAAC,MAAM,CAAC,QAAQ,CAAC;QACvB,OAAO,IAAI,CAAA;IACZ,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license MIT
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* Merge several async iterables, pulling from **all of them concurrently** and yielding each value as soon as it
|
|
8
|
+
* arrives (completion order, not source order). An error from any source rejects; remaining sources are returned
|
|
9
|
+
* (cancelled) on completion, error, or early break.
|
|
10
|
+
*/
|
|
11
|
+
export declare function mergeAsyncIterators<R>(sources: Array<AsyncIterable<R>>): AsyncIterableIterator<R>;
|
|
12
|
+
//# sourceMappingURL=merge-async-iterators.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"merge-async-iterators.d.ts","sourceRoot":"","sources":["../../lib/merge-async-iterators.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;;;GAIG;AACH,wBAAuB,mBAAmB,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,GAAG,qBAAqB,CAAC,CAAC,CAAC,CA2BxG"}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license MIT
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* Merge several async iterables, pulling from **all of them concurrently** and yielding each value as soon as it
|
|
8
|
+
* arrives (completion order, not source order). An error from any source rejects; remaining sources are returned
|
|
9
|
+
* (cancelled) on completion, error, or early break.
|
|
10
|
+
*/
|
|
11
|
+
export async function* mergeAsyncIterators(sources) {
|
|
12
|
+
const advance = (it) => it.next().then((result) => ({ it, result }));
|
|
13
|
+
const pending = new Map();
|
|
14
|
+
for (const source of sources) {
|
|
15
|
+
const it = source[Symbol.asyncIterator]();
|
|
16
|
+
pending.set(it, advance(it));
|
|
17
|
+
}
|
|
18
|
+
try {
|
|
19
|
+
while (pending.size > 0) {
|
|
20
|
+
const { it, result } = await Promise.race(pending.values());
|
|
21
|
+
if (result.done) {
|
|
22
|
+
pending.delete(it);
|
|
23
|
+
}
|
|
24
|
+
else {
|
|
25
|
+
yield result.value;
|
|
26
|
+
pending.set(it, advance(it));
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
finally {
|
|
31
|
+
// Stop any iterators we didn't drain (early break / error). Swallow rejections from their
|
|
32
|
+
// in-flight `next()` so they don't surface as unhandled.
|
|
33
|
+
for (const promise of pending.values())
|
|
34
|
+
void promise.catch(() => { });
|
|
35
|
+
await Promise.allSettled(Array.from(pending.keys(), (it) => it.return?.()));
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=merge-async-iterators.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"merge-async-iterators.js","sourceRoot":"","sources":["../../lib/merge-async-iterators.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;;;GAIG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,mBAAmB,CAAI,OAAgC;IAC7E,MAAM,OAAO,GAAG,CAAC,EAAoB,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,CAAA;IACtF,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkF,CAAA;IAEzG,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAA;QACzC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,CAAA;IAC7B,CAAC;IAED,IAAI,CAAC;QACJ,OAAO,OAAO,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;YAE3D,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBACjB,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;YACnB,CAAC;iBAAM,CAAC;gBACP,MAAM,MAAM,CAAC,KAAK,CAAA;gBAClB,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,CAAA;YAC7B,CAAC;QACF,CAAC;IACF,CAAC;YAAS,CAAC;QACV,0FAA0F;QAC1F,yDAAyD;QACzD,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,MAAM,EAAE;YAAE,KAAK,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QAEpE,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAA;IAC5E,CAAC;AACF,CAAC"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license MIT
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Transport-agnostic core of `parallelMap` — a record-dispatch worker pool. `runPool` keeps every
|
|
7
|
+
* worker busy mapping batches pulled from one shared source, with no `worker_threads` dependency so it
|
|
8
|
+
* unit-tests against fake workers.
|
|
9
|
+
*/
|
|
10
|
+
/** One pool slot. `process` ships a batch to its worker and resolves with that batch's results. */
|
|
11
|
+
export interface PoolWorker<T, R> {
|
|
12
|
+
process(batch: T[]): Promise<R[]>;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Drive a pool of `workers` over `source`: each worker loops — pull a batch, map it, emit results — until the source is
|
|
16
|
+
* exhausted. Results stream out in completion order (a worker's whole batch yields before it pulls the next, which
|
|
17
|
+
* bounds in-flight work to the pool size). A worker error propagates and tears the pool down.
|
|
18
|
+
*/
|
|
19
|
+
export declare function runPool<T, R>(workers: Array<PoolWorker<T, R>>, source: AsyncIterable<T> | Iterable<T>, batchSize: number): AsyncIterableIterator<R>;
|
|
20
|
+
//# sourceMappingURL=parallel-map-runtime.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parallel-map-runtime.d.ts","sourceRoot":"","sources":["../../lib/parallel-map-runtime.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,mGAAmG;AACnG,MAAM,WAAW,UAAU,CAAC,CAAC,EAAE,CAAC;IAC/B,OAAO,CAAC,KAAK,EAAE,CAAC,EAAE,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAAA;CACjC;AA8CD;;;;GAIG;AACH,wBAAgB,OAAO,CAAC,CAAC,EAAE,CAAC,EAC3B,OAAO,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAChC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,EACtC,SAAS,EAAE,MAAM,GACf,qBAAqB,CAAC,CAAC,CAAC,CAgB1B"}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license MIT
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Transport-agnostic core of `parallelMap` — a record-dispatch worker pool. `runPool` keeps every
|
|
7
|
+
* worker busy mapping batches pulled from one shared source, with no `worker_threads` dependency so it
|
|
8
|
+
* unit-tests against fake workers.
|
|
9
|
+
*/
|
|
10
|
+
import { mergeAsyncIterators } from "./merge-async-iterators.js";
|
|
11
|
+
/**
|
|
12
|
+
* Serialize pulls from `source` into batches of up to `batchSize`. Returns a `next()` that is safe to call concurrently
|
|
13
|
+
* (each call awaits the prior pull) — concurrent worker loops share one source without racing the underlying iterator.
|
|
14
|
+
* Resolves `null` once the source is exhausted.
|
|
15
|
+
*/
|
|
16
|
+
function makeBatcher(source, batchSize) {
|
|
17
|
+
const iterator = Symbol.asyncIterator in source
|
|
18
|
+
? source[Symbol.asyncIterator]()
|
|
19
|
+
: source[Symbol.iterator]();
|
|
20
|
+
let chain = Promise.resolve();
|
|
21
|
+
let exhausted = false;
|
|
22
|
+
return () => {
|
|
23
|
+
const pull = chain.then(async () => {
|
|
24
|
+
if (exhausted)
|
|
25
|
+
return null;
|
|
26
|
+
const batch = [];
|
|
27
|
+
while (batch.length < batchSize) {
|
|
28
|
+
const { value, done } = await iterator.next();
|
|
29
|
+
if (done) {
|
|
30
|
+
exhausted = true;
|
|
31
|
+
break;
|
|
32
|
+
}
|
|
33
|
+
batch.push(value);
|
|
34
|
+
}
|
|
35
|
+
return batch.length > 0 ? batch : null;
|
|
36
|
+
});
|
|
37
|
+
// Keep the chain alive regardless of this pull's outcome so the next call still serializes.
|
|
38
|
+
chain = pull.then(() => undefined, () => undefined);
|
|
39
|
+
return pull;
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Drive a pool of `workers` over `source`: each worker loops — pull a batch, map it, emit results — until the source is
|
|
44
|
+
* exhausted. Results stream out in completion order (a worker's whole batch yields before it pulls the next, which
|
|
45
|
+
* bounds in-flight work to the pool size). A worker error propagates and tears the pool down.
|
|
46
|
+
*/
|
|
47
|
+
export function runPool(workers, source, batchSize) {
|
|
48
|
+
const nextBatch = makeBatcher(source, batchSize);
|
|
49
|
+
async function* workerLoop(worker) {
|
|
50
|
+
for (;;) {
|
|
51
|
+
const batch = await nextBatch();
|
|
52
|
+
if (batch === null)
|
|
53
|
+
return;
|
|
54
|
+
const results = await worker.process(batch);
|
|
55
|
+
yield* results;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
return mergeAsyncIterators(workers.map((worker) => workerLoop(worker)));
|
|
59
|
+
}
|
|
60
|
+
//# sourceMappingURL=parallel-map-runtime.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parallel-map-runtime.js","sourceRoot":"","sources":["../../lib/parallel-map-runtime.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AAOhE;;;;GAIG;AACH,SAAS,WAAW,CAAI,MAAsC,EAAE,SAAiB;IAChF,MAAM,QAAQ,GACb,MAAM,CAAC,aAAa,IAAI,MAAM;QAC7B,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE;QAChC,CAAC,CAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAkC,CAAA;IAE9D,IAAI,KAAK,GAAqB,OAAO,CAAC,OAAO,EAAE,CAAA;IAC/C,IAAI,SAAS,GAAG,KAAK,CAAA;IAErB,OAAO,GAAG,EAAE;QACX,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YAClC,IAAI,SAAS;gBAAE,OAAO,IAAI,CAAA;YAE1B,MAAM,KAAK,GAAQ,EAAE,CAAA;YAErB,OAAO,KAAK,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;gBACjC,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;gBAE7C,IAAI,IAAI,EAAE,CAAC;oBACV,SAAS,GAAG,IAAI,CAAA;oBAChB,MAAK;gBACN,CAAC;gBAED,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAClB,CAAC;YAED,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAA;QACvC,CAAC,CAAC,CAAA;QAEF,4FAA4F;QAC5F,KAAK,GAAG,IAAI,CAAC,IAAI,CAChB,GAAG,EAAE,CAAC,SAAS,EACf,GAAG,EAAE,CAAC,SAAS,CACf,CAAA;QAED,OAAO,IAAI,CAAA;IACZ,CAAC,CAAA;AACF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,OAAO,CACtB,OAAgC,EAChC,MAAsC,EACtC,SAAiB;IAEjB,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;IAEhD,KAAK,SAAS,CAAC,CAAC,UAAU,CAAC,MAAwB;QAClD,SAAS,CAAC;YACT,MAAM,KAAK,GAAG,MAAM,SAAS,EAAE,CAAA;YAE/B,IAAI,KAAK,KAAK,IAAI;gBAAE,OAAM;YAE1B,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YAE3C,KAAK,CAAC,CAAC,OAAO,CAAA;QACf,CAAC;IACF,CAAC;IAED,OAAO,mBAAmB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;AACxE,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license MIT
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Runs inside a worker thread (spawned by parallelMap). Imports the handler module once (top-level =
|
|
7
|
+
* per-worker init), then maps each dispatched batch and posts the results back. Processes one batch at
|
|
8
|
+
* a time — the pool never dispatches the next until this batch's result returns.
|
|
9
|
+
*/
|
|
10
|
+
export {};
|
|
11
|
+
//# sourceMappingURL=parallel-map-worker-entry.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parallel-map-worker-entry.d.ts","sourceRoot":"","sources":["../../lib/parallel-map-worker-entry.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license MIT
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Runs inside a worker thread (spawned by parallelMap). Imports the handler module once (top-level =
|
|
7
|
+
* per-worker init), then maps each dispatched batch and posts the results back. Processes one batch at
|
|
8
|
+
* a time — the pool never dispatches the next until this batch's result returns.
|
|
9
|
+
*/
|
|
10
|
+
import { parentPort, workerData } from "node:worker_threads";
|
|
11
|
+
const data = workerData;
|
|
12
|
+
const port = parentPort;
|
|
13
|
+
let nextIndex = 0;
|
|
14
|
+
const handlerReady = import(data.handlerUrl).then((mod) => mod.handleItem ?? mod.default);
|
|
15
|
+
port.on("message", async (msg) => {
|
|
16
|
+
const message = msg;
|
|
17
|
+
if (message?.type !== "batch")
|
|
18
|
+
return;
|
|
19
|
+
try {
|
|
20
|
+
const handleItem = await handlerReady;
|
|
21
|
+
if (typeof handleItem !== "function") {
|
|
22
|
+
throw new Error(`Worker module ${data.handlerUrl} has no handleItem export.`);
|
|
23
|
+
}
|
|
24
|
+
const results = [];
|
|
25
|
+
const transfer = [];
|
|
26
|
+
for (const item of message.batch) {
|
|
27
|
+
const result = await handleItem(item, { index: nextIndex++ });
|
|
28
|
+
if (result === undefined)
|
|
29
|
+
continue;
|
|
30
|
+
results.push(result);
|
|
31
|
+
if (result instanceof Uint8Array)
|
|
32
|
+
transfer.push(result.buffer);
|
|
33
|
+
}
|
|
34
|
+
port.postMessage({ type: "result", results }, transfer);
|
|
35
|
+
}
|
|
36
|
+
catch (error) {
|
|
37
|
+
port.postMessage({ type: "error", message: error instanceof Error ? error.message : String(error) });
|
|
38
|
+
}
|
|
39
|
+
});
|
|
40
|
+
//# sourceMappingURL=parallel-map-worker-entry.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parallel-map-worker-entry.js","sourceRoot":"","sources":["../../lib/parallel-map-worker-entry.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AAS5D,MAAM,IAAI,GAAG,UAAwB,CAAA;AACrC,MAAM,IAAI,GAAG,UAAW,CAAA;AAExB,IAAI,SAAS,GAAG,CAAC,CAAA;AACjB,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAChD,CAAC,GAAgE,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,IAAI,GAAG,CAAC,OAAO,CACnG,CAAA;AAED,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,GAAY,EAAE,EAAE;IACzC,MAAM,OAAO,GAAG,GAA0C,CAAA;IAE1D,IAAI,OAAO,EAAE,IAAI,KAAK,OAAO;QAAE,OAAM;IAErC,IAAI,CAAC;QACJ,MAAM,UAAU,GAAG,MAAM,YAAY,CAAA;QAErC,IAAI,OAAO,UAAU,KAAK,UAAU,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CAAC,iBAAiB,IAAI,CAAC,UAAU,4BAA4B,CAAC,CAAA;QAC9E,CAAC;QAED,MAAM,OAAO,GAAc,EAAE,CAAA;QAC7B,MAAM,QAAQ,GAAkB,EAAE,CAAA;QAElC,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAClC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,CAAC,CAAA;YAE7D,IAAI,MAAM,KAAK,SAAS;gBAAE,SAAQ;YAElC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAEpB,IAAI,MAAM,YAAY,UAAU;gBAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,MAAqB,CAAC,CAAA;QAC9E,CAAC;QAED,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,QAAQ,CAAC,CAAA;IACxD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IACrG,CAAC;AACF,CAAC,CAAC,CAAA"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license MIT
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
/** Per-item handler a parallelMap worker module exports. `index` is per-worker monotonic. */
|
|
7
|
+
export type ParallelHandler<T = unknown, R = unknown> = (item: T, ctx: {
|
|
8
|
+
index: number;
|
|
9
|
+
}) => R | Uint8Array | undefined | Promise<R | Uint8Array | undefined>;
|
|
10
|
+
export interface ParallelMapOptions {
|
|
11
|
+
/** Worker module path/URL exporting `handleItem(item, ctx)`. Top-level code = per-worker init. */
|
|
12
|
+
worker: string | URL;
|
|
13
|
+
/**
|
|
14
|
+
* Number of worker threads in the pool. Keep it SMALL for I/O- or memory-bound handlers (per-row DB lookups, large
|
|
15
|
+
* shared datasets): throughput typically peaks at ~2–3 and _degrades_ past that as workers contend for the shared
|
|
16
|
+
* resource — only CPU-bound handlers scale toward the core count. Also bounded by RAM (each worker re-inits its
|
|
17
|
+
* deps). Sweep it for your workload rather than defaulting to `availableParallelism()`.
|
|
18
|
+
*/
|
|
19
|
+
concurrency: number;
|
|
20
|
+
/** Items per dispatched batch. @default 64 */
|
|
21
|
+
batchSize?: number;
|
|
22
|
+
/** Forwarded to every worker via `workerData.userData` (must be structured-cloneable). */
|
|
23
|
+
workerData?: unknown;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Map an async (or sync) iterable through a pool of worker threads. Main pulls items from `source`, batches them, and
|
|
27
|
+
* dispatches each batch to an idle worker running the `worker` handler module; results stream back as a single merged
|
|
28
|
+
* async iterator in **completion order** (not input order). A handler returning `undefined` drops that item; a
|
|
29
|
+
* `Uint8Array` is transferred zero-copy.
|
|
30
|
+
*
|
|
31
|
+
* **When it pays (measure — don't assume).** Threading only wins when per-item work is heavy relative to the ~µs
|
|
32
|
+
* cross-thread dispatch cost (structured-clone in + out). From real workloads: light per-row work — CSV/JSON parse,
|
|
33
|
+
* string normalize, `JSON.parse` — is _dispatch-bound_ and runs **0.3–0.9× of single-threaded (a net loss)**; ms-scale
|
|
34
|
+
* per-row work — model inference, geocoding, crypto/image ops — is where threads win. If the per-row cost isn't clearly
|
|
35
|
+
* milliseconds, benchmark against the single-threaded version before adopting.
|
|
36
|
+
*
|
|
37
|
+
* **Concurrency is not core count.** Heavy work that is I/O- or memory-bound (random reads into one large on-disk DB,
|
|
38
|
+
* anything sharing memory bandwidth) peaks _low_ — often ~2–3 workers — and then _degrades_, because the workers
|
|
39
|
+
* contend for the shared resource, not the CPU. Start small and sweep; don't reach for `availableParallelism()`. Only
|
|
40
|
+
* CPU-bound per-row work (pure compute) scales out toward the core count.
|
|
41
|
+
*
|
|
42
|
+
* All workers are terminated on completion, error, or early `return()`.
|
|
43
|
+
*/
|
|
44
|
+
export declare function parallelMap<T, R = unknown>(source: AsyncIterable<T> | Iterable<T>, options: ParallelMapOptions): AsyncIterableIterator<R>;
|
|
45
|
+
//# sourceMappingURL=parallel-map.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parallel-map.d.ts","sourceRoot":"","sources":["../../lib/parallel-map.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH,6FAA6F;AAC7F,MAAM,MAAM,eAAe,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,OAAO,IAAI,CACvD,IAAI,EAAE,CAAC,EACP,GAAG,EAAE;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,KAClB,CAAC,GAAG,UAAU,GAAG,SAAS,GAAG,OAAO,CAAC,CAAC,GAAG,UAAU,GAAG,SAAS,CAAC,CAAA;AAErE,MAAM,WAAW,kBAAkB;IAClC,kGAAkG;IAClG,MAAM,EAAE,MAAM,GAAG,GAAG,CAAA;IACpB;;;;;OAKG;IACH,WAAW,EAAE,MAAM,CAAA;IACnB,8CAA8C;IAC9C,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,0FAA0F;IAC1F,UAAU,CAAC,EAAE,OAAO,CAAA;CACpB;AAgCD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,WAAW,CAAC,CAAC,EAAE,CAAC,GAAG,OAAO,EACzC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,EACtC,OAAO,EAAE,kBAAkB,GACzB,qBAAqB,CAAC,CAAC,CAAC,CAwB1B"}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license MIT
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
import { Worker } from "node:worker_threads";
|
|
7
|
+
import { runPool } from "./parallel-map-runtime.js";
|
|
8
|
+
/** Wrap a worker as a pool slot: dispatch one batch, await its result. One batch in flight at a time. */
|
|
9
|
+
function workerHandle(worker) {
|
|
10
|
+
let pending = null;
|
|
11
|
+
const settle = (fn) => {
|
|
12
|
+
if (!pending)
|
|
13
|
+
return;
|
|
14
|
+
const p = pending;
|
|
15
|
+
pending = null;
|
|
16
|
+
fn(p);
|
|
17
|
+
};
|
|
18
|
+
worker.on("message", (msg) => {
|
|
19
|
+
if (msg?.type === "result")
|
|
20
|
+
settle((p) => p.resolve(msg.results));
|
|
21
|
+
else if (msg?.type === "error")
|
|
22
|
+
settle((p) => p.reject(new Error(msg.message)));
|
|
23
|
+
});
|
|
24
|
+
worker.on("error", (error) => settle((p) => p.reject(error instanceof Error ? error : new Error(String(error)))));
|
|
25
|
+
return {
|
|
26
|
+
process(batch) {
|
|
27
|
+
return new Promise((resolve, reject) => {
|
|
28
|
+
pending = { resolve, reject };
|
|
29
|
+
worker.postMessage({ type: "batch", batch });
|
|
30
|
+
});
|
|
31
|
+
},
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Map an async (or sync) iterable through a pool of worker threads. Main pulls items from `source`, batches them, and
|
|
36
|
+
* dispatches each batch to an idle worker running the `worker` handler module; results stream back as a single merged
|
|
37
|
+
* async iterator in **completion order** (not input order). A handler returning `undefined` drops that item; a
|
|
38
|
+
* `Uint8Array` is transferred zero-copy.
|
|
39
|
+
*
|
|
40
|
+
* **When it pays (measure — don't assume).** Threading only wins when per-item work is heavy relative to the ~µs
|
|
41
|
+
* cross-thread dispatch cost (structured-clone in + out). From real workloads: light per-row work — CSV/JSON parse,
|
|
42
|
+
* string normalize, `JSON.parse` — is _dispatch-bound_ and runs **0.3–0.9× of single-threaded (a net loss)**; ms-scale
|
|
43
|
+
* per-row work — model inference, geocoding, crypto/image ops — is where threads win. If the per-row cost isn't clearly
|
|
44
|
+
* milliseconds, benchmark against the single-threaded version before adopting.
|
|
45
|
+
*
|
|
46
|
+
* **Concurrency is not core count.** Heavy work that is I/O- or memory-bound (random reads into one large on-disk DB,
|
|
47
|
+
* anything sharing memory bandwidth) peaks _low_ — often ~2–3 workers — and then _degrades_, because the workers
|
|
48
|
+
* contend for the shared resource, not the CPU. Start small and sweep; don't reach for `availableParallelism()`. Only
|
|
49
|
+
* CPU-bound per-row work (pure compute) scales out toward the core count.
|
|
50
|
+
*
|
|
51
|
+
* All workers are terminated on completion, error, or early `return()`.
|
|
52
|
+
*/
|
|
53
|
+
export function parallelMap(source, options) {
|
|
54
|
+
const handlerUrl = options.worker instanceof URL ? options.worker.href : new URL(options.worker, `file://${process.cwd()}/`).href;
|
|
55
|
+
const concurrency = Math.max(1, Math.floor(options.concurrency));
|
|
56
|
+
const batchSize = options.batchSize ?? 64;
|
|
57
|
+
const entryUrl = new URL("./parallel-map-worker-entry.js", import.meta.url);
|
|
58
|
+
const workers = [];
|
|
59
|
+
async function* run() {
|
|
60
|
+
try {
|
|
61
|
+
const pool = Array.from({ length: concurrency }, () => {
|
|
62
|
+
const worker = new Worker(entryUrl, { workerData: { handlerUrl, userData: options.workerData } });
|
|
63
|
+
workers.push(worker);
|
|
64
|
+
return workerHandle(worker);
|
|
65
|
+
});
|
|
66
|
+
yield* runPool(pool, source, batchSize);
|
|
67
|
+
}
|
|
68
|
+
finally {
|
|
69
|
+
await Promise.all(workers.map((worker) => worker.terminate()));
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return run();
|
|
73
|
+
}
|
|
74
|
+
//# sourceMappingURL=parallel-map.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parallel-map.js","sourceRoot":"","sources":["../../lib/parallel-map.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAE5C,OAAO,EAAmB,OAAO,EAAE,MAAM,2BAA2B,CAAA;AAwBpE,yGAAyG;AACzG,SAAS,YAAY,CAAO,MAAc;IACzC,IAAI,OAAO,GAA+E,IAAI,CAAA;IAE9F,MAAM,MAAM,GAAG,CAAC,EAA4C,EAAE,EAAE;QAC/D,IAAI,CAAC,OAAO;YAAE,OAAM;QACpB,MAAM,CAAC,GAAG,OAAO,CAAA;QACjB,OAAO,GAAG,IAAI,CAAA;QACd,EAAE,CAAC,CAAC,CAAC,CAAA;IACN,CAAC,CAAA;IAED,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAA0E,EAAE,EAAE;QACnG,IAAI,GAAG,EAAE,IAAI,KAAK,QAAQ;YAAE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAA;aAC5D,IAAI,GAAG,EAAE,IAAI,KAAK,OAAO;YAAE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;IAChF,CAAC,CAAC,CAAA;IAEF,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAc,EAAE,EAAE,CACrC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAClF,CAAA;IAED,OAAO;QACN,OAAO,CAAC,KAAK;YACZ,OAAO,IAAI,OAAO,CAAM,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBAC3C,OAAO,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,CAAA;gBAC7B,MAAM,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAA;YAC7C,CAAC,CAAC,CAAA;QACH,CAAC;KACD,CAAA;AACF,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,WAAW,CAC1B,MAAsC,EACtC,OAA2B;IAE3B,MAAM,UAAU,GACf,OAAO,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,CAAA;IAC/G,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAA;IAChE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,EAAE,CAAA;IACzC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,gCAAgC,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAC3E,MAAM,OAAO,GAAa,EAAE,CAAA;IAE5B,KAAK,SAAS,CAAC,CAAC,GAAG;QAClB,IAAI,CAAC;YACJ,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE;gBACrD,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAA;gBACjG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;gBAEpB,OAAO,YAAY,CAAO,MAAM,CAAC,CAAA;YAClC,CAAC,CAAC,CAAA;YAEF,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,CAAC,CAAA;QACxC,CAAC;gBAAS,CAAC;YACV,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAA;QAC/D,CAAC;IACF,CAAC;IAED,OAAO,GAAG,EAAE,CAAA;AACb,CAAC"}
|
|
@@ -4,7 +4,9 @@
|
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
import type { CharacterSequenceInput } from "./CharacterSequence.js";
|
|
7
|
+
import { mergeAsyncIterators } from "./merge-async-iterators.js";
|
|
7
8
|
import type { AsyncDataResource } from "./shared.js";
|
|
9
|
+
export { mergeAsyncIterators };
|
|
8
10
|
export interface MinimalWorker {
|
|
9
11
|
on(event: "message", cb: (msg: unknown) => void): void;
|
|
10
12
|
on(event: "error", cb: (err: Error) => void): void;
|
|
@@ -16,14 +18,6 @@ export interface MinimalWorker {
|
|
|
16
18
|
* message or worker `error` rejects the iterator.
|
|
17
19
|
*/
|
|
18
20
|
export declare function workerToIterable<R>(worker: MinimalWorker, onBatchConsumed: () => void): AsyncIterableIterator<R>;
|
|
19
|
-
/**
|
|
20
|
-
* Merge several async iterables, pulling from **all of them concurrently** and yielding each value as
|
|
21
|
-
* soon as it arrives (completion order, not source order). This is what lets every segment worker make
|
|
22
|
-
* progress at once: a sequential `for…yield*` drain would consume one worker to completion while the
|
|
23
|
-
* rest stall at their in-flight window. An error from any source rejects; remaining sources are
|
|
24
|
-
* returned (cancelled) on completion, error, or early break.
|
|
25
|
-
*/
|
|
26
|
-
export declare function mergeAsyncIterators<R>(sources: Array<AsyncIterable<R>>): AsyncIterableIterator<R>;
|
|
27
21
|
export interface AsManyWorkersOptions {
|
|
28
22
|
/** Module path or URL exporting `handleRecord(bytes, ctx)`. Runs once per worker at import. */
|
|
29
23
|
worker: string | URL;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"segment-workers.d.ts","sourceRoot":"","sources":["../../lib/segment-workers.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;
|
|
1
|
+
{"version":3,"file":"segment-workers.d.ts","sourceRoot":"","sources":["../../lib/segment-workers.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AACpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AAEhE,OAAO,KAAK,EAAE,iBAAiB,EAAa,MAAM,aAAa,CAAA;AAE/D,OAAO,EAAE,mBAAmB,EAAE,CAAA;AAE9B,MAAM,WAAW,aAAa;IAC7B,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,GAAG,EAAE,OAAO,KAAK,IAAI,GAAG,IAAI,CAAA;IACtD,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI,CAAA;CAClD;AAID;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,IAAI,GAAG,qBAAqB,CAAC,CAAC,CAAC,CAmDhH;AAED,MAAM,WAAW,oBAAoB;IACpC,+FAA+F;IAC/F,MAAM,EAAE,MAAM,GAAG,GAAG,CAAA;IACpB,8CAA8C;IAC9C,SAAS,CAAC,EAAE,sBAAsB,CAAA;IAClC,yEAAyE;IACzE,WAAW,EAAE,MAAM,CAAA;IACnB,mFAAmF;IACnF,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,wCAAwC;IACxC,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,8EAA8E;IAC9E,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,2DAA2D;IAC3D,UAAU,CAAC,EAAE,OAAO,CAAA;CACpB;AAED;;;;GAIG;AACH,wBAAuB,iBAAiB,CAAC,CAAC,EACzC,MAAM,EAAE,iBAAiB,EACzB,OAAO,EAAE,oBAAoB,GAC3B,qBAAqB,CAAC,CAAC,CAAC,CA6C1B"}
|
|
@@ -4,7 +4,9 @@
|
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
import { Worker } from "node:worker_threads";
|
|
7
|
+
import { mergeAsyncIterators } from "./merge-async-iterators.js";
|
|
7
8
|
import { computeSegments } from "./segments.js";
|
|
9
|
+
export { mergeAsyncIterators };
|
|
8
10
|
/**
|
|
9
11
|
* Drain a worker's batched messages into an async iterator. Listeners attach **eagerly** (messages posted before
|
|
10
12
|
* iteration starts are buffered, not lost) and draining uses a `batches[] + head` pointer (no `Array.shift()`).
|
|
@@ -56,40 +58,6 @@ export function workerToIterable(worker, onBatchConsumed) {
|
|
|
56
58
|
}
|
|
57
59
|
return drain();
|
|
58
60
|
}
|
|
59
|
-
/**
|
|
60
|
-
* Merge several async iterables, pulling from **all of them concurrently** and yielding each value as
|
|
61
|
-
* soon as it arrives (completion order, not source order). This is what lets every segment worker make
|
|
62
|
-
* progress at once: a sequential `for…yield*` drain would consume one worker to completion while the
|
|
63
|
-
* rest stall at their in-flight window. An error from any source rejects; remaining sources are
|
|
64
|
-
* returned (cancelled) on completion, error, or early break.
|
|
65
|
-
*/
|
|
66
|
-
export async function* mergeAsyncIterators(sources) {
|
|
67
|
-
const advance = (it) => it.next().then((result) => ({ it, result }));
|
|
68
|
-
const pending = new Map();
|
|
69
|
-
for (const source of sources) {
|
|
70
|
-
const it = source[Symbol.asyncIterator]();
|
|
71
|
-
pending.set(it, advance(it));
|
|
72
|
-
}
|
|
73
|
-
try {
|
|
74
|
-
while (pending.size > 0) {
|
|
75
|
-
const { it, result } = await Promise.race(pending.values());
|
|
76
|
-
if (result.done) {
|
|
77
|
-
pending.delete(it);
|
|
78
|
-
}
|
|
79
|
-
else {
|
|
80
|
-
yield result.value;
|
|
81
|
-
pending.set(it, advance(it));
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
finally {
|
|
86
|
-
// Stop any iterators we didn't drain (early break / error). Swallow rejections from their
|
|
87
|
-
// in-flight `next()` so they don't surface as unhandled.
|
|
88
|
-
for (const promise of pending.values())
|
|
89
|
-
void promise.catch(() => { });
|
|
90
|
-
await Promise.allSettled(Array.from(pending.keys(), (it) => it.return?.()));
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
61
|
/**
|
|
94
62
|
* Spawn one worker per delimiter-aligned segment, each running the `worker` handler module over its own handle, and
|
|
95
63
|
* merge their results into a single async iterator. Results interleave across segments. Sends an `ack` per consumed
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"segment-workers.js","sourceRoot":"","sources":["../../lib/segment-workers.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAG5C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;
|
|
1
|
+
{"version":3,"file":"segment-workers.js","sourceRoot":"","sources":["../../lib/segment-workers.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAG5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAG/C,OAAO,EAAE,mBAAmB,EAAE,CAAA;AAS9B;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAI,MAAqB,EAAE,eAA2B;IACrF,MAAM,OAAO,GAAU,EAAE,CAAA;IACzB,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,IAAI,GAAG,KAAK,CAAA;IAChB,IAAI,KAAwB,CAAA;IAC5B,IAAI,IAA8B,CAAA;IAElC,MAAM,MAAM,GAAG,GAAG,EAAE;QACnB,IAAI,EAAE,EAAE,CAAA;QACR,IAAI,GAAG,SAAS,CAAA;IACjB,CAAC,CAAA;IAED,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,EAAE;QAC5B,MAAM,CAAC,GAAG,GAAuB,CAAA;QAEjC,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAA;aAC1C,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;YAAE,IAAI,GAAG,IAAI,CAAA;aAClC,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC7B,KAAK,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAA;YAC5B,IAAI,GAAG,IAAI,CAAA;QACZ,CAAC;QAED,MAAM,EAAE,CAAA;IACT,CAAC,CAAC,CAAA;IAEF,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;QAC1B,KAAK,GAAG,GAAG,CAAA;QACX,IAAI,GAAG,IAAI,CAAA;QACX,MAAM,EAAE,CAAA;IACT,CAAC,CAAC,CAAA;IAEF,KAAK,SAAS,CAAC,CAAC,KAAK;QACpB,SAAS,CAAC;YACT,IAAI,IAAI,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC3B,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,EAAE,CAAE,CAAA;gBAE9B,KAAK,MAAM,MAAM,IAAI,KAAK;oBAAE,MAAM,MAAM,CAAA;gBAExC,eAAe,EAAE,CAAA;gBACjB,SAAQ;YACT,CAAC;YAED,IAAI,KAAK;gBAAE,MAAM,KAAK,CAAA;YAEtB,IAAI,IAAI;gBAAE,OAAM;YAEhB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,IAAI,GAAG,OAAO,CAAC,CAAC,CAAA;QACvD,CAAC;IACF,CAAC;IAED,OAAO,KAAK,EAAE,CAAA;AACf,CAAC;AAmBD;;;;GAIG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,iBAAiB,CACvC,MAAyB,EACzB,OAA6B;IAE7B,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,YAAY,GAAG,CAAC,EAAE,CAAC;QAC5D,MAAM,IAAI,SAAS,CAAC,gFAAgF,CAAC,CAAA;IACtG,CAAC;IAED,MAAM,UAAU,GACf,OAAO,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,CAAA;IAC/G,MAAM,UAAU,GAAG,MAAM,YAAY,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAA;IAC/D,MAAM,QAAQ,GAAgB,MAAM,eAAe,CAAC,MAAM,EAAE;QAC3D,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,SAAS,EAAE,OAAO,CAAC,SAAS;KAC5B,CAAC,CAAA;IAEF,MAAM,OAAO,GAAa,EAAE,CAAA;IAC5B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,2BAA2B,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAEtE,IAAI,CAAC;QACJ,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,YAAY,EAAE,EAAE;YAC7D,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,QAAQ,EAAE;gBACnC,UAAU,EAAE;oBACX,MAAM,EAAE,UAAU;oBAClB,UAAU;oBACV,KAAK;oBACL,GAAG;oBACH,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,IAAI;oBACpC,YAAY;oBACZ,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,GAAG;oBACnC,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,CAAC;oBACrC,QAAQ,EAAE,OAAO,CAAC,UAAU;iBAC5B;aACD,CAAC,CAAA;YAEF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAEpB,OAAO,gBAAgB,CAAI,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAA;QACpE,CAAC,CAAC,CAAA;QAEF,gGAAgG;QAChG,6FAA6F;QAC7F,mDAAmD;QACnD,KAAK,CAAC,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAA;IACtC,CAAC;YAAS,CAAC;QACV,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAA;IACrD,CAAC;AACF,CAAC"}
|