spliterator 3.1.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/lib/AsyncSpliterator.d.ts +18 -2
- package/out/lib/AsyncSpliterator.d.ts.map +1 -1
- package/out/lib/AsyncSpliterator.js +93 -25
- package/out/lib/AsyncSpliterator.js.map +1 -1
- package/out/lib/CSVSpliterator.d.ts +15 -0
- package/out/lib/CSVSpliterator.d.ts.map +1 -1
- package/out/lib/CSVSpliterator.js +63 -15
- package/out/lib/CSVSpliterator.js.map +1 -1
- package/out/lib/JSONSpliterator.d.ts +13 -0
- package/out/lib/JSONSpliterator.d.ts.map +1 -1
- package/out/lib/JSONSpliterator.js +13 -0
- package/out/lib/JSONSpliterator.js.map +1 -1
- package/out/lib/Spliterator.d.ts.map +1 -1
- package/out/lib/Spliterator.js +37 -43
- package/out/lib/Spliterator.js.map +1 -1
- package/out/lib/parallel-map.d.ts +18 -3
- package/out/lib/parallel-map.d.ts.map +1 -1
- package/out/lib/parallel-map.js +12 -2
- package/out/lib/parallel-map.js.map +1 -1
- package/out/lib/shared.d.ts +8 -3
- package/out/lib/shared.d.ts.map +1 -1
- package/out/lib/shared.js.map +1 -1
- package/out/lib/writer.d.ts +15 -4
- package/out/lib/writer.d.ts.map +1 -1
- package/out/lib/writer.js +19 -3
- package/out/lib/writer.js.map +1 -1
- package/package.json +1 -1
|
@@ -49,12 +49,28 @@ export interface SpliteratorInit {
|
|
|
49
49
|
*/
|
|
50
50
|
take?: number;
|
|
51
51
|
/**
|
|
52
|
-
* Whether to
|
|
53
|
-
*
|
|
52
|
+
* Whether to treat double-quoted regions as opaque: a delimiter inside `"…"` does not split, and the emitted slices
|
|
53
|
+
* keep their quotes verbatim. Quote _stripping_ and `""` unescaping are the consumer's concern —
|
|
54
|
+
* {@linkcode CSVSpliterator} does both when this flag is set.
|
|
55
|
+
*
|
|
56
|
+
* The asynchronous engine requires a single-byte delimiter for this mode (every {@linkcode Delimiters} entry
|
|
57
|
+
* qualifies) — a multi-byte delimiter straddling a chunk boundary cannot be rescanned without re-toggling quote
|
|
58
|
+
* state.
|
|
54
59
|
*
|
|
55
60
|
* @default false
|
|
56
61
|
*/
|
|
57
62
|
enableQuoteHandling?: boolean;
|
|
63
|
+
/**
|
|
64
|
+
* Whether to treat a carriage return immediately preceding a delimiter as part of the delimiter — i.e. CRLF
|
|
65
|
+
* normalization when splitting on `\n`, matching `node:readline`'s `crlfDelay: Infinity`. Only a `\r` directly before
|
|
66
|
+
* an actual delimiter match is trimmed; a trailing `\r` at end-of-input is preserved.
|
|
67
|
+
*
|
|
68
|
+
* {@linkcode CSVSpliterator} defaults this to `true` for row splitting (RFC 4180 mandates CRLF row terminators);
|
|
69
|
+
* everywhere else the default is `false`.
|
|
70
|
+
*
|
|
71
|
+
* @default false
|
|
72
|
+
*/
|
|
73
|
+
crlf?: boolean;
|
|
58
74
|
/**
|
|
59
75
|
* Whether to emit debug information.
|
|
60
76
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AsyncSpliterator.d.ts","sourceRoot":"","sources":["../../lib/AsyncSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,cAAc,EAAE,KAAK,oBAAoB,EAAE,KAAK,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAG9F,OAAO,EAAqB,KAAK,sBAAsB,
|
|
1
|
+
{"version":3,"file":"AsyncSpliterator.d.ts","sourceRoot":"","sources":["../../lib/AsyncSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,cAAc,EAAE,KAAK,oBAAoB,EAAE,KAAK,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAG9F,OAAO,EAAqB,KAAK,sBAAsB,EAAc,MAAM,wBAAwB,CAAA;AAEnG,OAAO,EAAE,KAAK,oBAAoB,EAAqB,MAAM,sBAAsB,CAAA;AACnF,OAAO,EAAmB,KAAK,cAAc,EAAE,MAAM,eAAe,CAAA;AACpE,OAAO,KAAK,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAKnF;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B;;OAEG;IACH,SAAS,CAAC,EAAE,sBAAsB,CAAA;IAElC;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IAEjB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,OAAO,CAAA;IAEnB;;;;OAIG;IACH,IAAI,CAAC,EAAE,MAAM,CAAA;IAEb;;;;;;;;OAQG;IACH,IAAI,CAAC,EAAE,MAAM,CAAA;IAEb;;;;;;;;;;OAUG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAA;IAE7B;;;;;;;;;OASG;IACH,IAAI,CAAC,EAAE,OAAO,CAAA;IAEd;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,CAAA;CACf;AAED;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,eAAe;IAC5D;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IAEtB;;;;OAIG;IACH,WAAW,CAAC,EAAE,OAAO,CAAA;CACrB;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,QAAQ,GAAG,WAAW,GAAG,UAAU,CACvF,YAAW,qBAAqB,CAAC,CAAC,CAAC,EAAE,eAAe;;IAIpD;;;;;;;OAOG;IACH,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,kBAAkB,EAAE,IAAI,CAAC,EAAE,oBAAoB,GAAG,gBAAgB;IACtF;;;;;;;OAOG;IACH,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,iBAAiB,EAAE,IAAI,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAC9F;;;;;;;OAOG;IACH,MAAM,CAAC,IAAI,CACV,MAAM,EAAE,iBAAiB,GAAG,kBAAkB,EAC9C,IAAI,CAAC,EAAE,oBAAoB,GACzB,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB/C;;OAEG;IACU,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC;IAWnD;;OAEG;WACW,iBAAiB,CAAC,IAAI,EAAE,eAAe,GAAG,eAAe,CAAC,kBAAkB,EAAE,UAAU,CAAC;IAYvG;;;;;;OAMG;gBACS,MAAM,EAAE,kBAAkB,EAAE,IAAI,GAAE,oBAAyB;IAmV1D,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAgD/C;;;;OAIG;IACI,OAAO,IAAI,OAAO,CAAC,CAAC,EAAE,CAAC;IAI9B;;;;OAIG;IACI,cAAc,CAAC,OAAO,kCAAoB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAIrE;;;;;;OAMG;IACI,gBAAgB,IAAI,cAAc,CAAC,CAAC,CAAC;IAiB5C;;;;;;;;;;OAUG;IACI,WAAW,CAAC,CAAC,EAAE,SAAS,EAAE,oBAAoB,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,cAAc,CAAC,CAAC,CAAC;IAI5G;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC;IAItB,MAAM,IAAI,OAAO,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC;IAQzD;;;;OAIG;WACW,QAAQ,CAAC,MAAM,EAAE,iBAAiB,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAIhG;;;;;;;;OAQG;WACiB,MAAM,CAAC,MAAM,EAAE,iBAAiB,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAc3G;;;;;;;;;;OAUG;WACW,aAAa,CAAC,CAAC,GAAG,OAAO,EACtC,MAAM,EAAE,iBAAiB,EACzB,OAAO,EAAE,oBAAoB,GAC3B,qBAAqB,CAAC,CAAC,CAAC;CAQ3B"}
|
|
@@ -5,11 +5,12 @@
|
|
|
5
5
|
*/
|
|
6
6
|
import { ReadableStream } from "stream/web";
|
|
7
7
|
import { BufferController } from "./BufferController.js";
|
|
8
|
-
import { CharacterSequence } from "./CharacterSequence.js";
|
|
8
|
+
import { CharacterSequence, Delimiters } from "./CharacterSequence.js";
|
|
9
9
|
import { IndexQueue } from "./IndexQueue.js";
|
|
10
10
|
import { runSegmentWorkers } from "./segment-workers.js";
|
|
11
11
|
import { computeSegments } from "./segments.js";
|
|
12
12
|
const noop = () => void 0;
|
|
13
|
+
const sharedEncoder = new TextEncoder();
|
|
13
14
|
/**
|
|
14
15
|
* An asynchronous spliterator for reading delimited byte streams.
|
|
15
16
|
*
|
|
@@ -77,6 +78,11 @@ export class AsyncSpliterator {
|
|
|
77
78
|
this.#yieldDropCount = Math.max(0, init.drop ?? 0);
|
|
78
79
|
this.#yieldStopCount = Math.max(init.take ?? Infinity, 0) + this.#yieldDropCount;
|
|
79
80
|
this.#skipEmpty = init.skipEmpty ?? true;
|
|
81
|
+
this.#crlf = init.crlf ?? false;
|
|
82
|
+
this.#enableQuoteHandling = init.enableQuoteHandling ?? false;
|
|
83
|
+
if (this.#enableQuoteHandling && this.#needle.length !== 1) {
|
|
84
|
+
throw new RangeError("enableQuoteHandling requires a single-byte delimiter in the asynchronous engine — a multi-byte delimiter straddling a chunk boundary cannot be rescanned without re-toggling quote state.");
|
|
85
|
+
}
|
|
80
86
|
this.#controller = new BufferController({
|
|
81
87
|
initialBufferSize: this.#highWaterMark,
|
|
82
88
|
});
|
|
@@ -130,6 +136,34 @@ export class AsyncSpliterator {
|
|
|
130
136
|
* Whether to skip empty yields.
|
|
131
137
|
*/
|
|
132
138
|
#skipEmpty;
|
|
139
|
+
/**
|
|
140
|
+
* Whether to trim a carriage return immediately preceding each delimiter match.
|
|
141
|
+
*/
|
|
142
|
+
#crlf;
|
|
143
|
+
/**
|
|
144
|
+
* Whether delimiters inside double-quoted regions are ignored.
|
|
145
|
+
*/
|
|
146
|
+
#enableQuoteHandling;
|
|
147
|
+
/**
|
|
148
|
+
* The byte sequence for a double quote.
|
|
149
|
+
*/
|
|
150
|
+
#doubleQuoteSequence = new CharacterSequence('"');
|
|
151
|
+
/**
|
|
152
|
+
* Whether the scan cursor currently sits inside a double-quoted region. Only meaningful with
|
|
153
|
+
* {@linkcode SpliteratorInit.enableQuoteHandling} — the state must survive both `#fill` rounds and buffer compression,
|
|
154
|
+
* since a quoted region can span many chunks.
|
|
155
|
+
*/
|
|
156
|
+
#insideQuotes = false;
|
|
157
|
+
/**
|
|
158
|
+
* Raw buffer position up to which matches have been processed. Distinct from the enqueued range ends — those may be
|
|
159
|
+
* CRLF-trimmed, and quote toggles may sit beyond the last emitted field. Rebased on buffer compression.
|
|
160
|
+
*/
|
|
161
|
+
#searchCursor = 0;
|
|
162
|
+
/**
|
|
163
|
+
* Raw buffer position where the current (not yet terminated) slice begins — i.e. one past the last consumed
|
|
164
|
+
* delimiter. This is the compression boundary and the start of the EOF tail. Rebased on buffer compression.
|
|
165
|
+
*/
|
|
166
|
+
#pendingSliceStart = 0;
|
|
133
167
|
/**
|
|
134
168
|
* The high water mark for the buffer.
|
|
135
169
|
*
|
|
@@ -148,10 +182,6 @@ export class AsyncSpliterator {
|
|
|
148
182
|
// * The current byte index to perform read operations from.
|
|
149
183
|
// */
|
|
150
184
|
// #readPosition: number
|
|
151
|
-
/**
|
|
152
|
-
* The last byte range seen while draining the buffer.
|
|
153
|
-
*/
|
|
154
|
-
#lastByteRangeSeen;
|
|
155
185
|
/**
|
|
156
186
|
* Whether to output debug information.
|
|
157
187
|
*/
|
|
@@ -163,26 +193,60 @@ export class AsyncSpliterator {
|
|
|
163
193
|
//#endregion
|
|
164
194
|
//#region Private Methods
|
|
165
195
|
#log;
|
|
196
|
+
/**
|
|
197
|
+
* Trim a carriage return immediately preceding a delimiter match, when {@linkcode SpliteratorInit.crlf} is set.
|
|
198
|
+
* Affects only the emitted range's end — cursor advancement always uses the raw match offset.
|
|
199
|
+
*/
|
|
200
|
+
#trimEnd(start, end) {
|
|
201
|
+
if (this.#crlf && end > start && this.#controller.bytes[end - 1] === Delimiters.CarriageReturn) {
|
|
202
|
+
return end - 1;
|
|
203
|
+
}
|
|
204
|
+
return end;
|
|
205
|
+
}
|
|
166
206
|
/**
|
|
167
207
|
* Find and enqueue delimiter positions.
|
|
168
208
|
*
|
|
169
209
|
* The search is bounded by `bytesWritten` so we never scan into uninitialized buffer memory and never enqueue ranges
|
|
170
210
|
* whose `end` exceeds the valid byte length.
|
|
211
|
+
*
|
|
212
|
+
* Progress is tracked by `#searchCursor`/`#pendingSliceStart` rather than derived from enqueued range ends — those
|
|
213
|
+
* ends may be CRLF-trimmed, and (in quote mode) quote toggles may sit beyond the last emitted field.
|
|
171
214
|
*/
|
|
172
215
|
#search() {
|
|
173
|
-
const lastByteRange = this.#indices.peekLast();
|
|
174
|
-
let searchCursor = lastByteRange ? lastByteRange[1] + this.#needle.length : 0;
|
|
175
216
|
const searchEnd = this.#controller.bytesWritten;
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
217
|
+
if (!this.#enableQuoteHandling) {
|
|
218
|
+
while (this.#searchCursor <= searchEnd) {
|
|
219
|
+
const delimiterIndex = this.#needle.search(this.#controller.bytes, this.#searchCursor, searchEnd);
|
|
220
|
+
if (delimiterIndex === -1) {
|
|
221
|
+
this.#log(`No viable byte range found in buffer. Breaking.`, { searchCursor: this.#searchCursor });
|
|
222
|
+
break;
|
|
223
|
+
}
|
|
224
|
+
const byteRange = [this.#pendingSliceStart, this.#trimEnd(this.#pendingSliceStart, delimiterIndex)];
|
|
225
|
+
this.#log("Found byte range", byteRange);
|
|
226
|
+
this.#indices.enqueue(byteRange);
|
|
227
|
+
this.#searchCursor = delimiterIndex + this.#needle.length;
|
|
228
|
+
this.#pendingSliceStart = this.#searchCursor;
|
|
229
|
+
}
|
|
230
|
+
return;
|
|
231
|
+
}
|
|
232
|
+
// Quote-aware path: a delimiter inside a double-quoted region does not split. The quote
|
|
233
|
+
// state persists across fills and buffer compressions, since a quoted region may span
|
|
234
|
+
// many chunks. The delimiter is single-byte (enforced in the constructor), so a match can
|
|
235
|
+
// never straddle a chunk boundary and the cursor can safely advance to `searchEnd`.
|
|
236
|
+
const matches = this.#needle.searchMatches(this.#controller.bytes, this.#doubleQuoteSequence, this.#searchCursor, searchEnd);
|
|
237
|
+
for (const match of matches) {
|
|
238
|
+
if (match.patternId === 1) {
|
|
239
|
+
this.#insideQuotes = !this.#insideQuotes;
|
|
240
|
+
continue;
|
|
181
241
|
}
|
|
182
|
-
this.#
|
|
183
|
-
|
|
184
|
-
|
|
242
|
+
if (this.#insideQuotes)
|
|
243
|
+
continue;
|
|
244
|
+
const byteRange = [this.#pendingSliceStart, this.#trimEnd(this.#pendingSliceStart, match.offset)];
|
|
245
|
+
this.#log("Found byte range", byteRange);
|
|
246
|
+
this.#indices.enqueue(byteRange);
|
|
247
|
+
this.#pendingSliceStart = match.offset + this.#needle.length;
|
|
185
248
|
}
|
|
249
|
+
this.#searchCursor = searchEnd;
|
|
186
250
|
}
|
|
187
251
|
/**
|
|
188
252
|
* Read a chunk of data from the file.
|
|
@@ -191,8 +255,12 @@ export class AsyncSpliterator {
|
|
|
191
255
|
this.#lastReadResult = await this.#chunkReader.next();
|
|
192
256
|
if (this.#lastReadResult.done)
|
|
193
257
|
return;
|
|
258
|
+
// A string chunk (a `Readable` with an encoding set, a hand-rolled generator of strings)
|
|
259
|
+
// would coerce to garbage via `TypedArray.set` — encode it instead of silently mis-reading.
|
|
260
|
+
const chunk = this.#lastReadResult.value;
|
|
261
|
+
const bytes = typeof chunk === "string" ? sharedEncoder.encode(chunk) : chunk;
|
|
194
262
|
// Append the chunk to the buffer.
|
|
195
|
-
this.#controller.set(
|
|
263
|
+
this.#controller.set(bytes, this.#controller.bytesWritten);
|
|
196
264
|
}
|
|
197
265
|
/**
|
|
198
266
|
* Fill the buffer with data and search for delimiters.
|
|
@@ -200,11 +268,13 @@ export class AsyncSpliterator {
|
|
|
200
268
|
async #fill() {
|
|
201
269
|
if (this.#lastReadResult?.done) {
|
|
202
270
|
this.#log("Reached end of file. Preparing to finalize.");
|
|
203
|
-
// Enqueue a final slice covering the bytes between the last
|
|
271
|
+
// Enqueue a final slice covering the bytes between the last consumed delimiter and the
|
|
204
272
|
// end of the stream. The slice may be empty (when the stream ends exactly on a
|
|
205
273
|
// delimiter, or when the file is empty); we still enqueue it so that callers with
|
|
206
274
|
// `skipEmpty: false` see the trailing entry produced by `String.prototype.split`.
|
|
207
|
-
|
|
275
|
+
// `#pendingSliceStart` — not the last enqueued range's end — is the authority here:
|
|
276
|
+
// range ends may be CRLF-trimmed.
|
|
277
|
+
const tailStart = this.#pendingSliceStart;
|
|
208
278
|
const tailEnd = this.#controller.bytesWritten;
|
|
209
279
|
if (tailStart <= tailEnd) {
|
|
210
280
|
this.#log("Inserted trailing byte range", [tailStart, tailEnd]);
|
|
@@ -218,14 +288,13 @@ export class AsyncSpliterator {
|
|
|
218
288
|
}
|
|
219
289
|
// We're effectively done with that window of the buffer.
|
|
220
290
|
// So we can compress it to save memory.
|
|
221
|
-
const nextBufferStart = this.#
|
|
291
|
+
const nextBufferStart = this.#pendingSliceStart;
|
|
222
292
|
this.#log("Compressing buffer", { nextBufferStart });
|
|
223
293
|
this.#controller.compress(nextBufferStart);
|
|
224
|
-
// After compress, the buffer coordinate space has shifted
|
|
225
|
-
//
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
this.#lastByteRangeSeen = undefined;
|
|
294
|
+
// After compress, the buffer coordinate space has shifted — rebase the scan cursors.
|
|
295
|
+
// Quote state (`#insideQuotes`) is positionless and survives as-is.
|
|
296
|
+
this.#pendingSliceStart = 0;
|
|
297
|
+
this.#searchCursor = Math.max(0, this.#searchCursor - nextBufferStart);
|
|
229
298
|
while ((!this.#lastReadResult || !this.#lastReadResult.done) && this.#indices.byteLength < this.#highWaterMark) {
|
|
230
299
|
await this.#read();
|
|
231
300
|
this.#search();
|
|
@@ -280,7 +349,6 @@ export class AsyncSpliterator {
|
|
|
280
349
|
await this.#fill();
|
|
281
350
|
}
|
|
282
351
|
const currentByteRange = this.#indices.dequeue();
|
|
283
|
-
this.#lastByteRangeSeen = currentByteRange;
|
|
284
352
|
if (!currentByteRange) {
|
|
285
353
|
this.#done = true;
|
|
286
354
|
return this.#finalize();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AsyncSpliterator.js","sourceRoot":"","sources":["../../lib/AsyncSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,cAAc,EAAqD,MAAM,YAAY,CAAA;AAE9F,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AACxD,OAAO,EAAE,iBAAiB,EAA+B,MAAM,wBAAwB,CAAA;AACvF,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EAA6B,iBAAiB,EAAE,MAAM,sBAAsB,CAAA;AACnF,OAAO,EAAE,eAAe,EAAuB,MAAM,eAAe,CAAA;AAGpE,MAAM,IAAI,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC,CAAA;AAkFzB;;;;;;;;;;;;;;GAcG;AACH,MAAM,OAAO,gBAAgB;IAmC5B,MAAM,CAAC,IAAI,CACV,MAA8C,EAC9C,OAA6B,EAAE;QAE/B,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,aAAa,IAAI,MAAM,EAAE,CAAC;YAClE,OAAO,IAAI,gBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QAC1C,CAAC;QAED,OAAO,MAAM,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,mBAAmB,EAAE,EAAE,EAAE;YAC3E,MAAM,aAAa,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE;gBACvD,aAAa,EAAE,IAAI,CAAC,aAAa;aACjC,CAAC,CAAA;YAEF,OAAO,IAAI,gBAAgB,CAAC,aAAa,EAAE,IAAI,CAAC,CAAA;QACjD,CAAC,CAAC,CAAA;IACH,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QACjC,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;QACrB,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAA;QAExB,MAAM,IAAI,CAAC,YAAY,EAAE,CAAA;QAEzB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE,CAAA;QAC5C,CAAC;IACF,CAAC;IAED;;OAEG;IACI,MAAM,CAAC,iBAAiB,CAAC,IAAqB;QACpD,OAAO,IAAI,eAAe,CAAiC;YAC1D,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,UAAU;gBAChC,MAAM,WAAW,GAAG,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;gBAEtD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;oBACvC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;gBAC1B,CAAC;YACF,CAAC;SACD,CAAC,CAAA;IACH,CAAC;IAED;;;;;;OAMG;IACH,YAAY,MAA0B,EAAE,OAA6B,EAAE;QACtE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QACrB,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAA;QAElD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,WAAW,IAAI,KAAK,CAAA;QAE7C,IAAI,CAAC,OAAO,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QAEpD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,aAAa,IAAI,IAAI,GAAG,EAAE,CAAC,CAAA;QAExF,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAA;QAClD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,eAAe,CAAA;QAEhF,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAA;QAExC,IAAI,CAAC,WAAW,GAAG,IAAI,gBAAgB,CAAC;YACvC,iBAAiB,EAAE,IAAI,CAAC,cAAc;SACtC,CAAC,CAAA;QAEF,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,IAAI,KAAK,CAAA;QACjC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAC7D,CAAC;IAED,YAAY;IAEZ,4BAA4B;IAE5B;;OAEG;IACM,YAAY,CAAS;IAE9B;;OAEG;IACM,OAAO,CAAoB;IAEpC;;OAEG;IACM,YAAY,CAA2B;IAEhD;;OAEG;IACH,eAAe,CAAwC;IAEvD;;;;;;OAMG;IACM,QAAQ,GAAG,IAAI,UAAU,EAAE,CAAA;IAEpC;;OAEG;IACM,OAAO,CAAmB;IAEnC;;;OAGG;IACM,WAAW,CAAkB;IAEtC;;OAEG;IACM,eAAe,CAAQ;IAEhC;;OAEG;IACM,eAAe,CAAQ;IAEhC;;OAEG;IACM,UAAU,CAAS;IAE5B;;;;OAIG;IACM,cAAc,CAAQ;IAE/B;;OAEG;IACH,kBAAkB,GAAG,CAAC,CAAA;IAEtB;;OAEG;IACH,WAAW,GAAG,CAAC,CAAA;IAEf,MAAM;IACN,6DAA6D;IAC7D,MAAM;IACN,wBAAwB;IAExB;;OAEG;IACH,kBAAkB,CAAuB;IAEzC;;OAEG;IACH,MAAM,CAAS;IAEf;;OAEG;IACH,KAAK,GAAG,KAAK,CAAA;IAEb,YAAY;IAEZ,yBAAyB;IAEzB,IAAI,CAA0B;IAE9B;;;;;OAKG;IACH,OAAO;QACN,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAA;QAC9C,IAAI,YAAY,GAAG,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;QAC7E,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,CAAA;QAE/C,OAAO,YAAY,IAAI,SAAS,EAAE,CAAC;YAClC,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,YAAY,EAAE,SAAS,CAAC,CAAA;YAE3F,IAAI,cAAc,KAAK,CAAC,CAAC,EAAE,CAAC;gBAC3B,IAAI,CAAC,IAAI,CAAC,iDAAiD,EAAE,EAAE,YAAY,EAAE,CAAC,CAAA;gBAC9E,MAAK;YACN,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,CAAA;YAC7D,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,CAAA;YAErD,YAAY,GAAG,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;QACpD,CAAC;IACF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACV,IAAI,CAAC,eAAe,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAA;QAErD,IAAI,IAAI,CAAC,eAAe,CAAC,IAAI;YAAE,OAAM;QAErC,kCAAkC;QAClC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,CAAA;IAChF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACV,IAAI,IAAI,CAAC,eAAe,EAAE,IAAI,EAAE,CAAC;YAChC,IAAI,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAA;YACxD,sFAAsF;YACtF,+EAA+E;YAC/E,kFAAkF;YAClF,kFAAkF;YAClF,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;YAChG,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,CAAA;YAE7C,IAAI,SAAS,IAAI,OAAO,EAAE,CAAC;gBAC1B,IAAI,CAAC,IAAI,CAAC,8BAA8B,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAA;gBAC/D,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAA;YAC5C,CAAC;iBAAM,CAAC;gBACP,IAAI,CAAC,IAAI,CAAC,2CAA2C,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAA;YAC/E,CAAC;YAED,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;YAEjB,OAAM;QACP,CAAC;QAED,yDAAyD;QACzD,wCAAwC;QACxC,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;QAEtG,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,eAAe,EAAE,CAAC,CAAA;QACpD,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAA;QAE1C,0FAA0F;QAC1F,yFAAyF;QACzF,0FAA0F;QAC1F,4EAA4E;QAC5E,IAAI,CAAC,kBAAkB,GAAG,SAAS,CAAA;QAEnC,OAAO,CAAC,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;YAChH,MAAM,IAAI,CAAC,KAAK,EAAE,CAAA;YAClB,IAAI,CAAC,OAAO,EAAE,CAAA;QACf,CAAC;IACF,CAAC;IAED,oGAAoG;IACpG,KAAK,CAAC,YAAY;QACjB,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,EAAE,CAAA;QACnC,CAAC;QAAC,MAAM,CAAC;YACR,4CAA4C;QAC7C,CAAC;IACF,CAAC;IAED,KAAK,CAAC,SAAS;QACd,MAAM,IAAI,CAAC,YAAY,EAAE,CAAA;QAEzB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB;;;eAGG;YACH,MAAM,6BAA6B,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;YAElF,IAAI,CAAC,IAAI,CAAC;gBACT,iBAAiB,EAAE,IAAI,CAAC,kBAAkB;gBAC1C,UAAU,EAAE,IAAI,CAAC,WAAW;gBAC5B,6BAA6B;aAC7B,CAAC,CAAA;QACH,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE,CAAA;QAC5C,CAAC;QAED,OAAO;YACN,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,SAAS;SAChB,CAAA;IACF,CAAC;IAED,YAAY;IAEZ,0BAA0B;IAEnB,KAAK,CAAC,IAAI;QAChB,kFAAkF;QAClF,iEAAiE;QACjE,OAAO,IAAI,EAAE,CAAC;YACb,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,eAAe;gBAAE,OAAO,IAAI,CAAC,SAAS,EAAE,CAAA;YAEnF,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;gBAC9B,MAAM,IAAI,CAAC,KAAK,EAAE,CAAA;YACnB,CAAC;YAED,4FAA4F;YAC5F,wFAAwF;YACxF,mFAAmF;YACnF,IAAI,IAAI,CAAC,eAAe,EAAE,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC3E,MAAM,IAAI,CAAC,KAAK,EAAE,CAAA;YACnB,CAAC;YAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAA;YAChD,IAAI,CAAC,kBAAkB,GAAG,gBAAgB,CAAA;YAE1C,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACvB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;gBAEjB,OAAO,IAAI,CAAC,SAAS,EAAE,CAAA;YACxB,CAAC;YAED,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,gBAAgB,CAAA;YAErC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;YAEnD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC3C,SAAQ;YACT,CAAC;YAED,IAAI,CAAC,WAAW,EAAE,CAAA;YAElB,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;gBAC9C,SAAQ;YACT,CAAC;YAED,IAAI,CAAC,kBAAkB,IAAI,GAAG,GAAG,KAAK,CAAA;YAEtC,OAAO;gBACN,KAAK,EAAE,KAAqB;gBAC5B,IAAI,EAAE,KAAK;aACX,CAAA;QACF,CAAC;IACF,CAAC;IAED;;;;OAIG;IACI,OAAO;QACb,OAAO,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;IAC7B,CAAC;IAED;;;;OAIG;IACI,cAAc,CAAC,OAAO,GAAG,IAAI,WAAW,EAAE;QAChD,OAAO,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;IAC/D,CAAC;IAED;;;;;;OAMG;IACI,gBAAgB;QACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAA;QAE7C,OAAO,IAAI,cAAc,CAAC;YACzB,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE;gBAC1B,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;gBAE7C,IAAI,IAAI,EAAE,CAAC;oBACV,UAAU,CAAC,KAAK,EAAE,CAAA;gBACnB,CAAC;qBAAM,CAAC;oBACP,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;gBAC1B,CAAC;YACF,CAAC;YACD,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE;SAC3C,CAAC,CAAA;IACH,CAAC;IAED;;;;;;;;;;OAUG;IACI,WAAW,CAAI,SAAqC,EAAE,OAA2B;QACvF,OAAO,IAAI,CAAC,gBAAgB,EAAE,CAAC,WAAW,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IAC/D,CAAC;IAED;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC;QAC5B,OAAO,IAAI,CAAA;IACZ,CAAC;IAEM,MAAM;QACZ,OAAO,IAAI,CAAC,SAAS,EAAE,CAAA;IACxB,CAAC;IAED,YAAY;IAEZ,sBAAsB;IAEtB;;;;OAIG;IACI,MAAM,CAAC,QAAQ,CAAC,MAAyB,EAAE,OAAuB;QACxE,OAAO,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IACxC,CAAC;IAED;;;;;;;;OAQG;IACI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAyB,EAAE,OAAuB;QAC5E,MAAM,EAAE,mBAAmB,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAA;QACnE,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAEvD,OAAO,OAAO,CAAC,GAAG,CACjB,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE;YACnC,qEAAqE;YACrE,MAAM,aAAa,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,GAAG,CAAC,EAAE,CAAC,CAAA;YAEhF,OAAO,IAAI,gBAAgB,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAA;QAChG,CAAC,CAAC,CACF,CAAA;IACF,CAAC;IAED;;;;;;;;;;OAUG;IACI,MAAM,CAAC,aAAa,CAC1B,MAAyB,EACzB,OAA6B;QAE7B,+FAA+F;QAC/F,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,YAAY,GAAG,CAAC,EAAE,CAAC;YAC5D,MAAM,IAAI,SAAS,CAAC,gFAAgF,CAAC,CAAA;QACtG,CAAC;QAED,OAAO,iBAAiB,CAAI,MAAM,EAAE,OAAO,CAAC,CAAA;IAC7C,CAAC;CACD"}
|
|
1
|
+
{"version":3,"file":"AsyncSpliterator.js","sourceRoot":"","sources":["../../lib/AsyncSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,cAAc,EAAqD,MAAM,YAAY,CAAA;AAE9F,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AACxD,OAAO,EAAE,iBAAiB,EAA+B,UAAU,EAAE,MAAM,wBAAwB,CAAA;AACnG,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EAA6B,iBAAiB,EAAE,MAAM,sBAAsB,CAAA;AACnF,OAAO,EAAE,eAAe,EAAuB,MAAM,eAAe,CAAA;AAGpE,MAAM,IAAI,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC,CAAA;AACzB,MAAM,aAAa,GAAG,IAAI,WAAW,EAAE,CAAA;AAmGvC;;;;;;;;;;;;;;GAcG;AACH,MAAM,OAAO,gBAAgB;IAmC5B,MAAM,CAAC,IAAI,CACV,MAA8C,EAC9C,OAA6B,EAAE;QAE/B,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,aAAa,IAAI,MAAM,EAAE,CAAC;YAClE,OAAO,IAAI,gBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QAC1C,CAAC;QAED,OAAO,MAAM,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,mBAAmB,EAAE,EAAE,EAAE;YAC3E,MAAM,aAAa,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE;gBACvD,aAAa,EAAE,IAAI,CAAC,aAAa;aACjC,CAAC,CAAA;YAEF,OAAO,IAAI,gBAAgB,CAAC,aAAa,EAAE,IAAI,CAAC,CAAA;QACjD,CAAC,CAAC,CAAA;IACH,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QACjC,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;QACrB,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAA;QAExB,MAAM,IAAI,CAAC,YAAY,EAAE,CAAA;QAEzB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE,CAAA;QAC5C,CAAC;IACF,CAAC;IAED;;OAEG;IACI,MAAM,CAAC,iBAAiB,CAAC,IAAqB;QACpD,OAAO,IAAI,eAAe,CAAiC;YAC1D,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,UAAU;gBAChC,MAAM,WAAW,GAAG,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;gBAEtD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;oBACvC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;gBAC1B,CAAC;YACF,CAAC;SACD,CAAC,CAAA;IACH,CAAC;IAED;;;;;;OAMG;IACH,YAAY,MAA0B,EAAE,OAA6B,EAAE;QACtE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QACrB,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAA;QAElD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,WAAW,IAAI,KAAK,CAAA;QAE7C,IAAI,CAAC,OAAO,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QAEpD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,aAAa,IAAI,IAAI,GAAG,EAAE,CAAC,CAAA;QAExF,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAA;QAClD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,eAAe,CAAA;QAEhF,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAA;QACxC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,IAAI,KAAK,CAAA;QAC/B,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,mBAAmB,IAAI,KAAK,CAAA;QAE7D,IAAI,IAAI,CAAC,oBAAoB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5D,MAAM,IAAI,UAAU,CACnB,2LAA2L,CAC3L,CAAA;QACF,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,IAAI,gBAAgB,CAAC;YACvC,iBAAiB,EAAE,IAAI,CAAC,cAAc;SACtC,CAAC,CAAA;QAEF,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,IAAI,KAAK,CAAA;QACjC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAC7D,CAAC;IAED,YAAY;IAEZ,4BAA4B;IAE5B;;OAEG;IACM,YAAY,CAAS;IAE9B;;OAEG;IACM,OAAO,CAAoB;IAEpC;;OAEG;IACM,YAAY,CAAoC;IAEzD;;OAEG;IACH,eAAe,CAAiD;IAEhE;;;;;;OAMG;IACM,QAAQ,GAAG,IAAI,UAAU,EAAE,CAAA;IAEpC;;OAEG;IACM,OAAO,CAAmB;IAEnC;;;OAGG;IACM,WAAW,CAAkB;IAEtC;;OAEG;IACM,eAAe,CAAQ;IAEhC;;OAEG;IACM,eAAe,CAAQ;IAEhC;;OAEG;IACM,UAAU,CAAS;IAE5B;;OAEG;IACM,KAAK,CAAS;IAEvB;;OAEG;IACM,oBAAoB,CAAS;IAEtC;;OAEG;IACM,oBAAoB,GAAG,IAAI,iBAAiB,CAAC,GAAG,CAAC,CAAA;IAE1D;;;;OAIG;IACH,aAAa,GAAG,KAAK,CAAA;IAErB;;;OAGG;IACH,aAAa,GAAG,CAAC,CAAA;IAEjB;;;OAGG;IACH,kBAAkB,GAAG,CAAC,CAAA;IAEtB;;;;OAIG;IACM,cAAc,CAAQ;IAE/B;;OAEG;IACH,kBAAkB,GAAG,CAAC,CAAA;IAEtB;;OAEG;IACH,WAAW,GAAG,CAAC,CAAA;IAEf,MAAM;IACN,6DAA6D;IAC7D,MAAM;IACN,wBAAwB;IAExB;;OAEG;IACH,MAAM,CAAS;IAEf;;OAEG;IACH,KAAK,GAAG,KAAK,CAAA;IAEb,YAAY;IAEZ,yBAAyB;IAEzB,IAAI,CAA0B;IAE9B;;;OAGG;IACH,QAAQ,CAAC,KAAa,EAAE,GAAW;QAClC,IAAI,IAAI,CAAC,KAAK,IAAI,GAAG,GAAG,KAAK,IAAI,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,UAAU,CAAC,cAAc,EAAE,CAAC;YAChG,OAAO,GAAG,GAAG,CAAC,CAAA;QACf,CAAC;QAED,OAAO,GAAG,CAAA;IACX,CAAC;IAED;;;;;;;;OAQG;IACH,OAAO;QACN,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,CAAA;QAE/C,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAChC,OAAO,IAAI,CAAC,aAAa,IAAI,SAAS,EAAE,CAAC;gBACxC,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,aAAa,EAAE,SAAS,CAAC,CAAA;gBAEjG,IAAI,cAAc,KAAK,CAAC,CAAC,EAAE,CAAC;oBAC3B,IAAI,CAAC,IAAI,CAAC,iDAAiD,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC,CAAA;oBAClG,MAAK;gBACN,CAAC;gBAED,MAAM,SAAS,GAAc,CAAC,IAAI,CAAC,kBAAkB,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,kBAAkB,EAAE,cAAc,CAAC,CAAC,CAAA;gBAE9G,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,SAAS,CAAC,CAAA;gBACxC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;gBAEhC,IAAI,CAAC,aAAa,GAAG,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;gBACzD,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,aAAa,CAAA;YAC7C,CAAC;YAED,OAAM;QACP,CAAC;QAED,wFAAwF;QACxF,sFAAsF;QACtF,0FAA0F;QAC1F,oFAAoF;QACpF,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CACzC,IAAI,CAAC,WAAW,CAAC,KAAK,EACtB,IAAI,CAAC,oBAAoB,EACzB,IAAI,CAAC,aAAa,EAClB,SAAS,CACT,CAAA;QAED,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,KAAK,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;gBAC3B,IAAI,CAAC,aAAa,GAAG,CAAC,IAAI,CAAC,aAAa,CAAA;gBACxC,SAAQ;YACT,CAAC;YAED,IAAI,IAAI,CAAC,aAAa;gBAAE,SAAQ;YAEhC,MAAM,SAAS,GAAc,CAAC,IAAI,CAAC,kBAAkB,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,kBAAkB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;YAE5G,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,SAAS,CAAC,CAAA;YACxC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;YAEhC,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;QAC7D,CAAC;QAED,IAAI,CAAC,aAAa,GAAG,SAAS,CAAA;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACV,IAAI,CAAC,eAAe,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAA;QAErD,IAAI,IAAI,CAAC,eAAe,CAAC,IAAI;YAAE,OAAM;QAErC,yFAAyF;QACzF,4FAA4F;QAC5F,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAA;QACxC,MAAM,KAAK,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAA;QAE7E,kCAAkC;QAClC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,CAAA;IAC3D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACV,IAAI,IAAI,CAAC,eAAe,EAAE,IAAI,EAAE,CAAC;YAChC,IAAI,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAA;YACxD,uFAAuF;YACvF,+EAA+E;YAC/E,kFAAkF;YAClF,kFAAkF;YAClF,oFAAoF;YACpF,kCAAkC;YAClC,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAA;YACzC,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,CAAA;YAE7C,IAAI,SAAS,IAAI,OAAO,EAAE,CAAC;gBAC1B,IAAI,CAAC,IAAI,CAAC,8BAA8B,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAA;gBAC/D,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAA;YAC5C,CAAC;iBAAM,CAAC;gBACP,IAAI,CAAC,IAAI,CAAC,2CAA2C,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAA;YAC/E,CAAC;YAED,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;YAEjB,OAAM;QACP,CAAC;QAED,yDAAyD;QACzD,wCAAwC;QACxC,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,CAAA;QAE/C,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,eAAe,EAAE,CAAC,CAAA;QACpD,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAA;QAE1C,qFAAqF;QACrF,oEAAoE;QACpE,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAA;QAC3B,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,aAAa,GAAG,eAAe,CAAC,CAAA;QAEtE,OAAO,CAAC,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;YAChH,MAAM,IAAI,CAAC,KAAK,EAAE,CAAA;YAClB,IAAI,CAAC,OAAO,EAAE,CAAA;QACf,CAAC;IACF,CAAC;IAED,oGAAoG;IACpG,KAAK,CAAC,YAAY;QACjB,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,EAAE,CAAA;QACnC,CAAC;QAAC,MAAM,CAAC;YACR,4CAA4C;QAC7C,CAAC;IACF,CAAC;IAED,KAAK,CAAC,SAAS;QACd,MAAM,IAAI,CAAC,YAAY,EAAE,CAAA;QAEzB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB;;;eAGG;YACH,MAAM,6BAA6B,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;YAElF,IAAI,CAAC,IAAI,CAAC;gBACT,iBAAiB,EAAE,IAAI,CAAC,kBAAkB;gBAC1C,UAAU,EAAE,IAAI,CAAC,WAAW;gBAC5B,6BAA6B;aAC7B,CAAC,CAAA;QACH,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE,CAAA;QAC5C,CAAC;QAED,OAAO;YACN,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,SAAS;SAChB,CAAA;IACF,CAAC;IAED,YAAY;IAEZ,0BAA0B;IAEnB,KAAK,CAAC,IAAI;QAChB,kFAAkF;QAClF,iEAAiE;QACjE,OAAO,IAAI,EAAE,CAAC;YACb,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,eAAe;gBAAE,OAAO,IAAI,CAAC,SAAS,EAAE,CAAA;YAEnF,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;gBAC9B,MAAM,IAAI,CAAC,KAAK,EAAE,CAAA;YACnB,CAAC;YAED,4FAA4F;YAC5F,wFAAwF;YACxF,mFAAmF;YACnF,IAAI,IAAI,CAAC,eAAe,EAAE,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC3E,MAAM,IAAI,CAAC,KAAK,EAAE,CAAA;YACnB,CAAC;YAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAA;YAEhD,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACvB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;gBAEjB,OAAO,IAAI,CAAC,SAAS,EAAE,CAAA;YACxB,CAAC;YAED,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,gBAAgB,CAAA;YAErC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;YAEnD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC3C,SAAQ;YACT,CAAC;YAED,IAAI,CAAC,WAAW,EAAE,CAAA;YAElB,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;gBAC9C,SAAQ;YACT,CAAC;YAED,IAAI,CAAC,kBAAkB,IAAI,GAAG,GAAG,KAAK,CAAA;YAEtC,OAAO;gBACN,KAAK,EAAE,KAAqB;gBAC5B,IAAI,EAAE,KAAK;aACX,CAAA;QACF,CAAC;IACF,CAAC;IAED;;;;OAIG;IACI,OAAO;QACb,OAAO,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;IAC7B,CAAC;IAED;;;;OAIG;IACI,cAAc,CAAC,OAAO,GAAG,IAAI,WAAW,EAAE;QAChD,OAAO,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;IAC/D,CAAC;IAED;;;;;;OAMG;IACI,gBAAgB;QACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAA;QAE7C,OAAO,IAAI,cAAc,CAAC;YACzB,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE;gBAC1B,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;gBAE7C,IAAI,IAAI,EAAE,CAAC;oBACV,UAAU,CAAC,KAAK,EAAE,CAAA;gBACnB,CAAC;qBAAM,CAAC;oBACP,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;gBAC1B,CAAC;YACF,CAAC;YACD,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE;SAC3C,CAAC,CAAA;IACH,CAAC;IAED;;;;;;;;;;OAUG;IACI,WAAW,CAAI,SAAqC,EAAE,OAA2B;QACvF,OAAO,IAAI,CAAC,gBAAgB,EAAE,CAAC,WAAW,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IAC/D,CAAC;IAED;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC;QAC5B,OAAO,IAAI,CAAA;IACZ,CAAC;IAEM,MAAM;QACZ,OAAO,IAAI,CAAC,SAAS,EAAE,CAAA;IACxB,CAAC;IAED,YAAY;IAEZ,sBAAsB;IAEtB;;;;OAIG;IACI,MAAM,CAAC,QAAQ,CAAC,MAAyB,EAAE,OAAuB;QACxE,OAAO,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IACxC,CAAC;IAED;;;;;;;;OAQG;IACI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAyB,EAAE,OAAuB;QAC5E,MAAM,EAAE,mBAAmB,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAA;QACnE,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAEvD,OAAO,OAAO,CAAC,GAAG,CACjB,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE;YACnC,qEAAqE;YACrE,MAAM,aAAa,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,GAAG,CAAC,EAAE,CAAC,CAAA;YAEhF,OAAO,IAAI,gBAAgB,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAA;QAChG,CAAC,CAAC,CACF,CAAA;IACF,CAAC;IAED;;;;;;;;;;OAUG;IACI,MAAM,CAAC,aAAa,CAC1B,MAAyB,EACzB,OAA6B;QAE7B,+FAA+F;QAC/F,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,YAAY,GAAG,CAAC,EAAE,CAAC;YAC5D,MAAM,IAAI,SAAS,CAAC,gFAAgF,CAAC,CAAA;QACtG,CAAC;QAED,OAAO,iBAAiB,CAAI,MAAM,EAAE,OAAO,CAAC,CAAA;IAC7C,CAAC;CACD"}
|
|
@@ -37,6 +37,16 @@ export interface CSVSpliteratorInit extends SpliteratorInit {
|
|
|
37
37
|
* - `object` will emit each row as an object with the header names as keys.
|
|
38
38
|
* - `array` will emit each row as an array.
|
|
39
39
|
* - `entries` will emit each row as an array of key-value pairs.
|
|
40
|
+
*
|
|
41
|
+
* Note that {@linkcode CSVSpliteratorInit.header} defaults to `true` in every mode — the first row is consumed as the
|
|
42
|
+
* header even in `array` mode. Pass `header: false` for headerless data.
|
|
43
|
+
*
|
|
44
|
+
* When {@linkcode SpliteratorInit.enableQuoteHandling} is set, quoting is handled end-to-end: rows do not split on
|
|
45
|
+
* delimiters inside quotes (embedded newlines stay in their row), columns do not split on quoted column delimiters,
|
|
46
|
+
* wrapping quotes are stripped, and doubled quotes (`""`) unescape to `"`.
|
|
47
|
+
*
|
|
48
|
+
* {@linkcode SpliteratorInit.crlf} defaults to `true` here (unlike everywhere else): RFC 4180 mandates CRLF row
|
|
49
|
+
* terminators, so a correct CSV parser must accept them without leaking `\r` into the last column.
|
|
40
50
|
*/
|
|
41
51
|
mode?: CSVOutputMode;
|
|
42
52
|
/**
|
|
@@ -63,6 +73,11 @@ export interface CSVSpliteratorInit extends SpliteratorInit {
|
|
|
63
73
|
/**
|
|
64
74
|
* A static class spliterator for comma-separated values.
|
|
65
75
|
*
|
|
76
|
+
* **Performance:** the SIMD delimiter scan wins when scanning dominates — many rows, a few columns pulled out cheaply,
|
|
77
|
+
* streaming to bound memory. When per-row work is heavy (a full `JSON.parse`, expensive transforms) it can dominate the
|
|
78
|
+
* scan and erase the advantage; benchmark against a mature native parser before swapping an existing loop for speed.
|
|
79
|
+
* See {@link JSONSpliterator} for the measured case where per-row `JSON.parse` makes the streamed path a net loss.
|
|
80
|
+
*
|
|
66
81
|
* @see {@linkcode CSVSpliterator.from} for synchronous usage.
|
|
67
82
|
* @see {@linkcode CSVSpliterator.fromAsync} for asynchronous usage.
|
|
68
83
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CSVSpliterator.d.ts","sourceRoot":"","sources":["../../lib/CSVSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAqB,KAAK,sBAAsB,EAAc,MAAM,wBAAwB,CAAA;AACnG,OAAO,EAAE,KAAK,kBAAkB,EAAE,KAAK,iBAAiB,EAAW,MAAM,aAAa,CAAA;AACtF,OAAO,EAAE,KAAK,oBAAoB,EAAe,KAAK,eAAe,EAAE,MAAM,kBAAkB,CAAA;AAE/F;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,CAAA;AAC1D,MAAM,MAAM,cAAc,CAAC,CAAC,GAAG,OAAO,IAAI,CAAC,KAAK,EAAE,MAAM,KAAK,CAAC,CAAA;AAE9D,MAAM,MAAM,mBAAmB,CAAC,CAAC,GAAG,OAAO,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,CAAA;AACnG,MAAM,MAAM,oBAAoB,GAAG,MAAM,CAAC,MAAM,EAAE,cAAc,GAAG,SAAS,CAAC,CAAA;AAE7E,MAAM,MAAM,UAAU,CAAC,CAAC,GAAG,OAAO,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,aAAa,CAAC,EAAE,QAAQ,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;
|
|
1
|
+
{"version":3,"file":"CSVSpliterator.d.ts","sourceRoot":"","sources":["../../lib/CSVSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAqB,KAAK,sBAAsB,EAAc,MAAM,wBAAwB,CAAA;AACnG,OAAO,EAAE,KAAK,kBAAkB,EAAE,KAAK,iBAAiB,EAAW,MAAM,aAAa,CAAA;AACtF,OAAO,EAAE,KAAK,oBAAoB,EAAe,KAAK,eAAe,EAAE,MAAM,kBAAkB,CAAA;AAE/F;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,CAAA;AAC1D,MAAM,MAAM,cAAc,CAAC,CAAC,GAAG,OAAO,IAAI,CAAC,KAAK,EAAE,MAAM,KAAK,CAAC,CAAA;AAE9D,MAAM,MAAM,mBAAmB,CAAC,CAAC,GAAG,OAAO,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,CAAA;AACnG,MAAM,MAAM,oBAAoB,GAAG,MAAM,CAAC,MAAM,EAAE,cAAc,GAAG,SAAS,CAAC,CAAA;AAE7E,MAAM,MAAM,UAAU,CAAC,CAAC,GAAG,OAAO,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,aAAa,CAAC,EAAE,QAAQ,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;AA0DrH,MAAM,MAAM,2BAA2B,CAAC,CAAC,GAAG,MAAM,GAAG,MAAM,GAAG,SAAS,IAAI;IAC1E,CAAC,GAAG,EAAE,MAAM,GAAG,CAAC,CAAA;CAChB,CAAA;AAED;;;;;;GAMG;AACH,MAAM,MAAM,QAAQ,CAAC,CAAC,GAAG,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,CAAA;AAEhF,eAAO,MAAM,sBAAsB;;gCAGjB,QAAQ,CAAC,MAAM,CAAC,kBAAiB,QAAQ,CAAC,mBAAmB,CAAC,KAAQ,QAAQ,CAAC,OAAO,CAAC,EAAE;+BAShG,QAAQ,CAAC,MAAM,CAAC,kBACV,QAAQ,CAAC,mBAAmB,CAAC,KAC1C,2BAA2B,CAAC,OAAO,CAAC;CAYqB,CAAA;AAE7D,MAAM,WAAW,kBAAmB,SAAQ,eAAe;IAC1D;;;;;;;;;;;;;;;;OAgBG;IACH,IAAI,CAAC,EAAE,aAAa,CAAA;IAEpB;;;;OAIG;IACH,eAAe,CAAC,EAAE,sBAAsB,CAAA;IAExC;;;;;OAKG;IACH,aAAa,CAAC,EAAE,OAAO,CAAA;IAEvB;;;;OAIG;IACH,MAAM,CAAC,EAAE,OAAO,CAAA;IAEhB,YAAY,CAAC,EAAE,QAAQ,CAAC,mBAAmB,CAAC,GAAG,oBAAoB,CAAA;CACnE;AAED;;;;;;;;;;GAUG;AACH,8BAAsB,cAAc;IACnC;;;;OAIG;IACH,OAAc,eAAe,EAAE,MAAM,CAAmB;;IAMxD,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,2BAA2B,GAAG,2BAA2B,EAC9E,MAAM,EAAE,sBAAsB,EAC9B,OAAO,CAAC,EAAE,kBAAkB,GAAG;QAAE,IAAI,EAAE,QAAQ,CAAA;KAAE,GAC/C,SAAS,CAAC,CAAC,CAAC;IACf;;OAEG;IAEH,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,QAAQ,EAAE,GAAG,QAAQ,EAAE,EAC5C,MAAM,EAAE,sBAAsB,EAC9B,OAAO,CAAC,EAAE,kBAAkB,GAAG;QAAE,IAAI,EAAE,SAAS,CAAA;KAAE,GAChD,SAAS,CAAC,CAAC,CAAC;IACf;;;;OAIG;IACH,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,EACxC,MAAM,EAAE,sBAAsB,EAC9B,OAAO,CAAC,EAAE,kBAAkB,GAAG;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAC/C,SAAS,CAAC,CAAC,CAAC;IAyEf;;OAEG;IACH,MAAM,CAAC,SAAS,CAAC,CAAC,SAAS,2BAA2B,GAAG,2BAA2B,EACnF,MAAM,EAAE,iBAAiB,GAAG,kBAAkB,EAC9C,OAAO,CAAC,EAAE,kBAAkB,GAAG,oBAAoB,GAAG;QAAE,IAAI,EAAE,QAAQ,CAAA;KAAE,GACtE,cAAc,CAAC,CAAC,CAAC;IAEpB;;OAEG;IACH,MAAM,CAAC,SAAS,CAAC,CAAC,SAAS,QAAQ,EAAE,GAAG,QAAQ,EAAE,EACjD,MAAM,EAAE,iBAAiB,GAAG,kBAAkB,EAC9C,OAAO,CAAC,EAAE,kBAAkB,GAAG,oBAAoB,GAAG;QAAE,IAAI,EAAE,SAAS,CAAA;KAAE,GACvE,cAAc,CAAC,CAAC,CAAC;IACpB;;OAEG;IACH,MAAM,CAAC,SAAS,CAAC,CAAC,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,EAC7C,MAAM,EAAE,iBAAiB,GAAG,kBAAkB,EAC9C,OAAO,CAAC,EAAE,kBAAkB,GAAG,oBAAoB,GAAG;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GACtE,cAAc,CAAC,CAAC,CAAC;IACpB;;;;;;;;OAQG;IACH,MAAM,CAAC,SAAS,CACf,MAAM,EAAE,iBAAiB,GAAG,kBAAkB,EAC9C,IAAI,CAAC,EAAE,kBAAkB,GAAG,oBAAoB,GAC9C,cAAc,CAAC,OAAO,CAAC;CAgF1B"}
|
|
@@ -8,6 +8,45 @@ import { CharacterSequence, Delimiters } from "./CharacterSequence.js";
|
|
|
8
8
|
import { zipSync } from "./shared.js";
|
|
9
9
|
import { Spliterator } from "./Spliterator.js";
|
|
10
10
|
const identity = (value) => value;
|
|
11
|
+
const doubleQuoteSequence = new CharacterSequence('"');
|
|
12
|
+
/**
|
|
13
|
+
* Decode a single column, stripping wrapping quotes and unescaping doubled quotes (`""` → `"`) when quote handling is
|
|
14
|
+
* on. The unescape allocates only when the field was actually quoted.
|
|
15
|
+
*/
|
|
16
|
+
function decodeColumn(bytes, decoder, enableQuoteHandling) {
|
|
17
|
+
const value = decoder.decode(bytes);
|
|
18
|
+
if (enableQuoteHandling && value.length >= 2 && value.startsWith('"') && value.endsWith('"')) {
|
|
19
|
+
return value.slice(1, -1).replaceAll('""', '"');
|
|
20
|
+
}
|
|
21
|
+
return value;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Split one row's bytes into decoded column strings.
|
|
25
|
+
*
|
|
26
|
+
* Without quote handling this is a plain delimiter scan. With it, a column delimiter inside a double-quoted region does
|
|
27
|
+
* not split, and each field is unquoted/unescaped via {@linkcode decodeColumn}. Empty columns are always preserved — a
|
|
28
|
+
* 30-column row must stay 30 columns regardless of the caller's row-level `skipEmpty`.
|
|
29
|
+
*/
|
|
30
|
+
function splitRowColumns(row, columnDelimiter, decoder, enableQuoteHandling) {
|
|
31
|
+
if (!enableQuoteHandling) {
|
|
32
|
+
return columnDelimiter.searchAll(row).map(([start, end]) => decoder.decode(row.subarray(start, end)));
|
|
33
|
+
}
|
|
34
|
+
const columns = [];
|
|
35
|
+
let sliceStart = 0;
|
|
36
|
+
let insideQuotes = false;
|
|
37
|
+
for (const match of columnDelimiter.searchMatches(row, doubleQuoteSequence)) {
|
|
38
|
+
if (match.patternId === 1) {
|
|
39
|
+
insideQuotes = !insideQuotes;
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
if (insideQuotes)
|
|
43
|
+
continue;
|
|
44
|
+
columns.push(decodeColumn(row.subarray(sliceStart, match.offset), decoder, enableQuoteHandling));
|
|
45
|
+
sliceStart = match.offset + columnDelimiter.length;
|
|
46
|
+
}
|
|
47
|
+
columns.push(decodeColumn(row.subarray(sliceStart), decoder, enableQuoteHandling));
|
|
48
|
+
return columns;
|
|
49
|
+
}
|
|
11
50
|
export const CSVSpliteratorEmitters = {
|
|
12
51
|
array: null,
|
|
13
52
|
entries(columns, headerColumns = []) {
|
|
@@ -30,6 +69,11 @@ export const CSVSpliteratorEmitters = {
|
|
|
30
69
|
/**
|
|
31
70
|
* A static class spliterator for comma-separated values.
|
|
32
71
|
*
|
|
72
|
+
* **Performance:** the SIMD delimiter scan wins when scanning dominates — many rows, a few columns pulled out cheaply,
|
|
73
|
+
* streaming to bound memory. When per-row work is heavy (a full `JSON.parse`, expensive transforms) it can dominate the
|
|
74
|
+
* scan and erase the advantage; benchmark against a mature native parser before swapping an existing loop for speed.
|
|
75
|
+
* See {@link JSONSpliterator} for the measured case where per-row `JSON.parse` makes the streamed path a net loss.
|
|
76
|
+
*
|
|
33
77
|
* @see {@linkcode CSVSpliterator.from} for synchronous usage.
|
|
34
78
|
* @see {@linkcode CSVSpliterator.fromAsync} for asynchronous usage.
|
|
35
79
|
*/
|
|
@@ -51,23 +95,24 @@ export class CSVSpliterator {
|
|
|
51
95
|
static *from(source, init = {}) {
|
|
52
96
|
const {
|
|
53
97
|
// ---
|
|
54
|
-
header = true, transformers: transformersInput = [], normalizeKeys, mode = "array", columnDelimiter: columnDelimiterInput = this.ColumnDelimiter,
|
|
98
|
+
header = true, transformers: transformersInput = [], normalizeKeys, mode = "array", columnDelimiter: columnDelimiterInput = this.ColumnDelimiter, enableQuoteHandling = false,
|
|
99
|
+
// RFC 4180 mandates CRLF row terminators — accept them by default so the last column
|
|
100
|
+
// never carries a stray `\r` on Windows-lineage sources.
|
|
101
|
+
crlf = true, take = Infinity, drop = 0, ...rowInit } = init;
|
|
55
102
|
const emitter = CSVSpliteratorEmitters[mode];
|
|
56
103
|
let transformers = [];
|
|
57
104
|
let yieldCount = 0;
|
|
58
105
|
const yieldLimit = take + drop;
|
|
59
106
|
const decoder = new TextDecoder();
|
|
60
107
|
const columnDelimiter = new CharacterSequence(columnDelimiterInput ?? this.ColumnDelimiter);
|
|
61
|
-
//
|
|
62
|
-
// must
|
|
63
|
-
|
|
64
|
-
const columnSpliteratorInit = { delimiter: columnDelimiter, skipEmpty: false };
|
|
65
|
-
const rows = Spliterator.fromSync(source, rowInit);
|
|
108
|
+
// Quote handling applies at both levels: rows must not split on newlines inside quotes,
|
|
109
|
+
// columns must not split on quoted column delimiters.
|
|
110
|
+
const rows = Spliterator.fromSync(source, { ...rowInit, crlf, enableQuoteHandling });
|
|
66
111
|
if (header) {
|
|
67
112
|
const result = rows.next();
|
|
68
113
|
if (result.done)
|
|
69
114
|
return;
|
|
70
|
-
const columns =
|
|
115
|
+
const columns = splitRowColumns(result.value, columnDelimiter, decoder, enableQuoteHandling);
|
|
71
116
|
const headers = normalizeKeys ? normalizeColumnNames(columns) : columns;
|
|
72
117
|
if (Array.isArray(transformersInput)) {
|
|
73
118
|
transformers = Array.from(zipSync(headers, transformersInput), ([columnName, transformer]) => [
|
|
@@ -89,9 +134,9 @@ export class CSVSpliterator {
|
|
|
89
134
|
}
|
|
90
135
|
if (yieldCount >= yieldLimit)
|
|
91
136
|
break;
|
|
92
|
-
const
|
|
93
|
-
const columns = columnRanges.map(([s, e]) => decoder.decode(row.subarray(s, e)));
|
|
137
|
+
const columns = splitRowColumns(row, columnDelimiter, decoder, enableQuoteHandling);
|
|
94
138
|
yield emitter ? emitter(columns, transformers) : columns;
|
|
139
|
+
yieldCount++;
|
|
95
140
|
}
|
|
96
141
|
}
|
|
97
142
|
/**
|
|
@@ -106,20 +151,24 @@ export class CSVSpliterator {
|
|
|
106
151
|
static async *fromAsync(source, init = {}) {
|
|
107
152
|
const {
|
|
108
153
|
// ---
|
|
109
|
-
header = true, mode = "array", transformers: transformersInput = [], normalizeKeys = mode !== "array", columnDelimiter: columnDelimiterInput,
|
|
154
|
+
header = true, mode = "array", transformers: transformersInput = [], normalizeKeys = mode !== "array", columnDelimiter: columnDelimiterInput, enableQuoteHandling = false,
|
|
155
|
+
// RFC 4180 mandates CRLF row terminators — accept them by default so the last column
|
|
156
|
+
// never carries a stray `\r` on Windows-lineage sources.
|
|
157
|
+
crlf = true, take = Infinity, drop = 0, ...rowInit } = init;
|
|
110
158
|
const emitter = CSVSpliteratorEmitters[mode];
|
|
111
159
|
let transformers = [];
|
|
112
160
|
let yieldCount = 0;
|
|
113
161
|
const yieldLimit = take + drop;
|
|
114
162
|
const columnDelimiter = new CharacterSequence(columnDelimiterInput ?? this.ColumnDelimiter);
|
|
115
|
-
const columnSpliteratorInit = { delimiter: columnDelimiter, skipEmpty: false };
|
|
116
163
|
const decoder = new TextDecoder();
|
|
117
|
-
|
|
164
|
+
// Quote handling applies at both levels: rows must not split on newlines inside quotes,
|
|
165
|
+
// columns must not split on quoted column delimiters.
|
|
166
|
+
const rows = await Spliterator.from(source, { ...rowInit, crlf, enableQuoteHandling });
|
|
118
167
|
if (header) {
|
|
119
168
|
const result = await rows.next();
|
|
120
169
|
if (result.done)
|
|
121
170
|
return;
|
|
122
|
-
const columns =
|
|
171
|
+
const columns = splitRowColumns(result.value, columnDelimiter, decoder, enableQuoteHandling);
|
|
123
172
|
const headers = normalizeKeys ? normalizeColumnNames(columns) : columns;
|
|
124
173
|
if (Array.isArray(transformersInput)) {
|
|
125
174
|
transformers = Array.from(zipSync(headers, transformersInput), ([columnName, transformer]) => [
|
|
@@ -141,8 +190,7 @@ export class CSVSpliterator {
|
|
|
141
190
|
}
|
|
142
191
|
if (yieldCount >= yieldLimit)
|
|
143
192
|
break;
|
|
144
|
-
const
|
|
145
|
-
const columns = columnRanges.map(([s, e]) => decoder.decode(row.subarray(s, e)));
|
|
193
|
+
const columns = splitRowColumns(row, columnDelimiter, decoder, enableQuoteHandling);
|
|
146
194
|
yield emitter ? emitter(columns, transformers) : columns;
|
|
147
195
|
yieldCount++;
|
|
148
196
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CSVSpliterator.js","sourceRoot":"","sources":["../../lib/CSVSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAA;AAClD,OAAO,EAAE,iBAAiB,EAA+B,UAAU,EAAE,MAAM,wBAAwB,CAAA;AACnG,OAAO,EAAmD,OAAO,EAAE,MAAM,aAAa,CAAA;AACtF,OAAO,EAA6B,WAAW,EAAwB,MAAM,kBAAkB,CAAA;AAa/F,MAAM,QAAQ,GAA2B,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAA;
|
|
1
|
+
{"version":3,"file":"CSVSpliterator.js","sourceRoot":"","sources":["../../lib/CSVSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAA;AAClD,OAAO,EAAE,iBAAiB,EAA+B,UAAU,EAAE,MAAM,wBAAwB,CAAA;AACnG,OAAO,EAAmD,OAAO,EAAE,MAAM,aAAa,CAAA;AACtF,OAAO,EAA6B,WAAW,EAAwB,MAAM,kBAAkB,CAAA;AAa/F,MAAM,QAAQ,GAA2B,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAA;AAEzD,MAAM,mBAAmB,GAAG,IAAI,iBAAiB,CAAC,GAAG,CAAC,CAAA;AAEtD;;;GAGG;AACH,SAAS,YAAY,CAAC,KAAiB,EAAE,OAAoB,EAAE,mBAA4B;IAC1F,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;IAEnC,IAAI,mBAAmB,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC9F,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;IAChD,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;;;;GAMG;AACH,SAAS,eAAe,CACvB,GAAe,EACf,eAAkC,EAClC,OAAoB,EACpB,mBAA4B;IAE5B,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC1B,OAAO,eAAe,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAA;IACtG,CAAC;IAED,MAAM,OAAO,GAAa,EAAE,CAAA;IAC5B,IAAI,UAAU,GAAG,CAAC,CAAA;IAClB,IAAI,YAAY,GAAG,KAAK,CAAA;IAExB,KAAK,MAAM,KAAK,IAAI,eAAe,CAAC,aAAa,CAAC,GAAG,EAAE,mBAAmB,CAAC,EAAE,CAAC;QAC7E,IAAI,KAAK,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;YAC3B,YAAY,GAAG,CAAC,YAAY,CAAA;YAC5B,SAAQ;QACT,CAAC;QAED,IAAI,YAAY;YAAE,SAAQ;QAE1B,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAC,CAAA;QAChG,UAAU,GAAG,KAAK,CAAC,MAAM,GAAG,eAAe,CAAC,MAAM,CAAA;IACnD,CAAC;IAED,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAC,CAAA;IAElF,OAAO,OAAO,CAAA;AACf,CAAC;AAeD,MAAM,CAAC,MAAM,sBAAsB,GAAG;IACrC,KAAK,EAAE,IAAI;IAEX,OAAO,CAAC,OAAyB,EAAE,gBAA+C,EAAE;QACnF,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE;YAChF,MAAM,GAAG,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,CAAA;YAC5B,MAAM,SAAS,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAA;YAE9C,OAAO,CAAC,GAAG,IAAI,UAAU,GAAG,EAAE,EAAE,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,CAAA;QAC7D,CAAC,CAAC,CAAA;IACH,CAAC;IACD,MAAM,CACL,OAAyB,EACzB,gBAA+C,EAAE;QAEjD,MAAM,MAAM,GAAyC,EAAE,CAAA;QAEvD,KAAK,MAAM,CAAC,WAAW,EAAE,KAAK,EAAE,GAAG,CAAC,IAAI,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,EAAE,CAAC;YACzE,MAAM,GAAG,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,UAAU,GAAG,EAAE,CAAA;YAC/C,MAAM,SAAS,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAA;YAE9C,MAAM,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;QACrC,CAAC;QAED,OAAO,MAAM,CAAA;IACd,CAAC;CAC2D,CAAA;AA+C7D;;;;;;;;;;GAUG;AACH,MAAM,OAAgB,cAAc;IACnC;;;;OAIG;IACI,MAAM,CAAC,eAAe,GAAW,UAAU,CAAC,KAAK,CAAA;IAExD;QACC,MAAM,IAAI,SAAS,CAAC,0EAA0E,CAAC,CAAA;IAChG,CAAC;IAuBD;;;;OAIG;IACH,MAAM,CAAC,CAAC,IAAI,CAAC,MAA8B,EAAE,OAA2B,EAAE;QACzE,MAAM;QACL,MAAM;QACN,MAAM,GAAG,IAAI,EACb,YAAY,EAAE,iBAAiB,GAAG,EAAE,EACpC,aAAa,EACb,IAAI,GAAG,OAAO,EACd,eAAe,EAAE,oBAAoB,GAAG,IAAI,CAAC,eAAe,EAC5D,mBAAmB,GAAG,KAAK;QAC3B,qFAAqF;QACrF,yDAAyD;QACzD,IAAI,GAAG,IAAI,EACX,IAAI,GAAG,QAAQ,EACf,IAAI,GAAG,CAAC,EACR,GAAG,OAAO,EACV,GAAG,IAAI,CAAA;QAER,MAAM,OAAO,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAA;QAC5C,IAAI,YAAY,GAA0B,EAAE,CAAA;QAC5C,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,MAAM,UAAU,GAAG,IAAI,GAAG,IAAI,CAAA;QAE9B,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QACjC,MAAM,eAAe,GAAG,IAAI,iBAAiB,CAAC,oBAAoB,IAAI,IAAI,CAAC,eAAe,CAAC,CAAA;QAE3F,wFAAwF;QACxF,sDAAsD;QACtD,MAAM,IAAI,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,mBAAmB,EAAE,CAAC,CAAA;QAEpF,IAAI,MAAM,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;YAE1B,IAAI,MAAM,CAAC,IAAI;gBAAE,OAAM;YAEvB,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,CAAC,KAAK,EAAE,eAAe,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAA;YAC5F,MAAM,OAAO,GAAG,aAAa,CAAC,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;YAEvE,IAAI,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,CAAC;gBACtC,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,iBAAiB,CAAC,EAAE,CAAC,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC;oBAC7F,UAAW;oBACX,WAAW,IAAI,QAAQ;iBACvB,CAAC,CAAA;YACH,CAAC;iBAAM,CAAC;gBACP,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;oBACzC,MAAM,SAAS,GAAI,iBAA0C,CAAC,UAAU,CAAC,IAAI,QAAQ,CAAA;oBAErF,OAAO,CAAC,UAAU,EAAE,SAAS,CAAC,CAAA;gBAC/B,CAAC,CAAC,CAAA;YACH,CAAC;QACF,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,IAAI,UAAU,GAAG,IAAI,EAAE,CAAC;gBACvB,UAAU,EAAE,CAAA;gBACZ,SAAQ;YACT,CAAC;YAED,IAAI,UAAU,IAAI,UAAU;gBAAE,MAAK;YAEnC,MAAM,OAAO,GAAG,eAAe,CAAC,GAAG,EAAE,eAAe,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAA;YAEnF,MAAM,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;YAExD,UAAU,EAAE,CAAA;QACb,CAAC;IACF,CAAC;IAqCD;;;;;;;;OAQG;IACH,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CACtB,MAA8C,EAC9C,OAAkD,EAAE;QAEpD,MAAM;QACL,MAAM;QACN,MAAM,GAAG,IAAI,EACb,IAAI,GAAG,OAAO,EACd,YAAY,EAAE,iBAAiB,GAAG,EAAE,EACpC,aAAa,GAAG,IAAI,KAAK,OAAO,EAChC,eAAe,EAAE,oBAAoB,EACrC,mBAAmB,GAAG,KAAK;QAC3B,qFAAqF;QACrF,yDAAyD;QACzD,IAAI,GAAG,IAAI,EACX,IAAI,GAAG,QAAQ,EACf,IAAI,GAAG,CAAC,EACR,GAAG,OAAO,EACV,GAAG,IAAI,CAAA;QAER,MAAM,OAAO,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAA;QAC5C,IAAI,YAAY,GAA0B,EAAE,CAAA;QAC5C,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,MAAM,UAAU,GAAG,IAAI,GAAG,IAAI,CAAA;QAE9B,MAAM,eAAe,GAAG,IAAI,iBAAiB,CAAC,oBAAoB,IAAI,IAAI,CAAC,eAAe,CAAC,CAAA;QAE3F,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QAEjC,wFAAwF;QACxF,sDAAsD;QACtD,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,mBAAmB,EAAE,CAAC,CAAA;QAEtF,IAAI,MAAM,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAA;YAEhC,IAAI,MAAM,CAAC,IAAI;gBAAE,OAAM;YAEvB,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,CAAC,KAAK,EAAE,eAAe,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAA;YAC5F,MAAM,OAAO,GAAG,aAAa,CAAC,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;YAEvE,IAAI,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,CAAC;gBACtC,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,iBAAiB,CAAC,EAAE,CAAC,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC;oBAC7F,UAAW;oBACX,WAAW,IAAI,QAAQ;iBACvB,CAAC,CAAA;YACH,CAAC;iBAAM,CAAC;gBACP,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;oBACzC,MAAM,SAAS,GAAI,iBAA0C,CAAC,UAAU,CAAC,IAAI,QAAQ,CAAA;oBAErF,OAAO,CAAC,UAAU,EAAE,SAAS,CAAC,CAAA;gBAC/B,CAAC,CAAC,CAAA;YACH,CAAC;QACF,CAAC;QAED,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YAC9B,IAAI,UAAU,GAAG,IAAI,EAAE,CAAC;gBACvB,UAAU,EAAE,CAAA;gBACZ,SAAQ;YACT,CAAC;YAED,IAAI,UAAU,IAAI,UAAU;gBAAE,MAAK;YAEnC,MAAM,OAAO,GAAG,eAAe,CAAC,GAAG,EAAE,eAAe,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAA;YAEnF,MAAM,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;YAExD,UAAU,EAAE,CAAA;QACb,CAAC;IACF,CAAC"}
|
|
@@ -6,6 +6,19 @@
|
|
|
6
6
|
import type { CharacterSequenceInput } from "./CharacterSequence.js";
|
|
7
7
|
import type { AsyncDataResource } from "./shared.js";
|
|
8
8
|
import { type AsyncSpliteratorInit, type SpliteratorInit } from "./Spliterator.js";
|
|
9
|
+
/**
|
|
10
|
+
* Stream a delimited byte source and `JSON.parse` each row — one parsed value per line, for JSONL / NDJSON. Set the row
|
|
11
|
+
* delimiter via `options.delimiter` (there is no implicit default); `skipEmpty` drops blank rows.
|
|
12
|
+
*
|
|
13
|
+
* **Performance caveat — this path is `JSON.parse`-bound, not scan-bound.** The underlying SIMD newline scan is
|
|
14
|
+
* genuinely fast (scan-only measures ~1.75× a JS line-splitter on large files), but a full `JSON.parse` per row dwarfs
|
|
15
|
+
* the scan, so end-to-end this typically runs **~0.75× of Node's `readline` + `JSON.parse` — a net loss** — because the
|
|
16
|
+
* per-row decode + parse dominates and `readline`'s line emission is already well-tuned. Use this for API convenience,
|
|
17
|
+
* or when streaming to bound memory; do NOT swap a working `readline` loop to it expecting a speedup on parse-heavy
|
|
18
|
+
* JSONL. The scan advantage only shows when you _don't_ fully parse every row (segmentation, counting, extracting a
|
|
19
|
+
* couple of fields) — use {@link Spliterator} (raw byte ranges) or {@link TextSpliterator} for those. When unsure,
|
|
20
|
+
* benchmark.
|
|
21
|
+
*/
|
|
9
22
|
export declare abstract class JSONSpliterator {
|
|
10
23
|
constructor();
|
|
11
24
|
static from<T = unknown>(source: CharacterSequenceInput, options?: SpliteratorInit): Generator<T>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"JSONSpliterator.d.ts","sourceRoot":"","sources":["../../lib/JSONSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AACpE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AACpD,OAAO,EAAE,KAAK,oBAAoB,EAAe,KAAK,eAAe,EAAE,MAAM,kBAAkB,CAAA;AAE/F,8BAAsB,eAAe;;IAKpC,MAAM,CAAE,IAAI,CAAC,CAAC,GAAG,OAAO,EAAE,MAAM,EAAE,sBAAsB,EAAE,OAAO,GAAE,eAAoB,GAAG,SAAS,CAAC,CAAC,CAAC;IA0BtG;;;;OAIG;WACW,SAAS,CAAC,CAAC,GAAG,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,OAAO,GAAE,oBAAyB;CAwBlG"}
|
|
1
|
+
{"version":3,"file":"JSONSpliterator.d.ts","sourceRoot":"","sources":["../../lib/JSONSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AACpE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AACpD,OAAO,EAAE,KAAK,oBAAoB,EAAe,KAAK,eAAe,EAAE,MAAM,kBAAkB,CAAA;AAE/F;;;;;;;;;;;;GAYG;AACH,8BAAsB,eAAe;;IAKpC,MAAM,CAAE,IAAI,CAAC,CAAC,GAAG,OAAO,EAAE,MAAM,EAAE,sBAAsB,EAAE,OAAO,GAAE,eAAoB,GAAG,SAAS,CAAC,CAAC,CAAC;IA0BtG;;;;OAIG;WACW,SAAS,CAAC,CAAC,GAAG,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,OAAO,GAAE,oBAAyB;CAwBlG"}
|
|
@@ -4,6 +4,19 @@
|
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
import { Spliterator } from "./Spliterator.js";
|
|
7
|
+
/**
|
|
8
|
+
* Stream a delimited byte source and `JSON.parse` each row — one parsed value per line, for JSONL / NDJSON. Set the row
|
|
9
|
+
* delimiter via `options.delimiter` (there is no implicit default); `skipEmpty` drops blank rows.
|
|
10
|
+
*
|
|
11
|
+
* **Performance caveat — this path is `JSON.parse`-bound, not scan-bound.** The underlying SIMD newline scan is
|
|
12
|
+
* genuinely fast (scan-only measures ~1.75× a JS line-splitter on large files), but a full `JSON.parse` per row dwarfs
|
|
13
|
+
* the scan, so end-to-end this typically runs **~0.75× of Node's `readline` + `JSON.parse` — a net loss** — because the
|
|
14
|
+
* per-row decode + parse dominates and `readline`'s line emission is already well-tuned. Use this for API convenience,
|
|
15
|
+
* or when streaming to bound memory; do NOT swap a working `readline` loop to it expecting a speedup on parse-heavy
|
|
16
|
+
* JSONL. The scan advantage only shows when you _don't_ fully parse every row (segmentation, counting, extracting a
|
|
17
|
+
* couple of fields) — use {@link Spliterator} (raw byte ranges) or {@link TextSpliterator} for those. When unsure,
|
|
18
|
+
* benchmark.
|
|
19
|
+
*/
|
|
7
20
|
export class JSONSpliterator {
|
|
8
21
|
constructor() {
|
|
9
22
|
throw new TypeError("Static class cannot be instantiated. Did you mean `JSONSpliterator.from`?");
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"JSONSpliterator.js","sourceRoot":"","sources":["../../lib/JSONSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,OAAO,EAA6B,WAAW,EAAwB,MAAM,kBAAkB,CAAA;AAE/F,MAAM,OAAgB,eAAe;IACpC;QACC,MAAM,IAAI,SAAS,CAAC,2EAA2E,CAAC,CAAA;IACjG,CAAC;IAED,MAAM,CAAC,CAAC,IAAI,CAAc,MAA8B,EAAE,UAA2B,EAAE;QACtF,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QACjC,IAAI,SAAS,GAAG,CAAC,CAAA;QAEjB,MAAM,WAAW,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAEzD,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YAC/B,IAAI,MAAS,CAAA;YAEb,IAAI,CAAC;gBACJ,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;gBAEnC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAM,CAAA;YAClC,CAAC;YAAC,OAAO,WAAW,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,WAAW,CAAC,+BAA+B,SAAS,EAAE,CAAC,CAAA;gBACrE,KAAK,CAAC,KAAK,GAAG,WAAW,CAAA;gBAEzB,MAAM,KAAK,CAAA;YACZ,CAAC;YAED,MAAM,MAAM,CAAA;YAEZ,SAAS,EAAE,CAAA;QACZ,CAAC;IACF,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CAAc,MAAyB,EAAE,UAAgC,EAAE;QACjG,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QACjC,IAAI,SAAS,GAAG,CAAC,CAAA;QACjB,MAAM,WAAW,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAE3D,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YACrC,IAAI,MAAS,CAAA;YAEb,IAAI,CAAC;gBACJ,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;gBAEnC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAM,CAAA;YAClC,CAAC;YAAC,OAAO,WAAW,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,WAAW,CAAC,+BAA+B,SAAS,EAAE,CAAC,CAAA;gBACrE,KAAK,CAAC,KAAK,GAAG,WAAW,CAAA;gBAEzB,MAAM,KAAK,CAAA;YACZ,CAAC;YAED,MAAM,MAAM,CAAA;YAEZ,SAAS,EAAE,CAAA;QACZ,CAAC;IACF,CAAC;CACD"}
|
|
1
|
+
{"version":3,"file":"JSONSpliterator.js","sourceRoot":"","sources":["../../lib/JSONSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,OAAO,EAA6B,WAAW,EAAwB,MAAM,kBAAkB,CAAA;AAE/F;;;;;;;;;;;;GAYG;AACH,MAAM,OAAgB,eAAe;IACpC;QACC,MAAM,IAAI,SAAS,CAAC,2EAA2E,CAAC,CAAA;IACjG,CAAC;IAED,MAAM,CAAC,CAAC,IAAI,CAAc,MAA8B,EAAE,UAA2B,EAAE;QACtF,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QACjC,IAAI,SAAS,GAAG,CAAC,CAAA;QAEjB,MAAM,WAAW,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAEzD,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YAC/B,IAAI,MAAS,CAAA;YAEb,IAAI,CAAC;gBACJ,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;gBAEnC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAM,CAAA;YAClC,CAAC;YAAC,OAAO,WAAW,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,WAAW,CAAC,+BAA+B,SAAS,EAAE,CAAC,CAAA;gBACrE,KAAK,CAAC,KAAK,GAAG,WAAW,CAAA;gBAEzB,MAAM,KAAK,CAAA;YACZ,CAAC;YAED,MAAM,MAAM,CAAA;YAEZ,SAAS,EAAE,CAAA;QACZ,CAAC;IACF,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CAAc,MAAyB,EAAE,UAAgC,EAAE;QACjG,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QACjC,IAAI,SAAS,GAAG,CAAC,CAAA;QACjB,MAAM,WAAW,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAE3D,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YACrC,IAAI,MAAS,CAAA;YAEb,IAAI,CAAC;gBACJ,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;gBAEnC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAM,CAAA;YAClC,CAAC;YAAC,OAAO,WAAW,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,WAAW,CAAC,+BAA+B,SAAS,EAAE,CAAC,CAAA;gBACrE,KAAK,CAAC,KAAK,GAAG,WAAW,CAAA;gBAEzB,MAAM,KAAK,CAAA;YACZ,CAAC;YAED,MAAM,MAAM,CAAA;YAEZ,SAAS,EAAE,CAAA;QACZ,CAAC;IACF,CAAC;CACD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Spliterator.d.ts","sourceRoot":"","sources":["../../lib/Spliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,gBAAgB,EAAE,KAAK,oBAAoB,EAAE,KAAK,eAAe,EAAE,MAAM,uBAAuB,CAAA;AACzG,OAAO,
|
|
1
|
+
{"version":3,"file":"Spliterator.d.ts","sourceRoot":"","sources":["../../lib/Spliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,gBAAgB,EAAE,KAAK,oBAAoB,EAAE,KAAK,eAAe,EAAE,MAAM,uBAAuB,CAAA;AACzG,OAAO,EAKN,KAAK,sBAAsB,EAC3B,MAAM,wBAAwB,CAAA;AAE/B,OAAO,EAAoB,KAAK,kBAAkB,EAAE,KAAK,iBAAiB,EAAkB,MAAM,aAAa,CAAA;AAE/G,OAAO,EAAE,gBAAgB,EAAE,CAAA;AAE3B,YAAY,EAAE,oBAAoB,EAAE,eAAe,EAAE,CAAA;AAErD;;GAEG;AACH,qBAAa,WAAW,CAAC,CAAC,SAAS,UAAU,GAAG,QAAQ,GAAG,WAAW,GAAG,UAAU,CAClF,YAAW,gBAAgB,CAAC,CAAC,CAAC,EAAE,UAAU;;IAI1C;;;;;;;;OAQG;IACH,MAAM,CAAC,SAAS,+BAAwB;IAExC;;;;;OAKG;IACH,MAAM,CAAC,QAAQ,CAAC,CAAC,SAAS,sBAAsB,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,GAAE,eAAoB,GAAG,WAAW;IAIrG;;;;;;OAMG;IACH,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,GAAG,WAAW,GAAG,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,EAAE,eAAe,GAAG,WAAW;IAC5G;;;;;;;OAOG;IACH,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,kBAAkB,EAAE,IAAI,CAAC,EAAE,oBAAoB,GAAG,gBAAgB;IACtF;;;;;;;;OAQG;IACH,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,iBAAiB,EAAE,IAAI,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAC9F;;;;;;;;OAQG;IACH,MAAM,CAAC,IAAI,CACV,MAAM,EAAE,sBAAsB,GAAG,iBAAiB,GAAG,kBAAkB,EACvE,IAAI,CAAC,EAAE,eAAe,GAAG,oBAAoB,GAC3C,WAAW,GAAG,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAsC7D;;OAEG;WACW,iBAAiB,CAAC,IAAI,EAAE,eAAe,GAAG,eAAe,CAAC,UAAU,EAAE,UAAU,EAAE,CAAC;IAUjG;;OAEG;IACI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI;gBAInB,MAAM,EAAE,UAAU,EAAE,IAAI,GAAE,eAAoB;IAyP1D;;OAEG;IACI,IAAI,IAAI,cAAc,CAAC,CAAC,CAAC;IA4ChC;;;;;OAKG;IACI,OAAO,IAAI,CAAC,EAAE;IAIrB;;OAEG;IACI,cAAc,CAAC,OAAO,kCAAoB,GAAG,MAAM,EAAE;IAIrD,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,gBAAgB,CAAC,CAAC,CAAC;CAG/C"}
|
package/out/lib/Spliterator.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
import { AsyncSpliterator } from "./AsyncSpliterator.js";
|
|
7
|
-
import { CharacterSequence, debugAsVisibleCharacters, normalizeCharacterInput, } from "./CharacterSequence.js";
|
|
7
|
+
import { CharacterSequence, debugAsVisibleCharacters, Delimiters, normalizeCharacterInput, } from "./CharacterSequence.js";
|
|
8
8
|
import { IndexQueue } from "./IndexQueue.js";
|
|
9
9
|
import { isFileHandleLike } from "./shared.js";
|
|
10
10
|
export { AsyncSpliterator };
|
|
@@ -84,6 +84,7 @@ export class Spliterator {
|
|
|
84
84
|
this.#yieldByteEstimate = this.#source.byteLength - this.#readPosition;
|
|
85
85
|
this.#skipEmpty = init.skipEmpty ?? true;
|
|
86
86
|
this.#enableQuoteHandling = init.enableQuoteHandling ?? false;
|
|
87
|
+
this.#crlf = init.crlf ?? false;
|
|
87
88
|
this.#debug = init.debug ?? false;
|
|
88
89
|
this.#log = this.#debug ? console.debug.bind(console) : () => void 0;
|
|
89
90
|
}
|
|
@@ -110,6 +111,10 @@ export class Spliterator {
|
|
|
110
111
|
*/
|
|
111
112
|
#doubleQuoteSequence = new CharacterSequence('"');
|
|
112
113
|
#enableQuoteHandling;
|
|
114
|
+
/**
|
|
115
|
+
* Whether to trim a carriage return immediately preceding each delimiter match.
|
|
116
|
+
*/
|
|
117
|
+
#crlf;
|
|
113
118
|
/**
|
|
114
119
|
* How many yields to skip.
|
|
115
120
|
*/
|
|
@@ -199,9 +204,10 @@ export class Spliterator {
|
|
|
199
204
|
// resurfacing the entire delimiter run as one (non-empty) row.
|
|
200
205
|
this.#indices.enqueue([0, sourceByteLength]);
|
|
201
206
|
}
|
|
202
|
-
else
|
|
203
|
-
//
|
|
204
|
-
//
|
|
207
|
+
else {
|
|
208
|
+
// Emit an empty field for a trailing delimiter (match String.split). `#fill` always
|
|
209
|
+
// leaves the tail after the last consumed delimiter to this method, in both the plain
|
|
210
|
+
// and quote-aware paths — a source ending exactly on a delimiter has an empty tail.
|
|
205
211
|
const trailingDelimPos = sourceByteLength - this.#needle.length;
|
|
206
212
|
if (trailingDelimPos >= 0 && this.#needle.search(this.#source, trailingDelimPos) !== -1) {
|
|
207
213
|
this.#indices.enqueue([sourceByteLength, sourceByteLength]);
|
|
@@ -209,6 +215,16 @@ export class Spliterator {
|
|
|
209
215
|
}
|
|
210
216
|
this.#done = true;
|
|
211
217
|
}
|
|
218
|
+
/**
|
|
219
|
+
* Trim a carriage return immediately preceding a delimiter match, when {@linkcode SpliteratorInit.crlf} is set.
|
|
220
|
+
* Affects only the emitted range's end — `#readPosition` advancement always uses the raw match offset.
|
|
221
|
+
*/
|
|
222
|
+
#trimEnd(start, end) {
|
|
223
|
+
if (this.#crlf && end > start && this.#source[end - 1] === Delimiters.CarriageReturn) {
|
|
224
|
+
return end - 1;
|
|
225
|
+
}
|
|
226
|
+
return end;
|
|
227
|
+
}
|
|
212
228
|
/**
|
|
213
229
|
* Fill the buffer with data and search for delimiters.
|
|
214
230
|
*/
|
|
@@ -219,59 +235,37 @@ export class Spliterator {
|
|
|
219
235
|
const delimiterIndex = this.#needle.search(this.#source, this.#readPosition);
|
|
220
236
|
if (delimiterIndex === -1)
|
|
221
237
|
return;
|
|
222
|
-
this.#indices.enqueue([this.#readPosition, delimiterIndex]);
|
|
238
|
+
this.#indices.enqueue([this.#readPosition, this.#trimEnd(this.#readPosition, delimiterIndex)]);
|
|
223
239
|
this.#readPosition = delimiterIndex + this.#needle.length;
|
|
224
240
|
}
|
|
225
241
|
return;
|
|
226
242
|
}
|
|
243
|
+
// Quote-aware path: a delimiter inside a double-quoted region does not split, and the
|
|
244
|
+
// emitted slices keep their quotes verbatim — stripping and `""` unescaping belong to the
|
|
245
|
+
// consumer (CSVSpliterator does both). The tail after the last consumed delimiter is left
|
|
246
|
+
// to `#drain`, same as the plain path.
|
|
227
247
|
while (this.#readPosition < sourceByteLength && this.#indices.byteLength < this.#highWaterMark) {
|
|
228
248
|
const matches = this.#needle.searchMatches(this.#source, this.#doubleQuoteSequence, this.#readPosition, sourceByteLength);
|
|
229
249
|
if (matches.length === 0)
|
|
230
250
|
return;
|
|
231
251
|
let sliceStart = this.#readPosition;
|
|
232
252
|
let insideQuotes = false;
|
|
233
|
-
let quoteStart = -1;
|
|
234
253
|
for (const match of matches) {
|
|
235
254
|
if (match.patternId === 1) {
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
// Emit normal field before the opening quote
|
|
239
|
-
if (match.offset > sliceStart) {
|
|
240
|
-
this.#indices.enqueue([sliceStart, match.offset]);
|
|
241
|
-
}
|
|
242
|
-
insideQuotes = true;
|
|
243
|
-
quoteStart = match.offset;
|
|
244
|
-
}
|
|
245
|
-
else {
|
|
246
|
-
// Closing quote: emit quoted field, advance past quote
|
|
247
|
-
this.#indices.enqueue([quoteStart + 1, match.offset]);
|
|
248
|
-
insideQuotes = false;
|
|
249
|
-
sliceStart = match.offset + 1;
|
|
250
|
-
}
|
|
251
|
-
}
|
|
252
|
-
else {
|
|
253
|
-
// Delimiter found; one inside quotes is part of the field, so skip it.
|
|
254
|
-
if (insideQuotes)
|
|
255
|
-
continue;
|
|
256
|
-
// Emit content before this delimiter (empty if delimiter follows closing quote)
|
|
257
|
-
if (sliceStart < match.offset) {
|
|
258
|
-
this.#indices.enqueue([sliceStart, match.offset]);
|
|
259
|
-
}
|
|
260
|
-
sliceStart = match.offset + this.#needle.length;
|
|
255
|
+
insideQuotes = !insideQuotes;
|
|
256
|
+
continue;
|
|
261
257
|
}
|
|
258
|
+
// A delimiter inside quotes is part of the slice, so skip it.
|
|
259
|
+
if (insideQuotes)
|
|
260
|
+
continue;
|
|
261
|
+
this.#indices.enqueue([sliceStart, this.#trimEnd(sliceStart, match.offset)]);
|
|
262
|
+
sliceStart = match.offset + this.#needle.length;
|
|
262
263
|
}
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
this.#indices.enqueue([sliceStart, sourceByteLength]);
|
|
269
|
-
this.#readPosition = sourceByteLength;
|
|
270
|
-
// Case 2: Trailing delimiter (emit empty field to match String.split)
|
|
271
|
-
}
|
|
272
|
-
else if (sliceStart === sourceByteLength && this.#readPosition === sourceByteLength) {
|
|
273
|
-
this.#indices.enqueue([sourceByteLength, sourceByteLength]);
|
|
274
|
-
}
|
|
264
|
+
// No delimiter was consumed (e.g. an unclosed quote swallows the rest of the source) —
|
|
265
|
+
// leave the tail to `#drain` rather than rescanning the same matches forever.
|
|
266
|
+
if (sliceStart === this.#readPosition)
|
|
267
|
+
return;
|
|
268
|
+
this.#readPosition = sliceStart;
|
|
275
269
|
}
|
|
276
270
|
}
|
|
277
271
|
//#endregion
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Spliterator.js","sourceRoot":"","sources":["../../lib/Spliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,gBAAgB,EAAmD,MAAM,uBAAuB,CAAA;AACzG,OAAO,EACN,iBAAiB,EACjB,wBAAwB,EACxB,uBAAuB,GAEvB,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EAAE,gBAAgB,EAAmE,MAAM,aAAa,CAAA;AAE/G,OAAO,EAAE,gBAAgB,EAAE,CAAA;AAI3B;;GAEG;AACH,MAAM,OAAO,WAAW;IAGvB,mBAAmB;IAEnB;;;;;;;;OAQG;IACH,MAAM,CAAC,SAAS,GAAG,gBAAgB,CAAC,IAAI,CAAA;IAExC;;;;;OAKG;IACH,MAAM,CAAC,QAAQ,CAAmC,MAAS,EAAE,OAAwB,EAAE;QACtF,OAAO,IAAI,WAAW,CAAC,uBAAuB,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,CAAA;IAC9D,CAAC;IA0CD,MAAM,CAAC,IAAI,CACV,MAAuE,EACvE,OAA+C,EAAE;QAEjD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,aAAa,IAAI,MAAM,EAAE,CAAC;YAClE,OAAO,IAAI,gBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QAC1C,CAAC;QAED,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,YAAY,GAAG,IAAI,gBAAgB,CAAC,MAAM,CAAC,EAAE,CAAC;YACrF,OAAO,MAAM,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,mBAAmB,EAAE,EAAE,EAAE;gBAC3E,IAAI,aAAiC,CAAA;gBAErC,IAAI,CAAC;oBACJ,aAAa,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE;wBACjD,aAAa,EAAE,IAAI,CAAC,aAAa;qBACjC,CAAC,CAAA;gBACH,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,YAAY,GAAG,EAAE,CAAC;wBACzD,MAAM,OAAO,GAAG,IAAI,KAAK,CACxB,gHAAgH,CAChH,CAAA;wBAED,OAAO,CAAC,KAAK,GAAG,KAAK,CAAA;wBAErB,MAAM,OAAO,CAAA;oBACd,CAAC;oBAED,MAAM,KAAK,CAAA;gBACZ,CAAC;gBAED,OAAO,IAAI,gBAAgB,CAAC,aAAa,EAAE,IAAI,CAAC,CAAA;YACjD,CAAC,CAAC,CAAA;QACH,CAAC;QAED,OAAO,IAAI,WAAW,CAAC,uBAAuB,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,CAAA;IAC9D,CAAC;IAED;;OAEG;IACI,MAAM,CAAC,iBAAiB,CAAC,IAAqB;QACpD,OAAO,IAAI,eAAe,CAA2B;YACpD,SAAS,CAAC,KAAK,EAAE,UAAU;gBAC1B,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;gBAEhD,UAAU,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAA;YAC1C,CAAC;SACD,CAAC,CAAA;IACH,CAAC;IAED;;OAEG;IACI,CAAC,MAAM,CAAC,OAAO,CAAC;QACtB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;IACtB,CAAC;IAED,YAAY,MAAkB,EAAE,OAAwB,EAAE;QACzD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QAErB,IAAI,CAAC,OAAO,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QAEpD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAA;QAEvC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,CAAA;QAE7D,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAA;QAClD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,eAAe,CAAA;QAChF,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,aAAa,CAAA;QAEtE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAA;QACxC,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,mBAAmB,IAAI,KAAK,CAAA;
|
|
1
|
+
{"version":3,"file":"Spliterator.js","sourceRoot":"","sources":["../../lib/Spliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,gBAAgB,EAAmD,MAAM,uBAAuB,CAAA;AACzG,OAAO,EACN,iBAAiB,EACjB,wBAAwB,EACxB,UAAU,EACV,uBAAuB,GAEvB,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EAAE,gBAAgB,EAAmE,MAAM,aAAa,CAAA;AAE/G,OAAO,EAAE,gBAAgB,EAAE,CAAA;AAI3B;;GAEG;AACH,MAAM,OAAO,WAAW;IAGvB,mBAAmB;IAEnB;;;;;;;;OAQG;IACH,MAAM,CAAC,SAAS,GAAG,gBAAgB,CAAC,IAAI,CAAA;IAExC;;;;;OAKG;IACH,MAAM,CAAC,QAAQ,CAAmC,MAAS,EAAE,OAAwB,EAAE;QACtF,OAAO,IAAI,WAAW,CAAC,uBAAuB,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,CAAA;IAC9D,CAAC;IA0CD,MAAM,CAAC,IAAI,CACV,MAAuE,EACvE,OAA+C,EAAE;QAEjD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,aAAa,IAAI,MAAM,EAAE,CAAC;YAClE,OAAO,IAAI,gBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QAC1C,CAAC;QAED,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,YAAY,GAAG,IAAI,gBAAgB,CAAC,MAAM,CAAC,EAAE,CAAC;YACrF,OAAO,MAAM,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,mBAAmB,EAAE,EAAE,EAAE;gBAC3E,IAAI,aAAiC,CAAA;gBAErC,IAAI,CAAC;oBACJ,aAAa,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE;wBACjD,aAAa,EAAE,IAAI,CAAC,aAAa;qBACjC,CAAC,CAAA;gBACH,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,YAAY,GAAG,EAAE,CAAC;wBACzD,MAAM,OAAO,GAAG,IAAI,KAAK,CACxB,gHAAgH,CAChH,CAAA;wBAED,OAAO,CAAC,KAAK,GAAG,KAAK,CAAA;wBAErB,MAAM,OAAO,CAAA;oBACd,CAAC;oBAED,MAAM,KAAK,CAAA;gBACZ,CAAC;gBAED,OAAO,IAAI,gBAAgB,CAAC,aAAa,EAAE,IAAI,CAAC,CAAA;YACjD,CAAC,CAAC,CAAA;QACH,CAAC;QAED,OAAO,IAAI,WAAW,CAAC,uBAAuB,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,CAAA;IAC9D,CAAC;IAED;;OAEG;IACI,MAAM,CAAC,iBAAiB,CAAC,IAAqB;QACpD,OAAO,IAAI,eAAe,CAA2B;YACpD,SAAS,CAAC,KAAK,EAAE,UAAU;gBAC1B,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;gBAEhD,UAAU,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAA;YAC1C,CAAC;SACD,CAAC,CAAA;IACH,CAAC;IAED;;OAEG;IACI,CAAC,MAAM,CAAC,OAAO,CAAC;QACtB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;IACtB,CAAC;IAED,YAAY,MAAkB,EAAE,OAAwB,EAAE;QACzD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QAErB,IAAI,CAAC,OAAO,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QAEpD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAA;QAEvC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,CAAA;QAE7D,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAA;QAClD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,eAAe,CAAA;QAChF,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,aAAa,CAAA;QAEtE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAA;QACxC,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,mBAAmB,IAAI,KAAK,CAAA;QAC7D,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,IAAI,KAAK,CAAA;QAE/B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,IAAI,KAAK,CAAA;QACjC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAA;IACrE,CAAC;IAED,YAAY;IAEZ,4BAA4B;IAE5B;;OAEG;IACM,OAAO,CAAY;IAE5B;;;;;;OAMG;IACM,QAAQ,GAAG,IAAI,UAAU,EAAE,CAAA;IAEpC;;OAEG;IACM,OAAO,CAAmB;IAEnC;;OAEG;IACM,oBAAoB,GAAsB,IAAI,iBAAiB,CAAC,GAAG,CAAC,CAAA;IACpE,oBAAoB,CAAS;IAEtC;;OAEG;IACM,KAAK,CAAS;IAEvB;;OAEG;IACM,eAAe,CAAQ;IAEhC;;OAEG;IACM,eAAe,CAAQ;IAEhC;;;;OAIG;IACM,kBAAkB,CAAQ;IAEnC;;OAEG;IACM,UAAU,CAAS;IAE5B;;;;OAIG;IACM,cAAc,CAAQ;IAE/B;;OAEG;IACH,kBAAkB,GAAG,CAAC,CAAA;IAEtB;;OAEG;IACH,WAAW,GAAG,CAAC,CAAA;IAEf;;OAEG;IACH,aAAa,CAAQ;IAErB;;OAEG;IACH,kBAAkB,CAAuB;IAEzC;;OAEG;IACH,MAAM,CAAS;IAEf;;OAEG;IACH,KAAK,GAAG,KAAK,CAAA;IAEb,YAAY;IAEZ,yBAAyB;IAEzB,IAAI,CAA0B;IAE9B,SAAS;QACR,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB;;;eAGG;YACH,MAAM,6BAA6B,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;YAClF,MAAM,YAAY,GAAG,IAAI,CAAC,kBAAkB,GAAG,CAAC,IAAI,CAAC,kBAAkB,GAAG,6BAA6B,CAAC,CAAA;YAExG,IAAI,CAAC,IAAI,CAAC;gBACT,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU;gBACtC,YAAY,EAAE,IAAI,CAAC,aAAa;gBAChC,iBAAiB,EAAE,IAAI,CAAC,kBAAkB;gBAC1C,iBAAiB,EAAE,IAAI,CAAC,kBAAkB;gBAC1C,UAAU,EAAE,IAAI,CAAC,WAAW;gBAC5B,6BAA6B;gBAC7B,YAAY;aACZ,CAAC,CAAA;QACH,CAAC;QAED,OAAO;YACN,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,SAAS;SAChB,CAAA;IACF,CAAC;IAED,MAAM;QACL,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAA;QAEhD,IAAI,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAA;QAExD,qFAAqF;QACrF,MAAM,aAAa,GAAG,IAAI,CAAC,kBAAkB,CAAA;QAE7C,IAAI,IAAI,CAAC,aAAa,GAAG,gBAAgB,EAAE,CAAC;YAC3C,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,aAAa,EAAE,gBAAgB,CAAC,CAAC,CAAA;QAC9D,CAAC;aAAM,IAAI,IAAI,CAAC,WAAW,KAAK,CAAC,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;YAClE,sFAAsF;YACtF,mFAAmF;YACnF,iFAAiF;YACjF,+DAA+D;YAC/D,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC,CAAA;QAC7C,CAAC;aAAM,CAAC;YACP,oFAAoF;YACpF,sFAAsF;YACtF,oFAAoF;YACpF,MAAM,gBAAgB,GAAG,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;YAE/D,IAAI,gBAAgB,IAAI,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gBACzF,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,gBAAgB,EAAE,gBAAgB,CAAC,CAAC,CAAA;YAC5D,CAAC;QACF,CAAC;QAED,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;IAClB,CAAC;IAED;;;OAGG;IACH,QAAQ,CAAC,KAAa,EAAE,GAAW;QAClC,IAAI,IAAI,CAAC,KAAK,IAAI,GAAG,GAAG,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,UAAU,CAAC,cAAc,EAAE,CAAC;YACtF,OAAO,GAAG,GAAG,CAAC,CAAA;QACf,CAAC;QAED,OAAO,GAAG,CAAA;IACX,CAAC;IAED;;OAEG;IACH,KAAK;QACJ,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAA;QAEhD,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAChC,OAAO,IAAI,CAAC,aAAa,GAAG,gBAAgB,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;gBAChG,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,aAAa,CAAC,CAAA;gBAE5E,IAAI,cAAc,KAAK,CAAC,CAAC;oBAAE,OAAM;gBAEjC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC,CAAC,CAAA;gBAC9F,IAAI,CAAC,aAAa,GAAG,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;YAC1D,CAAC;YAED,OAAM;QACP,CAAC;QAED,sFAAsF;QACtF,0FAA0F;QAC1F,0FAA0F;QAC1F,uCAAuC;QACvC,OAAO,IAAI,CAAC,aAAa,GAAG,gBAAgB,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;YAChG,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CACzC,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,oBAAoB,EACzB,IAAI,CAAC,aAAa,EAClB,gBAAgB,CAChB,CAAA;YAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAM;YAEhC,IAAI,UAAU,GAAG,IAAI,CAAC,aAAa,CAAA;YACnC,IAAI,YAAY,GAAG,KAAK,CAAA;YAExB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC7B,IAAI,KAAK,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;oBAC3B,YAAY,GAAG,CAAC,YAAY,CAAA;oBAC5B,SAAQ;gBACT,CAAC;gBAED,8DAA8D;gBAC9D,IAAI,YAAY;oBAAE,SAAQ;gBAE1B,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;gBAC5E,UAAU,GAAG,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;YAChD,CAAC;YAED,uFAAuF;YACvF,8EAA8E;YAC9E,IAAI,UAAU,KAAK,IAAI,CAAC,aAAa;gBAAE,OAAM;YAE7C,IAAI,CAAC,aAAa,GAAG,UAAU,CAAA;QAChC,CAAC;IACF,CAAC;IAED,YAAY;IAEZ,0BAA0B;IAE1B;;OAEG;IACI,IAAI;QACV,kFAAkF;QAClF,iEAAiE;QACjE,OAAO,IAAI,EAAE,CAAC;YACb,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,eAAe;gBAAE,OAAO,IAAI,CAAC,SAAS,EAAE,CAAA;YAEnF,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI;gBAAE,IAAI,CAAC,KAAK,EAAE,CAAA;YAErC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACzB,IAAI,CAAC,MAAM,EAAE,CAAA;YACd,CAAC;YAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAA;YAChD,IAAI,CAAC,kBAAkB,GAAG,gBAAgB,CAAA;YAE1C,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACvB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;gBAEjB,OAAO,IAAI,CAAC,SAAS,EAAE,CAAA;YACxB,CAAC;YAED,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,gBAAgB,CAAA;YAErC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;YAE/C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC3C,SAAQ;YACT,CAAC;YAED,IAAI,CAAC,WAAW,EAAE,CAAA;YAElB,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;gBAC9C,SAAQ;YACT,CAAC;YAED,IAAI,CAAC,kBAAkB,IAAI,GAAG,GAAG,KAAK,CAAA;YAEtC,OAAO;gBACN,KAAK,EAAE,KAAqB;gBAC5B,IAAI,EAAE,KAAK;aACX,CAAA;QACF,CAAC;IACF,CAAC;IAED;;;;;OAKG;IACI,OAAO;QACb,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACxB,CAAC;IAED;;OAEG;IACI,cAAc,CAAC,OAAO,GAAG,IAAI,WAAW,EAAE;QAChD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;IAC1D,CAAC;IAEM,CAAC,MAAM,CAAC,QAAQ,CAAC;QACvB,OAAO,IAAI,CAAA;IACZ,CAAC"}
|
|
@@ -10,7 +10,12 @@ export type ParallelHandler<T = unknown, R = unknown> = (item: T, ctx: {
|
|
|
10
10
|
export interface ParallelMapOptions {
|
|
11
11
|
/** Worker module path/URL exporting `handleItem(item, ctx)`. Top-level code = per-worker init. */
|
|
12
12
|
worker: string | URL;
|
|
13
|
-
/**
|
|
13
|
+
/**
|
|
14
|
+
* Number of worker threads in the pool. Keep it SMALL for I/O- or memory-bound handlers (per-row DB lookups, large
|
|
15
|
+
* shared datasets): throughput typically peaks at ~2–3 and _degrades_ past that as workers contend for the shared
|
|
16
|
+
* resource — only CPU-bound handlers scale toward the core count. Also bounded by RAM (each worker re-inits its
|
|
17
|
+
* deps). Sweep it for your workload rather than defaulting to `availableParallelism()`.
|
|
18
|
+
*/
|
|
14
19
|
concurrency: number;
|
|
15
20
|
/** Items per dispatched batch. @default 64 */
|
|
16
21
|
batchSize?: number;
|
|
@@ -23,8 +28,18 @@ export interface ParallelMapOptions {
|
|
|
23
28
|
* async iterator in **completion order** (not input order). A handler returning `undefined` drops that item; a
|
|
24
29
|
* `Uint8Array` is transferred zero-copy.
|
|
25
30
|
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
31
|
+
* **When it pays (measure — don't assume).** Threading only wins when per-item work is heavy relative to the ~µs
|
|
32
|
+
* cross-thread dispatch cost (structured-clone in + out). From real workloads: light per-row work — CSV/JSON parse,
|
|
33
|
+
* string normalize, `JSON.parse` — is _dispatch-bound_ and runs **0.3–0.9× of single-threaded (a net loss)**; ms-scale
|
|
34
|
+
* per-row work — model inference, geocoding, crypto/image ops — is where threads win. If the per-row cost isn't clearly
|
|
35
|
+
* milliseconds, benchmark against the single-threaded version before adopting.
|
|
36
|
+
*
|
|
37
|
+
* **Concurrency is not core count.** Heavy work that is I/O- or memory-bound (random reads into one large on-disk DB,
|
|
38
|
+
* anything sharing memory bandwidth) peaks _low_ — often ~2–3 workers — and then _degrades_, because the workers
|
|
39
|
+
* contend for the shared resource, not the CPU. Start small and sweep; don't reach for `availableParallelism()`. Only
|
|
40
|
+
* CPU-bound per-row work (pure compute) scales out toward the core count.
|
|
41
|
+
*
|
|
42
|
+
* All workers are terminated on completion, error, or early `return()`.
|
|
28
43
|
*/
|
|
29
44
|
export declare function parallelMap<T, R = unknown>(source: AsyncIterable<T> | Iterable<T>, options: ParallelMapOptions): AsyncIterableIterator<R>;
|
|
30
45
|
//# sourceMappingURL=parallel-map.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parallel-map.d.ts","sourceRoot":"","sources":["../../lib/parallel-map.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH,6FAA6F;AAC7F,MAAM,MAAM,eAAe,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,OAAO,IAAI,CACvD,IAAI,EAAE,CAAC,EACP,GAAG,EAAE;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,KAClB,CAAC,GAAG,UAAU,GAAG,SAAS,GAAG,OAAO,CAAC,CAAC,GAAG,UAAU,GAAG,SAAS,CAAC,CAAA;AAErE,MAAM,WAAW,kBAAkB;IAClC,kGAAkG;IAClG,MAAM,EAAE,MAAM,GAAG,GAAG,CAAA;IACpB
|
|
1
|
+
{"version":3,"file":"parallel-map.d.ts","sourceRoot":"","sources":["../../lib/parallel-map.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH,6FAA6F;AAC7F,MAAM,MAAM,eAAe,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,OAAO,IAAI,CACvD,IAAI,EAAE,CAAC,EACP,GAAG,EAAE;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,KAClB,CAAC,GAAG,UAAU,GAAG,SAAS,GAAG,OAAO,CAAC,CAAC,GAAG,UAAU,GAAG,SAAS,CAAC,CAAA;AAErE,MAAM,WAAW,kBAAkB;IAClC,kGAAkG;IAClG,MAAM,EAAE,MAAM,GAAG,GAAG,CAAA;IACpB;;;;;OAKG;IACH,WAAW,EAAE,MAAM,CAAA;IACnB,8CAA8C;IAC9C,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,0FAA0F;IAC1F,UAAU,CAAC,EAAE,OAAO,CAAA;CACpB;AAgCD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,WAAW,CAAC,CAAC,EAAE,CAAC,GAAG,OAAO,EACzC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,EACtC,OAAO,EAAE,kBAAkB,GACzB,qBAAqB,CAAC,CAAC,CAAC,CAwB1B"}
|
package/out/lib/parallel-map.js
CHANGED
|
@@ -37,8 +37,18 @@ function workerHandle(worker) {
|
|
|
37
37
|
* async iterator in **completion order** (not input order). A handler returning `undefined` drops that item; a
|
|
38
38
|
* `Uint8Array` is transferred zero-copy.
|
|
39
39
|
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
40
|
+
* **When it pays (measure — don't assume).** Threading only wins when per-item work is heavy relative to the ~µs
|
|
41
|
+
* cross-thread dispatch cost (structured-clone in + out). From real workloads: light per-row work — CSV/JSON parse,
|
|
42
|
+
* string normalize, `JSON.parse` — is _dispatch-bound_ and runs **0.3–0.9× of single-threaded (a net loss)**; ms-scale
|
|
43
|
+
* per-row work — model inference, geocoding, crypto/image ops — is where threads win. If the per-row cost isn't clearly
|
|
44
|
+
* milliseconds, benchmark against the single-threaded version before adopting.
|
|
45
|
+
*
|
|
46
|
+
* **Concurrency is not core count.** Heavy work that is I/O- or memory-bound (random reads into one large on-disk DB,
|
|
47
|
+
* anything sharing memory bandwidth) peaks _low_ — often ~2–3 workers — and then _degrades_, because the workers
|
|
48
|
+
* contend for the shared resource, not the CPU. Start small and sweep; don't reach for `availableParallelism()`. Only
|
|
49
|
+
* CPU-bound per-row work (pure compute) scales out toward the core count.
|
|
50
|
+
*
|
|
51
|
+
* All workers are terminated on completion, error, or early `return()`.
|
|
42
52
|
*/
|
|
43
53
|
export function parallelMap(source, options) {
|
|
44
54
|
const handlerUrl = options.worker instanceof URL ? options.worker.href : new URL(options.worker, `file://${process.cwd()}/`).href;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parallel-map.js","sourceRoot":"","sources":["../../lib/parallel-map.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAE5C,OAAO,EAAmB,OAAO,EAAE,MAAM,2BAA2B,CAAA;
|
|
1
|
+
{"version":3,"file":"parallel-map.js","sourceRoot":"","sources":["../../lib/parallel-map.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAE5C,OAAO,EAAmB,OAAO,EAAE,MAAM,2BAA2B,CAAA;AAwBpE,yGAAyG;AACzG,SAAS,YAAY,CAAO,MAAc;IACzC,IAAI,OAAO,GAA+E,IAAI,CAAA;IAE9F,MAAM,MAAM,GAAG,CAAC,EAA4C,EAAE,EAAE;QAC/D,IAAI,CAAC,OAAO;YAAE,OAAM;QACpB,MAAM,CAAC,GAAG,OAAO,CAAA;QACjB,OAAO,GAAG,IAAI,CAAA;QACd,EAAE,CAAC,CAAC,CAAC,CAAA;IACN,CAAC,CAAA;IAED,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAA0E,EAAE,EAAE;QACnG,IAAI,GAAG,EAAE,IAAI,KAAK,QAAQ;YAAE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAA;aAC5D,IAAI,GAAG,EAAE,IAAI,KAAK,OAAO;YAAE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;IAChF,CAAC,CAAC,CAAA;IAEF,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAc,EAAE,EAAE,CACrC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAClF,CAAA;IAED,OAAO;QACN,OAAO,CAAC,KAAK;YACZ,OAAO,IAAI,OAAO,CAAM,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBAC3C,OAAO,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,CAAA;gBAC7B,MAAM,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAA;YAC7C,CAAC,CAAC,CAAA;QACH,CAAC;KACD,CAAA;AACF,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,WAAW,CAC1B,MAAsC,EACtC,OAA2B;IAE3B,MAAM,UAAU,GACf,OAAO,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,CAAA;IAC/G,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAA;IAChE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,EAAE,CAAA;IACzC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,gCAAgC,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAC3E,MAAM,OAAO,GAAa,EAAE,CAAA;IAE5B,KAAK,SAAS,CAAC,CAAC,GAAG;QAClB,IAAI,CAAC;YACJ,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE;gBACrD,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAA;gBACjG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;gBAEpB,OAAO,YAAY,CAAO,MAAM,CAAC,CAAA;YAClC,CAAC,CAAC,CAAA;YAEF,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,CAAC,CAAA;QACxC,CAAC;gBAAS,CAAC;YACV,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAA;QAC/D,CAAC;IACF,CAAC;IAED,OAAO,GAAG,EAAE,CAAA;AACb,CAAC"}
|
package/out/lib/shared.d.ts
CHANGED
|
@@ -114,9 +114,12 @@ export declare function applyReaderPolyfill<T extends FileResourceLike>(file: T)
|
|
|
114
114
|
/**
|
|
115
115
|
* An asynchronous iterable byte stream.
|
|
116
116
|
*
|
|
117
|
+
* String chunks are tolerated and UTF-8 encoded on arrival — a `fs.createReadStream(path, "utf8")` or Node `Readable`
|
|
118
|
+
* in string mode would otherwise coerce to garbage bytes via `TypedArray.set`.
|
|
119
|
+
*
|
|
117
120
|
* Note that as an iterable this will drained of all bytes when iterated over.
|
|
118
121
|
*/
|
|
119
|
-
export interface AsyncChunkIterator extends AsyncIterable<Uint8Array> {
|
|
122
|
+
export interface AsyncChunkIterator extends AsyncIterable<Uint8Array | string> {
|
|
120
123
|
/**
|
|
121
124
|
* Disposes of the byte stream, releasing any resources.
|
|
122
125
|
*
|
|
@@ -130,9 +133,11 @@ export interface AsyncChunkIterator extends AsyncIterable<Uint8Array> {
|
|
|
130
133
|
* - `string` representing a file path.
|
|
131
134
|
* - {@linkcode URL} object representing a file path.
|
|
132
135
|
* - {@linkcode FileHandleLike} object representing an open file handle.
|
|
133
|
-
* - {@linkcode AsyncChunkIterator} object representing an asynchronous byte stream
|
|
136
|
+
* - {@linkcode AsyncChunkIterator} object representing an asynchronous byte stream — anything async-iterable over bytes
|
|
137
|
+
* qualifies: a Node `Readable` (child-process stdout, a gunzip pipe), a web `ReadableStream` wrapped via
|
|
138
|
+
* `Symbol.asyncIterator`, or a hand-rolled generator.
|
|
134
139
|
*/
|
|
135
|
-
export type AsyncDataResource = string | URL | FileHandleLike;
|
|
140
|
+
export type AsyncDataResource = string | URL | FileHandleLike | AsyncChunkIterator;
|
|
136
141
|
/**
|
|
137
142
|
* A resource to a delimited byte stream, i.e., a file buffer, handle, or path.
|
|
138
143
|
*
|
package/out/lib/shared.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../lib/shared.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAE,KAAK,KAAK,EAAE,MAAM,SAAS,CAAA;AAEpC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAElD;;GAEG;AACH,MAAM,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,SAAS,CAAC,CAAA;AAEnE;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,IAAI,CAChC,UAAU,EACV,IAAI,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,GAAG,mBAAmB,GAAG,kBAAkB,CAC3E,CAAA;AAED;;GAEG;AACH,MAAM,MAAM,SAAS,GAAG;IACvB;;OAEG;IACH,KAAK,EAAE,MAAM;IACb;;OAEG;IACH,GAAG,EAAE,MAAM;CACX,CAAA;AAED;;GAEG;AACH,MAAM,MAAM,UAAU,GACnB,UAAU,GACV,iBAAiB,GACjB,WAAW,GACX,WAAW,GACX,SAAS,GACT,UAAU,GACV,UAAU,GACV,cAAc,GACd,aAAa,GACb,YAAY,GACZ,YAAY,CAAA;AAEf;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,cAAc,GAAG,UAAU,CAAA;AAExD;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,cAAc,CAExE;AAED;;;;GAIG;AACH,MAAM,MAAM,kBAAkB,CAAC,CAAC,IAAI,CAAC,SAAS,UAAU,GAAG,CAAC,GAAG,UAAU,CAAA;AAEzE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;;;;;OAMG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;CACjB;AAED;;;;;GAKG;AACH,MAAM,WAAW,eAAgB,SAAQ,aAAa,CAAC,UAAU,CAAC;IACjE;;OAEG;IACH,IAAI,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;IACrD,IAAI,CAAC,CAAC,SAAS,UAAU,EAAE,OAAO,EAAE,gBAAgB,GAAG;QAAE,MAAM,EAAE,CAAC,CAAA;KAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;CACjF;AAED;;;;GAIG;AACH,MAAM,WAAW,gBAAiB,SAAQ,IAAI;IAC7C;;;;;OAKG;IACH,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,gBAAgB,CAAA;IAEnD;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAElB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,CAAA;CAC3C;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,gBAAgB,CAE5E;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,CAAC,SAAS,gBAAgB,EAAE,IAAI,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,GAAG,eAAe,CAsB5G;AAED
|
|
1
|
+
{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../lib/shared.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAE,KAAK,KAAK,EAAE,MAAM,SAAS,CAAA;AAEpC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAElD;;GAEG;AACH,MAAM,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,SAAS,CAAC,CAAA;AAEnE;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,IAAI,CAChC,UAAU,EACV,IAAI,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,GAAG,mBAAmB,GAAG,kBAAkB,CAC3E,CAAA;AAED;;GAEG;AACH,MAAM,MAAM,SAAS,GAAG;IACvB;;OAEG;IACH,KAAK,EAAE,MAAM;IACb;;OAEG;IACH,GAAG,EAAE,MAAM;CACX,CAAA;AAED;;GAEG;AACH,MAAM,MAAM,UAAU,GACnB,UAAU,GACV,iBAAiB,GACjB,WAAW,GACX,WAAW,GACX,SAAS,GACT,UAAU,GACV,UAAU,GACV,cAAc,GACd,aAAa,GACb,YAAY,GACZ,YAAY,CAAA;AAEf;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,cAAc,GAAG,UAAU,CAAA;AAExD;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,cAAc,CAExE;AAED;;;;GAIG;AACH,MAAM,MAAM,kBAAkB,CAAC,CAAC,IAAI,CAAC,SAAS,UAAU,GAAG,CAAC,GAAG,UAAU,CAAA;AAEzE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;;;;;OAMG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;CACjB;AAED;;;;;GAKG;AACH,MAAM,WAAW,eAAgB,SAAQ,aAAa,CAAC,UAAU,CAAC;IACjE;;OAEG;IACH,IAAI,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;IACrD,IAAI,CAAC,CAAC,SAAS,UAAU,EAAE,OAAO,EAAE,gBAAgB,GAAG;QAAE,MAAM,EAAE,CAAC,CAAA;KAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;CACjF;AAED;;;;GAIG;AACH,MAAM,WAAW,gBAAiB,SAAQ,IAAI;IAC7C;;;;;OAKG;IACH,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,gBAAgB,CAAA;IAEnD;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAElB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,CAAA;CAC3C;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,gBAAgB,CAE5E;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,CAAC,SAAS,gBAAgB,EAAE,IAAI,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,GAAG,eAAe,CAsB5G;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,kBAAmB,SAAQ,aAAa,CAAC,UAAU,GAAG,MAAM,CAAC;IAC7E;;;;OAIG;IACH,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,CAAA;CAC3C;AAED;;;;;;;;;GASG;AACH,MAAM,MAAM,iBAAiB,GAAG,MAAM,GAAG,GAAG,GAAG,cAAc,GAAG,kBAAkB,CAAA;AAElF;;;;;GAKG;AACH,MAAM,MAAM,YAAY,GAAG,iBAAiB,GAAG,UAAU,CAAA;AAEzD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B,MAAM,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAAA;CACjC;AAED,MAAM,MAAM,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,GAAG,EAAE,MAAM,CAAC,CAAA;AAE5E,MAAM,MAAM,aAAa,CAAC,CAAC,IAAI,CAAC,SAAS,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAA;AAElF,wBAAgB,aAAa,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAErF;AACD;;;;;;;;;;GAUG;AACH,wBAAiB,OAAO,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAkBtF;AAED;;;;;;;;;;GAUG;AACH,wBAAuB,QAAQ,CAAC,CAAC,EAAE,CAAC,EACnC,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,EACjC,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,GAC/B,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,CAkBnE"}
|
package/out/lib/shared.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../lib/shared.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,iDAAiD;AACjD,OAAO,EAAc,MAAM,SAAS,CAAA;AAoDpC;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAc;IAC9C,OAAO,OAAO,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,IAAI,IAAI,KAAK,CAAC,CAAA;AACpE,CAAC;AAuED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAAc;IAChD,OAAO,OAAO,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,IAAI,KAAK,CAAC,CAAA;AACvE,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAA6B,IAAO;IACtE,IAAI,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;QACvD,OAAM;IACP,CAAC;IAED,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE;QACnB,KAAK,CAAC,IAAI,CAAuB,UAA6C,EAAE;YAC/E,MAAM,EAAE,QAAQ,EAAE,KAAK,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,CAAC,EAAE,GAAG,OAAO,CAAA;YACvE,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC,CAAA;YAElD,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,CAAC,KAAK,EAAE,CAA0B,CAAA;YAE/D,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBACpB,MAAM,WAAW,GAAG,OAAO,CAAC,MAA+B,CAAA;gBAC3D,WAAW,CAAC,GAAG,CAAC,KAAY,EAAE,MAAM,CAAC,CAAA;gBAErC,OAAO,WAAW,CAAA;YACnB,CAAC;YAED,OAAO,KAAK,CAAA;QACb,CAAC;KACD,CAAC,CAAA;AACH,CAAC;
|
|
1
|
+
{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../lib/shared.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,iDAAiD;AACjD,OAAO,EAAc,MAAM,SAAS,CAAA;AAoDpC;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAc;IAC9C,OAAO,OAAO,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,IAAI,IAAI,KAAK,CAAC,CAAA;AACpE,CAAC;AAuED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAAc;IAChD,OAAO,OAAO,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,IAAI,KAAK,CAAC,CAAA;AACvE,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAA6B,IAAO;IACtE,IAAI,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;QACvD,OAAM;IACP,CAAC;IAED,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE;QACnB,KAAK,CAAC,IAAI,CAAuB,UAA6C,EAAE;YAC/E,MAAM,EAAE,QAAQ,EAAE,KAAK,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,CAAC,EAAE,GAAG,OAAO,CAAA;YACvE,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC,CAAA;YAElD,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,CAAC,KAAK,EAAE,CAA0B,CAAA;YAE/D,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBACpB,MAAM,WAAW,GAAG,OAAO,CAAC,MAA+B,CAAA;gBAC3D,WAAW,CAAC,GAAG,CAAC,KAAY,EAAE,MAAM,CAAC,CAAA;gBAErC,OAAO,WAAW,CAAA;YACnB,CAAC;YAED,OAAO,KAAK,CAAA;QACb,CAAC;KACD,CAAC,CAAA;AACH,CAAC;AAkDD,MAAM,UAAU,aAAa,CAAO,MAAoB;IACvD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAgC,CAAA;AACzD,CAAC;AACD;;;;;;;;;;GAUG;AACH,MAAM,SAAS,CAAC,CAAC,OAAO,CAAO,CAAc,EAAE,CAAc;IAC5D,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAA;IACtC,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAA;IAEtC,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,OAAO,IAAI,EAAE,CAAC;QACb,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC,IAAI,EAAE,CAAA;QACvD,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC,IAAI,EAAE,CAAA;QAEvD,IAAI,KAAK,IAAI,KAAK,EAAE,CAAC;YACpB,MAAK;QACN,CAAC;QAED,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAA;QAE7B,KAAK,EAAE,CAAA;IACR,CAAC;AACF,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,QAAQ,CAC9B,CAAiC,EACjC,CAAiC;IAEjC,MAAM,SAAS,GAAG,MAAM,CAAC,aAAa,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAA;IAC9F,MAAM,SAAS,GAAG,MAAM,CAAC,aAAa,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAA;IAE9F,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,OAAO,IAAI,EAAE,CAAC;QACb,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAA;QAC7D,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAA;QAE7D,IAAI,KAAK,IAAI,KAAK,EAAE,CAAC;YACpB,MAAK;QACN,CAAC;QAED,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAA;QAE7B,KAAK,EAAE,CAAA;IACR,CAAC;AACF,CAAC"}
|
package/out/lib/writer.d.ts
CHANGED
|
@@ -6,22 +6,24 @@
|
|
|
6
6
|
import { type PathLike, WriteStream } from "node:fs";
|
|
7
7
|
/**
|
|
8
8
|
* Callback for writing a line to a newline-delimited file.
|
|
9
|
+
*
|
|
10
|
+
* The trailing newline is appended for you — pass the content alone.
|
|
9
11
|
*/
|
|
10
12
|
export interface WriteLineCallback {
|
|
11
13
|
/**
|
|
12
|
-
* Write a line to the file. The
|
|
14
|
+
* Write a line to the file, terminated with a newline. The content should not contain one itself.
|
|
13
15
|
*/
|
|
14
16
|
(content: string): Promise<void>;
|
|
15
17
|
/**
|
|
16
|
-
* Write a chunk to the file. The chunk should not contain
|
|
18
|
+
* Write a chunk to the file, terminated with a newline. The chunk should not contain one itself.
|
|
17
19
|
*/
|
|
18
20
|
(chunk: any, encoding: BufferEncoding): Promise<void>;
|
|
19
21
|
}
|
|
20
22
|
export interface NewlineWriter extends AsyncDisposable {
|
|
21
23
|
/**
|
|
22
|
-
* Write a line to the file.
|
|
24
|
+
* Write a line to the file, terminated with a newline.
|
|
23
25
|
*
|
|
24
|
-
* The content should not contain a newline character.
|
|
26
|
+
* The content should not contain a newline character — the writer supplies the delimiter.
|
|
25
27
|
*/
|
|
26
28
|
write: WriteLineCallback;
|
|
27
29
|
dispose: () => Promise<void>;
|
|
@@ -29,6 +31,15 @@ export interface NewlineWriter extends AsyncDisposable {
|
|
|
29
31
|
}
|
|
30
32
|
/**
|
|
31
33
|
* Creates a writer for newline-delimited files.
|
|
34
|
+
*
|
|
35
|
+
* Every {@linkcode NewlineWriter.write} terminates its content with `\n`, so the file is delimited (and round-trips
|
|
36
|
+
* through {@linkcode TextSpliterator}) without the caller appending anything:
|
|
37
|
+
*
|
|
38
|
+
* ```ts
|
|
39
|
+
* await using out = createNewlineWriter("rows.jsonl")
|
|
40
|
+
*
|
|
41
|
+
* for (const row of rows) await out.write(JSON.stringify(row))
|
|
42
|
+
* ```
|
|
32
43
|
*/
|
|
33
44
|
export declare function createNewlineWriter(filePath: PathLike): NewlineWriter;
|
|
34
45
|
//# sourceMappingURL=writer.d.ts.map
|
package/out/lib/writer.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../lib/writer.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAqB,KAAK,QAAQ,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AAEvE
|
|
1
|
+
{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../lib/writer.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAqB,KAAK,QAAQ,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AAEvE;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IACjC;;OAEG;IACH,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAChC;;OAEG;IACH,CAAC,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CACrD;AAED,MAAM,WAAW,aAAc,SAAQ,eAAe;IACrD;;;;OAIG;IACH,KAAK,EAAE,iBAAiB,CAAA;IACxB,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;IAC5B,MAAM,EAAE,WAAW,CAAA;CACnB;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,GAAG,aAAa,CA6CrE"}
|
package/out/lib/writer.js
CHANGED
|
@@ -6,18 +6,34 @@
|
|
|
6
6
|
import { createWriteStream, WriteStream } from "node:fs";
|
|
7
7
|
/**
|
|
8
8
|
* Creates a writer for newline-delimited files.
|
|
9
|
+
*
|
|
10
|
+
* Every {@linkcode NewlineWriter.write} terminates its content with `\n`, so the file is delimited (and round-trips
|
|
11
|
+
* through {@linkcode TextSpliterator}) without the caller appending anything:
|
|
12
|
+
*
|
|
13
|
+
* ```ts
|
|
14
|
+
* await using out = createNewlineWriter("rows.jsonl")
|
|
15
|
+
*
|
|
16
|
+
* for (const row of rows) await out.write(JSON.stringify(row))
|
|
17
|
+
* ```
|
|
9
18
|
*/
|
|
10
19
|
export function createNewlineWriter(filePath) {
|
|
11
20
|
const writer = createWriteStream(filePath);
|
|
12
|
-
const write = (content) => {
|
|
21
|
+
const write = (content, encoding) => {
|
|
13
22
|
return new Promise((resolve, reject) => {
|
|
14
|
-
|
|
23
|
+
const settle = (error) => {
|
|
15
24
|
if (error) {
|
|
16
25
|
reject(error);
|
|
17
26
|
return;
|
|
18
27
|
}
|
|
19
28
|
resolve();
|
|
20
|
-
}
|
|
29
|
+
};
|
|
30
|
+
// The delimiter goes out as its own chunk rather than concatenated onto `content`: the
|
|
31
|
+
// two-argument overload accepts a Buffer, and `buffer + "\n"` would stringify it via
|
|
32
|
+
// `toString()` and corrupt the bytes. Stream writes are ordered, so the newline always
|
|
33
|
+
// follows its content, and settling on the delimiter's callback resolves once the whole
|
|
34
|
+
// line has flushed.
|
|
35
|
+
writer.write(content, encoding ?? "utf8");
|
|
36
|
+
writer.write("\n", "utf8", settle);
|
|
21
37
|
});
|
|
22
38
|
};
|
|
23
39
|
const dispose = () => {
|
package/out/lib/writer.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"writer.js","sourceRoot":"","sources":["../../lib/writer.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,iBAAiB,EAAiB,WAAW,EAAE,MAAM,SAAS,CAAA;
|
|
1
|
+
{"version":3,"file":"writer.js","sourceRoot":"","sources":["../../lib/writer.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,iBAAiB,EAAiB,WAAW,EAAE,MAAM,SAAS,CAAA;AA6BvE;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,mBAAmB,CAAC,QAAkB;IACrD,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAA;IAE1C,MAAM,KAAK,GAAsB,CAAC,OAAY,EAAE,QAAyB,EAAE,EAAE;QAC5E,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACtC,MAAM,MAAM,GAAG,CAAC,KAA+B,EAAQ,EAAE;gBACxD,IAAI,KAAK,EAAE,CAAC;oBACX,MAAM,CAAC,KAAK,CAAC,CAAA;oBAEb,OAAM;gBACP,CAAC;gBAED,OAAO,EAAE,CAAA;YACV,CAAC,CAAA;YAED,uFAAuF;YACvF,qFAAqF;YACrF,uFAAuF;YACvF,wFAAwF;YACxF,oBAAoB;YACpB,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,QAAQ,IAAI,MAAM,CAAC,CAAA;YACzC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAA;QACnC,CAAC,CAAC,CAAA;IACH,CAAC,CAAA;IAED,MAAM,OAAO,GAAG,GAAG,EAAE;QACpB,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC5C,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gBACtB,IAAI,KAAK,EAAE,CAAC;oBACX,MAAM,CAAC,KAAK,CAAC,CAAA;oBAEb,OAAM;gBACP,CAAC;gBAED,OAAO,EAAE,CAAA;YACV,CAAC,CAAC,CAAA;QACH,CAAC,CAAC,CAAA;IACH,CAAC,CAAA;IAED,OAAO;QACN,KAAK;QACL,MAAM;QACN,OAAO;QACP,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,OAAO;KAC9B,CAAA;AACF,CAAC"}
|