spliterator 2.2.3 → 3.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -0
- package/out/index.d.ts +2 -1
- package/out/index.d.ts.map +1 -1
- package/out/index.js +2 -1
- package/out/index.js.map +1 -1
- package/out/lib/AsyncSpliterator.d.ts +35 -34
- package/out/lib/AsyncSpliterator.d.ts.map +1 -1
- package/out/lib/AsyncSpliterator.js +93 -84
- package/out/lib/AsyncSpliterator.js.map +1 -1
- package/out/lib/BufferController.d.ts +10 -12
- package/out/lib/BufferController.d.ts.map +1 -1
- package/out/lib/BufferController.js +10 -12
- package/out/lib/BufferController.js.map +1 -1
- package/out/lib/CSVSpliterator.d.ts +6 -5
- package/out/lib/CSVSpliterator.d.ts.map +1 -1
- package/out/lib/CSVSpliterator.js +8 -5
- package/out/lib/CSVSpliterator.js.map +1 -1
- package/out/lib/CharacterSequence.d.ts +15 -64
- package/out/lib/CharacterSequence.d.ts.map +1 -1
- package/out/lib/CharacterSequence.js +194 -88
- package/out/lib/CharacterSequence.js.map +1 -1
- package/out/lib/CompositeDataView.d.ts +4 -5
- package/out/lib/CompositeDataView.d.ts.map +1 -1
- package/out/lib/CompositeDataView.js +2 -3
- package/out/lib/CompositeDataView.js.map +1 -1
- package/out/lib/IndexQueue.d.ts +1 -1
- package/out/lib/IndexQueue.d.ts.map +1 -1
- package/out/lib/IndexQueue.js +40 -11
- package/out/lib/IndexQueue.js.map +1 -1
- package/out/lib/JSONSpliterator.d.ts +1 -1
- package/out/lib/JSONSpliterator.js +1 -1
- package/out/lib/PSVSpliterator.d.ts +1 -1
- package/out/lib/PSVSpliterator.js +1 -1
- package/out/lib/SlidingWindow.d.ts +1 -1
- package/out/lib/SlidingWindow.d.ts.map +1 -1
- package/out/lib/SlidingWindow.js +11 -10
- package/out/lib/SlidingWindow.js.map +1 -1
- package/out/lib/Spliterator.d.ts +2 -3
- package/out/lib/Spliterator.d.ts.map +1 -1
- package/out/lib/Spliterator.js +102 -114
- package/out/lib/Spliterator.js.map +1 -1
- package/out/lib/TSVSpliterator.d.ts +1 -1
- package/out/lib/TSVSpliterator.js +1 -1
- package/out/lib/TextSpliterator.d.ts +4 -3
- package/out/lib/TextSpliterator.d.ts.map +1 -1
- package/out/lib/TextSpliterator.js +3 -1
- package/out/lib/TextSpliterator.js.map +1 -1
- package/out/lib/casing.d.ts +2 -3
- package/out/lib/casing.d.ts.map +1 -1
- package/out/lib/casing.js +2 -3
- package/out/lib/casing.js.map +1 -1
- package/out/lib/segment-runtime.d.ts +27 -0
- package/out/lib/segment-runtime.d.ts.map +1 -0
- package/out/lib/segment-runtime.js +38 -0
- package/out/lib/segment-runtime.js.map +1 -0
- package/out/lib/segment-worker-entry.d.ts +10 -0
- package/out/lib/segment-worker-entry.d.ts.map +1 -0
- package/out/lib/segment-worker-entry.js +56 -0
- package/out/lib/segment-worker-entry.js.map +1 -0
- package/out/lib/segment-workers.d.ts +49 -0
- package/out/lib/segment-workers.d.ts.map +1 -0
- package/out/lib/segment-workers.js +138 -0
- package/out/lib/segment-workers.js.map +1 -0
- package/out/lib/segments.d.ts +25 -0
- package/out/lib/segments.d.ts.map +1 -0
- package/out/lib/segments.js +50 -0
- package/out/lib/segments.js.map +1 -0
- package/out/lib/shared.d.ts +6 -5
- package/out/lib/shared.d.ts.map +1 -1
- package/out/lib/shared.js +3 -1
- package/out/lib/shared.js.map +1 -1
- package/out/lib/take.d.ts +3 -3
- package/out/lib/take.d.ts.map +1 -1
- package/out/lib/take.js +3 -3
- package/out/lib/take.js.map +1 -1
- package/out/lib/wasm_base64.d.ts +16 -0
- package/out/lib/wasm_base64.d.ts.map +1 -0
- package/out/lib/wasm_base64.js +16 -0
- package/out/lib/wasm_base64.js.map +1 -0
- package/out/lib/wasm_module.d.ts +36 -0
- package/out/lib/wasm_module.d.ts.map +1 -0
- package/out/lib/wasm_module.js +33 -0
- package/out/lib/wasm_module.js.map +1 -0
- package/out/lib/writer.d.ts +1 -1
- package/out/lib/writer.d.ts.map +1 -1
- package/out/lib/writer.js +1 -1
- package/out/lib/writer.js.map +1 -1
- package/out/node/cli/commands/csv.d.ts +1 -1
- package/out/node/cli/commands/csv.d.ts.map +1 -1
- package/out/node/cli/commands/csv.js +1 -1
- package/out/node/cli/commands/csv.js.map +1 -1
- package/out/node/cli/commands/iterate.d.ts +1 -1
- package/out/node/cli/commands/iterate.d.ts.map +1 -1
- package/out/node/cli/commands/iterate.js +1 -1
- package/out/node/cli/commands/iterate.js.map +1 -1
- package/out/node/cli/index.d.ts +1 -1
- package/out/node/cli/index.js +1 -1
- package/out/node/cli/utils.d.ts +1 -1
- package/out/node/cli/utils.js +1 -1
- package/out/node/fs/index.d.ts +16 -4
- package/out/node/fs/index.d.ts.map +1 -1
- package/out/node/fs/index.js +28 -4
- package/out/node/fs/index.js.map +1 -1
- package/package.json +4 -2
package/out/lib/Spliterator.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @copyright Sister Software
|
|
3
|
-
* @license
|
|
3
|
+
* @license MIT
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
import { AsyncSpliterator } from "./AsyncSpliterator.js";
|
|
@@ -83,6 +83,7 @@ export class Spliterator {
|
|
|
83
83
|
this.#yieldStopCount = Math.max(init.take ?? Infinity, 0) + this.#yieldDropCount;
|
|
84
84
|
this.#yieldByteEstimate = this.#source.byteLength - this.#readPosition;
|
|
85
85
|
this.#skipEmpty = init.skipEmpty ?? true;
|
|
86
|
+
this.#enableQuoteHandling = init.enableQuoteHandling ?? false;
|
|
86
87
|
this.#debug = init.debug ?? false;
|
|
87
88
|
this.#log = this.#debug ? console.debug.bind(console) : () => void 0;
|
|
88
89
|
}
|
|
@@ -97,8 +98,7 @@ export class Spliterator {
|
|
|
97
98
|
*
|
|
98
99
|
* Note that indices are relative to the buffer, not the file.
|
|
99
100
|
*
|
|
100
|
-
* This means that the start index is always 0, and the end index is the byte length of the
|
|
101
|
-
* buffer.
|
|
101
|
+
* This means that the start index is always 0, and the end index is the byte length of the buffer.
|
|
102
102
|
*/
|
|
103
103
|
#indices = new IndexQueue();
|
|
104
104
|
/**
|
|
@@ -109,6 +109,7 @@ export class Spliterator {
|
|
|
109
109
|
* The byte sequence for a double quote.
|
|
110
110
|
*/
|
|
111
111
|
#doubleQuoteSequence = new CharacterSequence('"');
|
|
112
|
+
#enableQuoteHandling;
|
|
112
113
|
/**
|
|
113
114
|
* How many yields to skip.
|
|
114
115
|
*/
|
|
@@ -145,10 +146,6 @@ export class Spliterator {
|
|
|
145
146
|
* The current byte index to perform read operations from.
|
|
146
147
|
*/
|
|
147
148
|
#readPosition;
|
|
148
|
-
/**
|
|
149
|
-
* A bit of a hack to keep track of double quotes.
|
|
150
|
-
*/
|
|
151
|
-
#doubleQuoteStartIndex = -1;
|
|
152
149
|
/**
|
|
153
150
|
* The previous byte range seen while draining the buffer.
|
|
154
151
|
*/
|
|
@@ -167,8 +164,8 @@ export class Spliterator {
|
|
|
167
164
|
#finalize() {
|
|
168
165
|
if (this.#debug) {
|
|
169
166
|
/**
|
|
170
|
-
* The total number of bytes we expect to be omitted from the buffer. This is derived from the
|
|
171
|
-
*
|
|
167
|
+
* The total number of bytes we expect to be omitted from the buffer. This is derived from the number of of yields
|
|
168
|
+
* and the length of the delimiter.
|
|
172
169
|
*/
|
|
173
170
|
const expectedYieldedDelimitedBytes = (this.#yieldCount - 1) * this.#needle.length;
|
|
174
171
|
const omittedBytes = this.#yieldByteEstimate - (this.#yieldedByteLength + expectedYieldedDelimitedBytes);
|
|
@@ -192,26 +189,24 @@ export class Spliterator {
|
|
|
192
189
|
this.#log("Reached end of file. Preparing to finalize.");
|
|
193
190
|
// There's a few special cases we could get this far and not have drained the buffer.
|
|
194
191
|
const lastByteRange = this.#previousByteRange;
|
|
195
|
-
if (
|
|
196
|
-
|
|
197
|
-
// There's a special case where the last delimiter is at the end of the *file*.
|
|
198
|
-
// We need to determine if we're at the end of the *file* and there's no delimiter.
|
|
199
|
-
const possibleDelimiterIndex = this.#needle.search(this.#source, sourceByteLength - this.#needle.length);
|
|
200
|
-
if (possibleDelimiterIndex !== -1) {
|
|
201
|
-
this.#log("Inserted byte range at the last delimiter", [possibleDelimiterIndex, sourceByteLength]);
|
|
202
|
-
// We found a delimiter at the end of the file, so we enqueue the byte range.
|
|
203
|
-
this.#indices.enqueue([possibleDelimiterIndex + this.#needle.length, sourceByteLength]);
|
|
204
|
-
}
|
|
205
|
-
else {
|
|
206
|
-
this.#log("Inserted byte range at the last byte", [lastByteIndex, sourceByteLength]);
|
|
207
|
-
// The file didn't end with a delimiter, so we just enqueue the last byte range.
|
|
208
|
-
this.#indices.enqueue([lastByteIndex + this.#needle.length, sourceByteLength]);
|
|
209
|
-
}
|
|
192
|
+
if (this.#readPosition < sourceByteLength) {
|
|
193
|
+
this.#indices.enqueue([this.#readPosition, sourceByteLength]);
|
|
210
194
|
}
|
|
211
|
-
else if (this.#yieldCount === 0) {
|
|
212
|
-
//
|
|
195
|
+
else if (this.#yieldCount === 0 && lastByteRange === undefined) {
|
|
196
|
+
// Nothing was ever found or yielded (e.g. an empty source) — emit the whole buffer as
|
|
197
|
+
// a single field. A run of all-empty fields that `skipEmpty` dropped has a defined
|
|
198
|
+
// `lastByteRange`, so it falls through to the trailing-delimiter case instead of
|
|
199
|
+
// resurfacing the entire delimiter run as one (non-empty) row.
|
|
213
200
|
this.#indices.enqueue([0, sourceByteLength]);
|
|
214
201
|
}
|
|
202
|
+
else if (!this.#enableQuoteHandling) {
|
|
203
|
+
// Simple path: emit empty field for trailing delimiter (match String.split).
|
|
204
|
+
// Quote-aware path handles this inside #fill() via searchMatches.
|
|
205
|
+
const trailingDelimPos = sourceByteLength - this.#needle.length;
|
|
206
|
+
if (trailingDelimPos >= 0 && this.#needle.search(this.#source, trailingDelimPos) !== -1) {
|
|
207
|
+
this.#indices.enqueue([sourceByteLength, sourceByteLength]);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
215
210
|
this.#done = true;
|
|
216
211
|
}
|
|
217
212
|
/**
|
|
@@ -219,75 +214,64 @@ export class Spliterator {
|
|
|
219
214
|
*/
|
|
220
215
|
#fill() {
|
|
221
216
|
const sourceByteLength = this.#source.byteLength;
|
|
217
|
+
if (!this.#enableQuoteHandling) {
|
|
218
|
+
while (this.#readPosition < sourceByteLength && this.#indices.byteLength < this.#highWaterMark) {
|
|
219
|
+
const delimiterIndex = this.#needle.search(this.#source, this.#readPosition);
|
|
220
|
+
if (delimiterIndex === -1)
|
|
221
|
+
return;
|
|
222
|
+
this.#indices.enqueue([this.#readPosition, delimiterIndex]);
|
|
223
|
+
this.#readPosition = delimiterIndex + this.#needle.length;
|
|
224
|
+
}
|
|
225
|
+
return;
|
|
226
|
+
}
|
|
222
227
|
while (this.#readPosition < sourceByteLength && this.#indices.byteLength < this.#highWaterMark) {
|
|
223
|
-
const
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
let sliceEnd;
|
|
228
|
+
const matches = this.#needle.searchMatches(this.#source, this.#doubleQuoteSequence, this.#readPosition, sourceByteLength);
|
|
229
|
+
if (matches.length === 0)
|
|
230
|
+
return;
|
|
227
231
|
let sliceStart = this.#readPosition;
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
// this.#doubleQuoteStartIndex = -1
|
|
241
|
-
// } else {
|
|
242
|
-
// // console.log("Searching for double quote at read position", this.#readPosition)
|
|
243
|
-
// nextDoubleQuoteIndex = this.#doubleQuoteSequence.search(this.#source, this.#readPosition)
|
|
244
|
-
// if (nextDoubleQuoteIndex !== -1) {
|
|
245
|
-
// console.log("Found double quote at", nextDoubleQuoteIndex)
|
|
246
|
-
// this.#doubleQuoteStartIndex = nextDoubleQuoteIndex
|
|
247
|
-
// sliceStart = doubleQuoteStartIndex + 1
|
|
248
|
-
// delimiterSearchStart = nextDoubleQuoteIndex + 1
|
|
249
|
-
// } else {
|
|
250
|
-
// // console.log("No double quote found at read position")
|
|
251
|
-
// delimiterSearchStart = this.#readPosition
|
|
252
|
-
// }
|
|
253
|
-
// }
|
|
254
|
-
let delimiterIndex;
|
|
255
|
-
while (true) {
|
|
256
|
-
delimiterIndex = this.#needle.search(this.#source, delimiterSearchStart);
|
|
257
|
-
if (delimiterIndex === -1) {
|
|
258
|
-
// this.#log(`No delimiters left. Breaking.`)
|
|
259
|
-
return;
|
|
260
|
-
}
|
|
261
|
-
nextDoubleQuoteIndex = this.#doubleQuoteSequence.search(this.#source, delimiterSearchStart, delimiterIndex);
|
|
262
|
-
// Didn't find a double quote before the delimiter.
|
|
263
|
-
if (nextDoubleQuoteIndex === -1 && this.#doubleQuoteStartIndex === -1) {
|
|
264
|
-
// And we haven't found a double quote before.
|
|
265
|
-
sliceEnd = delimiterIndex;
|
|
266
|
-
this.#readPosition = sliceEnd + this.#needle.length;
|
|
267
|
-
break;
|
|
268
|
-
}
|
|
269
|
-
if (nextDoubleQuoteIndex !== -1) {
|
|
270
|
-
// We found a double quote before the delimiter...
|
|
271
|
-
if (this.#doubleQuoteStartIndex === -1) {
|
|
272
|
-
// But this is the first double quote we've found.
|
|
273
|
-
this.#doubleQuoteStartIndex = nextDoubleQuoteIndex;
|
|
274
|
-
delimiterSearchStart = delimiterIndex + 1;
|
|
275
|
-
continue;
|
|
232
|
+
let insideQuotes = false;
|
|
233
|
+
let quoteStart = -1;
|
|
234
|
+
for (const match of matches) {
|
|
235
|
+
if (match.patternId === 1) {
|
|
236
|
+
// Quote found
|
|
237
|
+
if (!insideQuotes) {
|
|
238
|
+
// Emit normal field before the opening quote
|
|
239
|
+
if (match.offset > sliceStart) {
|
|
240
|
+
this.#indices.enqueue([sliceStart, match.offset]);
|
|
241
|
+
}
|
|
242
|
+
insideQuotes = true;
|
|
243
|
+
quoteStart = match.offset;
|
|
276
244
|
}
|
|
277
245
|
else {
|
|
278
|
-
|
|
246
|
+
// Closing quote: emit quoted field, advance past quote
|
|
247
|
+
this.#indices.enqueue([quoteStart + 1, match.offset]);
|
|
248
|
+
insideQuotes = false;
|
|
249
|
+
sliceStart = match.offset + 1;
|
|
279
250
|
}
|
|
280
251
|
}
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
252
|
+
else {
|
|
253
|
+
// Delimiter found; one inside quotes is part of the field, so skip it.
|
|
254
|
+
if (insideQuotes)
|
|
255
|
+
continue;
|
|
256
|
+
// Emit content before this delimiter (empty if delimiter follows closing quote)
|
|
257
|
+
if (sliceStart < match.offset) {
|
|
258
|
+
this.#indices.enqueue([sliceStart, match.offset]);
|
|
259
|
+
}
|
|
260
|
+
sliceStart = match.offset + this.#needle.length;
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
const lastMatch = matches[matches.length - 1];
|
|
264
|
+
this.#readPosition = lastMatch.offset + (lastMatch.patternId === 0 ? this.#needle.length : 1);
|
|
265
|
+
// Emit trailing content after the last match
|
|
266
|
+
// Case 1: Content after last delimiter (final field, no trailing delimiter)
|
|
267
|
+
if (sliceStart < this.#readPosition) {
|
|
268
|
+
this.#indices.enqueue([sliceStart, sourceByteLength]);
|
|
269
|
+
this.#readPosition = sourceByteLength;
|
|
270
|
+
// Case 2: Trailing delimiter (emit empty field to match String.split)
|
|
271
|
+
}
|
|
272
|
+
else if (sliceStart === sourceByteLength && this.#readPosition === sourceByteLength) {
|
|
273
|
+
this.#indices.enqueue([sourceByteLength, sourceByteLength]);
|
|
286
274
|
}
|
|
287
|
-
const byteRange = [sliceStart, sliceEnd];
|
|
288
|
-
// this.#log("Found byte ranges", byteRange)
|
|
289
|
-
this.#log(`${byteRange} --> (${sliceStart - delimiterSearchStart})`, debugAsVisibleCharacters(this.#source.subarray(...byteRange)));
|
|
290
|
-
this.#indices.enqueue(byteRange);
|
|
291
275
|
}
|
|
292
276
|
}
|
|
293
277
|
//#endregion
|
|
@@ -296,33 +280,37 @@ export class Spliterator {
|
|
|
296
280
|
* Read the next byte range from the source.
|
|
297
281
|
*/
|
|
298
282
|
next() {
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
this.#
|
|
303
|
-
|
|
304
|
-
this.#
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
283
|
+
// Loop rather than recurse: a long run of skipped (empty or dropped) ranges would
|
|
284
|
+
// otherwise grow the call stack one frame per skip and overflow.
|
|
285
|
+
while (true) {
|
|
286
|
+
if (this.#done || this.#yieldCount >= this.#yieldStopCount)
|
|
287
|
+
return this.#finalize();
|
|
288
|
+
if (!this.#indices.size)
|
|
289
|
+
this.#fill();
|
|
290
|
+
if (!this.#indices.size) {
|
|
291
|
+
this.#drain();
|
|
292
|
+
}
|
|
293
|
+
const currentByteRange = this.#indices.dequeue();
|
|
294
|
+
this.#previousByteRange = currentByteRange;
|
|
295
|
+
if (!currentByteRange) {
|
|
296
|
+
this.#done = true;
|
|
297
|
+
return this.#finalize();
|
|
298
|
+
}
|
|
299
|
+
const [start, end] = currentByteRange;
|
|
300
|
+
const slice = this.#source.subarray(start, end);
|
|
301
|
+
if (slice.length === 0 && this.#skipEmpty) {
|
|
302
|
+
continue;
|
|
303
|
+
}
|
|
304
|
+
this.#yieldCount++;
|
|
305
|
+
if (this.#yieldCount <= this.#yieldDropCount) {
|
|
306
|
+
continue;
|
|
307
|
+
}
|
|
308
|
+
this.#yieldedByteLength += end - start;
|
|
309
|
+
return {
|
|
310
|
+
value: slice,
|
|
311
|
+
done: false,
|
|
312
|
+
};
|
|
320
313
|
}
|
|
321
|
-
this.#yieldedByteLength += end - start;
|
|
322
|
-
return {
|
|
323
|
-
value: slice,
|
|
324
|
-
done: false,
|
|
325
|
-
};
|
|
326
314
|
}
|
|
327
315
|
/**
|
|
328
316
|
* Collect all the byte ranges from the file.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Spliterator.js","sourceRoot":"","sources":["../../lib/Spliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,gBAAgB,EAAmD,MAAM,uBAAuB,CAAA;AACzG,OAAO,EACN,iBAAiB,EACjB,wBAAwB,EACxB,uBAAuB,GAEvB,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EAAE,gBAAgB,EAAmE,MAAM,aAAa,CAAA;AAE/G,OAAO,EAAE,gBAAgB,EAAE,CAAA;AAI3B;;GAEG;AACH,MAAM,OAAO,WAAW;IAGvB,mBAAmB;IAEnB;;;;;;;;OAQG;IACH,MAAM,CAAC,SAAS,GAAG,gBAAgB,CAAC,IAAI,CAAA;IAExC;;;;;OAKG;IACH,MAAM,CAAC,QAAQ,CAAmC,MAAS,EAAE,OAAwB,EAAE;QACtF,OAAO,IAAI,WAAW,CAAC,uBAAuB,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,CAAA;IAC9D,CAAC;
|
|
1
|
+
{"version":3,"file":"Spliterator.js","sourceRoot":"","sources":["../../lib/Spliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,gBAAgB,EAAmD,MAAM,uBAAuB,CAAA;AACzG,OAAO,EACN,iBAAiB,EACjB,wBAAwB,EACxB,uBAAuB,GAEvB,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EAAE,gBAAgB,EAAmE,MAAM,aAAa,CAAA;AAE/G,OAAO,EAAE,gBAAgB,EAAE,CAAA;AAI3B;;GAEG;AACH,MAAM,OAAO,WAAW;IAGvB,mBAAmB;IAEnB;;;;;;;;OAQG;IACH,MAAM,CAAC,SAAS,GAAG,gBAAgB,CAAC,IAAI,CAAA;IAExC;;;;;OAKG;IACH,MAAM,CAAC,QAAQ,CAAmC,MAAS,EAAE,OAAwB,EAAE;QACtF,OAAO,IAAI,WAAW,CAAC,uBAAuB,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,CAAA;IAC9D,CAAC;IA0CD,MAAM,CAAC,IAAI,CACV,MAAuE,EACvE,OAA+C,EAAE;QAEjD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,aAAa,IAAI,MAAM,EAAE,CAAC;YAClE,OAAO,IAAI,gBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QAC1C,CAAC;QAED,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,YAAY,GAAG,IAAI,gBAAgB,CAAC,MAAM,CAAC,EAAE,CAAC;YACrF,OAAO,MAAM,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,mBAAmB,EAAE,EAAE,EAAE;gBAC3E,IAAI,aAAiC,CAAA;gBAErC,IAAI,CAAC;oBACJ,aAAa,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE;wBACjD,aAAa,EAAE,IAAI,CAAC,aAAa;qBACjC,CAAC,CAAA;gBACH,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,YAAY,GAAG,EAAE,CAAC;wBACzD,MAAM,OAAO,GAAG,IAAI,KAAK,CACxB,gHAAgH,CAChH,CAAA;wBAED,OAAO,CAAC,KAAK,GAAG,KAAK,CAAA;wBAErB,MAAM,OAAO,CAAA;oBACd,CAAC;oBAED,MAAM,KAAK,CAAA;gBACZ,CAAC;gBAED,OAAO,IAAI,gBAAgB,CAAC,aAAa,EAAE,IAAI,CAAC,CAAA;YACjD,CAAC,CAAC,CAAA;QACH,CAAC;QAED,OAAO,IAAI,WAAW,CAAC,uBAAuB,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,CAAA;IAC9D,CAAC;IAED;;OAEG;IACI,MAAM,CAAC,iBAAiB,CAAC,IAAqB;QACpD,OAAO,IAAI,eAAe,CAA2B;YACpD,SAAS,CAAC,KAAK,EAAE,UAAU;gBAC1B,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;gBAEhD,UAAU,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAA;YAC1C,CAAC;SACD,CAAC,CAAA;IACH,CAAC;IAED;;OAEG;IACI,CAAC,MAAM,CAAC,OAAO,CAAC;QACtB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;IACtB,CAAC;IAED,YAAY,MAAkB,EAAE,OAAwB,EAAE;QACzD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QAErB,IAAI,CAAC,OAAO,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QAEpD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAA;QAEvC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,CAAA;QAE7D,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAA;QAClD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,eAAe,CAAA;QAChF,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,aAAa,CAAA;QAEtE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAA;QACxC,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,mBAAmB,IAAI,KAAK,CAAA;QAE7D,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,IAAI,KAAK,CAAA;QACjC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAA;IACrE,CAAC;IAED,YAAY;IAEZ,4BAA4B;IAE5B;;OAEG;IACM,OAAO,CAAY;IAE5B;;;;;;OAMG;IACM,QAAQ,GAAG,IAAI,UAAU,EAAE,CAAA;IAEpC;;OAEG;IACM,OAAO,CAAmB;IAEnC;;OAEG;IACM,oBAAoB,GAAsB,IAAI,iBAAiB,CAAC,GAAG,CAAC,CAAA;IACpE,oBAAoB,CAAS;IAEtC;;OAEG;IACM,eAAe,CAAQ;IAEhC;;OAEG;IACM,eAAe,CAAQ;IAEhC;;;;OAIG;IACM,kBAAkB,CAAQ;IAEnC;;OAEG;IACM,UAAU,CAAS;IAE5B;;;;OAIG;IACM,cAAc,CAAQ;IAE/B;;OAEG;IACH,kBAAkB,GAAG,CAAC,CAAA;IAEtB;;OAEG;IACH,WAAW,GAAG,CAAC,CAAA;IAEf;;OAEG;IACH,aAAa,CAAQ;IAErB;;OAEG;IACH,kBAAkB,CAAuB;IAEzC;;OAEG;IACH,MAAM,CAAS;IAEf;;OAEG;IACH,KAAK,GAAG,KAAK,CAAA;IAEb,YAAY;IAEZ,yBAAyB;IAEzB,IAAI,CAA0B;IAE9B,SAAS;QACR,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB;;;eAGG;YACH,MAAM,6BAA6B,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;YAClF,MAAM,YAAY,GAAG,IAAI,CAAC,kBAAkB,GAAG,CAAC,IAAI,CAAC,kBAAkB,GAAG,6BAA6B,CAAC,CAAA;YAExG,IAAI,CAAC,IAAI,CAAC;gBACT,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU;gBACtC,YAAY,EAAE,IAAI,CAAC,aAAa;gBAChC,iBAAiB,EAAE,IAAI,CAAC,kBAAkB;gBAC1C,iBAAiB,EAAE,IAAI,CAAC,kBAAkB;gBAC1C,UAAU,EAAE,IAAI,CAAC,WAAW;gBAC5B,6BAA6B;gBAC7B,YAAY;aACZ,CAAC,CAAA;QACH,CAAC;QAED,OAAO;YACN,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,SAAS;SAChB,CAAA;IACF,CAAC;IAED,MAAM;QACL,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAA;QAEhD,IAAI,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAA;QAExD,qFAAqF;QACrF,MAAM,aAAa,GAAG,IAAI,CAAC,kBAAkB,CAAA;QAE7C,IAAI,IAAI,CAAC,aAAa,GAAG,gBAAgB,EAAE,CAAC;YAC3C,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,aAAa,EAAE,gBAAgB,CAAC,CAAC,CAAA;QAC9D,CAAC;aAAM,IAAI,IAAI,CAAC,WAAW,KAAK,CAAC,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;YAClE,sFAAsF;YACtF,mFAAmF;YACnF,iFAAiF;YACjF,+DAA+D;YAC/D,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC,CAAA;QAC7C,CAAC;aAAM,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;YACvC,6EAA6E;YAC7E,kEAAkE;YAClE,MAAM,gBAAgB,GAAG,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;YAE/D,IAAI,gBAAgB,IAAI,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gBACzF,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,gBAAgB,EAAE,gBAAgB,CAAC,CAAC,CAAA;YAC5D,CAAC;QACF,CAAC;QAED,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;IAClB,CAAC;IAED;;OAEG;IACH,KAAK;QACJ,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAA;QAEhD,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAChC,OAAO,IAAI,CAAC,aAAa,GAAG,gBAAgB,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;gBAChG,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,aAAa,CAAC,CAAA;gBAE5E,IAAI,cAAc,KAAK,CAAC,CAAC;oBAAE,OAAM;gBAEjC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC,CAAA;gBAC3D,IAAI,CAAC,aAAa,GAAG,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;YAC1D,CAAC;YAED,OAAM;QACP,CAAC;QAED,OAAO,IAAI,CAAC,aAAa,GAAG,gBAAgB,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;YAChG,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CACzC,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,oBAAoB,EACzB,IAAI,CAAC,aAAa,EAClB,gBAAgB,CAChB,CAAA;YAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAM;YAEhC,IAAI,UAAU,GAAG,IAAI,CAAC,aAAa,CAAA;YACnC,IAAI,YAAY,GAAG,KAAK,CAAA;YACxB,IAAI,UAAU,GAAG,CAAC,CAAC,CAAA;YAEnB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC7B,IAAI,KAAK,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;oBAC3B,cAAc;oBACd,IAAI,CAAC,YAAY,EAAE,CAAC;wBACnB,6CAA6C;wBAC7C,IAAI,KAAK,CAAC,MAAM,GAAG,UAAU,EAAE,CAAC;4BAC/B,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;wBAClD,CAAC;wBACD,YAAY,GAAG,IAAI,CAAA;wBACnB,UAAU,GAAG,KAAK,CAAC,MAAM,CAAA;oBAC1B,CAAC;yBAAM,CAAC;wBACP,uDAAuD;wBACvD,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,UAAU,GAAG,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;wBACrD,YAAY,GAAG,KAAK,CAAA;wBACpB,UAAU,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAA;oBAC9B,CAAC;gBACF,CAAC;qBAAM,CAAC;oBACP,uEAAuE;oBACvE,IAAI,YAAY;wBAAE,SAAQ;oBAE1B,gFAAgF;oBAChF,IAAI,UAAU,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;wBAC/B,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;oBAClD,CAAC;oBACD,UAAU,GAAG,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;gBAChD,CAAC;YACF,CAAC;YAED,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAE,CAAA;YAC9C,IAAI,CAAC,aAAa,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,SAAS,CAAC,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;YAE7F,6CAA6C;YAC7C,4EAA4E;YAC5E,IAAI,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;gBACrC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAC,CAAA;gBACrD,IAAI,CAAC,aAAa,GAAG,gBAAgB,CAAA;gBACrC,sEAAsE;YACvE,CAAC;iBAAM,IAAI,UAAU,KAAK,gBAAgB,IAAI,IAAI,CAAC,aAAa,KAAK,gBAAgB,EAAE,CAAC;gBACvF,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,gBAAgB,EAAE,gBAAgB,CAAC,CAAC,CAAA;YAC5D,CAAC;QACF,CAAC;IACF,CAAC;IAED,YAAY;IAEZ,0BAA0B;IAE1B;;OAEG;IACI,IAAI;QACV,kFAAkF;QAClF,iEAAiE;QACjE,OAAO,IAAI,EAAE,CAAC;YACb,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,eAAe;gBAAE,OAAO,IAAI,CAAC,SAAS,EAAE,CAAA;YAEnF,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI;gBAAE,IAAI,CAAC,KAAK,EAAE,CAAA;YAErC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACzB,IAAI,CAAC,MAAM,EAAE,CAAA;YACd,CAAC;YAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAA;YAChD,IAAI,CAAC,kBAAkB,GAAG,gBAAgB,CAAA;YAE1C,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACvB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;gBAEjB,OAAO,IAAI,CAAC,SAAS,EAAE,CAAA;YACxB,CAAC;YAED,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,gBAAgB,CAAA;YAErC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;YAE/C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC3C,SAAQ;YACT,CAAC;YAED,IAAI,CAAC,WAAW,EAAE,CAAA;YAElB,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;gBAC9C,SAAQ;YACT,CAAC;YAED,IAAI,CAAC,kBAAkB,IAAI,GAAG,GAAG,KAAK,CAAA;YAEtC,OAAO;gBACN,KAAK,EAAE,KAAqB;gBAC5B,IAAI,EAAE,KAAK;aACX,CAAA;QACF,CAAC;IACF,CAAC;IAED;;;;;OAKG;IACI,OAAO;QACb,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACxB,CAAC;IAED;;OAEG;IACI,cAAc,CAAC,OAAO,GAAG,IAAI,WAAW,EAAE;QAChD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;IAC1D,CAAC;IAEM,CAAC,MAAM,CAAC,QAAQ,CAAC;QACvB,OAAO,IAAI,CAAA;IACZ,CAAC"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @copyright Sister Software
|
|
3
|
-
* @license
|
|
3
|
+
* @license MIT
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
import { type CharacterSequenceInput } from "./CharacterSequence.js";
|
|
@@ -15,8 +15,7 @@ export interface TextSpliteratorInit {
|
|
|
15
15
|
*/
|
|
16
16
|
encoding?: string;
|
|
17
17
|
/**
|
|
18
|
-
* Whether to throw an error when encountering invalid data, or to swap it with a replacement
|
|
19
|
-
* character.
|
|
18
|
+
* Whether to throw an error when encountering invalid data, or to swap it with a replacement character.
|
|
20
19
|
*/
|
|
21
20
|
fatal?: boolean;
|
|
22
21
|
/**
|
|
@@ -31,6 +30,7 @@ export declare abstract class TextSpliterator {
|
|
|
31
30
|
*
|
|
32
31
|
* @param source The source content to split.
|
|
33
32
|
* @param options The options to use when splitting the content.
|
|
33
|
+
*
|
|
34
34
|
* @yields Each slice of the source content.
|
|
35
35
|
* @see {@linkcode TextSpliterator.fromAsync} for asynchronous iteration with decoding.
|
|
36
36
|
* @see {@linkcode Spliterator.fromSync} for synchronous iteration without decoding.
|
|
@@ -41,6 +41,7 @@ export declare abstract class TextSpliterator {
|
|
|
41
41
|
*
|
|
42
42
|
* @param source The async data resource to split.
|
|
43
43
|
* @param options The options to use when splitting the content.
|
|
44
|
+
*
|
|
44
45
|
* @yields Each slice of the source content.
|
|
45
46
|
* @see {@linkcode TextSpliterator.from} for synchronous iteration with decoding.
|
|
46
47
|
* @see {@linkcode Spliterator.fromAsync} for asynchronous iteration without decoding.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"TextSpliterator.d.ts","sourceRoot":"","sources":["../../lib/TextSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,KAAK,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AACpE,OAAO,EAAE,KAAK,iBAAiB,EAAE,MAAM,aAAa,CAAA;AACpD,OAAO,EAAE,KAAK,oBAAoB,EAAe,KAAK,eAAe,EAAE,MAAM,kBAAkB,CAAA;AAE/F,MAAM,WAAW,mBAAmB;IACnC;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IAEjB
|
|
1
|
+
{"version":3,"file":"TextSpliterator.d.ts","sourceRoot":"","sources":["../../lib/TextSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,KAAK,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AACpE,OAAO,EAAE,KAAK,iBAAiB,EAAE,MAAM,aAAa,CAAA;AACpD,OAAO,EAAE,KAAK,oBAAoB,EAAe,KAAK,eAAe,EAAE,MAAM,kBAAkB,CAAA;AAE/F,MAAM,WAAW,mBAAmB;IACnC;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,CAAA;IAEf;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAA;CACnB;AAED,8BAAsB,eAAe;;IAKpC;;;;;;;;;OASG;IACH,MAAM,CAAE,IAAI,CACX,MAAM,EAAE,sBAAsB,EAC9B,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,OAAO,EAAE,GAAE,mBAAmB,GAAG,eAAoB,GACpF,SAAS,CAAC,MAAM,CAAC;IAwBpB;;;;;;;;;OASG;WACW,SAAS,CACtB,MAAM,EAAE,iBAAiB,EACzB,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,OAAO,EAAE,GAAE,mBAAmB,GAAG,oBAAyB,GACzF,cAAc,CAAC,MAAM,CAAC;CAsBzB"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @copyright Sister Software
|
|
3
|
-
* @license
|
|
3
|
+
* @license MIT
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
import {} from "./CharacterSequence.js";
|
|
@@ -15,6 +15,7 @@ export class TextSpliterator {
|
|
|
15
15
|
*
|
|
16
16
|
* @param source The source content to split.
|
|
17
17
|
* @param options The options to use when splitting the content.
|
|
18
|
+
*
|
|
18
19
|
* @yields Each slice of the source content.
|
|
19
20
|
* @see {@linkcode TextSpliterator.fromAsync} for asynchronous iteration with decoding.
|
|
20
21
|
* @see {@linkcode Spliterator.fromSync} for synchronous iteration without decoding.
|
|
@@ -42,6 +43,7 @@ export class TextSpliterator {
|
|
|
42
43
|
*
|
|
43
44
|
* @param source The async data resource to split.
|
|
44
45
|
* @param options The options to use when splitting the content.
|
|
46
|
+
*
|
|
45
47
|
* @yields Each slice of the source content.
|
|
46
48
|
* @see {@linkcode TextSpliterator.from} for synchronous iteration with decoding.
|
|
47
49
|
* @see {@linkcode Spliterator.fromAsync} for asynchronous iteration without decoding.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"TextSpliterator.js","sourceRoot":"","sources":["../../lib/TextSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAA+B,MAAM,wBAAwB,CAAA;AACpE,OAAO,EAA0B,MAAM,aAAa,CAAA;AACpD,OAAO,EAA6B,WAAW,EAAwB,MAAM,kBAAkB,CAAA;
|
|
1
|
+
{"version":3,"file":"TextSpliterator.js","sourceRoot":"","sources":["../../lib/TextSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAA+B,MAAM,wBAAwB,CAAA;AACpE,OAAO,EAA0B,MAAM,aAAa,CAAA;AACpD,OAAO,EAA6B,WAAW,EAAwB,MAAM,kBAAkB,CAAA;AAsB/F,MAAM,OAAgB,eAAe;IACpC;QACC,MAAM,IAAI,SAAS,CAAC,2EAA2E,CAAC,CAAA;IACjG,CAAC;IAED;;;;;;;;;OASG;IACH,MAAM,CAAC,CAAC,IAAI,CACX,MAA8B,EAC9B,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,OAAO,KAA4C,EAAE;QAEtF,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAA;QAC/D,IAAI,SAAS,GAAG,CAAC,CAAA;QAEjB,MAAM,WAAW,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAEzD,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YAC/B,IAAI,OAAe,CAAA;YAEnB,IAAI,CAAC;gBACJ,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YAC9B,CAAC;YAAC,OAAO,WAAW,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,WAAW,CAAC,gCAAgC,SAAS,EAAE,CAAC,CAAA;gBACtE,KAAK,CAAC,KAAK,GAAG,WAAW,CAAA;gBAEzB,MAAM,KAAK,CAAA;YACZ,CAAC;YAED,MAAM,OAAO,CAAA;YAEb,SAAS,EAAE,CAAA;QACZ,CAAC;IACF,CAAC;IAED;;;;;;;;;OASG;IACH,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CACtB,MAAyB,EACzB,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,OAAO,KAAiD,EAAE;QAE3F,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAA;QAC/D,IAAI,SAAS,GAAG,CAAC,CAAA;QACjB,MAAM,WAAW,GAAG,MAAM,WAAW,CAAC,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAEhE,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YACrC,IAAI,OAAe,CAAA;YAEnB,IAAI,CAAC;gBACJ,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YAC9B,CAAC;YAAC,OAAO,WAAW,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,WAAW,CAAC,gCAAgC,SAAS,EAAE,CAAC,CAAA;gBACtE,KAAK,CAAC,KAAK,GAAG,WAAW,CAAA;gBAEzB,MAAM,KAAK,CAAA;YACZ,CAAC;YAED,MAAM,OAAO,CAAA;YAEb,SAAS,EAAE,CAAA;QACZ,CAAC;IACF,CAAC;CACD"}
|
package/out/lib/casing.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @copyright Sister Software
|
|
3
|
-
* @license
|
|
3
|
+
* @license MIT
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
import type { CamelCase, SnakeCase } from "type-fest";
|
|
@@ -21,8 +21,7 @@ export declare function isUniformlyCased(input: string | null): boolean;
|
|
|
21
21
|
*/
|
|
22
22
|
export declare function smartCapitalCase(input: string): string;
|
|
23
23
|
/**
|
|
24
|
-
* Given an array of column names, normalize them to ensure they are unique and usable as object
|
|
25
|
-
* keys.
|
|
24
|
+
* Given an array of column names, normalize them to ensure they are unique and usable as object keys.
|
|
26
25
|
*/
|
|
27
26
|
export declare function normalizeColumnNames(columnHeaders: Iterable<string>): string[];
|
|
28
27
|
//# sourceMappingURL=casing.d.ts.map
|
package/out/lib/casing.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"casing.d.ts","sourceRoot":"","sources":["../../lib/casing.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AAErD;;GAEG;AACH,wBAAgB,cAAc,CAAC,CAAC,SAAS,MAAM,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,SAAS,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAiBnG;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,CAAC,SAAS,MAAM,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,SAAS,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAInG;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAE9D;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,
|
|
1
|
+
{"version":3,"file":"casing.d.ts","sourceRoot":"","sources":["../../lib/casing.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AAErD;;GAEG;AACH,wBAAgB,cAAc,CAAC,CAAC,SAAS,MAAM,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,SAAS,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAiBnG;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,CAAC,SAAS,MAAM,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,SAAS,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAInG;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAE9D;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAMtD;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,aAAa,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,MAAM,EAAE,CAkB9E"}
|
package/out/lib/casing.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @copyright Sister Software
|
|
3
|
-
* @license
|
|
3
|
+
* @license MIT
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
import { camelCase, capitalCase, snakeCase } from "change-case";
|
|
@@ -46,8 +46,7 @@ export function smartCapitalCase(input) {
|
|
|
46
46
|
return capitalCase(input);
|
|
47
47
|
}
|
|
48
48
|
/**
|
|
49
|
-
* Given an array of column names, normalize them to ensure they are unique and usable as object
|
|
50
|
-
* keys.
|
|
49
|
+
* Given an array of column names, normalize them to ensure they are unique and usable as object keys.
|
|
51
50
|
*/
|
|
52
51
|
export function normalizeColumnNames(columnHeaders) {
|
|
53
52
|
const columnInputCountMap = new Map();
|
package/out/lib/casing.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"casing.js","sourceRoot":"","sources":["../../lib/casing.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAG/D;;GAEG;AACH,MAAM,UAAU,cAAc,CAAmB,IAAO;IACvD,MAAM,cAAc,GAAG,IAAI;QAC1B,+DAA+D;SAC9D,OAAO,CAAC,eAAe,EAAE,IAAI,CAAC;SAC9B,IAAI,EAAE,CAAA;IAER,IAAI,cAAc,CAAC,WAAW,EAAE,KAAK,cAAc,EAAE,CAAC;QACrD,OACC,IAAI;YACH,sDAAsD;aACrD,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;YACxB,6EAA6E;aAC5E,OAAO,CAAC,QAAQ,EAAE,GAAG,CACvB,CAAA;IACF,CAAC;IAED,OAAO,SAAS,CAAC,cAAc,CAAQ,CAAA;AACxC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAmB,IAAO;IACvD,IAAI,IAAI,CAAC,WAAW,EAAE,KAAK,IAAI;QAAE,OAAO,IAAW,CAAA;IAEnD,OAAO,SAAS,CAAC,IAAI,CAAQ,CAAA;AAC9B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAoB;IACpD,OAAO,OAAO,CAAC,KAAK,IAAI,CAAC,KAAK,KAAK,KAAK,CAAC,WAAW,EAAE,IAAI,KAAK,KAAK,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAA;AAC1F,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC7C,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAA;
|
|
1
|
+
{"version":3,"file":"casing.js","sourceRoot":"","sources":["../../lib/casing.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAG/D;;GAEG;AACH,MAAM,UAAU,cAAc,CAAmB,IAAO;IACvD,MAAM,cAAc,GAAG,IAAI;QAC1B,+DAA+D;SAC9D,OAAO,CAAC,eAAe,EAAE,IAAI,CAAC;SAC9B,IAAI,EAAE,CAAA;IAER,IAAI,cAAc,CAAC,WAAW,EAAE,KAAK,cAAc,EAAE,CAAC;QACrD,OACC,IAAI;YACH,sDAAsD;aACrD,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;YACxB,6EAA6E;aAC5E,OAAO,CAAC,QAAQ,EAAE,GAAG,CACvB,CAAA;IACF,CAAC;IAED,OAAO,SAAS,CAAC,cAAc,CAAQ,CAAA;AACxC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAmB,IAAO;IACvD,IAAI,IAAI,CAAC,WAAW,EAAE,KAAK,IAAI;QAAE,OAAO,IAAW,CAAA;IAEnD,OAAO,SAAS,CAAC,IAAI,CAAQ,CAAA;AAC9B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAoB;IACpD,OAAO,OAAO,CAAC,KAAK,IAAI,CAAC,KAAK,KAAK,KAAK,CAAC,WAAW,EAAE,IAAI,KAAK,KAAK,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAA;AAC1F,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC7C,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAA;IAErC,IAAI,gBAAgB,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAA;IAEzC,OAAO,WAAW,CAAC,KAAK,CAAC,CAAA;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,aAA+B;IACnE,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAkB,CAAA;IACrD,MAAM,eAAe,GAAG,IAAI,GAAG,EAAU,CAAA;IACzC,MAAM,kBAAkB,GAAG,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,cAAc,CAAC,CAAA;IAE3E,KAAK,MAAM,YAAY,IAAI,kBAAkB,EAAE,CAAC;QAC/C,IAAI,eAAe,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;YACvC,MAAM,WAAW,GAAG,CAAC,mBAAmB,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;YACpE,mBAAmB,CAAC,GAAG,CAAC,YAAY,EAAE,WAAW,CAAC,CAAA;YAElD,MAAM,gBAAgB,GAAG,GAAG,YAAY,IAAI,WAAW,EAAE,CAAA;YACzD,eAAe,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAA;QACtC,CAAC;aAAM,CAAC;YACP,eAAe,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;QAClC,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;AACnC,CAAC"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license MIT
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
export type SegmentHandler = (record: Uint8Array, ctx: {
|
|
7
|
+
index: number;
|
|
8
|
+
segmentIndex: number;
|
|
9
|
+
}) => unknown | Promise<unknown>;
|
|
10
|
+
export interface RunSegmentIO {
|
|
11
|
+
records: AsyncIterable<Uint8Array>;
|
|
12
|
+
handleRecord: SegmentHandler;
|
|
13
|
+
segmentIndex: number;
|
|
14
|
+
batchSize: number;
|
|
15
|
+
maxInFlight: number;
|
|
16
|
+
post: (batch: unknown[], transfer: ArrayBuffer[]) => void;
|
|
17
|
+
waitForAck: () => Promise<void>;
|
|
18
|
+
inFlight: () => number;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Drive one segment: read records, run the handler, accumulate results into `batchSize` batches, transfer `Uint8Array`
|
|
22
|
+
* result buffers zero-copy, and respect a bounded in-flight window by awaiting an ack before exceeding `maxInFlight`
|
|
23
|
+
* outstanding batches. Results of `undefined` are skipped (the filter case). Transport-agnostic so it unit-tests
|
|
24
|
+
* without `worker_threads`.
|
|
25
|
+
*/
|
|
26
|
+
export declare function runSegment(io: RunSegmentIO): Promise<void>;
|
|
27
|
+
//# sourceMappingURL=segment-runtime.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"segment-runtime.d.ts","sourceRoot":"","sources":["../../lib/segment-runtime.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,MAAM,cAAc,GAAG,CAC5B,MAAM,EAAE,UAAU,EAClB,GAAG,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,KACxC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;AAE/B,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;IAClC,YAAY,EAAE,cAAc,CAAA;IAC5B,YAAY,EAAE,MAAM,CAAA;IACpB,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;IACnB,IAAI,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,IAAI,CAAA;IACzD,UAAU,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;IAC/B,QAAQ,EAAE,MAAM,MAAM,CAAA;CACtB;AAED;;;;;GAKG;AACH,wBAAsB,UAAU,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CA6BhE"}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license MIT
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* Drive one segment: read records, run the handler, accumulate results into `batchSize` batches, transfer `Uint8Array`
|
|
8
|
+
* result buffers zero-copy, and respect a bounded in-flight window by awaiting an ack before exceeding `maxInFlight`
|
|
9
|
+
* outstanding batches. Results of `undefined` are skipped (the filter case). Transport-agnostic so it unit-tests
|
|
10
|
+
* without `worker_threads`.
|
|
11
|
+
*/
|
|
12
|
+
export async function runSegment(io) {
|
|
13
|
+
let batch = [];
|
|
14
|
+
let transfer = [];
|
|
15
|
+
let index = 0;
|
|
16
|
+
const flush = async () => {
|
|
17
|
+
if (batch.length === 0)
|
|
18
|
+
return;
|
|
19
|
+
while (io.inFlight() >= io.maxInFlight)
|
|
20
|
+
await io.waitForAck();
|
|
21
|
+
io.post(batch, transfer);
|
|
22
|
+
batch = [];
|
|
23
|
+
transfer = [];
|
|
24
|
+
};
|
|
25
|
+
for await (const record of io.records) {
|
|
26
|
+
const result = await io.handleRecord(record, { index, segmentIndex: io.segmentIndex });
|
|
27
|
+
index++;
|
|
28
|
+
if (result === undefined)
|
|
29
|
+
continue;
|
|
30
|
+
batch.push(result);
|
|
31
|
+
if (result instanceof Uint8Array)
|
|
32
|
+
transfer.push(result.buffer);
|
|
33
|
+
if (batch.length >= io.batchSize)
|
|
34
|
+
await flush();
|
|
35
|
+
}
|
|
36
|
+
await flush();
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=segment-runtime.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"segment-runtime.js","sourceRoot":"","sources":["../../lib/segment-runtime.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAkBH;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,EAAgB;IAChD,IAAI,KAAK,GAAc,EAAE,CAAA;IACzB,IAAI,QAAQ,GAAkB,EAAE,CAAA;IAChC,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,MAAM,KAAK,GAAG,KAAK,IAAmB,EAAE;QACvC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAM;QAE9B,OAAO,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,WAAW;YAAE,MAAM,EAAE,CAAC,UAAU,EAAE,CAAA;QAE7D,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;QACxB,KAAK,GAAG,EAAE,CAAA;QACV,QAAQ,GAAG,EAAE,CAAA;IACd,CAAC,CAAA;IAED,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,YAAY,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,CAAC,YAAY,EAAE,CAAC,CAAA;QACtF,KAAK,EAAE,CAAA;QAEP,IAAI,MAAM,KAAK,SAAS;YAAE,SAAQ;QAElC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAElB,IAAI,MAAM,YAAY,UAAU;YAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,MAAqB,CAAC,CAAA;QAE7E,IAAI,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC,SAAS;YAAE,MAAM,KAAK,EAAE,CAAA;IAChD,CAAC;IAED,MAAM,KAAK,EAAE,CAAA;AACd,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license MIT
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Runs inside a worker thread (spawned by runSegmentWorkers). Reads its segment via its own handle,
|
|
7
|
+
* runs the user handler per record, and posts batched results to the parent with ack backpressure.
|
|
8
|
+
*/
|
|
9
|
+
export {};
|
|
10
|
+
//# sourceMappingURL=segment-worker-entry.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"segment-worker-entry.d.ts","sourceRoot":"","sources":["../../lib/segment-worker-entry.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG"}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license MIT
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Runs inside a worker thread (spawned by runSegmentWorkers). Reads its segment via its own handle,
|
|
7
|
+
* runs the user handler per record, and posts batched results to the parent with ack backpressure.
|
|
8
|
+
*/
|
|
9
|
+
import { parentPort, workerData } from "node:worker_threads";
|
|
10
|
+
import { AsyncSpliterator } from "./AsyncSpliterator.js";
|
|
11
|
+
import { runSegment } from "./segment-runtime.js";
|
|
12
|
+
async function main() {
|
|
13
|
+
const data = workerData;
|
|
14
|
+
const port = parentPort;
|
|
15
|
+
// Ack backpressure: the parent posts `"ack"` per consumed batch; we track outstanding batches.
|
|
16
|
+
let acked = 0;
|
|
17
|
+
let posted = 0;
|
|
18
|
+
let wakeAck;
|
|
19
|
+
port.on("message", (msg) => {
|
|
20
|
+
if (msg === "ack") {
|
|
21
|
+
acked++;
|
|
22
|
+
wakeAck?.();
|
|
23
|
+
wakeAck = undefined;
|
|
24
|
+
}
|
|
25
|
+
});
|
|
26
|
+
const mod = (await import(data.handlerUrl));
|
|
27
|
+
const handleRecord = mod.handleRecord ?? mod.default;
|
|
28
|
+
if (typeof handleRecord !== "function") {
|
|
29
|
+
port.postMessage({ type: "error", message: `Worker module ${data.handlerUrl} has no handleRecord export.` });
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
const { createChunkIterator } = await import("spliterator/node/fs");
|
|
33
|
+
const chunkIterator = await createChunkIterator(data.source, { start: data.start, end: data.end - 1 });
|
|
34
|
+
const records = new AsyncSpliterator(chunkIterator, { delimiter: data.delimiter, autoDispose: true });
|
|
35
|
+
try {
|
|
36
|
+
await runSegment({
|
|
37
|
+
records,
|
|
38
|
+
handleRecord,
|
|
39
|
+
segmentIndex: data.segmentIndex,
|
|
40
|
+
batchSize: data.batchSize,
|
|
41
|
+
maxInFlight: data.maxInFlight,
|
|
42
|
+
post: (batch, transfer) => {
|
|
43
|
+
posted++;
|
|
44
|
+
port.postMessage({ type: "batch", records: batch }, transfer);
|
|
45
|
+
},
|
|
46
|
+
waitForAck: () => new Promise((resolve) => (wakeAck = resolve)),
|
|
47
|
+
inFlight: () => posted - acked,
|
|
48
|
+
});
|
|
49
|
+
port.postMessage({ type: "done" });
|
|
50
|
+
}
|
|
51
|
+
catch (error) {
|
|
52
|
+
port.postMessage({ type: "error", message: error instanceof Error ? error.message : String(error) });
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
void main();
|
|
56
|
+
//# sourceMappingURL=segment-worker-entry.js.map
|