spliterator 2.2.2 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +21 -650
- package/README.md +42 -0
- package/out/index.d.ts +2 -1
- package/out/index.d.ts.map +1 -1
- package/out/index.js +2 -1
- package/out/index.js.map +1 -1
- package/out/lib/AsyncSpliterator.d.ts +33 -28
- package/out/lib/AsyncSpliterator.d.ts.map +1 -1
- package/out/lib/AsyncSpliterator.js +84 -70
- package/out/lib/AsyncSpliterator.js.map +1 -1
- package/out/lib/BufferController.d.ts +1 -1
- package/out/lib/BufferController.js +1 -1
- package/out/lib/CSVSpliterator.d.ts +1 -1
- package/out/lib/CSVSpliterator.d.ts.map +1 -1
- package/out/lib/CSVSpliterator.js +5 -3
- package/out/lib/CSVSpliterator.js.map +1 -1
- package/out/lib/CharacterSequence.d.ts +15 -64
- package/out/lib/CharacterSequence.d.ts.map +1 -1
- package/out/lib/CharacterSequence.js +194 -89
- package/out/lib/CharacterSequence.js.map +1 -1
- package/out/lib/CompositeDataView.d.ts +1 -1
- package/out/lib/CompositeDataView.js +1 -1
- package/out/lib/IndexQueue.d.ts +1 -1
- package/out/lib/IndexQueue.d.ts.map +1 -1
- package/out/lib/IndexQueue.js +40 -11
- package/out/lib/IndexQueue.js.map +1 -1
- package/out/lib/JSONSpliterator.d.ts +1 -1
- package/out/lib/JSONSpliterator.js +1 -1
- package/out/lib/PSVSpliterator.d.ts +1 -1
- package/out/lib/PSVSpliterator.js +1 -1
- package/out/lib/SlidingWindow.d.ts +1 -1
- package/out/lib/SlidingWindow.d.ts.map +1 -1
- package/out/lib/SlidingWindow.js +11 -10
- package/out/lib/SlidingWindow.js.map +1 -1
- package/out/lib/Spliterator.d.ts +1 -1
- package/out/lib/Spliterator.d.ts.map +1 -1
- package/out/lib/Spliterator.js +99 -110
- package/out/lib/Spliterator.js.map +1 -1
- package/out/lib/TSVSpliterator.d.ts +1 -1
- package/out/lib/TSVSpliterator.js +1 -1
- package/out/lib/TextSpliterator.d.ts +1 -1
- package/out/lib/TextSpliterator.js +1 -1
- package/out/lib/casing.d.ts +1 -1
- package/out/lib/casing.js +1 -1
- package/out/lib/segment-runtime.d.ts +27 -0
- package/out/lib/segment-runtime.d.ts.map +1 -0
- package/out/lib/segment-runtime.js +38 -0
- package/out/lib/segment-runtime.js.map +1 -0
- package/out/lib/segment-worker-entry.d.ts +10 -0
- package/out/lib/segment-worker-entry.d.ts.map +1 -0
- package/out/lib/segment-worker-entry.js +56 -0
- package/out/lib/segment-worker-entry.js.map +1 -0
- package/out/lib/segment-workers.d.ts +41 -0
- package/out/lib/segment-workers.d.ts.map +1 -0
- package/out/lib/segment-workers.js +105 -0
- package/out/lib/segment-workers.js.map +1 -0
- package/out/lib/segments.d.ts +25 -0
- package/out/lib/segments.d.ts.map +1 -0
- package/out/lib/segments.js +50 -0
- package/out/lib/segments.js.map +1 -0
- package/out/lib/shared.d.ts +1 -1
- package/out/lib/shared.js +1 -1
- package/out/lib/take.d.ts +1 -1
- package/out/lib/take.js +1 -1
- package/out/lib/wasm_base64.d.ts +16 -0
- package/out/lib/wasm_base64.d.ts.map +1 -0
- package/out/lib/wasm_base64.js +16 -0
- package/out/lib/wasm_base64.js.map +1 -0
- package/out/lib/wasm_module.d.ts +36 -0
- package/out/lib/wasm_module.d.ts.map +1 -0
- package/out/lib/wasm_module.js +33 -0
- package/out/lib/wasm_module.js.map +1 -0
- package/out/lib/writer.d.ts +1 -1
- package/out/lib/writer.js +1 -1
- package/out/node/cli/commands/csv.d.ts +1 -1
- package/out/node/cli/commands/csv.js +1 -1
- package/out/node/cli/commands/iterate.d.ts +1 -1
- package/out/node/cli/commands/iterate.js +1 -1
- package/out/node/cli/index.d.ts +1 -1
- package/out/node/cli/index.js +1 -1
- package/out/node/cli/utils.d.ts +1 -1
- package/out/node/cli/utils.js +1 -1
- package/out/node/fs/index.d.ts +14 -2
- package/out/node/fs/index.d.ts.map +1 -1
- package/out/node/fs/index.js +26 -2
- package/out/node/fs/index.js.map +1 -1
- package/package.json +5 -3
package/README.md
CHANGED
|
@@ -135,6 +135,48 @@ for await (const line of stream) {
|
|
|
135
135
|
}
|
|
136
136
|
```
|
|
137
137
|
|
|
138
|
+
### SIMD acceleration
|
|
139
|
+
|
|
140
|
+
Spliterator ships a small WebAssembly SIMD scanner that accelerates delimiter and quote scanning (roughly 5–6× over the JavaScript scanner for multi-byte delimiters, more for column splitting). It is embedded in the package — no extra files, fetches, or configuration.
|
|
141
|
+
|
|
142
|
+
The module loads **asynchronously**. Asynchronous parsing (`fromAsync`, streams) picks it up automatically once loaded. Purely synchronous parsing that finishes in a single tick would otherwise complete before the module is ready and transparently use the JavaScript scanner — to opt in, await it first:
|
|
143
|
+
|
|
144
|
+
```ts
|
|
145
|
+
import { CharacterSequence, CSVSpliterator } from "spliterator"
|
|
146
|
+
|
|
147
|
+
await CharacterSequence.whenReady() // resolves to true once the SIMD scanner is active
|
|
148
|
+
|
|
149
|
+
for (const row of CSVSpliterator.from(largeCsvString)) {
|
|
150
|
+
// ...now backed by the SIMD scanner
|
|
151
|
+
}
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
Correctness is identical either way; `whenReady()` only affects which scanner runs.
|
|
155
|
+
|
|
156
|
+
### Parallel parsing across threads
|
|
157
|
+
|
|
158
|
+
For one large file with a CPU-bound per-row transform, `AsyncSpliterator.asManyWorkers` splits the file into delimiter-aligned segments and runs a handler module across worker threads — each worker owns its own handle and reads only its segment. Results stream back to the main thread as a single async iterator, for a single-thread writer (a database, a JSONL file).
|
|
159
|
+
|
|
160
|
+
```ts
|
|
161
|
+
import { AsyncSpliterator } from "spliterator"
|
|
162
|
+
|
|
163
|
+
// transform.js (runs in each worker; top-level code is per-worker init):
|
|
164
|
+
// const dec = new TextDecoder(), enc = new TextEncoder()
|
|
165
|
+
// export function handleRecord(bytes) {
|
|
166
|
+
// return enc.encode(JSON.stringify(parse(dec.decode(bytes))) + "\n") // Uint8Array → zero-copy
|
|
167
|
+
// }
|
|
168
|
+
|
|
169
|
+
for await (const jsonLine of AsyncSpliterator.asManyWorkers<Uint8Array>("huge.csv", {
|
|
170
|
+
worker: new URL("./transform.js", import.meta.url),
|
|
171
|
+
delimiter: "\n",
|
|
172
|
+
concurrency: 8,
|
|
173
|
+
})) {
|
|
174
|
+
out.write(jsonLine) // single-thread writer on main
|
|
175
|
+
}
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
Need just the byte ranges to drive your own pool? `AsyncSpliterator.segments(path, { delimiter, concurrency })` returns them.
|
|
179
|
+
|
|
138
180
|
### Custom generators
|
|
139
181
|
|
|
140
182
|
While Spliterator includes premade exports for most use-cases, custom generators can be created via `Spliterator` and `AsyncSpliterator`. This class is a low-level interface that allows you to create your own generators for any kind of delimited content.
|
package/out/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @copyright Sister Software
|
|
3
|
-
* @license
|
|
3
|
+
* @license MIT
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
export * from "./lib/BufferController.js";
|
|
@@ -11,6 +11,7 @@ export * from "./lib/CSVSpliterator.js";
|
|
|
11
11
|
export * from "./lib/IndexQueue.js";
|
|
12
12
|
export * from "./lib/JSONSpliterator.js";
|
|
13
13
|
export * from "./lib/PSVSpliterator.js";
|
|
14
|
+
export * from "./lib/segments.js";
|
|
14
15
|
export * from "./lib/shared.js";
|
|
15
16
|
export * from "./lib/SlidingWindow.js";
|
|
16
17
|
export * from "./lib/Spliterator.js";
|
package/out/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,2BAA2B,CAAA;AACzC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,4BAA4B,CAAA;AAC1C,cAAc,4BAA4B,CAAA;AAC1C,cAAc,yBAAyB,CAAA;AACvC,cAAc,qBAAqB,CAAA;AACnC,cAAc,0BAA0B,CAAA;AACxC,cAAc,yBAAyB,CAAA;AACvC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,wBAAwB,CAAA;AACtC,cAAc,sBAAsB,CAAA;AACpC,cAAc,eAAe,CAAA;AAC7B,cAAc,0BAA0B,CAAA;AACxC,cAAc,yBAAyB,CAAA;AACvC,cAAc,iBAAiB,CAAA"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,2BAA2B,CAAA;AACzC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,4BAA4B,CAAA;AAC1C,cAAc,4BAA4B,CAAA;AAC1C,cAAc,yBAAyB,CAAA;AACvC,cAAc,qBAAqB,CAAA;AACnC,cAAc,0BAA0B,CAAA;AACxC,cAAc,yBAAyB,CAAA;AACvC,cAAc,mBAAmB,CAAA;AACjC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,wBAAwB,CAAA;AACtC,cAAc,sBAAsB,CAAA;AACpC,cAAc,eAAe,CAAA;AAC7B,cAAc,0BAA0B,CAAA;AACxC,cAAc,yBAAyB,CAAA;AACvC,cAAc,iBAAiB,CAAA"}
|
package/out/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @copyright Sister Software
|
|
3
|
-
* @license
|
|
3
|
+
* @license MIT
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
export * from "./lib/BufferController.js";
|
|
@@ -11,6 +11,7 @@ export * from "./lib/CSVSpliterator.js";
|
|
|
11
11
|
export * from "./lib/IndexQueue.js";
|
|
12
12
|
export * from "./lib/JSONSpliterator.js";
|
|
13
13
|
export * from "./lib/PSVSpliterator.js";
|
|
14
|
+
export * from "./lib/segments.js";
|
|
14
15
|
export * from "./lib/shared.js";
|
|
15
16
|
export * from "./lib/SlidingWindow.js";
|
|
16
17
|
export * from "./lib/Spliterator.js";
|
package/out/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,2BAA2B,CAAA;AACzC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,4BAA4B,CAAA;AAC1C,cAAc,4BAA4B,CAAA;AAC1C,cAAc,yBAAyB,CAAA;AACvC,cAAc,qBAAqB,CAAA;AACnC,cAAc,0BAA0B,CAAA;AACxC,cAAc,yBAAyB,CAAA;AACvC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,wBAAwB,CAAA;AACtC,cAAc,sBAAsB,CAAA;AACpC,cAAc,eAAe,CAAA;AAC7B,cAAc,0BAA0B,CAAA;AACxC,cAAc,yBAAyB,CAAA;AACvC,cAAc,iBAAiB,CAAA"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,2BAA2B,CAAA;AACzC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,4BAA4B,CAAA;AAC1C,cAAc,4BAA4B,CAAA;AAC1C,cAAc,yBAAyB,CAAA;AACvC,cAAc,qBAAqB,CAAA;AACnC,cAAc,0BAA0B,CAAA;AACxC,cAAc,yBAAyB,CAAA;AACvC,cAAc,mBAAmB,CAAA;AACjC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,wBAAwB,CAAA;AACtC,cAAc,sBAAsB,CAAA;AACpC,cAAc,eAAe,CAAA;AAC7B,cAAc,0BAA0B,CAAA;AACxC,cAAc,yBAAyB,CAAA;AACvC,cAAc,iBAAiB,CAAA"}
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @copyright Sister Software
|
|
3
|
-
* @license
|
|
3
|
+
* @license MIT
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
import { ReadableStream, type ReadableWritablePair, type StreamPipeOptions } from "stream/web";
|
|
7
7
|
import { type CharacterSequenceInput } from "./CharacterSequence.js";
|
|
8
|
-
import
|
|
8
|
+
import { type AsManyWorkersOptions } from "./segment-workers.js";
|
|
9
|
+
import { type SegmentOptions } from "./segments.js";
|
|
10
|
+
import type { AsyncChunkIterator, AsyncDataResource, ByteRange } from "./shared.js";
|
|
9
11
|
/**
|
|
10
12
|
* Initialization options for creating a spliterator.
|
|
11
13
|
*/
|
|
@@ -46,6 +48,13 @@ export interface SpliteratorInit {
|
|
|
46
48
|
* @default Infinity
|
|
47
49
|
*/
|
|
48
50
|
take?: number;
|
|
51
|
+
/**
|
|
52
|
+
* Whether to handle quoted fields (e.g., CSV). When enabled, delimiters inside double-quoted fields are ignored and
|
|
53
|
+
* quoted fields are emitted without quotes.
|
|
54
|
+
*
|
|
55
|
+
* @default false
|
|
56
|
+
*/
|
|
57
|
+
enableQuoteHandling?: boolean;
|
|
49
58
|
/**
|
|
50
59
|
* Whether to emit debug information.
|
|
51
60
|
*/
|
|
@@ -169,36 +178,32 @@ export declare class AsyncSpliterator<R extends Uint8Array | DataView | ArrayBuf
|
|
|
169
178
|
[Symbol.asyncIterator](): this;
|
|
170
179
|
return(): Promise<IteratorReturnResult<undefined>>;
|
|
171
180
|
/**
|
|
172
|
-
*
|
|
173
|
-
*
|
|
174
|
-
*
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
*
|
|
179
|
-
*
|
|
180
|
-
* You could add more people to the task by laying out all the pages, measuring the length and assigning each person a
|
|
181
|
-
* length of pages to traverse.
|
|
182
|
-
*
|
|
183
|
-
* There's a few ways to go about this:
|
|
184
|
-
*
|
|
185
|
-
* We could approach this in serial -- having the first worker start from page 1 and scanning until they find the
|
|
186
|
-
* beginning of the next chapter, handing off the range to the next worker. This is how `AsyncSpliterator` works.
|
|
181
|
+
* Compute delimiter-aligned `[start, end)` byte ranges dividing `source` into up to `concurrency` segments. The
|
|
182
|
+
* boundary primitive for parallel parsing — hand each range to a worker (with its own handle). See
|
|
183
|
+
* {@linkcode asManyWorkers} for the turnkey worker version.
|
|
184
|
+
*/
|
|
185
|
+
static segments(source: AsyncDataResource, options: SegmentOptions): Promise<ByteRange[]>;
|
|
186
|
+
/**
|
|
187
|
+
* Split `source` into delimiter-aligned segments and return one {@linkcode AsyncSpliterator} per segment. All share
|
|
188
|
+
* the event loop (no worker threads) — for moderate jobs or to overlap I/O. Returns at most `concurrency` instances.
|
|
187
189
|
*
|
|
188
|
-
*
|
|
189
|
-
*
|
|
190
|
-
* pages simultaneously, and settle up on the boundaries of the chapters they find.
|
|
190
|
+
* @param source - The data resource to read from.
|
|
191
|
+
* @param options - Delimiter, desired concurrency, and optional probe size.
|
|
191
192
|
*
|
|
192
|
-
* `AsyncSpliterator
|
|
193
|
-
|
|
193
|
+
* @returns One `AsyncSpliterator` per non-empty segment, possibly fewer than `concurrency`.
|
|
194
|
+
*/
|
|
195
|
+
static asMany(source: AsyncDataResource, options: SegmentOptions): Promise<AsyncSpliterator[]>;
|
|
196
|
+
/**
|
|
197
|
+
* Parse `source` across worker threads. Each worker owns a handle to a delimiter-aligned segment, runs the `worker`
|
|
198
|
+
* handler module per record, and streams results back to the main thread as one merged async iterator — for a
|
|
199
|
+
* single-thread writer. Results interleave across segments.
|
|
194
200
|
*
|
|
195
|
-
*
|
|
196
|
-
*
|
|
197
|
-
* @param concurrency - The _desired_ number of `AsyncSpliterator` instances to create.
|
|
201
|
+
* The handler returns a value (structured-cloned back), a `Uint8Array` (transferred zero-copy), or `undefined` (the
|
|
202
|
+
* record is skipped). The handler module's top-level code runs once per worker — load models/handles there.
|
|
198
203
|
*
|
|
199
|
-
* @
|
|
200
|
-
* @
|
|
204
|
+
* @param source A file path or URL (file handles cannot cross threads → `TypeError` otherwise).
|
|
205
|
+
* @param options Handler module, delimiter, concurrency, and batching/backpressure tuning.
|
|
201
206
|
*/
|
|
202
|
-
static
|
|
207
|
+
static asManyWorkers<R = unknown>(source: AsyncDataResource, options: AsManyWorkersOptions): AsyncIterableIterator<R>;
|
|
203
208
|
}
|
|
204
209
|
//# sourceMappingURL=AsyncSpliterator.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AsyncSpliterator.d.ts","sourceRoot":"","sources":["../../lib/AsyncSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,cAAc,EAAE,KAAK,oBAAoB,EAAE,KAAK,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAG9F,OAAO,EAAqB,KAAK,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AAEvF,OAAO,KAAK,EAAE,kBAAkB,EAAE,iBAAiB,
|
|
1
|
+
{"version":3,"file":"AsyncSpliterator.d.ts","sourceRoot":"","sources":["../../lib/AsyncSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,cAAc,EAAE,KAAK,oBAAoB,EAAE,KAAK,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAG9F,OAAO,EAAqB,KAAK,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AAEvF,OAAO,EAAE,KAAK,oBAAoB,EAAqB,MAAM,sBAAsB,CAAA;AACnF,OAAO,EAAmB,KAAK,cAAc,EAAE,MAAM,eAAe,CAAA;AACpE,OAAO,KAAK,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAInF;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B;;OAEG;IACH,SAAS,CAAC,EAAE,sBAAsB,CAAA;IAElC;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IAEjB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,OAAO,CAAA;IAEnB;;;;OAIG;IACH,IAAI,CAAC,EAAE,MAAM,CAAA;IAEb;;;;;;;;OAQG;IACH,IAAI,CAAC,EAAE,MAAM,CAAA;IAEb;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAA;IAE7B;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,CAAA;CACf;AAED;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,eAAe;IAC5D;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IAEtB;;;;OAIG;IACH,WAAW,CAAC,EAAE,OAAO,CAAA;CACrB;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,QAAQ,GAAG,WAAW,GAAG,UAAU,CACvF,YAAW,qBAAqB,CAAC,CAAC,CAAC,EAAE,eAAe;;IAIpD;;;;;;;OAOG;IACH,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,kBAAkB,EAAE,IAAI,CAAC,EAAE,oBAAoB,GAAG,gBAAgB;IACtF;;;;;;;OAOG;IACH,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,iBAAiB,EAAE,IAAI,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAC9F;;;;;;;OAOG;IACH,MAAM,CAAC,IAAI,CACV,MAAM,EAAE,iBAAiB,GAAG,kBAAkB,EAC9C,IAAI,CAAC,EAAE,oBAAoB,GACzB,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB/C;;OAEG;IACU,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC;IAWnD;;OAEG;WACW,iBAAiB,CAAC,IAAI,EAAE,eAAe,GAAG,eAAe,CAAC,kBAAkB,EAAE,UAAU,CAAC;IAYvG;;;;;;OAMG;gBACS,MAAM,EAAE,kBAAkB,EAAE,IAAI,GAAE,oBAAyB;IAuP1D,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAiD/C;;;;OAIG;IACI,OAAO,IAAI,OAAO,CAAC,CAAC,EAAE,CAAC;IAI9B;;;;OAIG;IACI,cAAc,CAAC,OAAO,kCAAoB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAIrE;;;;;;OAMG;IACI,gBAAgB,IAAI,cAAc,CAAC,CAAC,CAAC;IAiB5C;;;;;;;;;;OAUG;IACI,WAAW,CAAC,CAAC,EAAE,SAAS,EAAE,oBAAoB,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,cAAc,CAAC,CAAC,CAAC;IAI5G;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC;IAItB,MAAM,IAAI,OAAO,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC;IAQzD;;;;OAIG;WACW,QAAQ,CAAC,MAAM,EAAE,iBAAiB,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAIhG;;;;;;;;OAQG;WACiB,MAAM,CAAC,MAAM,EAAE,iBAAiB,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAc3G;;;;;;;;;;OAUG;WACW,aAAa,CAAC,CAAC,GAAG,OAAO,EACtC,MAAM,EAAE,iBAAiB,EACzB,OAAO,EAAE,oBAAoB,GAC3B,qBAAqB,CAAC,CAAC,CAAC;CAQ3B"}
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @copyright Sister Software
|
|
3
|
-
* @license
|
|
3
|
+
* @license MIT
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
import { ReadableStream } from "stream/web";
|
|
7
7
|
import { BufferController } from "./BufferController.js";
|
|
8
8
|
import { CharacterSequence } from "./CharacterSequence.js";
|
|
9
9
|
import { IndexQueue } from "./IndexQueue.js";
|
|
10
|
+
import { runSegmentWorkers } from "./segment-workers.js";
|
|
11
|
+
import { computeSegments } from "./segments.js";
|
|
10
12
|
const noop = () => void 0;
|
|
11
13
|
/**
|
|
12
14
|
* An asynchronous spliterator for reading delimited byte streams.
|
|
@@ -41,6 +43,7 @@ export class AsyncSpliterator {
|
|
|
41
43
|
async [Symbol.asyncDispose]() {
|
|
42
44
|
this.#indices.clear();
|
|
43
45
|
this.#controller.clear();
|
|
46
|
+
await this.#closeReader();
|
|
44
47
|
if (this.#autoDispose) {
|
|
45
48
|
await this.#source[Symbol.asyncDispose]?.();
|
|
46
49
|
}
|
|
@@ -228,7 +231,17 @@ export class AsyncSpliterator {
|
|
|
228
231
|
this.#search();
|
|
229
232
|
}
|
|
230
233
|
}
|
|
234
|
+
/** Signal the underlying chunk reader to stop, destroying an owned read stream (closing its fd). */
|
|
235
|
+
async #closeReader() {
|
|
236
|
+
try {
|
|
237
|
+
await this.#chunkReader.return?.();
|
|
238
|
+
}
|
|
239
|
+
catch {
|
|
240
|
+
// The reader may already be closed; ignore.
|
|
241
|
+
}
|
|
242
|
+
}
|
|
231
243
|
async #finalize() {
|
|
244
|
+
await this.#closeReader();
|
|
232
245
|
if (this.#debug) {
|
|
233
246
|
/**
|
|
234
247
|
* The total number of bytes we expect to be omitted from the buffer. This is derived from the number of of yields
|
|
@@ -252,37 +265,41 @@ export class AsyncSpliterator {
|
|
|
252
265
|
//#endregion
|
|
253
266
|
//#region Iterator Methods
|
|
254
267
|
async next() {
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
268
|
+
// Loop rather than recurse: a long run of skipped (empty or dropped) ranges would
|
|
269
|
+
// otherwise grow the call stack one frame per skip and overflow.
|
|
270
|
+
while (true) {
|
|
271
|
+
if (this.#done || this.#yieldCount >= this.#yieldStopCount)
|
|
272
|
+
return this.#finalize();
|
|
273
|
+
if (this.#indices.size === 0) {
|
|
274
|
+
await this.#fill();
|
|
275
|
+
}
|
|
276
|
+
// The first `#fill` may have read into EOF without entering the EOF branch (the branch only
|
|
277
|
+
// triggers when fill is called with `done` already set). In that case the trailing tail
|
|
278
|
+
// hasn't been enqueued yet, so call fill once more to let the EOF branch flush it.
|
|
279
|
+
if (this.#lastReadResult?.done && this.#indices.size === 0 && !this.#done) {
|
|
280
|
+
await this.#fill();
|
|
281
|
+
}
|
|
282
|
+
const currentByteRange = this.#indices.dequeue();
|
|
283
|
+
this.#lastByteRangeSeen = currentByteRange;
|
|
284
|
+
if (!currentByteRange) {
|
|
285
|
+
this.#done = true;
|
|
286
|
+
return this.#finalize();
|
|
287
|
+
}
|
|
288
|
+
const [start, end] = currentByteRange;
|
|
289
|
+
const slice = this.#controller.subarray(start, end);
|
|
290
|
+
if (slice.length === 0 && this.#skipEmpty) {
|
|
291
|
+
continue;
|
|
292
|
+
}
|
|
293
|
+
this.#yieldCount++;
|
|
294
|
+
if (this.#yieldCount <= this.#yieldDropCount) {
|
|
295
|
+
continue;
|
|
296
|
+
}
|
|
297
|
+
this.#yieldedByteLength += end - start;
|
|
298
|
+
return {
|
|
299
|
+
value: slice,
|
|
300
|
+
done: false,
|
|
301
|
+
};
|
|
280
302
|
}
|
|
281
|
-
this.#yieldedByteLength += end - start;
|
|
282
|
-
return {
|
|
283
|
-
value: slice,
|
|
284
|
-
done: false,
|
|
285
|
-
};
|
|
286
303
|
}
|
|
287
304
|
/**
|
|
288
305
|
* Collect all the byte ranges from the file.
|
|
@@ -348,51 +365,48 @@ export class AsyncSpliterator {
|
|
|
348
365
|
//#endregion
|
|
349
366
|
//#region Experimental
|
|
350
367
|
/**
|
|
351
|
-
*
|
|
352
|
-
*
|
|
353
|
-
*
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
*
|
|
360
|
-
*
|
|
361
|
-
*
|
|
362
|
-
* There's a few ways to go about this:
|
|
363
|
-
*
|
|
364
|
-
* We could approach this in serial -- having the first worker start from page 1 and scanning until they find the
|
|
365
|
-
* beginning of the next chapter, handing off the range to the next worker. This is how `AsyncSpliterator` works.
|
|
368
|
+
* Compute delimiter-aligned `[start, end)` byte ranges dividing `source` into up to `concurrency` segments. The
|
|
369
|
+
* boundary primitive for parallel parsing — hand each range to a worker (with its own handle). See
|
|
370
|
+
* {@linkcode asManyWorkers} for the turnkey worker version.
|
|
371
|
+
*/
|
|
372
|
+
static segments(source, options) {
|
|
373
|
+
return computeSegments(source, options);
|
|
374
|
+
}
|
|
375
|
+
/**
|
|
376
|
+
* Split `source` into delimiter-aligned segments and return one {@linkcode AsyncSpliterator} per segment. All share
|
|
377
|
+
* the event loop (no worker threads) — for moderate jobs or to overlap I/O. Returns at most `concurrency` instances.
|
|
366
378
|
*
|
|
367
|
-
*
|
|
368
|
-
*
|
|
369
|
-
* pages simultaneously, and settle up on the boundaries of the chapters they find.
|
|
379
|
+
* @param source - The data resource to read from.
|
|
380
|
+
* @param options - Delimiter, desired concurrency, and optional probe size.
|
|
370
381
|
*
|
|
371
|
-
* `AsyncSpliterator
|
|
372
|
-
|
|
382
|
+
* @returns One `AsyncSpliterator` per non-empty segment, possibly fewer than `concurrency`.
|
|
383
|
+
*/
|
|
384
|
+
static async asMany(source, options) {
|
|
385
|
+
const { createChunkIterator } = await import("spliterator/node/fs");
|
|
386
|
+
const segments = await computeSegments(source, options);
|
|
387
|
+
return Promise.all(segments.map(async ([start, end]) => {
|
|
388
|
+
// `end` is exclusive here; createChunkIterator's `end` is inclusive.
|
|
389
|
+
const chunkIterator = await createChunkIterator(source, { start, end: end - 1 });
|
|
390
|
+
return new AsyncSpliterator(chunkIterator, { delimiter: options.delimiter, autoDispose: true });
|
|
391
|
+
}));
|
|
392
|
+
}
|
|
393
|
+
/**
|
|
394
|
+
* Parse `source` across worker threads. Each worker owns a handle to a delimiter-aligned segment, runs the `worker`
|
|
395
|
+
* handler module per record, and streams results back to the main thread as one merged async iterator — for a
|
|
396
|
+
* single-thread writer. Results interleave across segments.
|
|
373
397
|
*
|
|
374
|
-
*
|
|
375
|
-
*
|
|
376
|
-
* @param concurrency - The _desired_ number of `AsyncSpliterator` instances to create.
|
|
398
|
+
* The handler returns a value (structured-cloned back), a `Uint8Array` (transferred zero-copy), or `undefined` (the
|
|
399
|
+
* record is skipped). The handler module's top-level code runs once per worker — load models/handles there.
|
|
377
400
|
*
|
|
378
|
-
* @
|
|
379
|
-
* @
|
|
401
|
+
* @param source A file path or URL (file handles cannot cross threads → `TypeError` otherwise).
|
|
402
|
+
* @param options Handler module, delimiter, concurrency, and batching/backpressure tuning.
|
|
380
403
|
*/
|
|
381
|
-
static
|
|
382
|
-
//
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
const spliterators = [];
|
|
386
|
-
const { createChunkIterator, readFileSize } = await import("spliterator/node/fs");
|
|
387
|
-
const fileSize = await readFileSize(source);
|
|
388
|
-
// const file = createChunkIterator(source, {
|
|
389
|
-
// start: 0,
|
|
390
|
-
// })
|
|
391
|
-
// The idea here is we're going to split the file into `concurrency` chunks.
|
|
392
|
-
if (Date.now()) {
|
|
393
|
-
throw new Error("Not implemented.");
|
|
404
|
+
static asManyWorkers(source, options) {
|
|
405
|
+
// Validate eagerly so callers fail fast (the generator body is otherwise lazy until consumed).
|
|
406
|
+
if (typeof source !== "string" && !(source instanceof URL)) {
|
|
407
|
+
throw new TypeError("asManyWorkers requires a file path or URL — file handles cannot cross threads.");
|
|
394
408
|
}
|
|
395
|
-
return
|
|
409
|
+
return runSegmentWorkers(source, options);
|
|
396
410
|
}
|
|
397
411
|
}
|
|
398
412
|
//# sourceMappingURL=AsyncSpliterator.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AsyncSpliterator.js","sourceRoot":"","sources":["../../lib/AsyncSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,cAAc,EAAqD,MAAM,YAAY,CAAA;AAE9F,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AACxD,OAAO,EAAE,iBAAiB,EAA+B,MAAM,wBAAwB,CAAA;AACvF,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;
|
|
1
|
+
{"version":3,"file":"AsyncSpliterator.js","sourceRoot":"","sources":["../../lib/AsyncSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,cAAc,EAAqD,MAAM,YAAY,CAAA;AAE9F,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AACxD,OAAO,EAAE,iBAAiB,EAA+B,MAAM,wBAAwB,CAAA;AACvF,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EAA6B,iBAAiB,EAAE,MAAM,sBAAsB,CAAA;AACnF,OAAO,EAAE,eAAe,EAAuB,MAAM,eAAe,CAAA;AAGpE,MAAM,IAAI,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC,CAAA;AAkFzB;;;;;;;;;;;;;;GAcG;AACH,MAAM,OAAO,gBAAgB;IAmC5B,MAAM,CAAC,IAAI,CACV,MAA8C,EAC9C,OAA6B,EAAE;QAE/B,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,aAAa,IAAI,MAAM,EAAE,CAAC;YAClE,OAAO,IAAI,gBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QAC1C,CAAC;QAED,OAAO,MAAM,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,mBAAmB,EAAE,EAAE,EAAE;YAC3E,MAAM,aAAa,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE;gBACvD,aAAa,EAAE,IAAI,CAAC,aAAa;aACjC,CAAC,CAAA;YAEF,OAAO,IAAI,gBAAgB,CAAC,aAAa,EAAE,IAAI,CAAC,CAAA;QACjD,CAAC,CAAC,CAAA;IACH,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QACjC,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;QACrB,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAA;QAExB,MAAM,IAAI,CAAC,YAAY,EAAE,CAAA;QAEzB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE,CAAA;QAC5C,CAAC;IACF,CAAC;IAED;;OAEG;IACI,MAAM,CAAC,iBAAiB,CAAC,IAAqB;QACpD,OAAO,IAAI,eAAe,CAAiC;YAC1D,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,UAAU;gBAChC,MAAM,WAAW,GAAG,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;gBAEtD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;oBACvC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;gBAC1B,CAAC;YACF,CAAC;SACD,CAAC,CAAA;IACH,CAAC;IAED;;;;;;OAMG;IACH,YAAY,MAA0B,EAAE,OAA6B,EAAE;QACtE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QACrB,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAA;QAElD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,WAAW,IAAI,KAAK,CAAA;QAE7C,IAAI,CAAC,OAAO,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QAEpD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,aAAa,IAAI,IAAI,GAAG,EAAE,CAAC,CAAA;QAExF,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAA;QAClD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,eAAe,CAAA;QAEhF,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAA;QAExC,IAAI,CAAC,WAAW,GAAG,IAAI,gBAAgB,CAAC;YACvC,iBAAiB,EAAE,IAAI,CAAC,cAAc;SACtC,CAAC,CAAA;QAEF,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,IAAI,KAAK,CAAA;QACjC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAC7D,CAAC;IAED,YAAY;IAEZ,4BAA4B;IAE5B;;OAEG;IACM,YAAY,CAAS;IAE9B;;OAEG;IACM,OAAO,CAAoB;IAEpC;;OAEG;IACM,YAAY,CAA2B;IAEhD;;OAEG;IACH,eAAe,CAAwC;IAEvD;;;;;;OAMG;IACM,QAAQ,GAAG,IAAI,UAAU,EAAE,CAAA;IAEpC;;OAEG;IACM,OAAO,CAAmB;IAEnC;;;OAGG;IACM,WAAW,CAAkB;IAEtC;;OAEG;IACM,eAAe,CAAQ;IAEhC;;OAEG;IACM,eAAe,CAAQ;IAEhC;;OAEG;IACM,UAAU,CAAS;IAE5B;;;;OAIG;IACM,cAAc,CAAQ;IAE/B;;OAEG;IACH,kBAAkB,GAAG,CAAC,CAAA;IAEtB;;OAEG;IACH,WAAW,GAAG,CAAC,CAAA;IAEf,MAAM;IACN,6DAA6D;IAC7D,MAAM;IACN,wBAAwB;IAExB;;OAEG;IACH,kBAAkB,CAAuB;IAEzC;;OAEG;IACH,MAAM,CAAS;IAEf;;OAEG;IACH,KAAK,GAAG,KAAK,CAAA;IAEb,YAAY;IAEZ,yBAAyB;IAEzB,IAAI,CAA0B;IAE9B;;;;;OAKG;IACH,OAAO;QACN,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAA;QAC9C,IAAI,YAAY,GAAG,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;QAC7E,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,CAAA;QAE/C,OAAO,YAAY,IAAI,SAAS,EAAE,CAAC;YAClC,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,YAAY,EAAE,SAAS,CAAC,CAAA;YAE3F,IAAI,cAAc,KAAK,CAAC,CAAC,EAAE,CAAC;gBAC3B,IAAI,CAAC,IAAI,CAAC,iDAAiD,EAAE,EAAE,YAAY,EAAE,CAAC,CAAA;gBAC9E,MAAK;YACN,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,CAAA;YAC7D,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,CAAA;YAErD,YAAY,GAAG,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;QACpD,CAAC;IACF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACV,IAAI,CAAC,eAAe,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAA;QAErD,IAAI,IAAI,CAAC,eAAe,CAAC,IAAI;YAAE,OAAM;QAErC,kCAAkC;QAClC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,CAAA;IAChF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACV,IAAI,IAAI,CAAC,eAAe,EAAE,IAAI,EAAE,CAAC;YAChC,IAAI,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAA;YACxD,sFAAsF;YACtF,+EAA+E;YAC/E,kFAAkF;YAClF,kFAAkF;YAClF,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;YAChG,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,CAAA;YAE7C,IAAI,SAAS,IAAI,OAAO,EAAE,CAAC;gBAC1B,IAAI,CAAC,IAAI,CAAC,8BAA8B,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAA;gBAC/D,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAA;YAC5C,CAAC;iBAAM,CAAC;gBACP,IAAI,CAAC,IAAI,CAAC,2CAA2C,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAA;YAC/E,CAAC;YAED,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;YAEjB,OAAM;QACP,CAAC;QAED,yDAAyD;QACzD,wCAAwC;QACxC,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;QAEtG,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,eAAe,EAAE,CAAC,CAAA;QACpD,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAA;QAE1C,0FAA0F;QAC1F,yFAAyF;QACzF,0FAA0F;QAC1F,4EAA4E;QAC5E,IAAI,CAAC,kBAAkB,GAAG,SAAS,CAAA;QAEnC,OAAO,CAAC,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;YAChH,MAAM,IAAI,CAAC,KAAK,EAAE,CAAA;YAClB,IAAI,CAAC,OAAO,EAAE,CAAA;QACf,CAAC;IACF,CAAC;IAED,oGAAoG;IACpG,KAAK,CAAC,YAAY;QACjB,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,EAAE,CAAA;QACnC,CAAC;QAAC,MAAM,CAAC;YACR,4CAA4C;QAC7C,CAAC;IACF,CAAC;IAED,KAAK,CAAC,SAAS;QACd,MAAM,IAAI,CAAC,YAAY,EAAE,CAAA;QAEzB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB;;;eAGG;YACH,MAAM,6BAA6B,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;YAElF,IAAI,CAAC,IAAI,CAAC;gBACT,iBAAiB,EAAE,IAAI,CAAC,kBAAkB;gBAC1C,UAAU,EAAE,IAAI,CAAC,WAAW;gBAC5B,6BAA6B;aAC7B,CAAC,CAAA;QACH,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE,CAAA;QAC5C,CAAC;QAED,OAAO;YACN,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,SAAS;SAChB,CAAA;IACF,CAAC;IAED,YAAY;IAEZ,0BAA0B;IAEnB,KAAK,CAAC,IAAI;QAChB,kFAAkF;QAClF,iEAAiE;QACjE,OAAO,IAAI,EAAE,CAAC;YACb,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,eAAe;gBAAE,OAAO,IAAI,CAAC,SAAS,EAAE,CAAA;YAEnF,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;gBAC9B,MAAM,IAAI,CAAC,KAAK,EAAE,CAAA;YACnB,CAAC;YAED,4FAA4F;YAC5F,wFAAwF;YACxF,mFAAmF;YACnF,IAAI,IAAI,CAAC,eAAe,EAAE,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC3E,MAAM,IAAI,CAAC,KAAK,EAAE,CAAA;YACnB,CAAC;YAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAA;YAChD,IAAI,CAAC,kBAAkB,GAAG,gBAAgB,CAAA;YAE1C,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACvB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;gBAEjB,OAAO,IAAI,CAAC,SAAS,EAAE,CAAA;YACxB,CAAC;YAED,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,gBAAgB,CAAA;YAErC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;YAEnD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC3C,SAAQ;YACT,CAAC;YAED,IAAI,CAAC,WAAW,EAAE,CAAA;YAElB,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;gBAC9C,SAAQ;YACT,CAAC;YAED,IAAI,CAAC,kBAAkB,IAAI,GAAG,GAAG,KAAK,CAAA;YAEtC,OAAO;gBACN,KAAK,EAAE,KAAqB;gBAC5B,IAAI,EAAE,KAAK;aACX,CAAA;QACF,CAAC;IACF,CAAC;IAED;;;;OAIG;IACI,OAAO;QACb,OAAO,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;IAC7B,CAAC;IAED;;;;OAIG;IACI,cAAc,CAAC,OAAO,GAAG,IAAI,WAAW,EAAE;QAChD,OAAO,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;IAC/D,CAAC;IAED;;;;;;OAMG;IACI,gBAAgB;QACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAA;QAE7C,OAAO,IAAI,cAAc,CAAC;YACzB,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE;gBAC1B,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;gBAE7C,IAAI,IAAI,EAAE,CAAC;oBACV,UAAU,CAAC,KAAK,EAAE,CAAA;gBACnB,CAAC;qBAAM,CAAC;oBACP,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;gBAC1B,CAAC;YACF,CAAC;YACD,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE;SAC3C,CAAC,CAAA;IACH,CAAC;IAED;;;;;;;;;;OAUG;IACI,WAAW,CAAI,SAAqC,EAAE,OAA2B;QACvF,OAAO,IAAI,CAAC,gBAAgB,EAAE,CAAC,WAAW,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IAC/D,CAAC;IAED;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC;QAC5B,OAAO,IAAI,CAAA;IACZ,CAAC;IAEM,MAAM;QACZ,OAAO,IAAI,CAAC,SAAS,EAAE,CAAA;IACxB,CAAC;IAED,YAAY;IAEZ,sBAAsB;IAEtB;;;;OAIG;IACI,MAAM,CAAC,QAAQ,CAAC,MAAyB,EAAE,OAAuB;QACxE,OAAO,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IACxC,CAAC;IAED;;;;;;;;OAQG;IACI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAyB,EAAE,OAAuB;QAC5E,MAAM,EAAE,mBAAmB,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAA;QACnE,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAEvD,OAAO,OAAO,CAAC,GAAG,CACjB,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE;YACnC,qEAAqE;YACrE,MAAM,aAAa,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,GAAG,CAAC,EAAE,CAAC,CAAA;YAEhF,OAAO,IAAI,gBAAgB,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAA;QAChG,CAAC,CAAC,CACF,CAAA;IACF,CAAC;IAED;;;;;;;;;;OAUG;IACI,MAAM,CAAC,aAAa,CAC1B,MAAyB,EACzB,OAA6B;QAE7B,+FAA+F;QAC/F,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,YAAY,GAAG,CAAC,EAAE,CAAC;YAC5D,MAAM,IAAI,SAAS,CAAC,gFAAgF,CAAC,CAAA;QACtG,CAAC;QAED,OAAO,iBAAiB,CAAI,MAAM,EAAE,OAAO,CAAC,CAAA;IAC7C,CAAC;CACD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CSVSpliterator.d.ts","sourceRoot":"","sources":["../../lib/CSVSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAqB,KAAK,sBAAsB,EAAc,MAAM,wBAAwB,CAAA;AACnG,OAAO,EAAE,KAAK,kBAAkB,EAAE,KAAK,iBAAiB,EAAW,MAAM,aAAa,CAAA;AACtF,OAAO,EAAE,KAAK,oBAAoB,EAAe,KAAK,eAAe,EAAE,MAAM,kBAAkB,CAAA;AAE/F;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,CAAA;AAC1D,MAAM,MAAM,cAAc,CAAC,CAAC,GAAG,OAAO,IAAI,CAAC,KAAK,EAAE,MAAM,KAAK,CAAC,CAAA;AAE9D,MAAM,MAAM,mBAAmB,CAAC,CAAC,GAAG,OAAO,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,CAAA;AACnG,MAAM,MAAM,oBAAoB,GAAG,MAAM,CAAC,MAAM,EAAE,cAAc,GAAG,SAAS,CAAC,CAAA;AAE7E,MAAM,MAAM,UAAU,CAAC,CAAC,GAAG,OAAO,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,aAAa,CAAC,EAAE,QAAQ,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;AAIrH,MAAM,MAAM,2BAA2B,CAAC,CAAC,GAAG,MAAM,GAAG,MAAM,GAAG,SAAS,IAAI;IAC1E,CAAC,GAAG,EAAE,MAAM,GAAG,CAAC,CAAA;CAChB,CAAA;AAED;;;;;;GAMG;AACH,MAAM,MAAM,QAAQ,CAAC,CAAC,GAAG,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,CAAA;AAEhF,eAAO,MAAM,sBAAsB;;gCAGjB,QAAQ,CAAC,MAAM,CAAC,kBAAiB,QAAQ,CAAC,mBAAmB,CAAC,KAAQ,QAAQ,CAAC,OAAO,CAAC,EAAE;+BAShG,QAAQ,CAAC,MAAM,CAAC,kBACV,QAAQ,CAAC,mBAAmB,CAAC,KAC1C,2BAA2B,CAAC,OAAO,CAAC;CAYqB,CAAA;AAE7D,MAAM,WAAW,kBAAmB,SAAQ,eAAe;IAC1D;;;;;;OAMG;IACH,IAAI,CAAC,EAAE,aAAa,CAAA;IAEpB;;;;OAIG;IACH,eAAe,CAAC,EAAE,sBAAsB,CAAA;IAExC;;;;;OAKG;IACH,aAAa,CAAC,EAAE,OAAO,CAAA;IAEvB;;;;OAIG;IACH,MAAM,CAAC,EAAE,OAAO,CAAA;IAEhB,YAAY,CAAC,EAAE,QAAQ,CAAC,mBAAmB,CAAC,GAAG,oBAAoB,CAAA;CACnE;AAED;;;;;GAKG;AACH,8BAAsB,cAAc;IACnC;;;;OAIG;IACH,OAAc,eAAe,EAAE,MAAM,CAAmB;;IAMxD,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,2BAA2B,GAAG,2BAA2B,EAC9E,MAAM,EAAE,sBAAsB,EAC9B,OAAO,CAAC,EAAE,kBAAkB,GAAG;QAAE,IAAI,EAAE,QAAQ,CAAA;KAAE,GAC/C,SAAS,CAAC,CAAC,CAAC;IACf;;OAEG;IAEH,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,QAAQ,EAAE,GAAG,QAAQ,EAAE,EAC5C,MAAM,EAAE,sBAAsB,EAC9B,OAAO,CAAC,EAAE,kBAAkB,GAAG;QAAE,IAAI,EAAE,SAAS,CAAA;KAAE,GAChD,SAAS,CAAC,CAAC,CAAC;IACf;;;;OAIG;IACH,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,EACxC,MAAM,EAAE,sBAAsB,EAC9B,OAAO,CAAC,EAAE,kBAAkB,GAAG;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAC/C,SAAS,CAAC,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"CSVSpliterator.d.ts","sourceRoot":"","sources":["../../lib/CSVSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAqB,KAAK,sBAAsB,EAAc,MAAM,wBAAwB,CAAA;AACnG,OAAO,EAAE,KAAK,kBAAkB,EAAE,KAAK,iBAAiB,EAAW,MAAM,aAAa,CAAA;AACtF,OAAO,EAAE,KAAK,oBAAoB,EAAe,KAAK,eAAe,EAAE,MAAM,kBAAkB,CAAA;AAE/F;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,CAAA;AAC1D,MAAM,MAAM,cAAc,CAAC,CAAC,GAAG,OAAO,IAAI,CAAC,KAAK,EAAE,MAAM,KAAK,CAAC,CAAA;AAE9D,MAAM,MAAM,mBAAmB,CAAC,CAAC,GAAG,OAAO,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,CAAA;AACnG,MAAM,MAAM,oBAAoB,GAAG,MAAM,CAAC,MAAM,EAAE,cAAc,GAAG,SAAS,CAAC,CAAA;AAE7E,MAAM,MAAM,UAAU,CAAC,CAAC,GAAG,OAAO,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,aAAa,CAAC,EAAE,QAAQ,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;AAIrH,MAAM,MAAM,2BAA2B,CAAC,CAAC,GAAG,MAAM,GAAG,MAAM,GAAG,SAAS,IAAI;IAC1E,CAAC,GAAG,EAAE,MAAM,GAAG,CAAC,CAAA;CAChB,CAAA;AAED;;;;;;GAMG;AACH,MAAM,MAAM,QAAQ,CAAC,CAAC,GAAG,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,CAAA;AAEhF,eAAO,MAAM,sBAAsB;;gCAGjB,QAAQ,CAAC,MAAM,CAAC,kBAAiB,QAAQ,CAAC,mBAAmB,CAAC,KAAQ,QAAQ,CAAC,OAAO,CAAC,EAAE;+BAShG,QAAQ,CAAC,MAAM,CAAC,kBACV,QAAQ,CAAC,mBAAmB,CAAC,KAC1C,2BAA2B,CAAC,OAAO,CAAC;CAYqB,CAAA;AAE7D,MAAM,WAAW,kBAAmB,SAAQ,eAAe;IAC1D;;;;;;OAMG;IACH,IAAI,CAAC,EAAE,aAAa,CAAA;IAEpB;;;;OAIG;IACH,eAAe,CAAC,EAAE,sBAAsB,CAAA;IAExC;;;;;OAKG;IACH,aAAa,CAAC,EAAE,OAAO,CAAA;IAEvB;;;;OAIG;IACH,MAAM,CAAC,EAAE,OAAO,CAAA;IAEhB,YAAY,CAAC,EAAE,QAAQ,CAAC,mBAAmB,CAAC,GAAG,oBAAoB,CAAA;CACnE;AAED;;;;;GAKG;AACH,8BAAsB,cAAc;IACnC;;;;OAIG;IACH,OAAc,eAAe,EAAE,MAAM,CAAmB;;IAMxD,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,2BAA2B,GAAG,2BAA2B,EAC9E,MAAM,EAAE,sBAAsB,EAC9B,OAAO,CAAC,EAAE,kBAAkB,GAAG;QAAE,IAAI,EAAE,QAAQ,CAAA;KAAE,GAC/C,SAAS,CAAC,CAAC,CAAC;IACf;;OAEG;IAEH,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,QAAQ,EAAE,GAAG,QAAQ,EAAE,EAC5C,MAAM,EAAE,sBAAsB,EAC9B,OAAO,CAAC,EAAE,kBAAkB,GAAG;QAAE,IAAI,EAAE,SAAS,CAAA;KAAE,GAChD,SAAS,CAAC,CAAC,CAAC;IACf;;;;OAIG;IACH,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,EACxC,MAAM,EAAE,sBAAsB,EAC9B,OAAO,CAAC,EAAE,kBAAkB,GAAG;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAC/C,SAAS,CAAC,CAAC,CAAC;IAsEf;;OAEG;IACH,MAAM,CAAC,SAAS,CAAC,CAAC,SAAS,2BAA2B,GAAG,2BAA2B,EACnF,MAAM,EAAE,iBAAiB,GAAG,kBAAkB,EAC9C,OAAO,CAAC,EAAE,kBAAkB,GAAG,oBAAoB,GAAG;QAAE,IAAI,EAAE,QAAQ,CAAA;KAAE,GACtE,cAAc,CAAC,CAAC,CAAC;IAEpB;;OAEG;IACH,MAAM,CAAC,SAAS,CAAC,CAAC,SAAS,QAAQ,EAAE,GAAG,QAAQ,EAAE,EACjD,MAAM,EAAE,iBAAiB,GAAG,kBAAkB,EAC9C,OAAO,CAAC,EAAE,kBAAkB,GAAG,oBAAoB,GAAG;QAAE,IAAI,EAAE,SAAS,CAAA;KAAE,GACvE,cAAc,CAAC,CAAC,CAAC;IACpB;;OAEG;IACH,MAAM,CAAC,SAAS,CAAC,CAAC,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,EAC7C,MAAM,EAAE,iBAAiB,GAAG,kBAAkB,EAC9C,OAAO,CAAC,EAAE,kBAAkB,GAAG,oBAAoB,GAAG;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GACtE,cAAc,CAAC,CAAC,CAAC;IACpB;;;;;;;;OAQG;IACH,MAAM,CAAC,SAAS,CACf,MAAM,EAAE,iBAAiB,GAAG,kBAAkB,EAC9C,IAAI,CAAC,EAAE,kBAAkB,GAAG,oBAAoB,GAC9C,cAAc,CAAC,OAAO,CAAC;CA4E1B"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @copyright Sister Software
|
|
3
|
-
* @license
|
|
3
|
+
* @license MIT
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
import { normalizeColumnNames } from "./casing.js";
|
|
@@ -89,7 +89,8 @@ export class CSVSpliterator {
|
|
|
89
89
|
}
|
|
90
90
|
if (yieldCount >= yieldLimit)
|
|
91
91
|
break;
|
|
92
|
-
const
|
|
92
|
+
const columnRanges = columnDelimiter.searchAll(row);
|
|
93
|
+
const columns = columnRanges.map(([s, e]) => decoder.decode(row.subarray(s, e)));
|
|
93
94
|
yield emitter ? emitter(columns, transformers) : columns;
|
|
94
95
|
}
|
|
95
96
|
}
|
|
@@ -140,7 +141,8 @@ export class CSVSpliterator {
|
|
|
140
141
|
}
|
|
141
142
|
if (yieldCount >= yieldLimit)
|
|
142
143
|
break;
|
|
143
|
-
const
|
|
144
|
+
const columnRanges = columnDelimiter.searchAll(row);
|
|
145
|
+
const columns = columnRanges.map(([s, e]) => decoder.decode(row.subarray(s, e)));
|
|
144
146
|
yield emitter ? emitter(columns, transformers) : columns;
|
|
145
147
|
yieldCount++;
|
|
146
148
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CSVSpliterator.js","sourceRoot":"","sources":["../../lib/CSVSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAA;AAClD,OAAO,EAAE,iBAAiB,EAA+B,UAAU,EAAE,MAAM,wBAAwB,CAAA;AACnG,OAAO,EAAmD,OAAO,EAAE,MAAM,aAAa,CAAA;AACtF,OAAO,EAA6B,WAAW,EAAwB,MAAM,kBAAkB,CAAA;AAa/F,MAAM,QAAQ,GAA2B,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAA;AAezD,MAAM,CAAC,MAAM,sBAAsB,GAAG;IACrC,KAAK,EAAE,IAAI;IAEX,OAAO,CAAC,OAAyB,EAAE,gBAA+C,EAAE;QACnF,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE;YAChF,MAAM,GAAG,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,CAAA;YAC5B,MAAM,SAAS,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAA;YAE9C,OAAO,CAAC,GAAG,IAAI,UAAU,GAAG,EAAE,EAAE,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,CAAA;QAC7D,CAAC,CAAC,CAAA;IACH,CAAC;IACD,MAAM,CACL,OAAyB,EACzB,gBAA+C,EAAE;QAEjD,MAAM,MAAM,GAAyC,EAAE,CAAA;QAEvD,KAAK,MAAM,CAAC,WAAW,EAAE,KAAK,EAAE,GAAG,CAAC,IAAI,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,EAAE,CAAC;YACzE,MAAM,GAAG,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,UAAU,GAAG,EAAE,CAAA;YAC/C,MAAM,SAAS,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAA;YAE9C,MAAM,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;QACrC,CAAC;QAED,OAAO,MAAM,CAAA;IACd,CAAC;CAC2D,CAAA;AAqC7D;;;;;GAKG;AACH,MAAM,OAAgB,cAAc;IACnC;;;;OAIG;IACI,MAAM,CAAC,eAAe,GAAW,UAAU,CAAC,KAAK,CAAA;IAExD;QACC,MAAM,IAAI,SAAS,CAAC,0EAA0E,CAAC,CAAA;IAChG,CAAC;IAuBD;;;;OAIG;IACH,MAAM,CAAC,CAAC,IAAI,CAAC,MAA8B,EAAE,OAA2B,EAAE;QACzE,MAAM;QACL,MAAM;QACN,MAAM,GAAG,IAAI,EACb,YAAY,EAAE,iBAAiB,GAAG,EAAE,EACpC,aAAa,EACb,IAAI,GAAG,OAAO,EACd,eAAe,EAAE,oBAAoB,GAAG,IAAI,CAAC,eAAe,EAC5D,IAAI,GAAG,QAAQ,EACf,IAAI,GAAG,CAAC,EACR,GAAG,OAAO,EACV,GAAG,IAAI,CAAA;QAER,MAAM,OAAO,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAA;QAC5C,IAAI,YAAY,GAA0B,EAAE,CAAA;QAC5C,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,MAAM,UAAU,GAAG,IAAI,GAAG,IAAI,CAAA;QAE9B,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QACjC,MAAM,eAAe,GAAG,IAAI,iBAAiB,CAAC,oBAAoB,IAAI,IAAI,CAAC,eAAe,CAAC,CAAA;QAC3F,oFAAoF;QACpF,+EAA+E;QAC/E,sFAAsF;QACtF,MAAM,qBAAqB,GAAoB,EAAE,SAAS,EAAE,eAAe,EAAE,SAAS,EAAE,KAAK,EAAE,CAAA;QAE/F,MAAM,IAAI,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAElD,IAAI,MAAM,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;YAE1B,IAAI,MAAM,CAAC,IAAI;gBAAE,OAAM;YAEvB,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,qBAAqB,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC,CAAA;YAC5F,MAAM,OAAO,GAAG,aAAa,CAAC,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;YAEvE,IAAI,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,CAAC;gBACtC,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,iBAAiB,CAAC,EAAE,CAAC,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC;oBAC7F,UAAW;oBACX,WAAW,IAAI,QAAQ;iBACvB,CAAC,CAAA;YACH,CAAC;iBAAM,CAAC;gBACP,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;oBACzC,MAAM,SAAS,GAAI,iBAA0C,CAAC,UAAU,CAAC,IAAI,QAAQ,CAAA;oBAErF,OAAO,CAAC,UAAU,EAAE,SAAS,CAAC,CAAA;gBAC/B,CAAC,CAAC,CAAA;YACH,CAAC;QACF,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,IAAI,UAAU,GAAG,IAAI,EAAE,CAAC;gBACvB,UAAU,EAAE,CAAA;gBACZ,SAAQ;YACT,CAAC;YAED,IAAI,UAAU,IAAI,UAAU;gBAAE,MAAK;YAEnC,MAAM,
|
|
1
|
+
{"version":3,"file":"CSVSpliterator.js","sourceRoot":"","sources":["../../lib/CSVSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAA;AAClD,OAAO,EAAE,iBAAiB,EAA+B,UAAU,EAAE,MAAM,wBAAwB,CAAA;AACnG,OAAO,EAAmD,OAAO,EAAE,MAAM,aAAa,CAAA;AACtF,OAAO,EAA6B,WAAW,EAAwB,MAAM,kBAAkB,CAAA;AAa/F,MAAM,QAAQ,GAA2B,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAA;AAezD,MAAM,CAAC,MAAM,sBAAsB,GAAG;IACrC,KAAK,EAAE,IAAI;IAEX,OAAO,CAAC,OAAyB,EAAE,gBAA+C,EAAE;QACnF,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE;YAChF,MAAM,GAAG,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,CAAA;YAC5B,MAAM,SAAS,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAA;YAE9C,OAAO,CAAC,GAAG,IAAI,UAAU,GAAG,EAAE,EAAE,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,CAAA;QAC7D,CAAC,CAAC,CAAA;IACH,CAAC;IACD,MAAM,CACL,OAAyB,EACzB,gBAA+C,EAAE;QAEjD,MAAM,MAAM,GAAyC,EAAE,CAAA;QAEvD,KAAK,MAAM,CAAC,WAAW,EAAE,KAAK,EAAE,GAAG,CAAC,IAAI,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,EAAE,CAAC;YACzE,MAAM,GAAG,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,UAAU,GAAG,EAAE,CAAA;YAC/C,MAAM,SAAS,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAA;YAE9C,MAAM,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;QACrC,CAAC;QAED,OAAO,MAAM,CAAA;IACd,CAAC;CAC2D,CAAA;AAqC7D;;;;;GAKG;AACH,MAAM,OAAgB,cAAc;IACnC;;;;OAIG;IACI,MAAM,CAAC,eAAe,GAAW,UAAU,CAAC,KAAK,CAAA;IAExD;QACC,MAAM,IAAI,SAAS,CAAC,0EAA0E,CAAC,CAAA;IAChG,CAAC;IAuBD;;;;OAIG;IACH,MAAM,CAAC,CAAC,IAAI,CAAC,MAA8B,EAAE,OAA2B,EAAE;QACzE,MAAM;QACL,MAAM;QACN,MAAM,GAAG,IAAI,EACb,YAAY,EAAE,iBAAiB,GAAG,EAAE,EACpC,aAAa,EACb,IAAI,GAAG,OAAO,EACd,eAAe,EAAE,oBAAoB,GAAG,IAAI,CAAC,eAAe,EAC5D,IAAI,GAAG,QAAQ,EACf,IAAI,GAAG,CAAC,EACR,GAAG,OAAO,EACV,GAAG,IAAI,CAAA;QAER,MAAM,OAAO,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAA;QAC5C,IAAI,YAAY,GAA0B,EAAE,CAAA;QAC5C,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,MAAM,UAAU,GAAG,IAAI,GAAG,IAAI,CAAA;QAE9B,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QACjC,MAAM,eAAe,GAAG,IAAI,iBAAiB,CAAC,oBAAoB,IAAI,IAAI,CAAC,eAAe,CAAC,CAAA;QAC3F,oFAAoF;QACpF,+EAA+E;QAC/E,sFAAsF;QACtF,MAAM,qBAAqB,GAAoB,EAAE,SAAS,EAAE,eAAe,EAAE,SAAS,EAAE,KAAK,EAAE,CAAA;QAE/F,MAAM,IAAI,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAElD,IAAI,MAAM,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;YAE1B,IAAI,MAAM,CAAC,IAAI;gBAAE,OAAM;YAEvB,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,qBAAqB,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC,CAAA;YAC5F,MAAM,OAAO,GAAG,aAAa,CAAC,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;YAEvE,IAAI,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,CAAC;gBACtC,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,iBAAiB,CAAC,EAAE,CAAC,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC;oBAC7F,UAAW;oBACX,WAAW,IAAI,QAAQ;iBACvB,CAAC,CAAA;YACH,CAAC;iBAAM,CAAC;gBACP,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;oBACzC,MAAM,SAAS,GAAI,iBAA0C,CAAC,UAAU,CAAC,IAAI,QAAQ,CAAA;oBAErF,OAAO,CAAC,UAAU,EAAE,SAAS,CAAC,CAAA;gBAC/B,CAAC,CAAC,CAAA;YACH,CAAC;QACF,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,IAAI,UAAU,GAAG,IAAI,EAAE,CAAC;gBACvB,UAAU,EAAE,CAAA;gBACZ,SAAQ;YACT,CAAC;YAED,IAAI,UAAU,IAAI,UAAU;gBAAE,MAAK;YAEnC,MAAM,YAAY,GAAG,eAAe,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA;YACnD,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;YAEhF,MAAM,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;QACzD,CAAC;IACF,CAAC;IAqCD;;;;;;;;OAQG;IACH,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CACtB,MAA8C,EAC9C,OAAkD,EAAE;QAEpD,MAAM;QACL,MAAM;QACN,MAAM,GAAG,IAAI,EACb,IAAI,GAAG,OAAO,EACd,YAAY,EAAE,iBAAiB,GAAG,EAAE,EACpC,aAAa,GAAG,IAAI,KAAK,OAAO,EAChC,eAAe,EAAE,oBAAoB,EACrC,IAAI,GAAG,QAAQ,EACf,IAAI,GAAG,CAAC,EACR,GAAG,OAAO,EACV,GAAG,IAAI,CAAA;QAER,MAAM,OAAO,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAA;QAC5C,IAAI,YAAY,GAA0B,EAAE,CAAA;QAC5C,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,MAAM,UAAU,GAAG,IAAI,GAAG,IAAI,CAAA;QAE9B,MAAM,eAAe,GAAG,IAAI,iBAAiB,CAAC,oBAAoB,IAAI,IAAI,CAAC,eAAe,CAAC,CAAA;QAC3F,MAAM,qBAAqB,GAAoB,EAAE,SAAS,EAAE,eAAe,EAAE,SAAS,EAAE,KAAK,EAAE,CAAA;QAE/F,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QAEjC,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAEpD,IAAI,MAAM,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAA;YAEhC,IAAI,MAAM,CAAC,IAAI;gBAAE,OAAM;YAEvB,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,qBAAqB,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC,CAAA;YAC5F,MAAM,OAAO,GAAG,aAAa,CAAC,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;YAEvE,IAAI,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,CAAC;gBACtC,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,iBAAiB,CAAC,EAAE,CAAC,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC;oBAC7F,UAAW;oBACX,WAAW,IAAI,QAAQ;iBACvB,CAAC,CAAA;YACH,CAAC;iBAAM,CAAC;gBACP,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;oBACzC,MAAM,SAAS,GAAI,iBAA0C,CAAC,UAAU,CAAC,IAAI,QAAQ,CAAA;oBAErF,OAAO,CAAC,UAAU,EAAE,SAAS,CAAC,CAAA;gBAC/B,CAAC,CAAC,CAAA;YACH,CAAC;QACF,CAAC;QAED,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YAC9B,IAAI,UAAU,GAAG,IAAI,EAAE,CAAC;gBACvB,UAAU,EAAE,CAAA;gBACZ,SAAQ;YACT,CAAC;YAED,IAAI,UAAU,IAAI,UAAU;gBAAE,MAAK;YAEnC,MAAM,YAAY,GAAG,eAAe,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA;YACnD,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;YAEhF,MAAM,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;YAExD,UAAU,EAAE,CAAA;QACb,CAAC;IACF,CAAC"}
|