spliterator 2.2.3 → 3.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -0
- package/out/index.d.ts +2 -1
- package/out/index.d.ts.map +1 -1
- package/out/index.js +2 -1
- package/out/index.js.map +1 -1
- package/out/lib/AsyncSpliterator.d.ts +35 -34
- package/out/lib/AsyncSpliterator.d.ts.map +1 -1
- package/out/lib/AsyncSpliterator.js +93 -84
- package/out/lib/AsyncSpliterator.js.map +1 -1
- package/out/lib/BufferController.d.ts +10 -12
- package/out/lib/BufferController.d.ts.map +1 -1
- package/out/lib/BufferController.js +10 -12
- package/out/lib/BufferController.js.map +1 -1
- package/out/lib/CSVSpliterator.d.ts +6 -5
- package/out/lib/CSVSpliterator.d.ts.map +1 -1
- package/out/lib/CSVSpliterator.js +8 -5
- package/out/lib/CSVSpliterator.js.map +1 -1
- package/out/lib/CharacterSequence.d.ts +15 -64
- package/out/lib/CharacterSequence.d.ts.map +1 -1
- package/out/lib/CharacterSequence.js +194 -88
- package/out/lib/CharacterSequence.js.map +1 -1
- package/out/lib/CompositeDataView.d.ts +4 -5
- package/out/lib/CompositeDataView.d.ts.map +1 -1
- package/out/lib/CompositeDataView.js +2 -3
- package/out/lib/CompositeDataView.js.map +1 -1
- package/out/lib/IndexQueue.d.ts +1 -1
- package/out/lib/IndexQueue.d.ts.map +1 -1
- package/out/lib/IndexQueue.js +40 -11
- package/out/lib/IndexQueue.js.map +1 -1
- package/out/lib/JSONSpliterator.d.ts +1 -1
- package/out/lib/JSONSpliterator.js +1 -1
- package/out/lib/PSVSpliterator.d.ts +1 -1
- package/out/lib/PSVSpliterator.js +1 -1
- package/out/lib/SlidingWindow.d.ts +1 -1
- package/out/lib/SlidingWindow.d.ts.map +1 -1
- package/out/lib/SlidingWindow.js +11 -10
- package/out/lib/SlidingWindow.js.map +1 -1
- package/out/lib/Spliterator.d.ts +2 -3
- package/out/lib/Spliterator.d.ts.map +1 -1
- package/out/lib/Spliterator.js +102 -114
- package/out/lib/Spliterator.js.map +1 -1
- package/out/lib/TSVSpliterator.d.ts +1 -1
- package/out/lib/TSVSpliterator.js +1 -1
- package/out/lib/TextSpliterator.d.ts +4 -3
- package/out/lib/TextSpliterator.d.ts.map +1 -1
- package/out/lib/TextSpliterator.js +3 -1
- package/out/lib/TextSpliterator.js.map +1 -1
- package/out/lib/casing.d.ts +2 -3
- package/out/lib/casing.d.ts.map +1 -1
- package/out/lib/casing.js +2 -3
- package/out/lib/casing.js.map +1 -1
- package/out/lib/segment-runtime.d.ts +27 -0
- package/out/lib/segment-runtime.d.ts.map +1 -0
- package/out/lib/segment-runtime.js +38 -0
- package/out/lib/segment-runtime.js.map +1 -0
- package/out/lib/segment-worker-entry.d.ts +10 -0
- package/out/lib/segment-worker-entry.d.ts.map +1 -0
- package/out/lib/segment-worker-entry.js +56 -0
- package/out/lib/segment-worker-entry.js.map +1 -0
- package/out/lib/segment-workers.d.ts +49 -0
- package/out/lib/segment-workers.d.ts.map +1 -0
- package/out/lib/segment-workers.js +138 -0
- package/out/lib/segment-workers.js.map +1 -0
- package/out/lib/segments.d.ts +25 -0
- package/out/lib/segments.d.ts.map +1 -0
- package/out/lib/segments.js +50 -0
- package/out/lib/segments.js.map +1 -0
- package/out/lib/shared.d.ts +6 -5
- package/out/lib/shared.d.ts.map +1 -1
- package/out/lib/shared.js +3 -1
- package/out/lib/shared.js.map +1 -1
- package/out/lib/take.d.ts +3 -3
- package/out/lib/take.d.ts.map +1 -1
- package/out/lib/take.js +3 -3
- package/out/lib/take.js.map +1 -1
- package/out/lib/wasm_base64.d.ts +16 -0
- package/out/lib/wasm_base64.d.ts.map +1 -0
- package/out/lib/wasm_base64.js +16 -0
- package/out/lib/wasm_base64.js.map +1 -0
- package/out/lib/wasm_module.d.ts +36 -0
- package/out/lib/wasm_module.d.ts.map +1 -0
- package/out/lib/wasm_module.js +33 -0
- package/out/lib/wasm_module.js.map +1 -0
- package/out/lib/writer.d.ts +1 -1
- package/out/lib/writer.d.ts.map +1 -1
- package/out/lib/writer.js +1 -1
- package/out/lib/writer.js.map +1 -1
- package/out/node/cli/commands/csv.d.ts +1 -1
- package/out/node/cli/commands/csv.d.ts.map +1 -1
- package/out/node/cli/commands/csv.js +1 -1
- package/out/node/cli/commands/csv.js.map +1 -1
- package/out/node/cli/commands/iterate.d.ts +1 -1
- package/out/node/cli/commands/iterate.d.ts.map +1 -1
- package/out/node/cli/commands/iterate.js +1 -1
- package/out/node/cli/commands/iterate.js.map +1 -1
- package/out/node/cli/index.d.ts +1 -1
- package/out/node/cli/index.js +1 -1
- package/out/node/cli/utils.d.ts +1 -1
- package/out/node/cli/utils.js +1 -1
- package/out/node/fs/index.d.ts +16 -4
- package/out/node/fs/index.d.ts.map +1 -1
- package/out/node/fs/index.js +28 -4
- package/out/node/fs/index.js.map +1 -1
- package/package.json +4 -2
package/README.md
CHANGED
|
@@ -135,6 +135,48 @@ for await (const line of stream) {
|
|
|
135
135
|
}
|
|
136
136
|
```
|
|
137
137
|
|
|
138
|
+
### SIMD acceleration
|
|
139
|
+
|
|
140
|
+
Spliterator ships a small WebAssembly SIMD scanner that accelerates delimiter and quote scanning (roughly 5–6× over the JavaScript scanner for multi-byte delimiters, more for column splitting). It is embedded in the package — no extra files, fetches, or configuration.
|
|
141
|
+
|
|
142
|
+
The module loads **asynchronously**. Asynchronous parsing (`fromAsync`, streams) picks it up automatically once loaded. Purely synchronous parsing that finishes in a single tick would otherwise complete before the module is ready and transparently use the JavaScript scanner — to opt in, await it first:
|
|
143
|
+
|
|
144
|
+
```ts
|
|
145
|
+
import { CharacterSequence, CSVSpliterator } from "spliterator"
|
|
146
|
+
|
|
147
|
+
await CharacterSequence.whenReady() // resolves to true once the SIMD scanner is active
|
|
148
|
+
|
|
149
|
+
for (const row of CSVSpliterator.from(largeCsvString)) {
|
|
150
|
+
// ...now backed by the SIMD scanner
|
|
151
|
+
}
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
Correctness is identical either way; `whenReady()` only affects which scanner runs.
|
|
155
|
+
|
|
156
|
+
### Parallel parsing across threads
|
|
157
|
+
|
|
158
|
+
For one large file with a CPU-bound per-row transform, `AsyncSpliterator.asManyWorkers` splits the file into delimiter-aligned segments and runs a handler module across worker threads — each worker owns its own handle and reads only its segment. Results stream back to the main thread as a single async iterator, for a single-thread writer (a database, a JSONL file).
|
|
159
|
+
|
|
160
|
+
```ts
|
|
161
|
+
import { AsyncSpliterator } from "spliterator"
|
|
162
|
+
|
|
163
|
+
// transform.js (runs in each worker; top-level code is per-worker init):
|
|
164
|
+
// const dec = new TextDecoder(), enc = new TextEncoder()
|
|
165
|
+
// export function handleRecord(bytes) {
|
|
166
|
+
// return enc.encode(JSON.stringify(parse(dec.decode(bytes))) + "\n") // Uint8Array → zero-copy
|
|
167
|
+
// }
|
|
168
|
+
|
|
169
|
+
for await (const jsonLine of AsyncSpliterator.asManyWorkers<Uint8Array>("huge.csv", {
|
|
170
|
+
worker: new URL("./transform.js", import.meta.url),
|
|
171
|
+
delimiter: "\n",
|
|
172
|
+
concurrency: 8,
|
|
173
|
+
})) {
|
|
174
|
+
out.write(jsonLine) // single-thread writer on main
|
|
175
|
+
}
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
Need just the byte ranges to drive your own pool? `AsyncSpliterator.segments(path, { delimiter, concurrency })` returns them.
|
|
179
|
+
|
|
138
180
|
### Custom generators
|
|
139
181
|
|
|
140
182
|
While Spliterator includes premade exports for most use-cases, custom generators can be created via `Spliterator` and `AsyncSpliterator`. This class is a low-level interface that allows you to create your own generators for any kind of delimited content.
|
package/out/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @copyright Sister Software
|
|
3
|
-
* @license
|
|
3
|
+
* @license MIT
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
export * from "./lib/BufferController.js";
|
|
@@ -11,6 +11,7 @@ export * from "./lib/CSVSpliterator.js";
|
|
|
11
11
|
export * from "./lib/IndexQueue.js";
|
|
12
12
|
export * from "./lib/JSONSpliterator.js";
|
|
13
13
|
export * from "./lib/PSVSpliterator.js";
|
|
14
|
+
export * from "./lib/segments.js";
|
|
14
15
|
export * from "./lib/shared.js";
|
|
15
16
|
export * from "./lib/SlidingWindow.js";
|
|
16
17
|
export * from "./lib/Spliterator.js";
|
package/out/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,2BAA2B,CAAA;AACzC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,4BAA4B,CAAA;AAC1C,cAAc,4BAA4B,CAAA;AAC1C,cAAc,yBAAyB,CAAA;AACvC,cAAc,qBAAqB,CAAA;AACnC,cAAc,0BAA0B,CAAA;AACxC,cAAc,yBAAyB,CAAA;AACvC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,wBAAwB,CAAA;AACtC,cAAc,sBAAsB,CAAA;AACpC,cAAc,eAAe,CAAA;AAC7B,cAAc,0BAA0B,CAAA;AACxC,cAAc,yBAAyB,CAAA;AACvC,cAAc,iBAAiB,CAAA"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,2BAA2B,CAAA;AACzC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,4BAA4B,CAAA;AAC1C,cAAc,4BAA4B,CAAA;AAC1C,cAAc,yBAAyB,CAAA;AACvC,cAAc,qBAAqB,CAAA;AACnC,cAAc,0BAA0B,CAAA;AACxC,cAAc,yBAAyB,CAAA;AACvC,cAAc,mBAAmB,CAAA;AACjC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,wBAAwB,CAAA;AACtC,cAAc,sBAAsB,CAAA;AACpC,cAAc,eAAe,CAAA;AAC7B,cAAc,0BAA0B,CAAA;AACxC,cAAc,yBAAyB,CAAA;AACvC,cAAc,iBAAiB,CAAA"}
|
package/out/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @copyright Sister Software
|
|
3
|
-
* @license
|
|
3
|
+
* @license MIT
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
export * from "./lib/BufferController.js";
|
|
@@ -11,6 +11,7 @@ export * from "./lib/CSVSpliterator.js";
|
|
|
11
11
|
export * from "./lib/IndexQueue.js";
|
|
12
12
|
export * from "./lib/JSONSpliterator.js";
|
|
13
13
|
export * from "./lib/PSVSpliterator.js";
|
|
14
|
+
export * from "./lib/segments.js";
|
|
14
15
|
export * from "./lib/shared.js";
|
|
15
16
|
export * from "./lib/SlidingWindow.js";
|
|
16
17
|
export * from "./lib/Spliterator.js";
|
package/out/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,2BAA2B,CAAA;AACzC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,4BAA4B,CAAA;AAC1C,cAAc,4BAA4B,CAAA;AAC1C,cAAc,yBAAyB,CAAA;AACvC,cAAc,qBAAqB,CAAA;AACnC,cAAc,0BAA0B,CAAA;AACxC,cAAc,yBAAyB,CAAA;AACvC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,wBAAwB,CAAA;AACtC,cAAc,sBAAsB,CAAA;AACpC,cAAc,eAAe,CAAA;AAC7B,cAAc,0BAA0B,CAAA;AACxC,cAAc,yBAAyB,CAAA;AACvC,cAAc,iBAAiB,CAAA"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,2BAA2B,CAAA;AACzC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,4BAA4B,CAAA;AAC1C,cAAc,4BAA4B,CAAA;AAC1C,cAAc,yBAAyB,CAAA;AACvC,cAAc,qBAAqB,CAAA;AACnC,cAAc,0BAA0B,CAAA;AACxC,cAAc,yBAAyB,CAAA;AACvC,cAAc,mBAAmB,CAAA;AACjC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,wBAAwB,CAAA;AACtC,cAAc,sBAAsB,CAAA;AACpC,cAAc,eAAe,CAAA;AAC7B,cAAc,0BAA0B,CAAA;AACxC,cAAc,yBAAyB,CAAA;AACvC,cAAc,iBAAiB,CAAA"}
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @copyright Sister Software
|
|
3
|
-
* @license
|
|
3
|
+
* @license MIT
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
import { ReadableStream, type ReadableWritablePair, type StreamPipeOptions } from "stream/web";
|
|
7
7
|
import { type CharacterSequenceInput } from "./CharacterSequence.js";
|
|
8
|
-
import
|
|
8
|
+
import { type AsManyWorkersOptions } from "./segment-workers.js";
|
|
9
|
+
import { type SegmentOptions } from "./segments.js";
|
|
10
|
+
import type { AsyncChunkIterator, AsyncDataResource, ByteRange } from "./shared.js";
|
|
9
11
|
/**
|
|
10
12
|
* Initialization options for creating a spliterator.
|
|
11
13
|
*/
|
|
@@ -46,6 +48,13 @@ export interface SpliteratorInit {
|
|
|
46
48
|
* @default Infinity
|
|
47
49
|
*/
|
|
48
50
|
take?: number;
|
|
51
|
+
/**
|
|
52
|
+
* Whether to handle quoted fields (e.g., CSV). When enabled, delimiters inside double-quoted fields are ignored and
|
|
53
|
+
* quoted fields are emitted without quotes.
|
|
54
|
+
*
|
|
55
|
+
* @default false
|
|
56
|
+
*/
|
|
57
|
+
enableQuoteHandling?: boolean;
|
|
49
58
|
/**
|
|
50
59
|
* Whether to emit debug information.
|
|
51
60
|
*/
|
|
@@ -56,8 +65,7 @@ export interface SpliteratorInit {
|
|
|
56
65
|
*/
|
|
57
66
|
export interface AsyncSpliteratorInit extends SpliteratorInit {
|
|
58
67
|
/**
|
|
59
|
-
* The buffer chunk size to read from the file, i.e. the high-water mark for the file read
|
|
60
|
-
* operation.
|
|
68
|
+
* The buffer chunk size to read from the file, i.e. the high-water mark for the file read operation.
|
|
61
69
|
*
|
|
62
70
|
* @default BlockSize * 16
|
|
63
71
|
* @minimum The delimiter byte length * 4 or block size of the file system. Whichever is greater.
|
|
@@ -80,7 +88,7 @@ export interface AsyncSpliteratorInit extends SpliteratorInit {
|
|
|
80
88
|
* const file = await AsyncSpliterator.from("data.csv")
|
|
81
89
|
*
|
|
82
90
|
* for await (const line of file) {
|
|
83
|
-
*
|
|
91
|
+
* console.log(line)
|
|
84
92
|
* }
|
|
85
93
|
* ```
|
|
86
94
|
*
|
|
@@ -170,39 +178,32 @@ export declare class AsyncSpliterator<R extends Uint8Array | DataView | ArrayBuf
|
|
|
170
178
|
[Symbol.asyncIterator](): this;
|
|
171
179
|
return(): Promise<IteratorReturnResult<undefined>>;
|
|
172
180
|
/**
|
|
173
|
-
*
|
|
174
|
-
*
|
|
175
|
-
*
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
*
|
|
180
|
-
*
|
|
181
|
-
*
|
|
182
|
-
* You could add more people to the task by laying out all the pages, measuring the length and
|
|
183
|
-
* assigning each person a length of pages to traverse.
|
|
184
|
-
*
|
|
185
|
-
* There's a few ways to go about this:
|
|
186
|
-
*
|
|
187
|
-
* We could approach this in serial -- having the first worker start from page 1 and scanning
|
|
188
|
-
* until they find the beginning of the next chapter, handing off the range to the next worker.
|
|
189
|
-
* This is how `AsyncSpliterator` works.
|
|
181
|
+
* Compute delimiter-aligned `[start, end)` byte ranges dividing `source` into up to `concurrency` segments. The
|
|
182
|
+
* boundary primitive for parallel parsing — hand each range to a worker (with its own handle). See
|
|
183
|
+
* {@linkcode asManyWorkers} for the turnkey worker version.
|
|
184
|
+
*/
|
|
185
|
+
static segments(source: AsyncDataResource, options: SegmentOptions): Promise<ByteRange[]>;
|
|
186
|
+
/**
|
|
187
|
+
* Split `source` into delimiter-aligned segments and return one {@linkcode AsyncSpliterator} per segment. All share
|
|
188
|
+
* the event loop (no worker threads) — for moderate jobs or to overlap I/O. Returns at most `concurrency` instances.
|
|
190
189
|
*
|
|
191
|
-
*
|
|
192
|
-
*
|
|
193
|
-
* able to scan their own _length_ of pages simultaneously, and settle up on the boundaries of the
|
|
194
|
-
* chapters they find.
|
|
190
|
+
* @param source - The data resource to read from.
|
|
191
|
+
* @param options - Delimiter, desired concurrency, and optional probe size.
|
|
195
192
|
*
|
|
196
|
-
* `AsyncSpliterator
|
|
197
|
-
|
|
193
|
+
* @returns One `AsyncSpliterator` per non-empty segment, possibly fewer than `concurrency`.
|
|
194
|
+
*/
|
|
195
|
+
static asMany(source: AsyncDataResource, options: SegmentOptions): Promise<AsyncSpliterator[]>;
|
|
196
|
+
/**
|
|
197
|
+
* Parse `source` across worker threads. Each worker owns a handle to a delimiter-aligned segment, runs the `worker`
|
|
198
|
+
* handler module per record, and streams results back to the main thread as one merged async iterator — for a
|
|
199
|
+
* single-thread writer. Results interleave across segments.
|
|
198
200
|
*
|
|
199
|
-
*
|
|
200
|
-
*
|
|
201
|
-
* @param concurrency - The _desired_ number of `AsyncSpliterator` instances to create.
|
|
201
|
+
* The handler returns a value (structured-cloned back), a `Uint8Array` (transferred zero-copy), or `undefined` (the
|
|
202
|
+
* record is skipped). The handler module's top-level code runs once per worker — load models/handles there.
|
|
202
203
|
*
|
|
203
|
-
* @
|
|
204
|
-
* @
|
|
204
|
+
* @param source A file path or URL (file handles cannot cross threads → `TypeError` otherwise).
|
|
205
|
+
* @param options Handler module, delimiter, concurrency, and batching/backpressure tuning.
|
|
205
206
|
*/
|
|
206
|
-
static
|
|
207
|
+
static asManyWorkers<R = unknown>(source: AsyncDataResource, options: AsManyWorkersOptions): AsyncIterableIterator<R>;
|
|
207
208
|
}
|
|
208
209
|
//# sourceMappingURL=AsyncSpliterator.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AsyncSpliterator.d.ts","sourceRoot":"","sources":["../../lib/AsyncSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,cAAc,EAAE,KAAK,oBAAoB,EAAE,KAAK,iBAAiB,EAAE,MAAM,YAAY,CAAA;
|
|
1
|
+
{"version":3,"file":"AsyncSpliterator.d.ts","sourceRoot":"","sources":["../../lib/AsyncSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,cAAc,EAAE,KAAK,oBAAoB,EAAE,KAAK,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAG9F,OAAO,EAAqB,KAAK,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AAEvF,OAAO,EAAE,KAAK,oBAAoB,EAAqB,MAAM,sBAAsB,CAAA;AACnF,OAAO,EAAmB,KAAK,cAAc,EAAE,MAAM,eAAe,CAAA;AACpE,OAAO,KAAK,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAInF;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B;;OAEG;IACH,SAAS,CAAC,EAAE,sBAAsB,CAAA;IAElC;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IAEjB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,OAAO,CAAA;IAEnB;;;;OAIG;IACH,IAAI,CAAC,EAAE,MAAM,CAAA;IAEb;;;;;;;;OAQG;IACH,IAAI,CAAC,EAAE,MAAM,CAAA;IAEb;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAA;IAE7B;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,CAAA;CACf;AAED;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,eAAe;IAC5D;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IAEtB;;;;OAIG;IACH,WAAW,CAAC,EAAE,OAAO,CAAA;CACrB;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,QAAQ,GAAG,WAAW,GAAG,UAAU,CACvF,YAAW,qBAAqB,CAAC,CAAC,CAAC,EAAE,eAAe;;IAIpD;;;;;;;OAOG;IACH,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,kBAAkB,EAAE,IAAI,CAAC,EAAE,oBAAoB,GAAG,gBAAgB;IACtF;;;;;;;OAOG;IACH,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,iBAAiB,EAAE,IAAI,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAC9F;;;;;;;OAOG;IACH,MAAM,CAAC,IAAI,CACV,MAAM,EAAE,iBAAiB,GAAG,kBAAkB,EAC9C,IAAI,CAAC,EAAE,oBAAoB,GACzB,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB/C;;OAEG;IACU,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC;IAWnD;;OAEG;WACW,iBAAiB,CAAC,IAAI,EAAE,eAAe,GAAG,eAAe,CAAC,kBAAkB,EAAE,UAAU,CAAC;IAYvG;;;;;;OAMG;gBACS,MAAM,EAAE,kBAAkB,EAAE,IAAI,GAAE,oBAAyB;IAuP1D,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAiD/C;;;;OAIG;IACI,OAAO,IAAI,OAAO,CAAC,CAAC,EAAE,CAAC;IAI9B;;;;OAIG;IACI,cAAc,CAAC,OAAO,kCAAoB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAIrE;;;;;;OAMG;IACI,gBAAgB,IAAI,cAAc,CAAC,CAAC,CAAC;IAiB5C;;;;;;;;;;OAUG;IACI,WAAW,CAAC,CAAC,EAAE,SAAS,EAAE,oBAAoB,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,cAAc,CAAC,CAAC,CAAC;IAI5G;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC;IAItB,MAAM,IAAI,OAAO,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC;IAQzD;;;;OAIG;WACW,QAAQ,CAAC,MAAM,EAAE,iBAAiB,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAIhG;;;;;;;;OAQG;WACiB,MAAM,CAAC,MAAM,EAAE,iBAAiB,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAc3G;;;;;;;;;;OAUG;WACW,aAAa,CAAC,CAAC,GAAG,OAAO,EACtC,MAAM,EAAE,iBAAiB,EACzB,OAAO,EAAE,oBAAoB,GAC3B,qBAAqB,CAAC,CAAC,CAAC;CAQ3B"}
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @copyright Sister Software
|
|
3
|
-
* @license
|
|
3
|
+
* @license MIT
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
import { ReadableStream } from "stream/web";
|
|
7
7
|
import { BufferController } from "./BufferController.js";
|
|
8
8
|
import { CharacterSequence } from "./CharacterSequence.js";
|
|
9
9
|
import { IndexQueue } from "./IndexQueue.js";
|
|
10
|
+
import { runSegmentWorkers } from "./segment-workers.js";
|
|
11
|
+
import { computeSegments } from "./segments.js";
|
|
10
12
|
const noop = () => void 0;
|
|
11
13
|
/**
|
|
12
14
|
* An asynchronous spliterator for reading delimited byte streams.
|
|
@@ -17,7 +19,7 @@ const noop = () => void 0;
|
|
|
17
19
|
* const file = await AsyncSpliterator.from("data.csv")
|
|
18
20
|
*
|
|
19
21
|
* for await (const line of file) {
|
|
20
|
-
*
|
|
22
|
+
* console.log(line)
|
|
21
23
|
* }
|
|
22
24
|
* ```
|
|
23
25
|
*
|
|
@@ -41,6 +43,7 @@ export class AsyncSpliterator {
|
|
|
41
43
|
async [Symbol.asyncDispose]() {
|
|
42
44
|
this.#indices.clear();
|
|
43
45
|
this.#controller.clear();
|
|
46
|
+
await this.#closeReader();
|
|
44
47
|
if (this.#autoDispose) {
|
|
45
48
|
await this.#source[Symbol.asyncDispose]?.();
|
|
46
49
|
}
|
|
@@ -103,8 +106,7 @@ export class AsyncSpliterator {
|
|
|
103
106
|
*
|
|
104
107
|
* Note that indices are relative to the buffer, not the file.
|
|
105
108
|
*
|
|
106
|
-
* This means that the start index is always 0, and the end index is the byte length of the
|
|
107
|
-
* buffer.
|
|
109
|
+
* This means that the start index is always 0, and the end index is the byte length of the buffer.
|
|
108
110
|
*/
|
|
109
111
|
#indices = new IndexQueue();
|
|
110
112
|
/**
|
|
@@ -112,8 +114,8 @@ export class AsyncSpliterator {
|
|
|
112
114
|
*/
|
|
113
115
|
#needle;
|
|
114
116
|
/**
|
|
115
|
-
* The buffer containing the current chunks of data. This will be resized as needed to accommodate
|
|
116
|
-
*
|
|
117
|
+
* The buffer containing the current chunks of data. This will be resized as needed to accommodate the file read
|
|
118
|
+
* operation.
|
|
117
119
|
*/
|
|
118
120
|
#controller;
|
|
119
121
|
/**
|
|
@@ -131,8 +133,7 @@ export class AsyncSpliterator {
|
|
|
131
133
|
/**
|
|
132
134
|
* The high water mark for the buffer.
|
|
133
135
|
*
|
|
134
|
-
* This defines the size of each read operation, as well as the total size of indices to keep in
|
|
135
|
-
* memory.
|
|
136
|
+
* This defines the size of each read operation, as well as the total size of indices to keep in memory.
|
|
136
137
|
*/
|
|
137
138
|
#highWaterMark;
|
|
138
139
|
/**
|
|
@@ -165,8 +166,8 @@ export class AsyncSpliterator {
|
|
|
165
166
|
/**
|
|
166
167
|
* Find and enqueue delimiter positions.
|
|
167
168
|
*
|
|
168
|
-
* The search is bounded by `bytesWritten` so we never scan into uninitialized buffer memory and
|
|
169
|
-
*
|
|
169
|
+
* The search is bounded by `bytesWritten` so we never scan into uninitialized buffer memory and never enqueue ranges
|
|
170
|
+
* whose `end` exceeds the valid byte length.
|
|
170
171
|
*/
|
|
171
172
|
#search() {
|
|
172
173
|
const lastByteRange = this.#indices.peekLast();
|
|
@@ -230,11 +231,21 @@ export class AsyncSpliterator {
|
|
|
230
231
|
this.#search();
|
|
231
232
|
}
|
|
232
233
|
}
|
|
234
|
+
/** Signal the underlying chunk reader to stop, destroying an owned read stream (closing its fd). */
|
|
235
|
+
async #closeReader() {
|
|
236
|
+
try {
|
|
237
|
+
await this.#chunkReader.return?.();
|
|
238
|
+
}
|
|
239
|
+
catch {
|
|
240
|
+
// The reader may already be closed; ignore.
|
|
241
|
+
}
|
|
242
|
+
}
|
|
233
243
|
async #finalize() {
|
|
244
|
+
await this.#closeReader();
|
|
234
245
|
if (this.#debug) {
|
|
235
246
|
/**
|
|
236
|
-
* The total number of bytes we expect to be omitted from the buffer. This is derived from the
|
|
237
|
-
*
|
|
247
|
+
* The total number of bytes we expect to be omitted from the buffer. This is derived from the number of of yields
|
|
248
|
+
* and the length of the delimiter.
|
|
238
249
|
*/
|
|
239
250
|
const expectedYieldedDelimitedBytes = (this.#yieldCount - 1) * this.#needle.length;
|
|
240
251
|
this.#log({
|
|
@@ -254,37 +265,41 @@ export class AsyncSpliterator {
|
|
|
254
265
|
//#endregion
|
|
255
266
|
//#region Iterator Methods
|
|
256
267
|
async next() {
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
268
|
+
// Loop rather than recurse: a long run of skipped (empty or dropped) ranges would
|
|
269
|
+
// otherwise grow the call stack one frame per skip and overflow.
|
|
270
|
+
while (true) {
|
|
271
|
+
if (this.#done || this.#yieldCount >= this.#yieldStopCount)
|
|
272
|
+
return this.#finalize();
|
|
273
|
+
if (this.#indices.size === 0) {
|
|
274
|
+
await this.#fill();
|
|
275
|
+
}
|
|
276
|
+
// The first `#fill` may have read into EOF without entering the EOF branch (the branch only
|
|
277
|
+
// triggers when fill is called with `done` already set). In that case the trailing tail
|
|
278
|
+
// hasn't been enqueued yet, so call fill once more to let the EOF branch flush it.
|
|
279
|
+
if (this.#lastReadResult?.done && this.#indices.size === 0 && !this.#done) {
|
|
280
|
+
await this.#fill();
|
|
281
|
+
}
|
|
282
|
+
const currentByteRange = this.#indices.dequeue();
|
|
283
|
+
this.#lastByteRangeSeen = currentByteRange;
|
|
284
|
+
if (!currentByteRange) {
|
|
285
|
+
this.#done = true;
|
|
286
|
+
return this.#finalize();
|
|
287
|
+
}
|
|
288
|
+
const [start, end] = currentByteRange;
|
|
289
|
+
const slice = this.#controller.subarray(start, end);
|
|
290
|
+
if (slice.length === 0 && this.#skipEmpty) {
|
|
291
|
+
continue;
|
|
292
|
+
}
|
|
293
|
+
this.#yieldCount++;
|
|
294
|
+
if (this.#yieldCount <= this.#yieldDropCount) {
|
|
295
|
+
continue;
|
|
296
|
+
}
|
|
297
|
+
this.#yieldedByteLength += end - start;
|
|
298
|
+
return {
|
|
299
|
+
value: slice,
|
|
300
|
+
done: false,
|
|
301
|
+
};
|
|
282
302
|
}
|
|
283
|
-
this.#yieldedByteLength += end - start;
|
|
284
|
-
return {
|
|
285
|
-
value: slice,
|
|
286
|
-
done: false,
|
|
287
|
-
};
|
|
288
303
|
}
|
|
289
304
|
/**
|
|
290
305
|
* Collect all the byte ranges from the file.
|
|
@@ -350,54 +365,48 @@ export class AsyncSpliterator {
|
|
|
350
365
|
//#endregion
|
|
351
366
|
//#region Experimental
|
|
352
367
|
/**
|
|
353
|
-
*
|
|
354
|
-
*
|
|
355
|
-
*
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
*
|
|
362
|
-
*
|
|
363
|
-
* assigning each person a length of pages to traverse.
|
|
364
|
-
*
|
|
365
|
-
* There's a few ways to go about this:
|
|
366
|
-
*
|
|
367
|
-
* We could approach this in serial -- having the first worker start from page 1 and scanning
|
|
368
|
-
* until they find the beginning of the next chapter, handing off the range to the next worker.
|
|
369
|
-
* This is how `AsyncSpliterator` works.
|
|
368
|
+
* Compute delimiter-aligned `[start, end)` byte ranges dividing `source` into up to `concurrency` segments. The
|
|
369
|
+
* boundary primitive for parallel parsing — hand each range to a worker (with its own handle). See
|
|
370
|
+
* {@linkcode asManyWorkers} for the turnkey worker version.
|
|
371
|
+
*/
|
|
372
|
+
static segments(source, options) {
|
|
373
|
+
return computeSegments(source, options);
|
|
374
|
+
}
|
|
375
|
+
/**
|
|
376
|
+
* Split `source` into delimiter-aligned segments and return one {@linkcode AsyncSpliterator} per segment. All share
|
|
377
|
+
* the event loop (no worker threads) — for moderate jobs or to overlap I/O. Returns at most `concurrency` instances.
|
|
370
378
|
*
|
|
371
|
-
*
|
|
372
|
-
*
|
|
373
|
-
* able to scan their own _length_ of pages simultaneously, and settle up on the boundaries of the
|
|
374
|
-
* chapters they find.
|
|
379
|
+
* @param source - The data resource to read from.
|
|
380
|
+
* @param options - Delimiter, desired concurrency, and optional probe size.
|
|
375
381
|
*
|
|
376
|
-
* `AsyncSpliterator
|
|
377
|
-
|
|
382
|
+
* @returns One `AsyncSpliterator` per non-empty segment, possibly fewer than `concurrency`.
|
|
383
|
+
*/
|
|
384
|
+
static async asMany(source, options) {
|
|
385
|
+
const { createChunkIterator } = await import("spliterator/node/fs");
|
|
386
|
+
const segments = await computeSegments(source, options);
|
|
387
|
+
return Promise.all(segments.map(async ([start, end]) => {
|
|
388
|
+
// `end` is exclusive here; createChunkIterator's `end` is inclusive.
|
|
389
|
+
const chunkIterator = await createChunkIterator(source, { start, end: end - 1 });
|
|
390
|
+
return new AsyncSpliterator(chunkIterator, { delimiter: options.delimiter, autoDispose: true });
|
|
391
|
+
}));
|
|
392
|
+
}
|
|
393
|
+
/**
|
|
394
|
+
* Parse `source` across worker threads. Each worker owns a handle to a delimiter-aligned segment, runs the `worker`
|
|
395
|
+
* handler module per record, and streams results back to the main thread as one merged async iterator — for a
|
|
396
|
+
* single-thread writer. Results interleave across segments.
|
|
378
397
|
*
|
|
379
|
-
*
|
|
380
|
-
*
|
|
381
|
-
* @param concurrency - The _desired_ number of `AsyncSpliterator` instances to create.
|
|
398
|
+
* The handler returns a value (structured-cloned back), a `Uint8Array` (transferred zero-copy), or `undefined` (the
|
|
399
|
+
* record is skipped). The handler module's top-level code runs once per worker — load models/handles there.
|
|
382
400
|
*
|
|
383
|
-
* @
|
|
384
|
-
* @
|
|
401
|
+
* @param source A file path or URL (file handles cannot cross threads → `TypeError` otherwise).
|
|
402
|
+
* @param options Handler module, delimiter, concurrency, and batching/backpressure tuning.
|
|
385
403
|
*/
|
|
386
|
-
static
|
|
387
|
-
//
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
const spliterators = [];
|
|
391
|
-
const { createChunkIterator, readFileSize } = await import("spliterator/node/fs");
|
|
392
|
-
const fileSize = await readFileSize(source);
|
|
393
|
-
// const file = createChunkIterator(source, {
|
|
394
|
-
// start: 0,
|
|
395
|
-
// })
|
|
396
|
-
// The idea here is we're going to split the file into `concurrency` chunks.
|
|
397
|
-
if (Date.now()) {
|
|
398
|
-
throw new Error("Not implemented.");
|
|
404
|
+
static asManyWorkers(source, options) {
|
|
405
|
+
// Validate eagerly so callers fail fast (the generator body is otherwise lazy until consumed).
|
|
406
|
+
if (typeof source !== "string" && !(source instanceof URL)) {
|
|
407
|
+
throw new TypeError("asManyWorkers requires a file path or URL — file handles cannot cross threads.");
|
|
399
408
|
}
|
|
400
|
-
return
|
|
409
|
+
return runSegmentWorkers(source, options);
|
|
401
410
|
}
|
|
402
411
|
}
|
|
403
412
|
//# sourceMappingURL=AsyncSpliterator.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AsyncSpliterator.js","sourceRoot":"","sources":["../../lib/AsyncSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,cAAc,EAAqD,MAAM,YAAY,CAAA;
|
|
1
|
+
{"version":3,"file":"AsyncSpliterator.js","sourceRoot":"","sources":["../../lib/AsyncSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,cAAc,EAAqD,MAAM,YAAY,CAAA;AAE9F,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AACxD,OAAO,EAAE,iBAAiB,EAA+B,MAAM,wBAAwB,CAAA;AACvF,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EAA6B,iBAAiB,EAAE,MAAM,sBAAsB,CAAA;AACnF,OAAO,EAAE,eAAe,EAAuB,MAAM,eAAe,CAAA;AAGpE,MAAM,IAAI,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC,CAAA;AAkFzB;;;;;;;;;;;;;;GAcG;AACH,MAAM,OAAO,gBAAgB;IAmC5B,MAAM,CAAC,IAAI,CACV,MAA8C,EAC9C,OAA6B,EAAE;QAE/B,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,aAAa,IAAI,MAAM,EAAE,CAAC;YAClE,OAAO,IAAI,gBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QAC1C,CAAC;QAED,OAAO,MAAM,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,mBAAmB,EAAE,EAAE,EAAE;YAC3E,MAAM,aAAa,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE;gBACvD,aAAa,EAAE,IAAI,CAAC,aAAa;aACjC,CAAC,CAAA;YAEF,OAAO,IAAI,gBAAgB,CAAC,aAAa,EAAE,IAAI,CAAC,CAAA;QACjD,CAAC,CAAC,CAAA;IACH,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QACjC,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;QACrB,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAA;QAExB,MAAM,IAAI,CAAC,YAAY,EAAE,CAAA;QAEzB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE,CAAA;QAC5C,CAAC;IACF,CAAC;IAED;;OAEG;IACI,MAAM,CAAC,iBAAiB,CAAC,IAAqB;QACpD,OAAO,IAAI,eAAe,CAAiC;YAC1D,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,UAAU;gBAChC,MAAM,WAAW,GAAG,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;gBAEtD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;oBACvC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;gBAC1B,CAAC;YACF,CAAC;SACD,CAAC,CAAA;IACH,CAAC;IAED;;;;;;OAMG;IACH,YAAY,MAA0B,EAAE,OAA6B,EAAE;QACtE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QACrB,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAA;QAElD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,WAAW,IAAI,KAAK,CAAA;QAE7C,IAAI,CAAC,OAAO,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QAEpD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,aAAa,IAAI,IAAI,GAAG,EAAE,CAAC,CAAA;QAExF,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAA;QAClD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,eAAe,CAAA;QAEhF,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAA;QAExC,IAAI,CAAC,WAAW,GAAG,IAAI,gBAAgB,CAAC;YACvC,iBAAiB,EAAE,IAAI,CAAC,cAAc;SACtC,CAAC,CAAA;QAEF,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,IAAI,KAAK,CAAA;QACjC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAC7D,CAAC;IAED,YAAY;IAEZ,4BAA4B;IAE5B;;OAEG;IACM,YAAY,CAAS;IAE9B;;OAEG;IACM,OAAO,CAAoB;IAEpC;;OAEG;IACM,YAAY,CAA2B;IAEhD;;OAEG;IACH,eAAe,CAAwC;IAEvD;;;;;;OAMG;IACM,QAAQ,GAAG,IAAI,UAAU,EAAE,CAAA;IAEpC;;OAEG;IACM,OAAO,CAAmB;IAEnC;;;OAGG;IACM,WAAW,CAAkB;IAEtC;;OAEG;IACM,eAAe,CAAQ;IAEhC;;OAEG;IACM,eAAe,CAAQ;IAEhC;;OAEG;IACM,UAAU,CAAS;IAE5B;;;;OAIG;IACM,cAAc,CAAQ;IAE/B;;OAEG;IACH,kBAAkB,GAAG,CAAC,CAAA;IAEtB;;OAEG;IACH,WAAW,GAAG,CAAC,CAAA;IAEf,MAAM;IACN,6DAA6D;IAC7D,MAAM;IACN,wBAAwB;IAExB;;OAEG;IACH,kBAAkB,CAAuB;IAEzC;;OAEG;IACH,MAAM,CAAS;IAEf;;OAEG;IACH,KAAK,GAAG,KAAK,CAAA;IAEb,YAAY;IAEZ,yBAAyB;IAEzB,IAAI,CAA0B;IAE9B;;;;;OAKG;IACH,OAAO;QACN,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAA;QAC9C,IAAI,YAAY,GAAG,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;QAC7E,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,CAAA;QAE/C,OAAO,YAAY,IAAI,SAAS,EAAE,CAAC;YAClC,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,YAAY,EAAE,SAAS,CAAC,CAAA;YAE3F,IAAI,cAAc,KAAK,CAAC,CAAC,EAAE,CAAC;gBAC3B,IAAI,CAAC,IAAI,CAAC,iDAAiD,EAAE,EAAE,YAAY,EAAE,CAAC,CAAA;gBAC9E,MAAK;YACN,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,CAAA;YAC7D,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,CAAA;YAErD,YAAY,GAAG,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;QACpD,CAAC;IACF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACV,IAAI,CAAC,eAAe,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAA;QAErD,IAAI,IAAI,CAAC,eAAe,CAAC,IAAI;YAAE,OAAM;QAErC,kCAAkC;QAClC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,CAAA;IAChF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACV,IAAI,IAAI,CAAC,eAAe,EAAE,IAAI,EAAE,CAAC;YAChC,IAAI,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAA;YACxD,sFAAsF;YACtF,+EAA+E;YAC/E,kFAAkF;YAClF,kFAAkF;YAClF,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;YAChG,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,CAAA;YAE7C,IAAI,SAAS,IAAI,OAAO,EAAE,CAAC;gBAC1B,IAAI,CAAC,IAAI,CAAC,8BAA8B,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAA;gBAC/D,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAA;YAC5C,CAAC;iBAAM,CAAC;gBACP,IAAI,CAAC,IAAI,CAAC,2CAA2C,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAA;YAC/E,CAAC;YAED,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;YAEjB,OAAM;QACP,CAAC;QAED,yDAAyD;QACzD,wCAAwC;QACxC,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;QAEtG,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,eAAe,EAAE,CAAC,CAAA;QACpD,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAA;QAE1C,0FAA0F;QAC1F,yFAAyF;QACzF,0FAA0F;QAC1F,4EAA4E;QAC5E,IAAI,CAAC,kBAAkB,GAAG,SAAS,CAAA;QAEnC,OAAO,CAAC,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;YAChH,MAAM,IAAI,CAAC,KAAK,EAAE,CAAA;YAClB,IAAI,CAAC,OAAO,EAAE,CAAA;QACf,CAAC;IACF,CAAC;IAED,oGAAoG;IACpG,KAAK,CAAC,YAAY;QACjB,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,EAAE,CAAA;QACnC,CAAC;QAAC,MAAM,CAAC;YACR,4CAA4C;QAC7C,CAAC;IACF,CAAC;IAED,KAAK,CAAC,SAAS;QACd,MAAM,IAAI,CAAC,YAAY,EAAE,CAAA;QAEzB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB;;;eAGG;YACH,MAAM,6BAA6B,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;YAElF,IAAI,CAAC,IAAI,CAAC;gBACT,iBAAiB,EAAE,IAAI,CAAC,kBAAkB;gBAC1C,UAAU,EAAE,IAAI,CAAC,WAAW;gBAC5B,6BAA6B;aAC7B,CAAC,CAAA;QACH,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE,CAAA;QAC5C,CAAC;QAED,OAAO;YACN,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,SAAS;SAChB,CAAA;IACF,CAAC;IAED,YAAY;IAEZ,0BAA0B;IAEnB,KAAK,CAAC,IAAI;QAChB,kFAAkF;QAClF,iEAAiE;QACjE,OAAO,IAAI,EAAE,CAAC;YACb,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,eAAe;gBAAE,OAAO,IAAI,CAAC,SAAS,EAAE,CAAA;YAEnF,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;gBAC9B,MAAM,IAAI,CAAC,KAAK,EAAE,CAAA;YACnB,CAAC;YAED,4FAA4F;YAC5F,wFAAwF;YACxF,mFAAmF;YACnF,IAAI,IAAI,CAAC,eAAe,EAAE,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC3E,MAAM,IAAI,CAAC,KAAK,EAAE,CAAA;YACnB,CAAC;YAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAA;YAChD,IAAI,CAAC,kBAAkB,GAAG,gBAAgB,CAAA;YAE1C,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACvB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;gBAEjB,OAAO,IAAI,CAAC,SAAS,EAAE,CAAA;YACxB,CAAC;YAED,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,gBAAgB,CAAA;YAErC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;YAEnD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC3C,SAAQ;YACT,CAAC;YAED,IAAI,CAAC,WAAW,EAAE,CAAA;YAElB,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;gBAC9C,SAAQ;YACT,CAAC;YAED,IAAI,CAAC,kBAAkB,IAAI,GAAG,GAAG,KAAK,CAAA;YAEtC,OAAO;gBACN,KAAK,EAAE,KAAqB;gBAC5B,IAAI,EAAE,KAAK;aACX,CAAA;QACF,CAAC;IACF,CAAC;IAED;;;;OAIG;IACI,OAAO;QACb,OAAO,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;IAC7B,CAAC;IAED;;;;OAIG;IACI,cAAc,CAAC,OAAO,GAAG,IAAI,WAAW,EAAE;QAChD,OAAO,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;IAC/D,CAAC;IAED;;;;;;OAMG;IACI,gBAAgB;QACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAA;QAE7C,OAAO,IAAI,cAAc,CAAC;YACzB,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE;gBAC1B,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;gBAE7C,IAAI,IAAI,EAAE,CAAC;oBACV,UAAU,CAAC,KAAK,EAAE,CAAA;gBACnB,CAAC;qBAAM,CAAC;oBACP,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;gBAC1B,CAAC;YACF,CAAC;YACD,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE;SAC3C,CAAC,CAAA;IACH,CAAC;IAED;;;;;;;;;;OAUG;IACI,WAAW,CAAI,SAAqC,EAAE,OAA2B;QACvF,OAAO,IAAI,CAAC,gBAAgB,EAAE,CAAC,WAAW,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IAC/D,CAAC;IAED;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC;QAC5B,OAAO,IAAI,CAAA;IACZ,CAAC;IAEM,MAAM;QACZ,OAAO,IAAI,CAAC,SAAS,EAAE,CAAA;IACxB,CAAC;IAED,YAAY;IAEZ,sBAAsB;IAEtB;;;;OAIG;IACI,MAAM,CAAC,QAAQ,CAAC,MAAyB,EAAE,OAAuB;QACxE,OAAO,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IACxC,CAAC;IAED;;;;;;;;OAQG;IACI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAyB,EAAE,OAAuB;QAC5E,MAAM,EAAE,mBAAmB,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAA;QACnE,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAEvD,OAAO,OAAO,CAAC,GAAG,CACjB,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE;YACnC,qEAAqE;YACrE,MAAM,aAAa,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,GAAG,CAAC,EAAE,CAAC,CAAA;YAEhF,OAAO,IAAI,gBAAgB,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAA;QAChG,CAAC,CAAC,CACF,CAAA;IACF,CAAC;IAED;;;;;;;;;;OAUG;IACI,MAAM,CAAC,aAAa,CAC1B,MAAyB,EACzB,OAA6B;QAE7B,+FAA+F;QAC/F,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,YAAY,GAAG,CAAC,EAAE,CAAC;YAC5D,MAAM,IAAI,SAAS,CAAC,gFAAgF,CAAC,CAAA;QACtG,CAAC;QAED,OAAO,iBAAiB,CAAI,MAAM,EAAE,OAAO,CAAC,CAAA;IAC7C,CAAC;CACD"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @copyright Sister Software
|
|
3
|
-
* @license
|
|
3
|
+
* @license MIT
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
/**
|
|
@@ -34,16 +34,14 @@ export declare class BufferController {
|
|
|
34
34
|
/**
|
|
35
35
|
* Modify the buffer in place, keeping only the bytes between the start and end indices.
|
|
36
36
|
*
|
|
37
|
-
* This is performed after we're confident that any data preceeding `start` or following `end` is
|
|
38
|
-
* no longer needed.
|
|
37
|
+
* This is performed after we're confident that any data preceeding `start` or following `end` is no longer needed.
|
|
39
38
|
*
|
|
40
|
-
* `bytesWritten` is rebased onto the new window so it always reflects the count of valid bytes
|
|
41
|
-
*
|
|
42
|
-
* underlying allocation is larger.
|
|
39
|
+
* `bytesWritten` is rebased onto the new window so it always reflects the count of valid bytes present after
|
|
40
|
+
* compression. Bytes outside the kept window are dropped from the count even if the underlying allocation is larger.
|
|
43
41
|
*
|
|
44
42
|
* @param start - The starting byte index of which bytes to keep.
|
|
45
|
-
* @param end - The ending byte index of which bytes to keep. Defaults to the current buffer
|
|
46
|
-
*
|
|
43
|
+
* @param end - The ending byte index of which bytes to keep. Defaults to the current buffer length. Values past the
|
|
44
|
+
* buffer length are clamped.
|
|
47
45
|
*/
|
|
48
46
|
compress(start?: number, end?: number): void;
|
|
49
47
|
/**
|
|
@@ -54,8 +52,8 @@ export declare class BufferController {
|
|
|
54
52
|
*/
|
|
55
53
|
clear(begin?: number, end?: number): void;
|
|
56
54
|
/**
|
|
57
|
-
* Gets a new Uint8Array view of the ArrayBuffer store for this array, referencing the elements at
|
|
58
|
-
*
|
|
55
|
+
* Gets a new Uint8Array view of the ArrayBuffer store for this array, referencing the elements at begin, inclusive,
|
|
56
|
+
* up to end, exclusive.
|
|
59
57
|
*
|
|
60
58
|
* @param begin — The index of the beginning of the array.
|
|
61
59
|
* @param end — The index of the end of the array
|
|
@@ -66,8 +64,8 @@ export declare class BufferController {
|
|
|
66
64
|
/**
|
|
67
65
|
* Sets a value or an array of values.
|
|
68
66
|
*
|
|
69
|
-
* Unlike the `TypedArray.set` method, this method will grow the buffer if the offset is greater
|
|
70
|
-
*
|
|
67
|
+
* Unlike the `TypedArray.set` method, this method will grow the buffer if the offset is greater than the current byte
|
|
68
|
+
* length.
|
|
71
69
|
*
|
|
72
70
|
* @param array — A typed or untyped array of values to set.
|
|
73
71
|
* @param offset — The index in the current array at which the values are to be written.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"BufferController.d.ts","sourceRoot":"","sources":["../../lib/BufferController.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;GAEG;AACH,qBAAa,gBAAgB;;IAM5B;;;;OAIG;IACH,KAAK,EAAE,UAAU,CAAA;IAEjB;;;;OAIG;IACH,YAAY,EAAE,MAAM,CAAA;IAEpB;;OAEG;IACH,IAAW,iBAAiB,IAAI,MAAM,CAErC;gBAEW,IAAI,EAAE;QAAE,iBAAiB,EAAE,MAAM,CAAA;KAAE;IAM/C;;OAEG;IACI,IAAI,CAAC,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI;IAa7C
|
|
1
|
+
{"version":3,"file":"BufferController.d.ts","sourceRoot":"","sources":["../../lib/BufferController.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;GAEG;AACH,qBAAa,gBAAgB;;IAM5B;;;;OAIG;IACH,KAAK,EAAE,UAAU,CAAA;IAEjB;;;;OAIG;IACH,YAAY,EAAE,MAAM,CAAA;IAEpB;;OAEG;IACH,IAAW,iBAAiB,IAAI,MAAM,CAErC;gBAEW,IAAI,EAAE;QAAE,iBAAiB,EAAE,MAAM,CAAA;KAAE;IAM/C;;OAEG;IACI,IAAI,CAAC,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI;IAa7C;;;;;;;;;;;OAWG;IACI,QAAQ,CAAC,KAAK,GAAE,MAAU,EAAE,GAAG,GAAE,MAA0B,GAAG,IAAI;IASzE;;;;;OAKG;IACI,KAAK,CAAC,KAAK,GAAE,MAAU,EAAE,GAAG,GAAE,MAA0B,GAAG,IAAI;IAKtE;;;;;;;;OAQG;IACI,QAAQ,CAAC,KAAK,GAAE,MAAU,EAAE,GAAG,GAAE,MAA0B,GAAG,UAAU;IAY/E;;;;;;;;;;OAUG;IACI,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,EAAE,MAAM,GAAE,MAAU,GAAG,MAAM;CAehE"}
|