spliterator 1.5.2 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -1
- package/out/index.d.ts +0 -2
- package/out/index.d.ts.map +1 -1
- package/out/index.js +0 -2
- package/out/index.js.map +1 -1
- package/out/lib/AsyncSpliterator.d.ts +92 -11
- package/out/lib/AsyncSpliterator.d.ts.map +1 -1
- package/out/lib/AsyncSpliterator.js +153 -81
- package/out/lib/AsyncSpliterator.js.map +1 -1
- package/out/lib/BufferController.d.ts +12 -0
- package/out/lib/BufferController.d.ts.map +1 -1
- package/out/lib/BufferController.js +22 -0
- package/out/lib/BufferController.js.map +1 -1
- package/out/lib/CSVSpliterator.d.ts +30 -24
- package/out/lib/CSVSpliterator.d.ts.map +1 -1
- package/out/lib/CSVSpliterator.js +83 -33
- package/out/lib/CSVSpliterator.js.map +1 -1
- package/out/lib/CharacterSequence.d.ts +5 -15
- package/out/lib/CharacterSequence.d.ts.map +1 -1
- package/out/lib/CharacterSequence.js +15 -49
- package/out/lib/CharacterSequence.js.map +1 -1
- package/out/lib/JSONSpliterator.js +1 -1
- package/out/lib/JSONSpliterator.js.map +1 -1
- package/out/lib/SlidingWindow.d.ts +1 -47
- package/out/lib/SlidingWindow.d.ts.map +1 -1
- package/out/lib/SlidingWindow.js +0 -114
- package/out/lib/SlidingWindow.js.map +1 -1
- package/out/lib/Spliterator.d.ts +55 -5
- package/out/lib/Spliterator.d.ts.map +1 -1
- package/out/lib/Spliterator.js +138 -41
- package/out/lib/Spliterator.js.map +1 -1
- package/out/lib/shared.d.ts +21 -9
- package/out/lib/shared.d.ts.map +1 -1
- package/out/lib/shared.js +6 -6
- package/out/lib/shared.js.map +1 -1
- package/out/node/cli/commands/csv.d.ts +44 -0
- package/out/node/cli/commands/csv.d.ts.map +1 -0
- package/out/node/cli/commands/csv.js +78 -0
- package/out/node/cli/commands/csv.js.map +1 -0
- package/out/node/cli/commands/iterate.d.ts +37 -0
- package/out/node/cli/commands/iterate.d.ts.map +1 -0
- package/out/node/cli/commands/iterate.js +54 -0
- package/out/node/cli/commands/iterate.js.map +1 -0
- package/out/node/cli/index.d.ts +9 -0
- package/out/node/cli/index.d.ts.map +1 -0
- package/out/node/cli/index.js +34 -0
- package/out/node/cli/index.js.map +1 -0
- package/out/node/cli/utils.d.ts +52 -0
- package/out/node/cli/utils.d.ts.map +1 -0
- package/out/node/cli/utils.js +70 -0
- package/out/node/cli/utils.js.map +1 -0
- package/out/node/fs/index.d.ts +58 -0
- package/out/node/fs/index.d.ts.map +1 -0
- package/out/node/fs/index.js +142 -0
- package/out/node/fs/index.js.map +1 -0
- package/package.json +9 -4
- package/out/lib/DelimitedChunkReader.d.ts +0 -67
- package/out/lib/DelimitedChunkReader.d.ts.map +0 -1
- package/out/lib/DelimitedChunkReader.js +0 -113
- package/out/lib/DelimitedChunkReader.js.map +0 -1
- package/out/lib/DelimiterTransformer.d.ts +0 -27
- package/out/lib/DelimiterTransformer.d.ts.map +0 -1
- package/out/lib/DelimiterTransformer.js +0 -26
- package/out/lib/DelimiterTransformer.js.map +0 -1
- package/out/lib/node/fs.d.ts +0 -98
- package/out/lib/node/fs.d.ts.map +0 -1
- package/out/lib/node/fs.js +0 -162
- package/out/lib/node/fs.js.map +0 -1
package/README.md
CHANGED
|
@@ -80,7 +80,23 @@ for await (const columns of reader) {
|
|
|
80
80
|
}
|
|
81
81
|
```
|
|
82
82
|
|
|
83
|
-
##
|
|
83
|
+
## CLI Usage
|
|
84
|
+
|
|
85
|
+
Spliterator also includes a CLI tool that can be used to stream delimited content from the command line, transform it, filter it, and more.
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
spliterator csv people.csv people.jsonl
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
The CLI also supports reading from standard input:
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
cat people.csv | spliterator csv people.jsonl
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
For information on all available commands, run `spliterator --help`.
|
|
98
|
+
|
|
99
|
+
## Advanced Usage
|
|
84
100
|
|
|
85
101
|
Spliterator includes a collection of low-level classes and interfaces that can be used to create custom generators for any kind of delimited content.
|
|
86
102
|
|
package/out/index.d.ts
CHANGED
|
@@ -8,8 +8,6 @@ export * from "./lib/casing.js";
|
|
|
8
8
|
export * from "./lib/CharacterSequence.js";
|
|
9
9
|
export * from "./lib/CompositeDataView.js";
|
|
10
10
|
export * from "./lib/CSVSpliterator.js";
|
|
11
|
-
export * from "./lib/DelimitedChunkReader.js";
|
|
12
|
-
export * from "./lib/DelimiterTransformer.js";
|
|
13
11
|
export * from "./lib/IndexQueue.js";
|
|
14
12
|
export * from "./lib/JSONSpliterator.js";
|
|
15
13
|
export * from "./lib/shared.js";
|
package/out/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,2BAA2B,CAAA;AACzC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,4BAA4B,CAAA;AAC1C,cAAc,4BAA4B,CAAA;AAC1C,cAAc,yBAAyB,CAAA;AACvC,cAAc
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,2BAA2B,CAAA;AACzC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,4BAA4B,CAAA;AAC1C,cAAc,4BAA4B,CAAA;AAC1C,cAAc,yBAAyB,CAAA;AACvC,cAAc,qBAAqB,CAAA;AACnC,cAAc,0BAA0B,CAAA;AACxC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,wBAAwB,CAAA;AACtC,cAAc,sBAAsB,CAAA;AACpC,cAAc,eAAe,CAAA;AAC7B,cAAc,0BAA0B,CAAA;AACxC,cAAc,iBAAiB,CAAA"}
|
package/out/index.js
CHANGED
|
@@ -8,8 +8,6 @@ export * from "./lib/casing.js";
|
|
|
8
8
|
export * from "./lib/CharacterSequence.js";
|
|
9
9
|
export * from "./lib/CompositeDataView.js";
|
|
10
10
|
export * from "./lib/CSVSpliterator.js";
|
|
11
|
-
export * from "./lib/DelimitedChunkReader.js";
|
|
12
|
-
export * from "./lib/DelimiterTransformer.js";
|
|
13
11
|
export * from "./lib/IndexQueue.js";
|
|
14
12
|
export * from "./lib/JSONSpliterator.js";
|
|
15
13
|
export * from "./lib/shared.js";
|
package/out/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,2BAA2B,CAAA;AACzC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,4BAA4B,CAAA;AAC1C,cAAc,4BAA4B,CAAA;AAC1C,cAAc,yBAAyB,CAAA;AACvC,cAAc
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,2BAA2B,CAAA;AACzC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,4BAA4B,CAAA;AAC1C,cAAc,4BAA4B,CAAA;AAC1C,cAAc,yBAAyB,CAAA;AACvC,cAAc,qBAAqB,CAAA;AACnC,cAAc,0BAA0B,CAAA;AACxC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,wBAAwB,CAAA;AACtC,cAAc,sBAAsB,CAAA;AACpC,cAAc,eAAe,CAAA;AAC7B,cAAc,0BAA0B,CAAA;AACxC,cAAc,iBAAiB,CAAA"}
|
|
@@ -3,8 +3,9 @@
|
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
|
+
import { ReadableStream, ReadableWritablePair, StreamPipeOptions } from "stream/web";
|
|
6
7
|
import { CharacterSequenceInput } from "./CharacterSequence.js";
|
|
7
|
-
import {
|
|
8
|
+
import { AsyncChunkIterator, AsyncDataResource } from "./shared.js";
|
|
8
9
|
/**
|
|
9
10
|
* Initialization options for creating a spliterator.
|
|
10
11
|
*/
|
|
@@ -64,11 +65,11 @@ export interface AsyncSpliteratorInit extends SpliteratorInit {
|
|
|
64
65
|
*/
|
|
65
66
|
highWaterMark?: number;
|
|
66
67
|
/**
|
|
67
|
-
* Whether to
|
|
68
|
+
* Whether to automatically dispose of the source once the iterator is done.
|
|
68
69
|
*
|
|
69
70
|
* @default true
|
|
70
71
|
*/
|
|
71
|
-
|
|
72
|
+
autoDispose?: boolean;
|
|
72
73
|
}
|
|
73
74
|
/**
|
|
74
75
|
* An asynchronous spliterator for reading delimited byte streams.
|
|
@@ -85,43 +86,123 @@ export interface AsyncSpliteratorInit extends SpliteratorInit {
|
|
|
85
86
|
*
|
|
86
87
|
* @see {@linkcode Spliterator} for the synchronous version.
|
|
87
88
|
*/
|
|
88
|
-
export declare class AsyncSpliterator implements AsyncIterableIterator<
|
|
89
|
+
export declare class AsyncSpliterator<R extends DataView | ArrayBuffer = Uint8Array> implements AsyncIterableIterator<R>, AsyncDisposable {
|
|
89
90
|
#private;
|
|
90
91
|
/**
|
|
91
|
-
* Create a new delimited generator from
|
|
92
|
+
* Create a new delimited generator from an asynchronous byte stream.
|
|
92
93
|
*
|
|
93
|
-
*
|
|
94
|
+
* @param source - The data resource to read from.
|
|
95
|
+
* @param init - The initialization options for the generator.
|
|
96
|
+
*
|
|
97
|
+
* @returns A new generator instance, yielding byte ranges.
|
|
98
|
+
*/
|
|
99
|
+
static from(source: AsyncChunkIterator, init?: AsyncSpliteratorInit): AsyncSpliterator;
|
|
100
|
+
/**
|
|
101
|
+
* Create a new delimited generator from a resource such as a file handle or URL.
|
|
94
102
|
*
|
|
95
103
|
* @param source - The data resource to read from.
|
|
96
104
|
* @param init - The initialization options for the generator.
|
|
105
|
+
*
|
|
106
|
+
* @returns A new generator instance, yielding byte ranges.
|
|
97
107
|
*/
|
|
98
108
|
static from(source: AsyncDataResource, init?: AsyncSpliteratorInit): Promise<AsyncSpliterator>;
|
|
109
|
+
/**
|
|
110
|
+
* Create a new delimited generator from a resource such as a file handle, URL, or byte stream.
|
|
111
|
+
*
|
|
112
|
+
* @param source - The data resource to read from.
|
|
113
|
+
* @param init - The initialization options for the generator.
|
|
114
|
+
*
|
|
115
|
+
* @returns A new generator instance, yielding byte ranges.
|
|
116
|
+
*/
|
|
117
|
+
static from(source: AsyncDataResource | AsyncChunkIterator, init?: AsyncSpliteratorInit): AsyncSpliterator | Promise<AsyncSpliterator>;
|
|
99
118
|
/**
|
|
100
119
|
* Dispose of the spliterator, closing the file handle if necessary.
|
|
101
120
|
*/
|
|
102
121
|
[Symbol.asyncDispose](): Promise<void>;
|
|
103
122
|
/**
|
|
104
|
-
* Create a new delimited generator from a
|
|
123
|
+
* Create a new delimited generator from a data resource.
|
|
124
|
+
*/
|
|
125
|
+
static toTransformStream(init: SpliteratorInit): TransformStream<AsyncChunkIterator, Uint8Array>;
|
|
126
|
+
/**
|
|
127
|
+
* Create a new delimited generator from an asynchronous byte stream.
|
|
105
128
|
*
|
|
106
|
-
* @param
|
|
129
|
+
* @param source - The file to read from.
|
|
107
130
|
* @param init - The initialization options for the generator.
|
|
108
131
|
* @see {@linkcode AsyncSpliterator.from} to create a new generator from a data resource.
|
|
109
132
|
*/
|
|
110
|
-
constructor(
|
|
111
|
-
next(): Promise<IteratorResult<
|
|
133
|
+
constructor(source: AsyncChunkIterator, init?: AsyncSpliteratorInit);
|
|
134
|
+
next(): Promise<IteratorResult<R>>;
|
|
112
135
|
/**
|
|
113
136
|
* Collect all the byte ranges from the file.
|
|
114
137
|
*
|
|
115
138
|
* **This method will read the entire file into memory.**
|
|
116
139
|
*/
|
|
117
|
-
toArray(): Promise<
|
|
140
|
+
toArray(): Promise<R[]>;
|
|
118
141
|
/**
|
|
119
142
|
* Collect all the byte ranges from the file as a string.
|
|
120
143
|
*
|
|
121
144
|
* **This method will read the entire file into memory.**
|
|
122
145
|
*/
|
|
123
146
|
toDecodedArray(decoder?: import("util").TextDecoder): Promise<string[]>;
|
|
147
|
+
/**
|
|
148
|
+
* Wraps the spliterator in a readable stream.
|
|
149
|
+
*
|
|
150
|
+
* This is useful for piping the spliterator to other streams.
|
|
151
|
+
*
|
|
152
|
+
* @returns A readable stream of byte ranges.
|
|
153
|
+
*/
|
|
154
|
+
toReadableStream(): ReadableStream<R>;
|
|
155
|
+
/**
|
|
156
|
+
* Pipes the spliterator to a writable stream.
|
|
157
|
+
*
|
|
158
|
+
* ```ts
|
|
159
|
+
* const spliterator = await AsyncSpliterator.from("data.csv")
|
|
160
|
+
*
|
|
161
|
+
* const textStream = spliterator.pipeTo(new TextDecoderStream())
|
|
162
|
+
* ```
|
|
163
|
+
*
|
|
164
|
+
* @see {@linkcode toReadableStream} to first convert the spliterator to a readable stream.
|
|
165
|
+
*/
|
|
166
|
+
pipeThrough<T>(transform: ReadableWritablePair<T, R>, options?: StreamPipeOptions): ReadableStream<T>;
|
|
167
|
+
/**
|
|
168
|
+
* Iterate over the byte ranges in the file, yielding each range. between delimiters.
|
|
169
|
+
*/
|
|
124
170
|
[Symbol.asyncIterator](): this;
|
|
125
171
|
return(): Promise<IteratorReturnResult<undefined>>;
|
|
172
|
+
/**
|
|
173
|
+
* Given a file handle containing delimited data and a desired slice count, returns an array of
|
|
174
|
+
* slices of the buffer between delimiters.
|
|
175
|
+
*
|
|
176
|
+
* This is an advanced function so an analogy is provided:
|
|
177
|
+
*
|
|
178
|
+
* Suppose you had to manually search through a very large book page by page to find where each
|
|
179
|
+
* chapter begins and ends. For a book with 1,000,000 pages, a single person would take a long
|
|
180
|
+
* time to go through it all.
|
|
181
|
+
*
|
|
182
|
+
* You could add more people to the task by laying out all the pages, measuring the length and
|
|
183
|
+
* assigning each person a length of pages to traverse.
|
|
184
|
+
*
|
|
185
|
+
* There's a few ways to go about this:
|
|
186
|
+
*
|
|
187
|
+
* We could approach this in serial -- having the first worker start from page 1 and scanning
|
|
188
|
+
* until they find the beginning of the next chapter, handing off the range to the next worker.
|
|
189
|
+
* This is how `AsyncSpliterator` works.
|
|
190
|
+
*
|
|
191
|
+
* However this is inefficient because no matter how many workers we have, they must wait for the
|
|
192
|
+
* previous worker to finish before they can start. Ideally, a desired number of workers would be
|
|
193
|
+
* able to scan their own _length_ of pages simultaneously, and settle up on the boundaries of the
|
|
194
|
+
* chapters they find.
|
|
195
|
+
*
|
|
196
|
+
* `AsyncSpliterator.asMany` is like the second approach, spltting the book into mostly equal
|
|
197
|
+
* lengths and having each worker scan their own length of pages.
|
|
198
|
+
*
|
|
199
|
+
* @param source - The file handle to read from.
|
|
200
|
+
* @param delimiter - The character to delimit by. Typically a newline or comma.
|
|
201
|
+
* @param concurrency - The _desired_ number of `AsyncSpliterator` instances to create.
|
|
202
|
+
*
|
|
203
|
+
* @returns An array of `AsyncSpliterator` instances, possibly less than the desired concurrency.
|
|
204
|
+
* @internal
|
|
205
|
+
*/
|
|
206
|
+
static asMany(source: AsyncDataResource, delimiter: CharacterSequenceInput, concurrency: number): Promise<AsyncSpliterator[]>;
|
|
126
207
|
}
|
|
127
208
|
//# sourceMappingURL=AsyncSpliterator.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AsyncSpliterator.d.ts","sourceRoot":"","sources":["../../lib/AsyncSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;
|
|
1
|
+
{"version":3,"file":"AsyncSpliterator.d.ts","sourceRoot":"","sources":["../../lib/AsyncSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAEpF,OAAO,EAAqB,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AAElF,OAAO,EAAE,kBAAkB,EAAE,iBAAiB,EAAkB,MAAM,aAAa,CAAA;AAEnF;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B;;OAEG;IACH,SAAS,CAAC,EAAE,sBAAsB,CAAA;IAElC;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IAEjB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,OAAO,CAAA;IAEnB;;;;OAIG;IACH,IAAI,CAAC,EAAE,MAAM,CAAA;IAEb;;;;;;;;OAQG;IACH,IAAI,CAAC,EAAE,MAAM,CAAA;IAEb;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,CAAA;CACf;AAED;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,eAAe;IAC5D;;;;;;;OAOG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IAEtB;;;;OAIG;IACH,WAAW,CAAC,EAAE,OAAO,CAAA;CACrB;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,gBAAgB,CAAC,CAAC,SAAS,QAAQ,GAAG,WAAW,GAAG,UAAU,CAC1E,YAAW,qBAAqB,CAAC,CAAC,CAAC,EAAE,eAAe;;IAIpD;;;;;;;OAOG;IACH,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,kBAAkB,EAAE,IAAI,CAAC,EAAE,oBAAoB,GAAG,gBAAgB;IACtF;;;;;;;OAOG;IACH,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,iBAAiB,EAAE,IAAI,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAC9F;;;;;;;OAOG;IACH,MAAM,CAAC,IAAI,CACV,MAAM,EAAE,iBAAiB,GAAG,kBAAkB,EAC9C,IAAI,CAAC,EAAE,oBAAoB,GACzB,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB/C;;OAEG;IACU,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC;IASnD;;OAEG;WACW,iBAAiB,CAAC,IAAI,EAAE,eAAe,GAAG,eAAe,CAAC,kBAAkB,EAAE,UAAU,CAAC;IAYvG;;;;;;OAMG;gBACS,MAAM,EAAE,kBAAkB,EAAE,IAAI,GAAE,oBAAyB;IA2P1D,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAqC/C;;;;OAIG;IACI,OAAO,IAAI,OAAO,CAAC,CAAC,EAAE,CAAC;IAI9B;;;;OAIG;IACI,cAAc,CAAC,OAAO,6BAAoB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAIrE;;;;;;OAMG;IACI,gBAAgB,IAAI,cAAc,CAAC,CAAC,CAAC;IAiB5C;;;;;;;;;;OAUG;IACI,WAAW,CAAC,CAAC,EAAE,SAAS,EAAE,oBAAoB,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,cAAc,CAAC,CAAC,CAAC;IAI5G;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC;IAItB,MAAM,IAAI,OAAO,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC;IAQzD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAiCG;WACiB,MAAM,CACzB,MAAM,EAAE,iBAAiB,EACzB,SAAS,EAAE,sBAAsB,EACjC,WAAW,EAAE,MAAM,GACjB,OAAO,CAAC,gBAAgB,EAAE,CAAC;CAqB9B"}
|
|
@@ -3,10 +3,10 @@
|
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
|
+
import { ReadableStream } from "stream/web";
|
|
6
7
|
import { BufferController } from "./BufferController.js";
|
|
7
8
|
import { CharacterSequence } from "./CharacterSequence.js";
|
|
8
9
|
import { IndexQueue } from "./IndexQueue.js";
|
|
9
|
-
import { applyReaderPolyfill, isFileResourceLike, } from "./shared.js";
|
|
10
10
|
/**
|
|
11
11
|
* An asynchronous spliterator for reading delimited byte streams.
|
|
12
12
|
*
|
|
@@ -23,25 +23,16 @@ import { applyReaderPolyfill, isFileResourceLike, } from "./shared.js";
|
|
|
23
23
|
* @see {@linkcode Spliterator} for the synchronous version.
|
|
24
24
|
*/
|
|
25
25
|
export class AsyncSpliterator {
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
*
|
|
30
|
-
* Unlike the constructor, this method can be used to create a generator from a file path or URL.
|
|
31
|
-
*
|
|
32
|
-
* @param source - The data resource to read from.
|
|
33
|
-
* @param init - The initialization options for the generator.
|
|
34
|
-
*/
|
|
35
|
-
static async from(source, init = {}) {
|
|
36
|
-
let file;
|
|
37
|
-
if (isFileResourceLike(source)) {
|
|
38
|
-
file = source;
|
|
26
|
+
static from(source, init = {}) {
|
|
27
|
+
if (typeof source === "object" && Symbol.asyncIterator in source) {
|
|
28
|
+
return new AsyncSpliterator(source, init);
|
|
39
29
|
}
|
|
40
|
-
|
|
41
|
-
const
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
30
|
+
return import("spliterator/node/fs").then(async ({ createChunkIterator }) => {
|
|
31
|
+
const chunkIterator = await createChunkIterator(source, {
|
|
32
|
+
highWaterMark: init.highWaterMark,
|
|
33
|
+
});
|
|
34
|
+
return new AsyncSpliterator(chunkIterator, init);
|
|
35
|
+
});
|
|
45
36
|
}
|
|
46
37
|
/**
|
|
47
38
|
* Dispose of the spliterator, closing the file handle if necessary.
|
|
@@ -49,31 +40,41 @@ export class AsyncSpliterator {
|
|
|
49
40
|
async [Symbol.asyncDispose]() {
|
|
50
41
|
this.#indices.clear();
|
|
51
42
|
this.#controller.clear();
|
|
52
|
-
if (this.#
|
|
53
|
-
await this.#
|
|
43
|
+
if (this.#autoDispose) {
|
|
44
|
+
await this.#source[Symbol.asyncDispose]?.();
|
|
54
45
|
}
|
|
55
46
|
}
|
|
56
47
|
/**
|
|
57
|
-
* Create a new delimited generator from a
|
|
48
|
+
* Create a new delimited generator from a data resource.
|
|
49
|
+
*/
|
|
50
|
+
static toTransformStream(init) {
|
|
51
|
+
return new TransformStream({
|
|
52
|
+
async transform(chunk, controller) {
|
|
53
|
+
const spliterator = await AsyncSpliterator.from(chunk, init);
|
|
54
|
+
for await (const slice of spliterator) {
|
|
55
|
+
controller.enqueue(slice);
|
|
56
|
+
}
|
|
57
|
+
},
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Create a new delimited generator from an asynchronous byte stream.
|
|
58
62
|
*
|
|
59
|
-
* @param
|
|
63
|
+
* @param source - The file to read from.
|
|
60
64
|
* @param init - The initialization options for the generator.
|
|
61
65
|
* @see {@linkcode AsyncSpliterator.from} to create a new generator from a data resource.
|
|
62
66
|
*/
|
|
63
|
-
constructor(
|
|
64
|
-
|
|
65
|
-
this.#
|
|
66
|
-
this.#
|
|
67
|
+
constructor(source, init = {}) {
|
|
68
|
+
this.#source = source;
|
|
69
|
+
this.#chunkReader = source[Symbol.asyncIterator]();
|
|
70
|
+
this.#autoDispose = init.autoDispose ?? false;
|
|
67
71
|
this.#needle = new CharacterSequence(init.delimiter);
|
|
68
|
-
this.#
|
|
69
|
-
this.#totalByteSize = file.size;
|
|
70
|
-
this.#yieldByteEstimate = this.#totalByteSize - this.#readPosition;
|
|
71
|
-
this.#highWaterMark = Math.max(this.#needle.length * 4, init.highWaterMark ?? 4096);
|
|
72
|
+
this.#highWaterMark = Math.max(this.#needle.length * 4, init.highWaterMark ?? 4096 * 16);
|
|
72
73
|
this.#yieldDropCount = Math.max(0, init.drop ?? 0);
|
|
73
74
|
this.#yieldStopCount = Math.max(init.take ?? Infinity, 0) + this.#yieldDropCount;
|
|
74
75
|
this.#skipEmpty = init.skipEmpty ?? true;
|
|
75
76
|
this.#controller = new BufferController({
|
|
76
|
-
initialBufferSize:
|
|
77
|
+
initialBufferSize: this.#highWaterMark,
|
|
77
78
|
});
|
|
78
79
|
this.#debug = init.debug ?? false;
|
|
79
80
|
this.#log = this.#debug ? console.debug.bind(console) : () => void 0;
|
|
@@ -81,13 +82,21 @@ export class AsyncSpliterator {
|
|
|
81
82
|
//#endregion
|
|
82
83
|
//#region Private Properties
|
|
83
84
|
/**
|
|
84
|
-
*
|
|
85
|
+
* Whether to call `Symbol.asyncDispose` on the source once the iterator is done.
|
|
85
86
|
*/
|
|
86
|
-
#
|
|
87
|
+
#autoDispose;
|
|
87
88
|
/**
|
|
88
|
-
*
|
|
89
|
+
* The source of the data.
|
|
89
90
|
*/
|
|
90
|
-
#
|
|
91
|
+
#source;
|
|
92
|
+
/**
|
|
93
|
+
* The chunk reader for the file.
|
|
94
|
+
*/
|
|
95
|
+
#chunkReader;
|
|
96
|
+
/**
|
|
97
|
+
* The last chunk of data read from the source.
|
|
98
|
+
*/
|
|
99
|
+
#lastReadResult;
|
|
91
100
|
/**
|
|
92
101
|
* A queue of index tuples marking the start and end of delimiter positions.
|
|
93
102
|
*
|
|
@@ -106,16 +115,6 @@ export class AsyncSpliterator {
|
|
|
106
115
|
* the file read operation.
|
|
107
116
|
*/
|
|
108
117
|
#controller;
|
|
109
|
-
/**
|
|
110
|
-
* The total byte size of the file.
|
|
111
|
-
*/
|
|
112
|
-
#totalByteSize;
|
|
113
|
-
/**
|
|
114
|
-
* The total number of bytes we expect to yield.
|
|
115
|
-
*
|
|
116
|
-
* We use this to ensure that we don't miss any data when the file is read in chunks.
|
|
117
|
-
*/
|
|
118
|
-
#yieldByteEstimate;
|
|
119
118
|
/**
|
|
120
119
|
* How many yields to skip.
|
|
121
120
|
*/
|
|
@@ -143,10 +142,10 @@ export class AsyncSpliterator {
|
|
|
143
142
|
* The total number of yields.
|
|
144
143
|
*/
|
|
145
144
|
#yieldCount = 0;
|
|
146
|
-
/**
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
#readPosition
|
|
145
|
+
// /**
|
|
146
|
+
// * The current byte index to perform read operations from.
|
|
147
|
+
// */
|
|
148
|
+
// #readPosition: number
|
|
150
149
|
/**
|
|
151
150
|
* The last byte range seen while draining the buffer.
|
|
152
151
|
*/
|
|
@@ -183,34 +182,17 @@ export class AsyncSpliterator {
|
|
|
183
182
|
* Read a chunk of data from the file.
|
|
184
183
|
*/
|
|
185
184
|
async #read() {
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
const offset = this.#controller.bytesWritten;
|
|
192
|
-
// Before reading we grow the buffer to ensure that we have enough space.
|
|
193
|
-
const nextBufferSize = Math.max(this.#controller.byteLengthMinimum, offset + nextReadLength);
|
|
194
|
-
this.#controller.grow(nextBufferSize);
|
|
195
|
-
this.#log("Before read", { cursor: this.#readPosition, nextReadLength, nextBufferSize, offset });
|
|
196
|
-
await this.#file.read({
|
|
197
|
-
buffer: this.#controller.bytes,
|
|
198
|
-
offset,
|
|
199
|
-
position: this.#readPosition,
|
|
200
|
-
length: nextReadLength,
|
|
201
|
-
});
|
|
202
|
-
this.#controller.bytesWritten += nextReadLength;
|
|
203
|
-
// this.#log(
|
|
204
|
-
// "Inspecting buffer",
|
|
205
|
-
// debugAsVisibleCharacters(this.#controller.bytes.subarray(0, this.#controller.bytesWritten))
|
|
206
|
-
// )
|
|
207
|
-
this.#readPosition += nextReadLength;
|
|
185
|
+
this.#lastReadResult = await this.#chunkReader.next();
|
|
186
|
+
if (this.#lastReadResult.done)
|
|
187
|
+
return;
|
|
188
|
+
// Append the chunk to the buffer.
|
|
189
|
+
this.#controller.set(this.#lastReadResult.value, this.#controller.bytesWritten);
|
|
208
190
|
}
|
|
209
191
|
/**
|
|
210
192
|
* Fill the buffer with data and search for delimiters.
|
|
211
193
|
*/
|
|
212
194
|
async #fill() {
|
|
213
|
-
if (this.#
|
|
195
|
+
if (this.#lastReadResult?.done) {
|
|
214
196
|
this.#log("Reached end of file. Preparing to finalize.");
|
|
215
197
|
// There's a few special cases we could get this far and not have drained the buffer.
|
|
216
198
|
const lastByteRange = this.#lastByteRangeSeen;
|
|
@@ -250,7 +232,7 @@ export class AsyncSpliterator {
|
|
|
250
232
|
const nextBufferStart = this.#lastByteRangeSeen ? this.#lastByteRangeSeen[1] + this.#needle.length : 0;
|
|
251
233
|
this.#log("Compressing buffer", { nextBufferStart });
|
|
252
234
|
this.#controller.compress(nextBufferStart);
|
|
253
|
-
while (this.#
|
|
235
|
+
while ((!this.#lastReadResult || !this.#lastReadResult.done) && this.#indices.byteLength < this.#highWaterMark) {
|
|
254
236
|
await this.#read();
|
|
255
237
|
this.#search();
|
|
256
238
|
}
|
|
@@ -262,19 +244,14 @@ export class AsyncSpliterator {
|
|
|
262
244
|
* number of of yields and the length of the delimiter.
|
|
263
245
|
*/
|
|
264
246
|
const expectedYieldedDelimitedBytes = (this.#yieldCount - 1) * this.#needle.length;
|
|
265
|
-
const omittedBytes = this.#yieldByteEstimate - (this.#yieldedByteLength + expectedYieldedDelimitedBytes);
|
|
266
247
|
this.#log({
|
|
267
|
-
totalByteSize: this.#totalByteSize,
|
|
268
|
-
readPosition: this.#readPosition,
|
|
269
|
-
yieldByteEstimate: this.#yieldByteEstimate,
|
|
270
248
|
yieldedByteLength: this.#yieldedByteLength,
|
|
271
249
|
yieldCount: this.#yieldCount,
|
|
272
250
|
expectedYieldedDelimitedBytes,
|
|
273
|
-
omittedBytes,
|
|
274
251
|
});
|
|
275
252
|
}
|
|
276
|
-
if (this.#
|
|
277
|
-
await this.#
|
|
253
|
+
if (this.#autoDispose) {
|
|
254
|
+
await this.#source[Symbol.asyncDispose]?.();
|
|
278
255
|
}
|
|
279
256
|
return {
|
|
280
257
|
done: true,
|
|
@@ -291,6 +268,10 @@ export class AsyncSpliterator {
|
|
|
291
268
|
}
|
|
292
269
|
const currentByteRange = this.#indices.dequeue();
|
|
293
270
|
this.#lastByteRangeSeen = currentByteRange;
|
|
271
|
+
if (!currentByteRange) {
|
|
272
|
+
this.#done = true;
|
|
273
|
+
return this.#finalize();
|
|
274
|
+
}
|
|
294
275
|
const [start, end] = currentByteRange;
|
|
295
276
|
const slice = this.#controller.subarray(start, end);
|
|
296
277
|
if (slice.length === 0 && this.#skipEmpty) {
|
|
@@ -322,11 +303,102 @@ export class AsyncSpliterator {
|
|
|
322
303
|
toDecodedArray(decoder = new TextDecoder()) {
|
|
323
304
|
return Array.fromAsync(this, (bytes) => decoder.decode(bytes));
|
|
324
305
|
}
|
|
306
|
+
/**
|
|
307
|
+
* Wraps the spliterator in a readable stream.
|
|
308
|
+
*
|
|
309
|
+
* This is useful for piping the spliterator to other streams.
|
|
310
|
+
*
|
|
311
|
+
* @returns A readable stream of byte ranges.
|
|
312
|
+
*/
|
|
313
|
+
toReadableStream() {
|
|
314
|
+
const iterator = this[Symbol.asyncIterator]();
|
|
315
|
+
return new ReadableStream({
|
|
316
|
+
pull: async (controller) => {
|
|
317
|
+
const { done, value } = await iterator.next();
|
|
318
|
+
if (done) {
|
|
319
|
+
controller.close();
|
|
320
|
+
}
|
|
321
|
+
else {
|
|
322
|
+
controller.enqueue(value);
|
|
323
|
+
}
|
|
324
|
+
},
|
|
325
|
+
cancel: () => this[Symbol.asyncDispose]?.(),
|
|
326
|
+
});
|
|
327
|
+
}
|
|
328
|
+
/**
|
|
329
|
+
* Pipes the spliterator to a writable stream.
|
|
330
|
+
*
|
|
331
|
+
* ```ts
|
|
332
|
+
* const spliterator = await AsyncSpliterator.from("data.csv")
|
|
333
|
+
*
|
|
334
|
+
* const textStream = spliterator.pipeTo(new TextDecoderStream())
|
|
335
|
+
* ```
|
|
336
|
+
*
|
|
337
|
+
* @see {@linkcode toReadableStream} to first convert the spliterator to a readable stream.
|
|
338
|
+
*/
|
|
339
|
+
pipeThrough(transform, options) {
|
|
340
|
+
return this.toReadableStream().pipeThrough(transform, options);
|
|
341
|
+
}
|
|
342
|
+
/**
|
|
343
|
+
* Iterate over the byte ranges in the file, yielding each range. between delimiters.
|
|
344
|
+
*/
|
|
325
345
|
[Symbol.asyncIterator]() {
|
|
326
346
|
return this;
|
|
327
347
|
}
|
|
328
348
|
return() {
|
|
329
349
|
return this.#finalize();
|
|
330
350
|
}
|
|
351
|
+
//#endregion
|
|
352
|
+
//#region Experimental
|
|
353
|
+
/**
|
|
354
|
+
* Given a file handle containing delimited data and a desired slice count, returns an array of
|
|
355
|
+
* slices of the buffer between delimiters.
|
|
356
|
+
*
|
|
357
|
+
* This is an advanced function so an analogy is provided:
|
|
358
|
+
*
|
|
359
|
+
* Suppose you had to manually search through a very large book page by page to find where each
|
|
360
|
+
* chapter begins and ends. For a book with 1,000,000 pages, a single person would take a long
|
|
361
|
+
* time to go through it all.
|
|
362
|
+
*
|
|
363
|
+
* You could add more people to the task by laying out all the pages, measuring the length and
|
|
364
|
+
* assigning each person a length of pages to traverse.
|
|
365
|
+
*
|
|
366
|
+
* There's a few ways to go about this:
|
|
367
|
+
*
|
|
368
|
+
* We could approach this in serial -- having the first worker start from page 1 and scanning
|
|
369
|
+
* until they find the beginning of the next chapter, handing off the range to the next worker.
|
|
370
|
+
* This is how `AsyncSpliterator` works.
|
|
371
|
+
*
|
|
372
|
+
* However this is inefficient because no matter how many workers we have, they must wait for the
|
|
373
|
+
* previous worker to finish before they can start. Ideally, a desired number of workers would be
|
|
374
|
+
* able to scan their own _length_ of pages simultaneously, and settle up on the boundaries of the
|
|
375
|
+
* chapters they find.
|
|
376
|
+
*
|
|
377
|
+
* `AsyncSpliterator.asMany` is like the second approach, spltting the book into mostly equal
|
|
378
|
+
* lengths and having each worker scan their own length of pages.
|
|
379
|
+
*
|
|
380
|
+
* @param source - The file handle to read from.
|
|
381
|
+
* @param delimiter - The character to delimit by. Typically a newline or comma.
|
|
382
|
+
* @param concurrency - The _desired_ number of `AsyncSpliterator` instances to create.
|
|
383
|
+
*
|
|
384
|
+
* @returns An array of `AsyncSpliterator` instances, possibly less than the desired concurrency.
|
|
385
|
+
* @internal
|
|
386
|
+
*/
|
|
387
|
+
static async asMany(source, delimiter, concurrency) {
|
|
388
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
389
|
+
concurrency = Math.max(1, concurrency);
|
|
390
|
+
const needle = new CharacterSequence(delimiter);
|
|
391
|
+
const spliterators = [];
|
|
392
|
+
const { createChunkIterator, readFileSize } = await import("spliterator/node/fs");
|
|
393
|
+
const fileSize = await readFileSize(source);
|
|
394
|
+
// const file = createChunkIterator(source, {
|
|
395
|
+
// start: 0,
|
|
396
|
+
// })
|
|
397
|
+
// The idea here is we're going to split the file into `concurrency` chunks.
|
|
398
|
+
if (Date.now()) {
|
|
399
|
+
throw new Error("Not implemented.");
|
|
400
|
+
}
|
|
401
|
+
return spliterators;
|
|
402
|
+
}
|
|
331
403
|
}
|
|
332
404
|
//# sourceMappingURL=AsyncSpliterator.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AsyncSpliterator.js","sourceRoot":"","sources":["../../lib/AsyncSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AACxD,OAAO,EAAE,iBAAiB,EAA0B,MAAM,wBAAwB,CAAA;AAClF,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;
|
|
1
|
+
{"version":3,"file":"AsyncSpliterator.js","sourceRoot":"","sources":["../../lib/AsyncSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,cAAc,EAA2C,MAAM,YAAY,CAAA;AACpF,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AACxD,OAAO,EAAE,iBAAiB,EAA0B,MAAM,wBAAwB,CAAA;AAClF,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AA4E5C;;;;;;;;;;;;;;GAcG;AACH,MAAM,OAAO,gBAAgB;IAmC5B,MAAM,CAAC,IAAI,CACV,MAA8C,EAC9C,OAA6B,EAAE;QAE/B,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,aAAa,IAAI,MAAM,EAAE,CAAC;YAClE,OAAO,IAAI,gBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QAC1C,CAAC;QAED,OAAO,MAAM,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,mBAAmB,EAAE,EAAE,EAAE;YAC3E,MAAM,aAAa,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE;gBACvD,aAAa,EAAE,IAAI,CAAC,aAAa;aACjC,CAAC,CAAA;YAEF,OAAO,IAAI,gBAAgB,CAAC,aAAa,EAAE,IAAI,CAAC,CAAA;QACjD,CAAC,CAAC,CAAA;IACH,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QACjC,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;QACrB,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAA;QAExB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE,CAAA;QAC5C,CAAC;IACF,CAAC;IAED;;OAEG;IACI,MAAM,CAAC,iBAAiB,CAAC,IAAqB;QACpD,OAAO,IAAI,eAAe,CAAiC;YAC1D,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,UAAU;gBAChC,MAAM,WAAW,GAAG,MAAM,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;gBAE5D,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;oBACvC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;gBAC1B,CAAC;YACF,CAAC;SACD,CAAC,CAAA;IACH,CAAC;IAED;;;;;;OAMG;IACH,YAAY,MAA0B,EAAE,OAA6B,EAAE;QACtE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QACrB,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAA;QAElD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,WAAW,IAAI,KAAK,CAAA;QAE7C,IAAI,CAAC,OAAO,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QAEpD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,aAAa,IAAI,IAAI,GAAG,EAAE,CAAC,CAAA;QAExF,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAA;QAClD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,eAAe,CAAA;QAEhF,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAA;QAExC,IAAI,CAAC,WAAW,GAAG,IAAI,gBAAgB,CAAC;YACvC,iBAAiB,EAAE,IAAI,CAAC,cAAc;SACtC,CAAC,CAAA;QAEF,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,IAAI,KAAK,CAAA;QACjC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAA;IACrE,CAAC;IAED,YAAY;IAEZ,4BAA4B;IAE5B;;OAEG;IACM,YAAY,CAAS;IAE9B;;OAEG;IACM,OAAO,CAAoB;IAEpC;;OAEG;IACM,YAAY,CAA2B;IAEhD;;OAEG;IACH,eAAe,CAAwC;IAEvD;;;;;;;OAOG;IACM,QAAQ,GAAG,IAAI,UAAU,EAAE,CAAA;IAEpC;;OAEG;IACM,OAAO,CAAmB;IAEnC;;;OAGG;IACM,WAAW,CAAkB;IAEtC;;OAEG;IACM,eAAe,CAAQ;IAEhC;;OAEG;IACM,eAAe,CAAQ;IAEhC;;OAEG;IACM,UAAU,CAAS;IAE5B;;;;;OAKG;IACM,cAAc,CAAQ;IAE/B;;OAEG;IACH,kBAAkB,GAAG,CAAC,CAAA;IAEtB;;OAEG;IACH,WAAW,GAAG,CAAC,CAAA;IAEf,MAAM;IACN,6DAA6D;IAC7D,MAAM;IACN,wBAAwB;IAExB;;OAEG;IACH,kBAAkB,CAAuB;IAEzC;;OAEG;IACH,MAAM,CAAS;IAEf;;OAEG;IACH,KAAK,GAAG,KAAK,CAAA;IAEb,YAAY;IAEZ,yBAAyB;IAEzB,IAAI,CAA0B;IAE9B;;OAEG;IACH,OAAO;QACN,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAA;QAC9C,IAAI,YAAY,GAAG,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;QAE7E,OAAO,YAAY,IAAI,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,CAAC;YACtD,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,YAAY,CAAC,CAAA;YAEhF,IAAI,cAAc,KAAK,CAAC,CAAC,EAAE,CAAC;gBAC3B,IAAI,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAA;gBAC5D,MAAK;YACN,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,CAAA;YAC7D,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,CAAA;YAErD,YAAY,GAAG,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;QACpD,CAAC;IACF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACV,IAAI,CAAC,eAAe,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAA;QAErD,IAAI,IAAI,CAAC,eAAe,CAAC,IAAI;YAAE,OAAM;QAErC,kCAAkC;QAClC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,CAAA;IAChF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACV,IAAI,IAAI,CAAC,eAAe,EAAE,IAAI,EAAE,CAAC;YAChC,IAAI,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAA;YACxD,qFAAqF;YACrF,MAAM,aAAa,GAAG,IAAI,CAAC,kBAAkB,CAAA;YAE7C,IAAI,aAAa,EAAE,CAAC;gBACnB,MAAM,aAAa,GAAG,aAAa,CAAC,CAAC,CAAC,CAAA;gBAEtC,+EAA+E;gBAE/E,IAAI,aAAa,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,CAAC;oBACnD,mFAAmF;oBACnF,MAAM,sBAAsB,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CACjD,IAAI,CAAC,WAAW,CAAC,KAAK,EACtB,IAAI,CAAC,WAAW,CAAC,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CACnD,CAAA;oBAED,IAAI,sBAAsB,KAAK,CAAC,CAAC,EAAE,CAAC;wBACnC,IAAI,CAAC,IAAI,CAAC,2CAA2C,EAAE;4BACtD,sBAAsB;4BACtB,IAAI,CAAC,WAAW,CAAC,YAAY;yBAC7B,CAAC,CAAA;wBACF,6EAA6E;wBAC7E,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,sBAAsB,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAA;oBACrG,CAAC;yBAAM,CAAC;wBACP,IAAI,CAAC,IAAI,CAAC,sCAAsC,EAAE,CAAC,aAAa,EAAE,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAA;wBACjG,gFAAgF;wBAChF,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAA;oBAC5F,CAAC;gBACF,CAAC;qBAAM,CAAC;oBACP,IAAI,CAAC,IAAI,CACR,0EAA0E,aAAa,sCAAsC,IAAI,CAAC,WAAW,CAAC,YAAY,GAAG,CAC7J,CAAA;gBACF,CAAC;YACF,CAAC;iBAAM,IAAI,IAAI,CAAC,WAAW,KAAK,CAAC,EAAE,CAAC;gBACnC,kFAAkF;gBAClF,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAA;YAC1D,CAAC;YAED,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;YAEjB,OAAM;QACP,CAAC;QAED,yDAAyD;QACzD,wCAAwC;QACxC,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;QAEtG,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,eAAe,EAAE,CAAC,CAAA;QACpD,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAA;QAE1C,OAAO,CAAC,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;YAChH,MAAM,IAAI,CAAC,KAAK,EAAE,CAAA;YAClB,IAAI,CAAC,OAAO,EAAE,CAAA;QACf,CAAC;IACF,CAAC;IAED,KAAK,CAAC,SAAS;QACd,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB;;;eAGG;YACH,MAAM,6BAA6B,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;YAElF,IAAI,CAAC,IAAI,CAAC;gBACT,iBAAiB,EAAE,IAAI,CAAC,kBAAkB;gBAC1C,UAAU,EAAE,IAAI,CAAC,WAAW;gBAC5B,6BAA6B;aAC7B,CAAC,CAAA;QACH,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE,CAAA;QAC5C,CAAC;QAED,OAAO;YACN,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,SAAS;SAChB,CAAA;IACF,CAAC;IAED,YAAY;IAEZ,0BAA0B;IAEnB,KAAK,CAAC,IAAI;QAChB,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,eAAe;YAAE,OAAO,IAAI,CAAC,SAAS,EAAE,CAAA;QAEnF,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC9B,MAAM,IAAI,CAAC,KAAK,EAAE,CAAA;QACnB,CAAC;QAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAA;QAChD,IAAI,CAAC,kBAAkB,GAAG,gBAAgB,CAAA;QAE1C,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACvB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;YACjB,OAAO,IAAI,CAAC,SAAS,EAAE,CAAA;QACxB,CAAC;QAED,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,gBAAgB,CAAA;QAErC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;QAEnD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAC3C,OAAO,IAAI,CAAC,IAAI,EAAE,CAAA;QACnB,CAAC;QAED,IAAI,CAAC,WAAW,EAAE,CAAA;QAElB,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YAC9C,OAAO,IAAI,CAAC,IAAI,EAAE,CAAA;QACnB,CAAC;QAED,IAAI,CAAC,kBAAkB,IAAI,GAAG,GAAG,KAAK,CAAA;QAEtC,OAAO;YACN,KAAK,EAAE,KAAqB;YAC5B,IAAI,EAAE,KAAK;SACX,CAAA;IACF,CAAC;IAED;;;;OAIG;IACI,OAAO;QACb,OAAO,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;IAC7B,CAAC;IAED;;;;OAIG;IACI,cAAc,CAAC,OAAO,GAAG,IAAI,WAAW,EAAE;QAChD,OAAO,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;IAC/D,CAAC;IAED;;;;;;OAMG;IACI,gBAAgB;QACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAA;QAE7C,OAAO,IAAI,cAAc,CAAC;YACzB,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE;gBAC1B,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;gBAE7C,IAAI,IAAI,EAAE,CAAC;oBACV,UAAU,CAAC,KAAK,EAAE,CAAA;gBACnB,CAAC;qBAAM,CAAC;oBACP,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;gBAC1B,CAAC;YACF,CAAC;YACD,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE;SAC3C,CAAC,CAAA;IACH,CAAC;IAED;;;;;;;;;;OAUG;IACI,WAAW,CAAI,SAAqC,EAAE,OAA2B;QACvF,OAAO,IAAI,CAAC,gBAAgB,EAAE,CAAC,WAAW,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IAC/D,CAAC;IAED;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC;QAC5B,OAAO,IAAI,CAAA;IACZ,CAAC;IAEM,MAAM;QACZ,OAAO,IAAI,CAAC,SAAS,EAAE,CAAA;IACxB,CAAC;IAED,YAAY;IAEZ,sBAAsB;IAEtB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAiCG;IACI,MAAM,CAAC,KAAK,CAAC,MAAM,CACzB,MAAyB,EACzB,SAAiC,EACjC,WAAmB;QAEnB,6DAA6D;QAC7D,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,CAAA;QACtC,MAAM,MAAM,GAAG,IAAI,iBAAiB,CAAC,SAAS,CAAC,CAAA;QAE/C,MAAM,YAAY,GAAuB,EAAE,CAAA;QAE3C,MAAM,EAAE,mBAAmB,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAA;QACjF,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC,CAAA;QAC3C,6CAA6C;QAC7C,aAAa;QACb,KAAK;QAEL,6EAA6E;QAE7E,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAA;QACpC,CAAC;QAED,OAAO,YAAY,CAAA;IACpB,CAAC;CACD"}
|
|
@@ -58,5 +58,17 @@ export declare class BufferController {
|
|
|
58
58
|
* @throws If the end index is greater than the current byte length.
|
|
59
59
|
*/
|
|
60
60
|
subarray(begin?: number, end?: number): Uint8Array;
|
|
61
|
+
/**
|
|
62
|
+
* Sets a value or an array of values.
|
|
63
|
+
*
|
|
64
|
+
* Unlike the `TypedArray.set` method, this method will grow the buffer if the offset is greater
|
|
65
|
+
* than the current byte length.
|
|
66
|
+
*
|
|
67
|
+
* @param array — A typed or untyped array of values to set.
|
|
68
|
+
* @param offset — The index in the current array at which the values are to be written.
|
|
69
|
+
*
|
|
70
|
+
* @returns The number of bytes written.
|
|
71
|
+
*/
|
|
72
|
+
set(array: ArrayLike<number>, offset?: number): number;
|
|
61
73
|
}
|
|
62
74
|
//# sourceMappingURL=BufferController.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"BufferController.d.ts","sourceRoot":"","sources":["../../lib/BufferController.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;GAEG;AACH,qBAAa,gBAAgB;;IAM5B;;;;OAIG;IACH,KAAK,EAAE,UAAU,CAAA;IAEjB;;;;OAIG;IACH,YAAY,EAAE,MAAM,CAAA;IAEpB;;OAEG;IACH,IAAW,iBAAiB,IAAI,MAAM,CAErC;gBAEW,IAAI,EAAE;QAAE,iBAAiB,EAAE,MAAM,CAAA;KAAE;IAM/C;;OAEG;IACI,IAAI,CAAC,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI;IAa7C;;;;;;;;OAQG;IACI,QAAQ,CAAC,KAAK,GAAE,MAAU,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI;IAStD;;;;;OAKG;IACI,KAAK,CAAC,KAAK,GAAE,MAAU,EAAE,GAAG,GAAE,MAA0B,GAAG,IAAI;IAKtE;;;;;;;;OAQG;IACI,QAAQ,CAAC,KAAK,GAAE,MAAU,EAAE,GAAG,GAAE,MAA0B,GAAG,UAAU;
|
|
1
|
+
{"version":3,"file":"BufferController.d.ts","sourceRoot":"","sources":["../../lib/BufferController.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;GAEG;AACH,qBAAa,gBAAgB;;IAM5B;;;;OAIG;IACH,KAAK,EAAE,UAAU,CAAA;IAEjB;;;;OAIG;IACH,YAAY,EAAE,MAAM,CAAA;IAEpB;;OAEG;IACH,IAAW,iBAAiB,IAAI,MAAM,CAErC;gBAEW,IAAI,EAAE;QAAE,iBAAiB,EAAE,MAAM,CAAA;KAAE;IAM/C;;OAEG;IACI,IAAI,CAAC,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI;IAa7C;;;;;;;;OAQG;IACI,QAAQ,CAAC,KAAK,GAAE,MAAU,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI;IAStD;;;;;OAKG;IACI,KAAK,CAAC,KAAK,GAAE,MAAU,EAAE,GAAG,GAAE,MAA0B,GAAG,IAAI;IAKtE;;;;;;;;OAQG;IACI,QAAQ,CAAC,KAAK,GAAE,MAAU,EAAE,GAAG,GAAE,MAA0B,GAAG,UAAU;IAY/E;;;;;;;;;;OAUG;IACI,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,EAAE,MAAM,GAAE,MAAU,GAAG,MAAM;CAehE"}
|