spliterator 6.2.0 → 6.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/lib/core/AsyncSpliterator.d.ts +1 -1
- package/out/lib/core/AsyncSpliterator.d.ts.map +1 -1
- package/out/lib/core/AsyncSpliterator.js +2 -2
- package/out/lib/core/AsyncSpliterator.js.map +1 -1
- package/out/lib/core/SlidingWindow.d.ts +1 -1
- package/out/lib/core/SlidingWindow.d.ts.map +1 -1
- package/out/lib/core/SlidingWindow.js.map +1 -1
- package/out/lib/core/Spliterator.d.ts +1 -1
- package/out/lib/core/Spliterator.d.ts.map +1 -1
- package/out/lib/core/Spliterator.js +1 -1
- package/out/lib/core/Spliterator.js.map +1 -1
- package/out/lib/formats/CSVSpliterator.d.ts +2 -2
- package/out/lib/formats/CSVSpliterator.d.ts.map +1 -1
- package/out/lib/formats/CSVSpliterator.js +92 -1
- package/out/lib/formats/CSVSpliterator.js.map +1 -1
- package/out/lib/formats/XLSXSpliterator.d.ts +1 -1
- package/out/lib/formats/XLSXSpliterator.d.ts.map +1 -1
- package/package.json +2 -3
- package/out/lib/AsyncSequence.d.ts +0 -180
- package/out/lib/AsyncSequence.d.ts.map +0 -1
- package/out/lib/AsyncSequence.js +0 -432
- package/out/lib/AsyncSequence.js.map +0 -1
- package/out/lib/AsyncSpliterator.d.ts +0 -227
- package/out/lib/AsyncSpliterator.d.ts.map +0 -1
- package/out/lib/AsyncSpliterator.js +0 -510
- package/out/lib/AsyncSpliterator.js.map +0 -1
- package/out/lib/BufferController.d.ts +0 -77
- package/out/lib/BufferController.d.ts.map +0 -1
- package/out/lib/BufferController.js +0 -118
- package/out/lib/BufferController.js.map +0 -1
- package/out/lib/CSVSpliterator.d.ts +0 -131
- package/out/lib/CSVSpliterator.d.ts.map +0 -1
- package/out/lib/CSVSpliterator.js +0 -172
- package/out/lib/CSVSpliterator.js.map +0 -1
- package/out/lib/CharacterSequence.d.ts +0 -72
- package/out/lib/CharacterSequence.d.ts.map +0 -1
- package/out/lib/CharacterSequence.js +0 -347
- package/out/lib/CharacterSequence.js.map +0 -1
- package/out/lib/CompositeDataView.d.ts +0 -111
- package/out/lib/CompositeDataView.d.ts.map +0 -1
- package/out/lib/CompositeDataView.js +0 -241
- package/out/lib/CompositeDataView.js.map +0 -1
- package/out/lib/IndexQueue.d.ts +0 -48
- package/out/lib/IndexQueue.d.ts.map +0 -1
- package/out/lib/IndexQueue.js +0 -111
- package/out/lib/IndexQueue.js.map +0 -1
- package/out/lib/JSONSpliterator.d.ts +0 -44
- package/out/lib/JSONSpliterator.d.ts.map +0 -1
- package/out/lib/JSONSpliterator.js +0 -77
- package/out/lib/JSONSpliterator.js.map +0 -1
- package/out/lib/PSVSpliterator.d.ts +0 -21
- package/out/lib/PSVSpliterator.d.ts.map +0 -1
- package/out/lib/PSVSpliterator.js +0 -22
- package/out/lib/PSVSpliterator.js.map +0 -1
- package/out/lib/SlidingWindow.d.ts +0 -47
- package/out/lib/SlidingWindow.d.ts.map +0 -1
- package/out/lib/SlidingWindow.js +0 -63
- package/out/lib/SlidingWindow.js.map +0 -1
- package/out/lib/Spliterator.d.ts +0 -96
- package/out/lib/Spliterator.d.ts.map +0 -1
- package/out/lib/Spliterator.js +0 -330
- package/out/lib/Spliterator.js.map +0 -1
- package/out/lib/TSVSpliterator.d.ts +0 -21
- package/out/lib/TSVSpliterator.d.ts.map +0 -1
- package/out/lib/TSVSpliterator.js +0 -22
- package/out/lib/TSVSpliterator.js.map +0 -1
- package/out/lib/TextSpliterator.d.ts +0 -53
- package/out/lib/TextSpliterator.d.ts.map +0 -1
- package/out/lib/TextSpliterator.js +0 -67
- package/out/lib/TextSpliterator.js.map +0 -1
- package/out/lib/XLSXSpliterator.d.ts +0 -143
- package/out/lib/XLSXSpliterator.d.ts.map +0 -1
- package/out/lib/XLSXSpliterator.js +0 -161
- package/out/lib/XLSXSpliterator.js.map +0 -1
- package/out/lib/adaptive-source.d.ts +0 -39
- package/out/lib/adaptive-source.d.ts.map +0 -1
- package/out/lib/adaptive-source.js +0 -95
- package/out/lib/adaptive-source.js.map +0 -1
- package/out/lib/casing.d.ts +0 -27
- package/out/lib/casing.d.ts.map +0 -1
- package/out/lib/casing.js +0 -68
- package/out/lib/casing.js.map +0 -1
- package/out/lib/chunks.d.ts +0 -20
- package/out/lib/chunks.d.ts.map +0 -1
- package/out/lib/chunks.js +0 -36
- package/out/lib/chunks.js.map +0 -1
- package/out/lib/merge-async-iterators.d.ts +0 -12
- package/out/lib/merge-async-iterators.d.ts.map +0 -1
- package/out/lib/merge-async-iterators.js +0 -39
- package/out/lib/merge-async-iterators.js.map +0 -1
- package/out/lib/parallel-map-runtime.d.ts +0 -22
- package/out/lib/parallel-map-runtime.d.ts.map +0 -1
- package/out/lib/parallel-map-runtime.js +0 -60
- package/out/lib/parallel-map-runtime.js.map +0 -1
- package/out/lib/parallel-map-worker-entry.d.ts +0 -11
- package/out/lib/parallel-map-worker-entry.d.ts.map +0 -1
- package/out/lib/parallel-map-worker-entry.js +0 -41
- package/out/lib/parallel-map-worker-entry.js.map +0 -1
- package/out/lib/parallel-map-workers.d.ts +0 -57
- package/out/lib/parallel-map-workers.d.ts.map +0 -1
- package/out/lib/parallel-map-workers.js +0 -82
- package/out/lib/parallel-map-workers.js.map +0 -1
- package/out/lib/parallel-map.d.ts +0 -28
- package/out/lib/parallel-map.d.ts.map +0 -1
- package/out/lib/parallel-map.js +0 -30
- package/out/lib/parallel-map.js.map +0 -1
- package/out/lib/pivot.d.ts +0 -30
- package/out/lib/pivot.d.ts.map +0 -1
- package/out/lib/pivot.js +0 -47
- package/out/lib/pivot.js.map +0 -1
- package/out/lib/row-emitters.d.ts +0 -35
- package/out/lib/row-emitters.d.ts.map +0 -1
- package/out/lib/row-emitters.js +0 -50
- package/out/lib/row-emitters.js.map +0 -1
- package/out/lib/segment-runtime.d.ts +0 -27
- package/out/lib/segment-runtime.d.ts.map +0 -1
- package/out/lib/segment-runtime.js +0 -41
- package/out/lib/segment-runtime.js.map +0 -1
- package/out/lib/segment-worker-entry.d.ts +0 -10
- package/out/lib/segment-worker-entry.d.ts.map +0 -1
- package/out/lib/segment-worker-entry.js +0 -59
- package/out/lib/segment-worker-entry.js.map +0 -1
- package/out/lib/segment-workers.d.ts +0 -56
- package/out/lib/segment-workers.d.ts.map +0 -1
- package/out/lib/segment-workers.js +0 -110
- package/out/lib/segment-workers.js.map +0 -1
- package/out/lib/segments.d.ts +0 -31
- package/out/lib/segments.d.ts.map +0 -1
- package/out/lib/segments.js +0 -52
- package/out/lib/segments.js.map +0 -1
- package/out/lib/shared.d.ts +0 -155
- package/out/lib/shared.d.ts.map +0 -1
- package/out/lib/shared.js +0 -41
- package/out/lib/shared.js.map +0 -1
- package/out/lib/take.d.ts +0 -42
- package/out/lib/take.d.ts.map +0 -1
- package/out/lib/take.js +0 -101
- package/out/lib/take.js.map +0 -1
- package/out/lib/utils.d.ts +0 -37
- package/out/lib/utils.d.ts.map +0 -1
- package/out/lib/utils.js +0 -39
- package/out/lib/utils.js.map +0 -1
- package/out/lib/wasm_base64.d.ts +0 -19
- package/out/lib/wasm_base64.d.ts.map +0 -1
- package/out/lib/wasm_base64.js +0 -19
- package/out/lib/wasm_base64.js.map +0 -1
- package/out/lib/wasm_module.d.ts +0 -57
- package/out/lib/wasm_module.d.ts.map +0 -1
- package/out/lib/wasm_module.js +0 -34
- package/out/lib/wasm_module.js.map +0 -1
- package/out/lib/writer.d.ts +0 -45
- package/out/lib/writer.d.ts.map +0 -1
- package/out/lib/writer.js +0 -57
- package/out/lib/writer.js.map +0 -1
- package/out/lib/zip.d.ts +0 -33
- package/out/lib/zip.d.ts.map +0 -1
- package/out/lib/zip.js +0 -59
- package/out/lib/zip.js.map +0 -1
- package/out/node/cli/parse-args.d.ts +0 -120
- package/out/node/cli/parse-args.d.ts.map +0 -1
- package/out/node/cli/parse-args.js +0 -118
- package/out/node/cli/parse-args.js.map +0 -1
|
@@ -1,510 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @copyright Sister Software
|
|
3
|
-
* @license MIT
|
|
4
|
-
* @author Teffen Ellis, et al.
|
|
5
|
-
*/
|
|
6
|
-
import { ReadableStream } from "stream/web";
|
|
7
|
-
import { BufferController } from "./BufferController.js";
|
|
8
|
-
import { CharacterSequence, Delimiters } from "./CharacterSequence.js";
|
|
9
|
-
import { IndexQueue } from "./IndexQueue.js";
|
|
10
|
-
import { runSegmentWorkers } from "./segment-workers.js";
|
|
11
|
-
import { computeSegments } from "./segments.js";
|
|
12
|
-
const noop = () => void 0;
|
|
13
|
-
const sharedEncoder = new TextEncoder();
|
|
14
|
-
/**
|
|
15
|
-
* An asynchronous spliterator for reading delimited byte streams.
|
|
16
|
-
*
|
|
17
|
-
* ```ts
|
|
18
|
-
* import { AsyncSpliterator } from "spliterator"
|
|
19
|
-
*
|
|
20
|
-
* const file = await AsyncSpliterator.from("data.csv")
|
|
21
|
-
*
|
|
22
|
-
* for await (const line of file) {
|
|
23
|
-
* console.log(line)
|
|
24
|
-
* }
|
|
25
|
-
* ```
|
|
26
|
-
*
|
|
27
|
-
* @see {@linkcode Spliterator} for the synchronous version.
|
|
28
|
-
*/
|
|
29
|
-
export class AsyncSpliterator {
|
|
30
|
-
static from(source, init = {}) {
|
|
31
|
-
if (typeof source === "object" && Symbol.asyncIterator in source) {
|
|
32
|
-
return new AsyncSpliterator(source, init);
|
|
33
|
-
}
|
|
34
|
-
return import("spliterator/node/fs").then(async ({ createChunkIterator }) => {
|
|
35
|
-
const chunkIterator = await createChunkIterator(source, {
|
|
36
|
-
highWaterMark: init.highWaterMark,
|
|
37
|
-
});
|
|
38
|
-
return new AsyncSpliterator(chunkIterator, init);
|
|
39
|
-
});
|
|
40
|
-
}
|
|
41
|
-
/**
|
|
42
|
-
* Dispose of the spliterator, closing the file handle if necessary.
|
|
43
|
-
*/
|
|
44
|
-
async [Symbol.asyncDispose]() {
|
|
45
|
-
this.#indices.clear();
|
|
46
|
-
this.#controller.clear();
|
|
47
|
-
await this.#closeReader();
|
|
48
|
-
if (this.#autoDispose) {
|
|
49
|
-
await this.#source[Symbol.asyncDispose]?.();
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
/**
|
|
53
|
-
* Create a new delimited generator from a data resource.
|
|
54
|
-
*/
|
|
55
|
-
static toTransformStream(init) {
|
|
56
|
-
return new TransformStream({
|
|
57
|
-
async transform(chunk, controller) {
|
|
58
|
-
const spliterator = AsyncSpliterator.from(chunk, init);
|
|
59
|
-
for await (const slice of spliterator) {
|
|
60
|
-
controller.enqueue(slice);
|
|
61
|
-
}
|
|
62
|
-
},
|
|
63
|
-
});
|
|
64
|
-
}
|
|
65
|
-
/**
|
|
66
|
-
* Create a new delimited generator from an asynchronous byte stream.
|
|
67
|
-
*
|
|
68
|
-
* @param source - The file to read from.
|
|
69
|
-
* @param init - The initialization options for the generator.
|
|
70
|
-
* @see {@linkcode AsyncSpliterator.from} to create a new generator from a data resource.
|
|
71
|
-
*/
|
|
72
|
-
constructor(source, init = {}) {
|
|
73
|
-
this.#source = source;
|
|
74
|
-
this.#chunkReader = source[Symbol.asyncIterator]();
|
|
75
|
-
this.#autoDispose = init.autoDispose ?? false;
|
|
76
|
-
this.#needle = new CharacterSequence(init.delimiter);
|
|
77
|
-
this.#highWaterMark = Math.max(this.#needle.length * 4, init.highWaterMark ?? 4096 * 16);
|
|
78
|
-
this.#yieldDropCount = Math.max(0, init.drop ?? 0);
|
|
79
|
-
this.#yieldStopCount = Math.max(init.take ?? Infinity, 0) + this.#yieldDropCount;
|
|
80
|
-
this.#skipEmpty = init.skipEmpty ?? true;
|
|
81
|
-
this.#crlf = init.crlf ?? false;
|
|
82
|
-
this.#enableQuoteHandling = init.enableQuoteHandling ?? false;
|
|
83
|
-
if (this.#enableQuoteHandling && this.#needle.length !== 1) {
|
|
84
|
-
throw new RangeError("enableQuoteHandling requires a single-byte delimiter in the asynchronous engine — a multi-byte delimiter straddling a chunk boundary cannot be rescanned without re-toggling quote state.");
|
|
85
|
-
}
|
|
86
|
-
this.#controller = new BufferController({
|
|
87
|
-
initialBufferSize: this.#highWaterMark,
|
|
88
|
-
});
|
|
89
|
-
this.#debug = init.debug ?? false;
|
|
90
|
-
this.#log = this.#debug ? console.debug.bind(console) : noop;
|
|
91
|
-
}
|
|
92
|
-
//#endregion
|
|
93
|
-
//#region Private Properties
|
|
94
|
-
/**
|
|
95
|
-
* Whether to call `Symbol.asyncDispose` on the source once the iterator is done.
|
|
96
|
-
*/
|
|
97
|
-
#autoDispose;
|
|
98
|
-
/**
|
|
99
|
-
* The source of the data.
|
|
100
|
-
*/
|
|
101
|
-
#source;
|
|
102
|
-
/**
|
|
103
|
-
* The chunk reader for the file.
|
|
104
|
-
*/
|
|
105
|
-
#chunkReader;
|
|
106
|
-
/**
|
|
107
|
-
* The last chunk of data read from the source.
|
|
108
|
-
*/
|
|
109
|
-
#lastReadResult;
|
|
110
|
-
/**
|
|
111
|
-
* A queue of index tuples marking the start and end of delimiter positions.
|
|
112
|
-
*
|
|
113
|
-
* Note that indices are relative to the buffer, not the file.
|
|
114
|
-
*
|
|
115
|
-
* This means that the start index is always 0, and the end index is the byte length of the buffer.
|
|
116
|
-
*/
|
|
117
|
-
#indices = new IndexQueue();
|
|
118
|
-
/**
|
|
119
|
-
* The byte sequence to search for, i.e. an encoded delimiter.
|
|
120
|
-
*/
|
|
121
|
-
#needle;
|
|
122
|
-
/**
|
|
123
|
-
* The buffer containing the current chunks of data. This will be resized as needed to accommodate the file read
|
|
124
|
-
* operation.
|
|
125
|
-
*/
|
|
126
|
-
#controller;
|
|
127
|
-
/**
|
|
128
|
-
* How many yields to skip.
|
|
129
|
-
*/
|
|
130
|
-
#yieldDropCount;
|
|
131
|
-
/**
|
|
132
|
-
* How many yields to allow before stopping.
|
|
133
|
-
*/
|
|
134
|
-
#yieldStopCount;
|
|
135
|
-
/**
|
|
136
|
-
* Whether to skip empty yields.
|
|
137
|
-
*/
|
|
138
|
-
#skipEmpty;
|
|
139
|
-
/**
|
|
140
|
-
* Whether to trim a carriage return immediately preceding each delimiter match.
|
|
141
|
-
*/
|
|
142
|
-
#crlf;
|
|
143
|
-
/**
|
|
144
|
-
* Whether delimiters inside double-quoted regions are ignored.
|
|
145
|
-
*/
|
|
146
|
-
#enableQuoteHandling;
|
|
147
|
-
/**
|
|
148
|
-
* The byte sequence for a double quote.
|
|
149
|
-
*/
|
|
150
|
-
#doubleQuoteSequence = new CharacterSequence('"');
|
|
151
|
-
/**
|
|
152
|
-
* Whether the scan cursor currently sits inside a double-quoted region. Only meaningful with
|
|
153
|
-
* {@linkcode SpliteratorInit.enableQuoteHandling} — the state must survive both `#fill` rounds and buffer compression,
|
|
154
|
-
* since a quoted region can span many chunks.
|
|
155
|
-
*/
|
|
156
|
-
#insideQuotes = false;
|
|
157
|
-
/**
|
|
158
|
-
* Raw buffer position up to which matches have been processed. Distinct from the enqueued range ends — those may be
|
|
159
|
-
* CRLF-trimmed, and quote toggles may sit beyond the last emitted field. Rebased on buffer compression.
|
|
160
|
-
*/
|
|
161
|
-
#searchCursor = 0;
|
|
162
|
-
/**
|
|
163
|
-
* Raw buffer position where the current (not yet terminated) slice begins — i.e. one past the last consumed
|
|
164
|
-
* delimiter. This is the compression boundary and the start of the EOF tail. Rebased on buffer compression.
|
|
165
|
-
*/
|
|
166
|
-
#pendingSliceStart = 0;
|
|
167
|
-
/**
|
|
168
|
-
* The high water mark for the buffer.
|
|
169
|
-
*
|
|
170
|
-
* This defines the size of each read operation, as well as the total size of indices to keep in memory.
|
|
171
|
-
*/
|
|
172
|
-
#highWaterMark;
|
|
173
|
-
/**
|
|
174
|
-
* The total number of bytes we've yielded.
|
|
175
|
-
*/
|
|
176
|
-
#yieldedByteLength = 0;
|
|
177
|
-
/**
|
|
178
|
-
* The total number of yields.
|
|
179
|
-
*/
|
|
180
|
-
#yieldCount = 0;
|
|
181
|
-
// /**
|
|
182
|
-
// * The current byte index to perform read operations from.
|
|
183
|
-
// */
|
|
184
|
-
// #readPosition: number
|
|
185
|
-
/**
|
|
186
|
-
* Whether to output debug information.
|
|
187
|
-
*/
|
|
188
|
-
#debug;
|
|
189
|
-
/**
|
|
190
|
-
* Whether the iterator is done.
|
|
191
|
-
*/
|
|
192
|
-
#done = false;
|
|
193
|
-
//#endregion
|
|
194
|
-
//#region Private Methods
|
|
195
|
-
#log;
|
|
196
|
-
/**
|
|
197
|
-
* Trim a carriage return immediately preceding a delimiter match, when {@linkcode SpliteratorInit.crlf} is set.
|
|
198
|
-
* Affects only the emitted range's end — cursor advancement always uses the raw match offset.
|
|
199
|
-
*/
|
|
200
|
-
#trimEnd(start, end) {
|
|
201
|
-
if (this.#crlf && end > start && this.#controller.bytes[end - 1] === Delimiters.CarriageReturn) {
|
|
202
|
-
return end - 1;
|
|
203
|
-
}
|
|
204
|
-
return end;
|
|
205
|
-
}
|
|
206
|
-
/**
|
|
207
|
-
* Find and enqueue delimiter positions.
|
|
208
|
-
*
|
|
209
|
-
* The search is bounded by `bytesWritten` so we never scan into uninitialized buffer memory and never enqueue ranges
|
|
210
|
-
* whose `end` exceeds the valid byte length.
|
|
211
|
-
*
|
|
212
|
-
* Progress is tracked by `#searchCursor`/`#pendingSliceStart` rather than derived from enqueued range ends — those
|
|
213
|
-
* ends may be CRLF-trimmed, and (in quote mode) quote toggles may sit beyond the last emitted field.
|
|
214
|
-
*/
|
|
215
|
-
#search() {
|
|
216
|
-
const searchEnd = this.#controller.bytesWritten;
|
|
217
|
-
const quotePattern = this.#enableQuoteHandling ? this.#doubleQuoteSequence : undefined;
|
|
218
|
-
// The streaming fast path carries range construction and quote state through one bounded
|
|
219
|
-
// WASM call. A full result batch resumes at the unconsumed delimiter, so dense inputs are
|
|
220
|
-
// neither truncated nor rescanned from the beginning.
|
|
221
|
-
while (this.#searchCursor < searchEnd) {
|
|
222
|
-
const scan = this.#needle.scanRanges(this.#controller.bytes, {
|
|
223
|
-
scanCursor: this.#searchCursor,
|
|
224
|
-
pendingSliceStart: this.#pendingSliceStart,
|
|
225
|
-
insideQuotes: this.#insideQuotes,
|
|
226
|
-
}, searchEnd, quotePattern);
|
|
227
|
-
if (!scan)
|
|
228
|
-
break;
|
|
229
|
-
for (let i = 0; i < scan.count; i++) {
|
|
230
|
-
const start = scan.ranges[i * 2];
|
|
231
|
-
const end = this.#trimEnd(start, scan.ranges[i * 2 + 1]);
|
|
232
|
-
const byteRange = [start, end];
|
|
233
|
-
this.#log("Found byte range", byteRange);
|
|
234
|
-
this.#indices.enqueue(byteRange);
|
|
235
|
-
}
|
|
236
|
-
const previousCursor = this.#searchCursor;
|
|
237
|
-
this.#searchCursor = scan.scanCursor;
|
|
238
|
-
this.#pendingSliceStart = scan.pendingSliceStart;
|
|
239
|
-
this.#insideQuotes = scan.insideQuotes;
|
|
240
|
-
if (this.#searchCursor >= searchEnd)
|
|
241
|
-
return;
|
|
242
|
-
if (this.#searchCursor <= previousCursor)
|
|
243
|
-
break;
|
|
244
|
-
}
|
|
245
|
-
if (!this.#enableQuoteHandling) {
|
|
246
|
-
while (this.#searchCursor <= searchEnd) {
|
|
247
|
-
const delimiterIndex = this.#needle.search(this.#controller.bytes, this.#searchCursor, searchEnd);
|
|
248
|
-
if (delimiterIndex === -1) {
|
|
249
|
-
this.#log(`No viable byte range found in buffer. Breaking.`, { searchCursor: this.#searchCursor });
|
|
250
|
-
break;
|
|
251
|
-
}
|
|
252
|
-
const byteRange = [this.#pendingSliceStart, this.#trimEnd(this.#pendingSliceStart, delimiterIndex)];
|
|
253
|
-
this.#log("Found byte range", byteRange);
|
|
254
|
-
this.#indices.enqueue(byteRange);
|
|
255
|
-
this.#searchCursor = delimiterIndex + this.#needle.length;
|
|
256
|
-
this.#pendingSliceStart = this.#searchCursor;
|
|
257
|
-
}
|
|
258
|
-
return;
|
|
259
|
-
}
|
|
260
|
-
// Quote-aware path: a delimiter inside a double-quoted region does not split. The quote
|
|
261
|
-
// state persists across fills and buffer compressions, since a quoted region may span
|
|
262
|
-
// many chunks. The delimiter is single-byte (enforced in the constructor), so a match can
|
|
263
|
-
// never straddle a chunk boundary and the cursor can safely advance to `searchEnd`.
|
|
264
|
-
const matches = this.#needle.searchMatches(this.#controller.bytes, this.#doubleQuoteSequence, this.#searchCursor, searchEnd);
|
|
265
|
-
for (const match of matches) {
|
|
266
|
-
if (match.patternId === 1) {
|
|
267
|
-
this.#insideQuotes = !this.#insideQuotes;
|
|
268
|
-
continue;
|
|
269
|
-
}
|
|
270
|
-
if (this.#insideQuotes)
|
|
271
|
-
continue;
|
|
272
|
-
const byteRange = [this.#pendingSliceStart, this.#trimEnd(this.#pendingSliceStart, match.offset)];
|
|
273
|
-
this.#log("Found byte range", byteRange);
|
|
274
|
-
this.#indices.enqueue(byteRange);
|
|
275
|
-
this.#pendingSliceStart = match.offset + this.#needle.length;
|
|
276
|
-
}
|
|
277
|
-
this.#searchCursor = searchEnd;
|
|
278
|
-
}
|
|
279
|
-
/**
|
|
280
|
-
* Read a chunk of data from the file.
|
|
281
|
-
*/
|
|
282
|
-
async #read() {
|
|
283
|
-
this.#lastReadResult = await this.#chunkReader.next();
|
|
284
|
-
if (this.#lastReadResult.done)
|
|
285
|
-
return;
|
|
286
|
-
// A string chunk (a `Readable` with an encoding set, a hand-rolled generator of strings)
|
|
287
|
-
// would coerce to garbage via `TypedArray.set` — encode it instead of silently mis-reading.
|
|
288
|
-
const chunk = this.#lastReadResult.value;
|
|
289
|
-
const bytes = typeof chunk === "string" ? sharedEncoder.encode(chunk) : chunk;
|
|
290
|
-
// Append the chunk to the buffer.
|
|
291
|
-
this.#controller.set(bytes, this.#controller.bytesWritten);
|
|
292
|
-
}
|
|
293
|
-
/**
|
|
294
|
-
* Fill the buffer with data and search for delimiters.
|
|
295
|
-
*/
|
|
296
|
-
async #fill() {
|
|
297
|
-
if (this.#lastReadResult?.done) {
|
|
298
|
-
this.#log("Reached end of file. Preparing to finalize.");
|
|
299
|
-
// Enqueue a final slice covering the bytes between the last consumed delimiter and the
|
|
300
|
-
// end of the stream. The slice may be empty (when the stream ends exactly on a
|
|
301
|
-
// delimiter, or when the file is empty); we still enqueue it so that callers with
|
|
302
|
-
// `skipEmpty: false` see the trailing entry produced by `String.prototype.split`.
|
|
303
|
-
// `#pendingSliceStart` — not the last enqueued range's end — is the authority here:
|
|
304
|
-
// range ends may be CRLF-trimmed.
|
|
305
|
-
const tailStart = this.#pendingSliceStart;
|
|
306
|
-
const tailEnd = this.#controller.bytesWritten;
|
|
307
|
-
if (tailStart <= tailEnd) {
|
|
308
|
-
this.#log("Inserted trailing byte range", [tailStart, tailEnd]);
|
|
309
|
-
this.#indices.enqueue([tailStart, tailEnd]);
|
|
310
|
-
}
|
|
311
|
-
else {
|
|
312
|
-
this.#log("Skipped trailing range — start beyond end", { tailStart, tailEnd });
|
|
313
|
-
}
|
|
314
|
-
this.#done = true;
|
|
315
|
-
return;
|
|
316
|
-
}
|
|
317
|
-
// We're effectively done with that window of the buffer.
|
|
318
|
-
// So we can compress it to save memory.
|
|
319
|
-
const nextBufferStart = this.#pendingSliceStart;
|
|
320
|
-
this.#log("Compressing buffer", { nextBufferStart });
|
|
321
|
-
this.#controller.compress(nextBufferStart);
|
|
322
|
-
// After compress, the buffer coordinate space has shifted — rebase the scan cursors.
|
|
323
|
-
// Quote state (`#insideQuotes`) is positionless and survives as-is.
|
|
324
|
-
this.#pendingSliceStart = 0;
|
|
325
|
-
this.#searchCursor = Math.max(0, this.#searchCursor - nextBufferStart);
|
|
326
|
-
while ((!this.#lastReadResult || !this.#lastReadResult.done) && this.#indices.byteLength < this.#highWaterMark) {
|
|
327
|
-
await this.#read();
|
|
328
|
-
this.#search();
|
|
329
|
-
}
|
|
330
|
-
}
|
|
331
|
-
/**
|
|
332
|
-
* Signal the underlying chunk reader to stop, destroying an owned read stream (closing its fd).
|
|
333
|
-
*/
|
|
334
|
-
async #closeReader() {
|
|
335
|
-
try {
|
|
336
|
-
await this.#chunkReader.return?.();
|
|
337
|
-
}
|
|
338
|
-
catch {
|
|
339
|
-
// The reader may already be closed; ignore.
|
|
340
|
-
}
|
|
341
|
-
}
|
|
342
|
-
async #finalize() {
|
|
343
|
-
await this.#closeReader();
|
|
344
|
-
if (this.#debug) {
|
|
345
|
-
/**
|
|
346
|
-
* The total number of bytes we expect to be omitted from the buffer. This is derived from the number of of yields
|
|
347
|
-
* and the length of the delimiter.
|
|
348
|
-
*/
|
|
349
|
-
const expectedYieldedDelimitedBytes = (this.#yieldCount - 1) * this.#needle.length;
|
|
350
|
-
this.#log({
|
|
351
|
-
yieldedByteLength: this.#yieldedByteLength,
|
|
352
|
-
yieldCount: this.#yieldCount,
|
|
353
|
-
expectedYieldedDelimitedBytes,
|
|
354
|
-
});
|
|
355
|
-
}
|
|
356
|
-
if (this.#autoDispose) {
|
|
357
|
-
await this.#source[Symbol.asyncDispose]?.();
|
|
358
|
-
}
|
|
359
|
-
return {
|
|
360
|
-
done: true,
|
|
361
|
-
value: undefined,
|
|
362
|
-
};
|
|
363
|
-
}
|
|
364
|
-
//#endregion
|
|
365
|
-
//#region Iterator Methods
|
|
366
|
-
async next() {
|
|
367
|
-
// Loop rather than recurse: a long run of skipped (empty or dropped) ranges would
|
|
368
|
-
// otherwise grow the call stack one frame per skip and overflow.
|
|
369
|
-
while (true) {
|
|
370
|
-
if (this.#done || this.#yieldCount >= this.#yieldStopCount)
|
|
371
|
-
return this.#finalize();
|
|
372
|
-
if (!this.#indices.size) {
|
|
373
|
-
await this.#fill();
|
|
374
|
-
}
|
|
375
|
-
// The first `#fill` may have read into EOF without entering the EOF branch (the branch only
|
|
376
|
-
// triggers when fill is called with `done` already set). In that case the trailing tail
|
|
377
|
-
// hasn't been enqueued yet, so call fill once more to let the EOF branch flush it.
|
|
378
|
-
if (this.#lastReadResult?.done && !this.#indices.size && !this.#done) {
|
|
379
|
-
await this.#fill();
|
|
380
|
-
}
|
|
381
|
-
const currentByteRange = this.#indices.dequeue();
|
|
382
|
-
if (!currentByteRange) {
|
|
383
|
-
this.#done = true;
|
|
384
|
-
return this.#finalize();
|
|
385
|
-
}
|
|
386
|
-
const [start, end] = currentByteRange;
|
|
387
|
-
const slice = this.#controller.subarray(start, end);
|
|
388
|
-
if (!slice.length && this.#skipEmpty) {
|
|
389
|
-
continue;
|
|
390
|
-
}
|
|
391
|
-
this.#yieldCount++;
|
|
392
|
-
if (this.#yieldCount <= this.#yieldDropCount) {
|
|
393
|
-
continue;
|
|
394
|
-
}
|
|
395
|
-
this.#yieldedByteLength += end - start;
|
|
396
|
-
return {
|
|
397
|
-
value: slice,
|
|
398
|
-
done: false,
|
|
399
|
-
};
|
|
400
|
-
}
|
|
401
|
-
}
|
|
402
|
-
/**
|
|
403
|
-
* Collect all the byte ranges from the file.
|
|
404
|
-
*
|
|
405
|
-
* **This method will read the entire file into memory.**
|
|
406
|
-
*/
|
|
407
|
-
toArray() {
|
|
408
|
-
return Array.fromAsync(this);
|
|
409
|
-
}
|
|
410
|
-
/**
|
|
411
|
-
* Collect all the byte ranges from the file as a string.
|
|
412
|
-
*
|
|
413
|
-
* **This method will read the entire file into memory.**
|
|
414
|
-
*/
|
|
415
|
-
toDecodedArray(decoder = new TextDecoder()) {
|
|
416
|
-
return Array.fromAsync(this, (bytes) => decoder.decode(bytes));
|
|
417
|
-
}
|
|
418
|
-
/**
|
|
419
|
-
* Wraps the spliterator in a readable stream.
|
|
420
|
-
*
|
|
421
|
-
* This is useful for piping the spliterator to other streams.
|
|
422
|
-
*
|
|
423
|
-
* @returns A readable stream of byte ranges.
|
|
424
|
-
*/
|
|
425
|
-
toReadableStream() {
|
|
426
|
-
const iterator = this[Symbol.asyncIterator]();
|
|
427
|
-
return new ReadableStream({
|
|
428
|
-
pull: async (controller) => {
|
|
429
|
-
const { done, value } = await iterator.next();
|
|
430
|
-
if (done) {
|
|
431
|
-
controller.close();
|
|
432
|
-
}
|
|
433
|
-
else {
|
|
434
|
-
controller.enqueue(value);
|
|
435
|
-
}
|
|
436
|
-
},
|
|
437
|
-
cancel: () => this[Symbol.asyncDispose]?.(),
|
|
438
|
-
});
|
|
439
|
-
}
|
|
440
|
-
/**
|
|
441
|
-
* Pipes the spliterator to a writable stream.
|
|
442
|
-
*
|
|
443
|
-
* ```ts
|
|
444
|
-
* const spliterator = await AsyncSpliterator.from("data.csv")
|
|
445
|
-
*
|
|
446
|
-
* const textStream = spliterator.pipeTo(new TextDecoderStream())
|
|
447
|
-
* ```
|
|
448
|
-
*
|
|
449
|
-
* @see {@linkcode toReadableStream} to first convert the spliterator to a readable stream.
|
|
450
|
-
*/
|
|
451
|
-
pipeThrough(transform, options) {
|
|
452
|
-
return this.toReadableStream().pipeThrough(transform, options);
|
|
453
|
-
}
|
|
454
|
-
/**
|
|
455
|
-
* Iterate over the byte ranges in the file, yielding each range. between delimiters.
|
|
456
|
-
*/
|
|
457
|
-
[Symbol.asyncIterator]() {
|
|
458
|
-
return this;
|
|
459
|
-
}
|
|
460
|
-
return() {
|
|
461
|
-
return this.#finalize();
|
|
462
|
-
}
|
|
463
|
-
//#endregion
|
|
464
|
-
//#region Experimental
|
|
465
|
-
/**
|
|
466
|
-
* Compute delimiter-aligned `[start, end)` byte ranges dividing `source` into up to `concurrency` segments. The
|
|
467
|
-
* boundary primitive for parallel parsing — hand each range to a worker (with its own handle). See
|
|
468
|
-
* {@linkcode asManyWorkers} for the turnkey worker version.
|
|
469
|
-
*/
|
|
470
|
-
static segments(source, options) {
|
|
471
|
-
return computeSegments(source, options);
|
|
472
|
-
}
|
|
473
|
-
/**
|
|
474
|
-
* Split `source` into delimiter-aligned segments and return one {@linkcode AsyncSpliterator} per segment. All share
|
|
475
|
-
* the event loop (no worker threads) — for moderate jobs or to overlap I/O. Returns at most `concurrency` instances.
|
|
476
|
-
*
|
|
477
|
-
* @param source - The data resource to read from.
|
|
478
|
-
* @param options - Delimiter, desired concurrency, and optional probe size.
|
|
479
|
-
*
|
|
480
|
-
* @returns One `AsyncSpliterator` per non-empty segment, possibly fewer than `concurrency`.
|
|
481
|
-
*/
|
|
482
|
-
static async asMany(source, options) {
|
|
483
|
-
const { createChunkIterator } = await import("spliterator/node/fs");
|
|
484
|
-
const segments = await computeSegments(source, options);
|
|
485
|
-
return Promise.all(segments.map(async ([start, end]) => {
|
|
486
|
-
// `end` is exclusive here; createChunkIterator's `end` is inclusive.
|
|
487
|
-
const chunkIterator = await createChunkIterator(source, { start, end: end - 1 });
|
|
488
|
-
return new AsyncSpliterator(chunkIterator, { delimiter: options.delimiter, autoDispose: true });
|
|
489
|
-
}));
|
|
490
|
-
}
|
|
491
|
-
/**
|
|
492
|
-
* Parse `source` across worker threads. Each worker owns a handle to a delimiter-aligned segment, runs the `worker`
|
|
493
|
-
* handler module per record, and streams results back to the main thread as one merged async iterator — for a
|
|
494
|
-
* single-thread writer. Results interleave across segments.
|
|
495
|
-
*
|
|
496
|
-
* The handler returns a value (structured-cloned back), a `Uint8Array` (transferred zero-copy), or `undefined` (the
|
|
497
|
-
* record is skipped). The handler module's top-level code runs once per worker — load models/handles there.
|
|
498
|
-
*
|
|
499
|
-
* @param source A file path or URL (file handles cannot cross threads → `TypeError` otherwise).
|
|
500
|
-
* @param options Handler module, delimiter, concurrency, and batching/backpressure tuning.
|
|
501
|
-
*/
|
|
502
|
-
static asManyWorkers(source, options) {
|
|
503
|
-
// Validate eagerly so callers fail fast (the generator body is otherwise lazy until consumed).
|
|
504
|
-
if (typeof source !== "string" && !(source instanceof URL)) {
|
|
505
|
-
throw new TypeError("asManyWorkers requires a file path or URL — file handles cannot cross threads.");
|
|
506
|
-
}
|
|
507
|
-
return runSegmentWorkers(source, options);
|
|
508
|
-
}
|
|
509
|
-
}
|
|
510
|
-
//# sourceMappingURL=AsyncSpliterator.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"AsyncSpliterator.js","sourceRoot":"","sources":["../../lib/AsyncSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,cAAc,EAAqD,MAAM,YAAY,CAAA;AAE9F,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AACxD,OAAO,EAAE,iBAAiB,EAA+B,UAAU,EAAE,MAAM,wBAAwB,CAAA;AACnG,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EAA6B,iBAAiB,EAAE,MAAM,sBAAsB,CAAA;AACnF,OAAO,EAAE,eAAe,EAAuB,MAAM,eAAe,CAAA;AAGpE,MAAM,IAAI,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC,CAAA;AACzB,MAAM,aAAa,GAAG,IAAI,WAAW,EAAE,CAAA;AAqGvC;;;;;;;;;;;;;;GAcG;AACH,MAAM,OAAO,gBAAgB;IAmC5B,MAAM,CAAC,IAAI,CACV,MAA8C,EAC9C,IAAI,GAAyB,EAAE;QAE/B,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,aAAa,IAAI,MAAM,EAAE,CAAC;YAClE,OAAO,IAAI,gBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QAC1C,CAAC;QAED,OAAO,MAAM,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,mBAAmB,EAAE,EAAE,EAAE;YAC3E,MAAM,aAAa,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE;gBACvD,aAAa,EAAE,IAAI,CAAC,aAAa;aACjC,CAAC,CAAA;YAEF,OAAO,IAAI,gBAAgB,CAAC,aAAa,EAAE,IAAI,CAAC,CAAA;QACjD,CAAC,CAAC,CAAA;IACH,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QACjC,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;QACrB,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAA;QAExB,MAAM,IAAI,CAAC,YAAY,EAAE,CAAA;QAEzB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE,CAAA;QAC5C,CAAC;IACF,CAAC;IAED;;OAEG;IACI,MAAM,CAAC,iBAAiB,CAAC,IAAqB;QACpD,OAAO,IAAI,eAAe,CAAiC;YAC1D,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,UAAU;gBAChC,MAAM,WAAW,GAAG,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;gBAEtD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;oBACvC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;gBAC1B,CAAC;YACF,CAAC;SACD,CAAC,CAAA;IACH,CAAC;IAED;;;;;;OAMG;IACH,YAAY,MAA0B,EAAE,IAAI,GAAyB,EAAE;QACtE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QACrB,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAA;QAElD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,WAAW,IAAI,KAAK,CAAA;QAE7C,IAAI,CAAC,OAAO,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QAEpD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,aAAa,IAAI,IAAI,GAAG,EAAE,CAAC,CAAA;QAExF,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAA;QAClD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,eAAe,CAAA;QAEhF,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAA;QACxC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,IAAI,KAAK,CAAA;QAC/B,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,mBAAmB,IAAI,KAAK,CAAA;QAE7D,IAAI,IAAI,CAAC,oBAAoB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5D,MAAM,IAAI,UAAU,CACnB,2LAA2L,CAC3L,CAAA;QACF,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,IAAI,gBAAgB,CAAC;YACvC,iBAAiB,EAAE,IAAI,CAAC,cAAc;SACtC,CAAC,CAAA;QAEF,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,IAAI,KAAK,CAAA;QACjC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAC7D,CAAC;IAED,YAAY;IAEZ,4BAA4B;IAE5B;;OAEG;IACM,YAAY,CAAS;IAE9B;;OAEG;IACM,OAAO,CAAoB;IAEpC;;OAEG;IACM,YAAY,CAAoC;IAEzD;;OAEG;IACH,eAAe,CAAiD;IAEhE;;;;;;OAMG;IACM,QAAQ,GAAG,IAAI,UAAU,EAAE,CAAA;IAEpC;;OAEG;IACM,OAAO,CAAmB;IAEnC;;;OAGG;IACM,WAAW,CAAkB;IAEtC;;OAEG;IACM,eAAe,CAAQ;IAEhC;;OAEG;IACM,eAAe,CAAQ;IAEhC;;OAEG;IACM,UAAU,CAAS;IAE5B;;OAEG;IACM,KAAK,CAAS;IAEvB;;OAEG;IACM,oBAAoB,CAAS;IAEtC;;OAEG;IACM,oBAAoB,GAAG,IAAI,iBAAiB,CAAC,GAAG,CAAC,CAAA;IAE1D;;;;OAIG;IACH,aAAa,GAAG,KAAK,CAAA;IAErB;;;OAGG;IACH,aAAa,GAAG,CAAC,CAAA;IAEjB;;;OAGG;IACH,kBAAkB,GAAG,CAAC,CAAA;IAEtB;;;;OAIG;IACM,cAAc,CAAQ;IAE/B;;OAEG;IACH,kBAAkB,GAAG,CAAC,CAAA;IAEtB;;OAEG;IACH,WAAW,GAAG,CAAC,CAAA;IAEf,MAAM;IACN,6DAA6D;IAC7D,MAAM;IACN,wBAAwB;IAExB;;OAEG;IACH,MAAM,CAAS;IAEf;;OAEG;IACH,KAAK,GAAG,KAAK,CAAA;IAEb,YAAY;IAEZ,yBAAyB;IAEzB,IAAI,CAA0B;IAE9B;;;OAGG;IACH,QAAQ,CAAC,KAAa,EAAE,GAAW;QAClC,IAAI,IAAI,CAAC,KAAK,IAAI,GAAG,GAAG,KAAK,IAAI,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,UAAU,CAAC,cAAc,EAAE,CAAC;YAChG,OAAO,GAAG,GAAG,CAAC,CAAA;QACf,CAAC;QAED,OAAO,GAAG,CAAA;IACX,CAAC;IAED;;;;;;;;OAQG;IACH,OAAO;QACN,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,CAAA;QAC/C,MAAM,YAAY,GAAG,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,SAAS,CAAA;QAEtF,yFAAyF;QACzF,0FAA0F;QAC1F,sDAAsD;QACtD,OAAO,IAAI,CAAC,aAAa,GAAG,SAAS,EAAE,CAAC;YACvC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CACnC,IAAI,CAAC,WAAW,CAAC,KAAK,EACtB;gBACC,UAAU,EAAE,IAAI,CAAC,aAAa;gBAC9B,iBAAiB,EAAE,IAAI,CAAC,kBAAkB;gBAC1C,YAAY,EAAE,IAAI,CAAC,aAAa;aAChC,EACD,SAAS,EACT,YAAY,CACZ,CAAA;YAED,IAAI,CAAC,IAAI;gBAAE,MAAK;YAEhB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,CAAA;gBACjC,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAE,CAAC,CAAA;gBACzD,MAAM,SAAS,GAAc,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;gBAEzC,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,SAAS,CAAC,CAAA;gBACxC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;YACjC,CAAC;YAED,MAAM,cAAc,GAAG,IAAI,CAAC,aAAa,CAAA;YACzC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,UAAU,CAAA;YACpC,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,iBAAiB,CAAA;YAChD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,YAAY,CAAA;YAEtC,IAAI,IAAI,CAAC,aAAa,IAAI,SAAS;gBAAE,OAAM;YAE3C,IAAI,IAAI,CAAC,aAAa,IAAI,cAAc;gBAAE,MAAK;QAChD,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAChC,OAAO,IAAI,CAAC,aAAa,IAAI,SAAS,EAAE,CAAC;gBACxC,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,aAAa,EAAE,SAAS,CAAC,CAAA;gBAEjG,IAAI,cAAc,KAAK,CAAC,CAAC,EAAE,CAAC;oBAC3B,IAAI,CAAC,IAAI,CAAC,iDAAiD,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC,CAAA;oBAElG,MAAK;gBACN,CAAC;gBAED,MAAM,SAAS,GAAc,CAAC,IAAI,CAAC,kBAAkB,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,kBAAkB,EAAE,cAAc,CAAC,CAAC,CAAA;gBAE9G,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,SAAS,CAAC,CAAA;gBACxC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;gBAEhC,IAAI,CAAC,aAAa,GAAG,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;gBACzD,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,aAAa,CAAA;YAC7C,CAAC;YAED,OAAM;QACP,CAAC;QAED,wFAAwF;QACxF,sFAAsF;QACtF,0FAA0F;QAC1F,oFAAoF;QACpF,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CACzC,IAAI,CAAC,WAAW,CAAC,KAAK,EACtB,IAAI,CAAC,oBAAoB,EACzB,IAAI,CAAC,aAAa,EAClB,SAAS,CACT,CAAA;QAED,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,KAAK,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;gBAC3B,IAAI,CAAC,aAAa,GAAG,CAAC,IAAI,CAAC,aAAa,CAAA;gBAExC,SAAQ;YACT,CAAC;YAED,IAAI,IAAI,CAAC,aAAa;gBAAE,SAAQ;YAEhC,MAAM,SAAS,GAAc,CAAC,IAAI,CAAC,kBAAkB,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,kBAAkB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;YAE5G,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,SAAS,CAAC,CAAA;YACxC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;YAEhC,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;QAC7D,CAAC;QAED,IAAI,CAAC,aAAa,GAAG,SAAS,CAAA;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACV,IAAI,CAAC,eAAe,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAA;QAErD,IAAI,IAAI,CAAC,eAAe,CAAC,IAAI;YAAE,OAAM;QAErC,yFAAyF;QACzF,4FAA4F;QAC5F,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAA;QACxC,MAAM,KAAK,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAA;QAE7E,kCAAkC;QAClC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,CAAA;IAC3D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACV,IAAI,IAAI,CAAC,eAAe,EAAE,IAAI,EAAE,CAAC;YAChC,IAAI,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAA;YACxD,uFAAuF;YACvF,+EAA+E;YAC/E,kFAAkF;YAClF,kFAAkF;YAClF,oFAAoF;YACpF,kCAAkC;YAClC,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAA;YACzC,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,CAAA;YAE7C,IAAI,SAAS,IAAI,OAAO,EAAE,CAAC;gBAC1B,IAAI,CAAC,IAAI,CAAC,8BAA8B,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAA;gBAC/D,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAA;YAC5C,CAAC;iBAAM,CAAC;gBACP,IAAI,CAAC,IAAI,CAAC,2CAA2C,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAA;YAC/E,CAAC;YAED,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;YAEjB,OAAM;QACP,CAAC;QAED,yDAAyD;QACzD,wCAAwC;QACxC,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,CAAA;QAE/C,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,eAAe,EAAE,CAAC,CAAA;QACpD,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAA;QAE1C,qFAAqF;QACrF,oEAAoE;QACpE,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAA;QAC3B,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,aAAa,GAAG,eAAe,CAAC,CAAA;QAEtE,OAAO,CAAC,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;YAChH,MAAM,IAAI,CAAC,KAAK,EAAE,CAAA;YAClB,IAAI,CAAC,OAAO,EAAE,CAAA;QACf,CAAC;IACF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY;QACjB,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,EAAE,CAAA;QACnC,CAAC;QAAC,MAAM,CAAC;YACR,4CAA4C;QAC7C,CAAC;IACF,CAAC;IAED,KAAK,CAAC,SAAS;QACd,MAAM,IAAI,CAAC,YAAY,EAAE,CAAA;QAEzB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB;;;eAGG;YACH,MAAM,6BAA6B,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;YAElF,IAAI,CAAC,IAAI,CAAC;gBACT,iBAAiB,EAAE,IAAI,CAAC,kBAAkB;gBAC1C,UAAU,EAAE,IAAI,CAAC,WAAW;gBAC5B,6BAA6B;aAC7B,CAAC,CAAA;QACH,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE,CAAA;QAC5C,CAAC;QAED,OAAO;YACN,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,SAAS;SAChB,CAAA;IACF,CAAC;IAED,YAAY;IAEZ,0BAA0B;IAEnB,KAAK,CAAC,IAAI;QAChB,kFAAkF;QAClF,iEAAiE;QACjE,OAAO,IAAI,EAAE,CAAC;YACb,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,eAAe;gBAAE,OAAO,IAAI,CAAC,SAAS,EAAE,CAAA;YAEnF,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACzB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAA;YACnB,CAAC;YAED,4FAA4F;YAC5F,wFAAwF;YACxF,mFAAmF;YACnF,IAAI,IAAI,CAAC,eAAe,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACtE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAA;YACnB,CAAC;YAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAA;YAEhD,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACvB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;gBAEjB,OAAO,IAAI,CAAC,SAAS,EAAE,CAAA;YACxB,CAAC;YAED,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,gBAAgB,CAAA;YAErC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;YAEnD,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACtC,SAAQ;YACT,CAAC;YAED,IAAI,CAAC,WAAW,EAAE,CAAA;YAElB,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;gBAC9C,SAAQ;YACT,CAAC;YAED,IAAI,CAAC,kBAAkB,IAAI,GAAG,GAAG,KAAK,CAAA;YAEtC,OAAO;gBACN,KAAK,EAAE,KAAqB;gBAC5B,IAAI,EAAE,KAAK;aACX,CAAA;QACF,CAAC;IACF,CAAC;IAED;;;;OAIG;IACI,OAAO;QACb,OAAO,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;IAC7B,CAAC;IAED;;;;OAIG;IACI,cAAc,CAAC,OAAO,GAAG,IAAI,WAAW,EAAE;QAChD,OAAO,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;IAC/D,CAAC;IAED;;;;;;OAMG;IACI,gBAAgB;QACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAA;QAE7C,OAAO,IAAI,cAAc,CAAC;YACzB,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE;gBAC1B,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;gBAE7C,IAAI,IAAI,EAAE,CAAC;oBACV,UAAU,CAAC,KAAK,EAAE,CAAA;gBACnB,CAAC;qBAAM,CAAC;oBACP,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;gBAC1B,CAAC;YACF,CAAC;YACD,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE;SAC3C,CAAC,CAAA;IACH,CAAC;IAED;;;;;;;;;;OAUG;IACI,WAAW,CAAI,SAAqC,EAAE,OAA2B;QACvF,OAAO,IAAI,CAAC,gBAAgB,EAAE,CAAC,WAAW,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IAC/D,CAAC;IAED;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC;QAC5B,OAAO,IAAI,CAAA;IACZ,CAAC;IAEM,MAAM;QACZ,OAAO,IAAI,CAAC,SAAS,EAAE,CAAA;IACxB,CAAC;IAED,YAAY;IAEZ,sBAAsB;IAEtB;;;;OAIG;IACI,MAAM,CAAC,QAAQ,CAAC,MAAyB,EAAE,OAAuB;QACxE,OAAO,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IACxC,CAAC;IAED;;;;;;;;OAQG;IACI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAyB,EAAE,OAAuB;QAC5E,MAAM,EAAE,mBAAmB,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAA;QACnE,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAEvD,OAAO,OAAO,CAAC,GAAG,CACjB,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE;YACnC,qEAAqE;YACrE,MAAM,aAAa,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,GAAG,CAAC,EAAE,CAAC,CAAA;YAEhF,OAAO,IAAI,gBAAgB,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAA;QAChG,CAAC,CAAC,CACF,CAAA;IACF,CAAC;IAED;;;;;;;;;;OAUG;IACI,MAAM,CAAC,aAAa,CAC1B,MAAyB,EACzB,OAA6B;QAE7B,+FAA+F;QAC/F,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,YAAY,GAAG,CAAC,EAAE,CAAC;YAC5D,MAAM,IAAI,SAAS,CAAC,gFAAgF,CAAC,CAAA;QACtG,CAAC;QAED,OAAO,iBAAiB,CAAI,MAAM,EAAE,OAAO,CAAC,CAAA;IAC7C,CAAC;CACD"}
|
|
@@ -1,77 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @copyright Sister Software
|
|
3
|
-
* @license MIT
|
|
4
|
-
* @author Teffen Ellis, et al.
|
|
5
|
-
*/
|
|
6
|
-
/**
|
|
7
|
-
* A class to manage a buffer of bytes, growing and shrinking as needed.
|
|
8
|
-
*/
|
|
9
|
-
export declare class BufferController {
|
|
10
|
-
#private;
|
|
11
|
-
/**
|
|
12
|
-
* The underlying buffer containing the data.
|
|
13
|
-
*
|
|
14
|
-
* This property is mutable for performance reasons. Consumers should treat it as immutable.
|
|
15
|
-
*/
|
|
16
|
-
bytes: Uint8Array;
|
|
17
|
-
/**
|
|
18
|
-
* The total number of bytes currently written to the buffer.
|
|
19
|
-
*
|
|
20
|
-
* This property is mutable for performance reasons. Consumers should treat it as immutable.
|
|
21
|
-
*/
|
|
22
|
-
bytesWritten: number;
|
|
23
|
-
/**
|
|
24
|
-
* The minimum byte length of the buffer.
|
|
25
|
-
*/
|
|
26
|
-
get byteLengthMinimum(): number;
|
|
27
|
-
constructor(init: {
|
|
28
|
-
initialBufferSize: number;
|
|
29
|
-
});
|
|
30
|
-
/**
|
|
31
|
-
* Grow the buffer to the desired byte length, typically double the current byte length.
|
|
32
|
-
*/
|
|
33
|
-
grow(desiredByteLength?: number): void;
|
|
34
|
-
/**
|
|
35
|
-
* Modify the buffer in place, keeping only the bytes between the start and end indices.
|
|
36
|
-
*
|
|
37
|
-
* This is performed after we're confident that any data preceeding `start` or following `end` is no longer needed.
|
|
38
|
-
*
|
|
39
|
-
* `bytesWritten` is rebased onto the new window so it always reflects the count of valid bytes present after
|
|
40
|
-
* compression. Bytes outside the kept window are dropped from the count even if the underlying allocation is larger.
|
|
41
|
-
*
|
|
42
|
-
* @param start - The starting byte index of which bytes to keep.
|
|
43
|
-
* @param end - The ending byte index of which bytes to keep. Defaults to the current buffer length. Values past the
|
|
44
|
-
* buffer length are clamped.
|
|
45
|
-
*/
|
|
46
|
-
compress(start?: number, end?: number): void;
|
|
47
|
-
/**
|
|
48
|
-
* Clear the buffer, zeroing out all bytes.
|
|
49
|
-
*
|
|
50
|
-
* @param begin The starting byte index from which to clear.
|
|
51
|
-
* @param end The ending byte index to clear.
|
|
52
|
-
*/
|
|
53
|
-
clear(begin?: number, end?: number): void;
|
|
54
|
-
/**
|
|
55
|
-
* Gets a new Uint8Array view of the ArrayBuffer store for this array, referencing the elements at begin, inclusive,
|
|
56
|
-
* up to end, exclusive.
|
|
57
|
-
*
|
|
58
|
-
* @param begin — The index of the beginning of the array.
|
|
59
|
-
* @param end — The index of the end of the array
|
|
60
|
-
* @throws If the start index is greater than the end index.
|
|
61
|
-
* @throws If the end index is greater than the current byte length.
|
|
62
|
-
*/
|
|
63
|
-
subarray(begin?: number, end?: number): Uint8Array;
|
|
64
|
-
/**
|
|
65
|
-
* Sets a value or an array of values.
|
|
66
|
-
*
|
|
67
|
-
* Unlike the `TypedArray.set` method, this method will grow the buffer if the offset is greater than the current byte
|
|
68
|
-
* length.
|
|
69
|
-
*
|
|
70
|
-
* @param array — A typed or untyped array of values to set.
|
|
71
|
-
* @param offset — The index in the current array at which the values are to be written.
|
|
72
|
-
*
|
|
73
|
-
* @returns The number of bytes written.
|
|
74
|
-
*/
|
|
75
|
-
set(array: ArrayLike<number>, offset?: number): number;
|
|
76
|
-
}
|
|
77
|
-
//# sourceMappingURL=BufferController.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"BufferController.d.ts","sourceRoot":"","sources":["../../lib/BufferController.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;GAEG;AACH,qBAAa,gBAAgB;;IAM5B;;;;OAIG;IACH,KAAK,EAAE,UAAU,CAAA;IAEjB;;;;OAIG;IACH,YAAY,EAAE,MAAM,CAAA;IAEpB;;OAEG;IACH,IAAW,iBAAiB,IAAI,MAAM,CAErC;IAED,YAAY,IAAI,EAAE;QAAE,iBAAiB,EAAE,MAAM,CAAA;KAAE,EAI9C;IAED;;OAEG;IACI,IAAI,CAAC,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAW5C;IAED;;;;;;;;;;;OAWG;IACI,QAAQ,CAAC,KAAK,SAAI,EAAE,GAAG,GAAE,MAA0B,GAAG,IAAI,CAOhE;IAED;;;;;OAKG;IACI,KAAK,CAAC,KAAK,SAAI,EAAE,GAAG,GAAE,MAA0B,GAAG,IAAI,CAG7D;IAED;;;;;;;;OAQG;IACI,QAAQ,CAAC,KAAK,SAAI,EAAE,GAAG,GAAE,MAA0B,GAAG,UAAU,CAUtE;IAED;;;;;;;;;;OAUG;IACI,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,EAAE,MAAM,SAAI,GAAG,MAAM,CAcvD;CACD"}
|