spliterator 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +650 -0
- package/README.md +119 -0
- package/out/index.d.ts +21 -0
- package/out/index.d.ts.map +1 -0
- package/out/index.js +21 -0
- package/out/index.js.map +1 -0
- package/out/lib/AsyncSpliterator.d.ts +127 -0
- package/out/lib/AsyncSpliterator.d.ts.map +1 -0
- package/out/lib/AsyncSpliterator.js +332 -0
- package/out/lib/AsyncSpliterator.js.map +1 -0
- package/out/lib/BufferController.d.ts +62 -0
- package/out/lib/BufferController.d.ts.map +1 -0
- package/out/lib/BufferController.js +93 -0
- package/out/lib/BufferController.js.map +1 -0
- package/out/lib/CSVSpliterator.d.ts +104 -0
- package/out/lib/CSVSpliterator.d.ts.map +1 -0
- package/out/lib/CSVSpliterator.js +83 -0
- package/out/lib/CSVSpliterator.js.map +1 -0
- package/out/lib/CharacterSequence.d.ts +103 -0
- package/out/lib/CharacterSequence.d.ts.map +1 -0
- package/out/lib/CharacterSequence.js +184 -0
- package/out/lib/CharacterSequence.js.map +1 -0
- package/out/lib/CompositeDataView.d.ts +104 -0
- package/out/lib/CompositeDataView.d.ts.map +1 -0
- package/out/lib/CompositeDataView.js +242 -0
- package/out/lib/CompositeDataView.js.map +1 -0
- package/out/lib/DelimitedChunkReader.d.ts +67 -0
- package/out/lib/DelimitedChunkReader.d.ts.map +1 -0
- package/out/lib/DelimitedChunkReader.js +113 -0
- package/out/lib/DelimitedChunkReader.js.map +1 -0
- package/out/lib/DelimiterTransformer.d.ts +27 -0
- package/out/lib/DelimiterTransformer.d.ts.map +1 -0
- package/out/lib/DelimiterTransformer.js +26 -0
- package/out/lib/DelimiterTransformer.js.map +1 -0
- package/out/lib/IndexQueue.d.ts +48 -0
- package/out/lib/IndexQueue.d.ts.map +1 -0
- package/out/lib/IndexQueue.js +80 -0
- package/out/lib/IndexQueue.js.map +1 -0
- package/out/lib/JSONSpliterator.d.ts +19 -0
- package/out/lib/JSONSpliterator.d.ts.map +1 -0
- package/out/lib/JSONSpliterator.js +55 -0
- package/out/lib/JSONSpliterator.js.map +1 -0
- package/out/lib/SlidingWindow.d.ts +93 -0
- package/out/lib/SlidingWindow.d.ts.map +1 -0
- package/out/lib/SlidingWindow.js +176 -0
- package/out/lib/SlidingWindow.js.map +1 -0
- package/out/lib/Spliterator.d.ts +51 -0
- package/out/lib/Spliterator.d.ts.map +1 -0
- package/out/lib/Spliterator.js +238 -0
- package/out/lib/Spliterator.js.map +1 -0
- package/out/lib/TextSpliterator.d.ts +40 -0
- package/out/lib/TextSpliterator.d.ts.map +1 -0
- package/out/lib/TextSpliterator.js +56 -0
- package/out/lib/TextSpliterator.js.map +1 -0
- package/out/lib/casing.d.ts +28 -0
- package/out/lib/casing.d.ts.map +1 -0
- package/out/lib/casing.js +69 -0
- package/out/lib/casing.js.map +1 -0
- package/out/lib/node/fs.d.ts +98 -0
- package/out/lib/node/fs.d.ts.map +1 -0
- package/out/lib/node/fs.js +162 -0
- package/out/lib/node/fs.js.map +1 -0
- package/out/lib/shared.d.ts +167 -0
- package/out/lib/shared.d.ts.map +1 -0
- package/out/lib/shared.js +92 -0
- package/out/lib/shared.js.map +1 -0
- package/out/lib/take.d.ts +26 -0
- package/out/lib/take.d.ts.map +1 -0
- package/out/lib/take.js +71 -0
- package/out/lib/take.js.map +1 -0
- package/out/lib/writer.d.ts +34 -0
- package/out/lib/writer.d.ts.map +1 -0
- package/out/lib/writer.js +41 -0
- package/out/lib/writer.js.map +1 -0
- package/package.json +100 -0
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
import { TypedArray } from "./shared.js";
|
|
7
|
+
/**
|
|
8
|
+
* A generic mutable typed array.
|
|
9
|
+
*/
|
|
10
|
+
export interface MutableTypedArray<T extends number | bigint = number | bigint> {
|
|
11
|
+
readonly length: number;
|
|
12
|
+
readonly [n: number]: T;
|
|
13
|
+
set(array: ArrayLike<T>, offset?: number): void;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* A constructor for a mutable typed array.
|
|
17
|
+
*/
|
|
18
|
+
export type MutableTypedArrayConstructor<T extends number | bigint = number | bigint> = new (length: number) => MutableTypedArray<T>;
|
|
19
|
+
/**
|
|
20
|
+
* Given several contiguous byte arrays, wraps them as if they were flattened into a single buffer.
|
|
21
|
+
*/
|
|
22
|
+
export declare class CompositeDataView<Chunk extends TypedArray = TypedArray> implements Iterable<Chunk[0]>, RelativeIndexable<Chunk[0]> {
|
|
23
|
+
#private;
|
|
24
|
+
constructor(chunks?: Chunk[]);
|
|
25
|
+
/**
|
|
26
|
+
* The combined length of all chunks.
|
|
27
|
+
*
|
|
28
|
+
* @see {@linkcode size} for the number of chunks in the composite buffer.
|
|
29
|
+
*/
|
|
30
|
+
get byteLength(): number;
|
|
31
|
+
/**
|
|
32
|
+
* The number of chunks in the composite buffer.
|
|
33
|
+
*
|
|
34
|
+
* @see {@linkcode byteLength} for the total length of the composite buffer.
|
|
35
|
+
*/
|
|
36
|
+
get size(): number;
|
|
37
|
+
get chunks(): readonly Chunk[];
|
|
38
|
+
/**
|
|
39
|
+
* The underlying buffer of the composite buffer.
|
|
40
|
+
*
|
|
41
|
+
* Note that this is not memoized and will create a new buffer each time it is accessed.
|
|
42
|
+
*/
|
|
43
|
+
get buffer(): Chunk;
|
|
44
|
+
get [Symbol.toStringTag](): string;
|
|
45
|
+
[Symbol.iterator](): Iterator<Chunk[0]>;
|
|
46
|
+
/**
|
|
47
|
+
* Returns the value at the given index, as if the chunks were flattened into a single buffer.
|
|
48
|
+
*/
|
|
49
|
+
at(index: number): Chunk[0];
|
|
50
|
+
/**
|
|
51
|
+
* Push a chunk onto the composite buffer.
|
|
52
|
+
*/
|
|
53
|
+
append(chunk: Chunk): number;
|
|
54
|
+
/**
|
|
55
|
+
* Prepend a chunk before the first chunk in the composite buffer.
|
|
56
|
+
*
|
|
57
|
+
* @returns The new length of the composite buffer.
|
|
58
|
+
*/
|
|
59
|
+
prepend(chunk: Chunk): number;
|
|
60
|
+
/**
|
|
61
|
+
* Remove the most recent chunk from the composite buffer.
|
|
62
|
+
*/
|
|
63
|
+
removeLast(): Chunk | undefined;
|
|
64
|
+
/**
|
|
65
|
+
* Remove the first chunk from the composite buffer.
|
|
66
|
+
*/
|
|
67
|
+
removeFirst(): Chunk | undefined;
|
|
68
|
+
/**
|
|
69
|
+
* Returns a view into the underlying chunks without copying data.
|
|
70
|
+
*/
|
|
71
|
+
subarray(
|
|
72
|
+
/**
|
|
73
|
+
* Element to begin at. The offset is inclusive.
|
|
74
|
+
*/
|
|
75
|
+
begin?: number,
|
|
76
|
+
/**
|
|
77
|
+
* Element to end at. The offset is exclusive. If not provided, the view extends to the end of
|
|
78
|
+
* the buffer.
|
|
79
|
+
*/
|
|
80
|
+
end?: number): Chunk;
|
|
81
|
+
/**
|
|
82
|
+
* Flattens the composite buffer into a single buffer.
|
|
83
|
+
*/
|
|
84
|
+
flat(): Chunk;
|
|
85
|
+
/**
|
|
86
|
+
* Returns a shallow copy of the composite buffer.
|
|
87
|
+
*
|
|
88
|
+
* @see {@linkcode subarray} for a view into the underlying chunks.
|
|
89
|
+
*/
|
|
90
|
+
slice(
|
|
91
|
+
/**
|
|
92
|
+
* Zero-based index at which to begin extraction.
|
|
93
|
+
*
|
|
94
|
+
* If this value is negative, it is treated as `byteLength + begin`.
|
|
95
|
+
*/
|
|
96
|
+
start?: number,
|
|
97
|
+
/**
|
|
98
|
+
* Zero-based index before which to end extraction.
|
|
99
|
+
*
|
|
100
|
+
* If this value is negative, it is treated as `byteLength + end`.
|
|
101
|
+
*/
|
|
102
|
+
end?: number): Chunk;
|
|
103
|
+
}
|
|
104
|
+
//# sourceMappingURL=CompositeDataView.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"CompositeDataView.d.ts","sourceRoot":"","sources":["../../lib/CompositeDataView.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAExC;;GAEG;AACH,MAAM,WAAW,iBAAiB,CAAC,CAAC,SAAS,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM;IAC7E,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,EAAE,CAAC,EAAE,MAAM,GAAG,CAAC,CAAA;IACvB,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAC/C;AAED;;GAEG;AACH,MAAM,MAAM,4BAA4B,CAAC,CAAC,SAAS,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,IAAI,KACvF,MAAM,EAAE,MAAM,KACV,iBAAiB,CAAC,CAAC,CAAC,CAAA;AAUzB;;GAEG;AACH,qBAAa,iBAAiB,CAAC,KAAK,SAAS,UAAU,GAAG,UAAU,CACnE,YAAW,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;;gBAoE9C,MAAM,CAAC,EAAE,KAAK,EAAE;IAQ5B;;;;OAIG;IACH,IAAW,UAAU,IAAI,MAAM,CAE9B;IAED;;;;OAIG;IACH,IAAW,IAAI,IAAI,MAAM,CAExB;IAED,IAAW,MAAM,IAAI,SAAS,KAAK,EAAE,CAEpC;IAED;;;;OAIG;IACH,IAAW,MAAM,IAAI,KAAK,CAEzB;IAED,IAAW,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,MAAM,CAExC;IAEO,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAM/C;;OAEG;IACI,EAAE,CAAC,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC;IAUlC;;OAEG;IACI,MAAM,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM;IAUnC;;;;OAIG;IACI,OAAO,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM;IAOpC;;OAEG;IACI,UAAU,IAAI,KAAK,GAAG,SAAS;IAWtC;;OAEG;IACI,WAAW,IAAI,KAAK,GAAG,SAAS;IAWvC;;OAEG;IACI,QAAQ;IACd;;OAEG;IACH,KAAK,GAAE,MAAU;IACjB;;;OAGG;IACH,GAAG,GAAE,MAAiC,GACpC,KAAK;IAiDR;;OAEG;IACI,IAAI,IAAI,KAAK;IAIpB;;;;OAIG;IACI,KAAK;IACX;;;;OAIG;IACH,KAAK,GAAE,MAAU;IACjB;;;;OAIG;IACH,GAAG,GAAE,MAAiC,GACpC,KAAK;CAWR"}
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* Given several contiguous byte arrays, wraps them as if they were flattened into a single buffer.
|
|
8
|
+
*/
|
|
9
|
+
export class CompositeDataView {
|
|
10
|
+
/**
|
|
11
|
+
* The chunks that make up the composite buffer.
|
|
12
|
+
*/
|
|
13
|
+
#chunks = [];
|
|
14
|
+
/**
|
|
15
|
+
* Cached length of the composite buffer.
|
|
16
|
+
*/
|
|
17
|
+
#memoizedByteLength = 0;
|
|
18
|
+
/**
|
|
19
|
+
* Cached chunk offset information for binary search.
|
|
20
|
+
*/
|
|
21
|
+
#chunkOffsets = [];
|
|
22
|
+
/**
|
|
23
|
+
* Helper method to infer the constructor of the chunks.
|
|
24
|
+
*/
|
|
25
|
+
#inferChunkConstructor() {
|
|
26
|
+
const first = this.#chunks[0];
|
|
27
|
+
if (!first) {
|
|
28
|
+
throw new TypeError("No chunks in composite buffer, cannot infer constructor");
|
|
29
|
+
}
|
|
30
|
+
return first.constructor;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Binary search to find the chunk containing the given index.
|
|
34
|
+
*/
|
|
35
|
+
#findChunkForIndex(index) {
|
|
36
|
+
let left = 0;
|
|
37
|
+
let right = this.#chunkOffsets.length - 1;
|
|
38
|
+
while (left <= right) {
|
|
39
|
+
const mid = Math.floor((left + right) / 2);
|
|
40
|
+
const chunkInfo = this.#chunkOffsets[mid];
|
|
41
|
+
if (index >= chunkInfo.offset && index < chunkInfo.offset + chunkInfo.length) {
|
|
42
|
+
return { chunk: this.#chunks[mid], chunkInfo };
|
|
43
|
+
}
|
|
44
|
+
if (index < chunkInfo.offset) {
|
|
45
|
+
right = mid - 1;
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
left = mid + 1;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
return null;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Update chunk offset information after modification.
|
|
55
|
+
*/
|
|
56
|
+
#updateChunkOffsets() {
|
|
57
|
+
let currentOffset = 0;
|
|
58
|
+
this.#chunkOffsets = this.#chunks.map((chunk) => {
|
|
59
|
+
const info = { offset: currentOffset, length: chunk.length };
|
|
60
|
+
currentOffset += chunk.length;
|
|
61
|
+
return info;
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
constructor(chunks) {
|
|
65
|
+
if (chunks) {
|
|
66
|
+
this.#chunks = chunks;
|
|
67
|
+
this.#memoizedByteLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
|
|
68
|
+
this.#updateChunkOffsets();
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* The combined length of all chunks.
|
|
73
|
+
*
|
|
74
|
+
* @see {@linkcode size} for the number of chunks in the composite buffer.
|
|
75
|
+
*/
|
|
76
|
+
get byteLength() {
|
|
77
|
+
return this.#memoizedByteLength;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* The number of chunks in the composite buffer.
|
|
81
|
+
*
|
|
82
|
+
* @see {@linkcode byteLength} for the total length of the composite buffer.
|
|
83
|
+
*/
|
|
84
|
+
get size() {
|
|
85
|
+
return this.#chunks.length;
|
|
86
|
+
}
|
|
87
|
+
get chunks() {
|
|
88
|
+
return this.#chunks;
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* The underlying buffer of the composite buffer.
|
|
92
|
+
*
|
|
93
|
+
* Note that this is not memoized and will create a new buffer each time it is accessed.
|
|
94
|
+
*/
|
|
95
|
+
get buffer() {
|
|
96
|
+
return this.flat();
|
|
97
|
+
}
|
|
98
|
+
get [Symbol.toStringTag]() {
|
|
99
|
+
return "CompositeDataView";
|
|
100
|
+
}
|
|
101
|
+
*[Symbol.iterator]() {
|
|
102
|
+
for (const chunk of this.#chunks) {
|
|
103
|
+
yield* chunk;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Returns the value at the given index, as if the chunks were flattened into a single buffer.
|
|
108
|
+
*/
|
|
109
|
+
at(index) {
|
|
110
|
+
const result = this.#findChunkForIndex(index);
|
|
111
|
+
if (!result) {
|
|
112
|
+
throw new RangeError("Index out of range");
|
|
113
|
+
}
|
|
114
|
+
const { chunk, chunkInfo } = result;
|
|
115
|
+
return chunk[index - chunkInfo.offset];
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Push a chunk onto the composite buffer.
|
|
119
|
+
*/
|
|
120
|
+
append(chunk) {
|
|
121
|
+
this.#chunks.push(chunk);
|
|
122
|
+
this.#memoizedByteLength += chunk.length;
|
|
123
|
+
this.#updateChunkOffsets();
|
|
124
|
+
return this.#memoizedByteLength;
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Prepend a chunk before the first chunk in the composite buffer.
|
|
128
|
+
*
|
|
129
|
+
* @returns The new length of the composite buffer.
|
|
130
|
+
*/
|
|
131
|
+
prepend(chunk) {
|
|
132
|
+
this.#chunks.unshift(chunk);
|
|
133
|
+
this.#memoizedByteLength += chunk.length;
|
|
134
|
+
this.#updateChunkOffsets();
|
|
135
|
+
return this.#memoizedByteLength;
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Remove the most recent chunk from the composite buffer.
|
|
139
|
+
*/
|
|
140
|
+
removeLast() {
|
|
141
|
+
const chunk = this.#chunks.pop();
|
|
142
|
+
if (chunk) {
|
|
143
|
+
this.#memoizedByteLength -= chunk.length;
|
|
144
|
+
this.#updateChunkOffsets();
|
|
145
|
+
}
|
|
146
|
+
return chunk;
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* Remove the first chunk from the composite buffer.
|
|
150
|
+
*/
|
|
151
|
+
removeFirst() {
|
|
152
|
+
const chunk = this.#chunks.shift();
|
|
153
|
+
if (chunk) {
|
|
154
|
+
this.#memoizedByteLength -= chunk.length;
|
|
155
|
+
this.#updateChunkOffsets();
|
|
156
|
+
}
|
|
157
|
+
return chunk;
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Returns a view into the underlying chunks without copying data.
|
|
161
|
+
*/
|
|
162
|
+
subarray(
|
|
163
|
+
/**
|
|
164
|
+
* Element to begin at. The offset is inclusive.
|
|
165
|
+
*/
|
|
166
|
+
begin = 0,
|
|
167
|
+
/**
|
|
168
|
+
* Element to end at. The offset is exclusive. If not provided, the view extends to the end of
|
|
169
|
+
* the buffer.
|
|
170
|
+
*/
|
|
171
|
+
end = this.#memoizedByteLength) {
|
|
172
|
+
if (begin < 0) {
|
|
173
|
+
throw new RangeError("Invalid subarray range: begin is negative");
|
|
174
|
+
}
|
|
175
|
+
if (end < 0) {
|
|
176
|
+
throw new RangeError("Invalid subarray range: end is negative");
|
|
177
|
+
}
|
|
178
|
+
// const actualEnd = end === undefined ? this.#memoizedByteLength : end
|
|
179
|
+
const startChunk = this.#findChunkForIndex(begin);
|
|
180
|
+
const endChunk = this.#findChunkForIndex(end - 1);
|
|
181
|
+
if (!startChunk || !endChunk) {
|
|
182
|
+
throw new RangeError("Invalid subarray range: Start: " + begin + ", End: " + end);
|
|
183
|
+
}
|
|
184
|
+
// If the range is within a single chunk, return a view into that chunk
|
|
185
|
+
if (startChunk.chunk === endChunk.chunk) {
|
|
186
|
+
const relativeStart = begin - startChunk.chunkInfo.offset;
|
|
187
|
+
const relativeEnd = end - startChunk.chunkInfo.offset;
|
|
188
|
+
return startChunk.chunk.subarray(relativeStart, relativeEnd);
|
|
189
|
+
}
|
|
190
|
+
// Otherwise, we need to create a new buffer and copy the data
|
|
191
|
+
const Constructor = this.#inferChunkConstructor();
|
|
192
|
+
const length = end - begin;
|
|
193
|
+
const result = new Constructor(length);
|
|
194
|
+
let writeOffset = 0;
|
|
195
|
+
let currentChunkIndex = this.#chunks.indexOf(startChunk.chunk);
|
|
196
|
+
while (currentChunkIndex < this.#chunks.length) {
|
|
197
|
+
const currentChunk = this.#chunks[currentChunkIndex];
|
|
198
|
+
const chunkInfo = this.#chunkOffsets[currentChunkIndex];
|
|
199
|
+
const readStart = currentChunk === startChunk.chunk ? begin - chunkInfo.offset : 0;
|
|
200
|
+
const readEnd = currentChunk === endChunk.chunk ? end - chunkInfo.offset : currentChunk.length;
|
|
201
|
+
result.set(currentChunk.subarray(readStart, readEnd), writeOffset);
|
|
202
|
+
writeOffset += readEnd - readStart;
|
|
203
|
+
if (currentChunk === endChunk.chunk)
|
|
204
|
+
break;
|
|
205
|
+
currentChunkIndex++;
|
|
206
|
+
}
|
|
207
|
+
return result;
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* Flattens the composite buffer into a single buffer.
|
|
211
|
+
*/
|
|
212
|
+
flat() {
|
|
213
|
+
return this.subarray();
|
|
214
|
+
}
|
|
215
|
+
/**
|
|
216
|
+
* Returns a shallow copy of the composite buffer.
|
|
217
|
+
*
|
|
218
|
+
* @see {@linkcode subarray} for a view into the underlying chunks.
|
|
219
|
+
*/
|
|
220
|
+
slice(
|
|
221
|
+
/**
|
|
222
|
+
* Zero-based index at which to begin extraction.
|
|
223
|
+
*
|
|
224
|
+
* If this value is negative, it is treated as `byteLength + begin`.
|
|
225
|
+
*/
|
|
226
|
+
start = 0,
|
|
227
|
+
/**
|
|
228
|
+
* Zero-based index before which to end extraction.
|
|
229
|
+
*
|
|
230
|
+
* If this value is negative, it is treated as `byteLength + end`.
|
|
231
|
+
*/
|
|
232
|
+
end = this.#memoizedByteLength) {
|
|
233
|
+
if (start < 0) {
|
|
234
|
+
start = Math.max(0, this.#memoizedByteLength + start);
|
|
235
|
+
}
|
|
236
|
+
if (end < 0) {
|
|
237
|
+
end = Math.max(0, this.#memoizedByteLength + end);
|
|
238
|
+
}
|
|
239
|
+
return this.subarray(start, end);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
//# sourceMappingURL=CompositeDataView.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"CompositeDataView.js","sourceRoot":"","sources":["../../lib/CompositeDataView.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AA4BH;;GAEG;AACH,MAAM,OAAO,iBAAiB;IAG7B;;OAEG;IACH,OAAO,GAAY,EAAE,CAAA;IAErB;;OAEG;IACH,mBAAmB,GAAG,CAAC,CAAA;IAEvB;;OAEG;IACH,aAAa,GAAgB,EAAE,CAAA;IAE/B;;OAEG;IACH,sBAAsB;QACrB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;QAC7B,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,MAAM,IAAI,SAAS,CAAC,yDAAyD,CAAC,CAAA;QAC/E,CAAC;QACD,OAAO,KAAK,CAAC,WAAqD,CAAA;IACnE,CAAC;IAED;;OAEG;IACH,kBAAkB,CAAC,KAAa;QAC/B,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,IAAI,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAA;QAEzC,OAAO,IAAI,IAAI,KAAK,EAAE,CAAC;YACtB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAA;YAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAE,CAAA;YAE1C,IAAI,KAAK,IAAI,SAAS,CAAC,MAAM,IAAI,KAAK,GAAG,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC;gBAC9E,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAE,EAAE,SAAS,EAAE,CAAA;YAChD,CAAC;YAED,IAAI,KAAK,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC;gBAC9B,KAAK,GAAG,GAAG,GAAG,CAAC,CAAA;YAChB,CAAC;iBAAM,CAAC;gBACP,IAAI,GAAG,GAAG,GAAG,CAAC,CAAA;YACf,CAAC;QACF,CAAC;QAED,OAAO,IAAI,CAAA;IACZ,CAAC;IAED;;OAEG;IACH,mBAAmB;QAClB,IAAI,aAAa,GAAG,CAAC,CAAA;QAErB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YAC/C,MAAM,IAAI,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAA;YAE5D,aAAa,IAAI,KAAK,CAAC,MAAM,CAAA;YAE7B,OAAO,IAAI,CAAA;QACZ,CAAC,CAAC,CAAA;IACH,CAAC;IAED,YAAY,MAAgB;QAC3B,IAAI,MAAM,EAAE,CAAC;YACZ,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;YACrB,IAAI,CAAC,mBAAmB,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;YAC/E,IAAI,CAAC,mBAAmB,EAAE,CAAA;QAC3B,CAAC;IACF,CAAC;IAED;;;;OAIG;IACH,IAAW,UAAU;QACpB,OAAO,IAAI,CAAC,mBAAmB,CAAA;IAChC,CAAC;IAED;;;;OAIG;IACH,IAAW,IAAI;QACd,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;IAC3B,CAAC;IAED,IAAW,MAAM;QAChB,OAAO,IAAI,CAAC,OAAO,CAAA;IACpB,CAAC;IAED;;;;OAIG;IACH,IAAW,MAAM;QAChB,OAAO,IAAI,CAAC,IAAI,EAAE,CAAA;IACnB,CAAC;IAED,IAAW,CAAC,MAAM,CAAC,WAAW,CAAC;QAC9B,OAAO,mBAAmB,CAAA;IAC3B,CAAC;IAEM,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC;QACxB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,KAAK,CAAC,CAAC,KAAK,CAAA;QACb,CAAC;IACF,CAAC;IAED;;OAEG;IACI,EAAE,CAAC,KAAa;QACtB,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAA;QAC7C,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,MAAM,IAAI,UAAU,CAAC,oBAAoB,CAAC,CAAA;QAC3C,CAAC;QAED,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,MAAM,CAAA;QACnC,OAAO,KAAK,CAAC,KAAK,GAAG,SAAS,CAAC,MAAM,CAAE,CAAA;IACxC,CAAC;IAED;;OAEG;IACI,MAAM,CAAC,KAAY;QACzB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAExB,IAAI,CAAC,mBAAmB,IAAI,KAAK,CAAC,MAAM,CAAA;QAExC,IAAI,CAAC,mBAAmB,EAAE,CAAA;QAE1B,OAAO,IAAI,CAAC,mBAAmB,CAAA;IAChC,CAAC;IAED;;;;OAIG;IACI,OAAO,CAAC,KAAY;QAC1B,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAC3B,IAAI,CAAC,mBAAmB,IAAI,KAAK,CAAC,MAAM,CAAA;QACxC,IAAI,CAAC,mBAAmB,EAAE,CAAA;QAC1B,OAAO,IAAI,CAAC,mBAAmB,CAAA;IAChC,CAAC;IAED;;OAEG;IACI,UAAU;QAChB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAA;QAEhC,IAAI,KAAK,EAAE,CAAC;YACX,IAAI,CAAC,mBAAmB,IAAI,KAAK,CAAC,MAAM,CAAA;YAExC,IAAI,CAAC,mBAAmB,EAAE,CAAA;QAC3B,CAAC;QACD,OAAO,KAAK,CAAA;IACb,CAAC;IAED;;OAEG;IACI,WAAW;QACjB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAA;QAElC,IAAI,KAAK,EAAE,CAAC;YACX,IAAI,CAAC,mBAAmB,IAAI,KAAK,CAAC,MAAM,CAAA;YACxC,IAAI,CAAC,mBAAmB,EAAE,CAAA;QAC3B,CAAC;QAED,OAAO,KAAK,CAAA;IACb,CAAC;IAED;;OAEG;IACI,QAAQ;IACd;;OAEG;IACH,QAAgB,CAAC;IACjB;;;OAGG;IACH,MAAc,IAAI,CAAC,mBAAmB;QAEtC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACf,MAAM,IAAI,UAAU,CAAC,2CAA2C,CAAC,CAAA;QAClE,CAAC;QAED,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;YACb,MAAM,IAAI,UAAU,CAAC,yCAAyC,CAAC,CAAA;QAChE,CAAC;QAED,uEAAuE;QACvE,MAAM,UAAU,GAAG,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAA;QACjD,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,GAAG,CAAC,CAAC,CAAA;QAEjD,IAAI,CAAC,UAAU,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC9B,MAAM,IAAI,UAAU,CAAC,iCAAiC,GAAG,KAAK,GAAG,SAAS,GAAG,GAAG,CAAC,CAAA;QAClF,CAAC;QAED,uEAAuE;QACvE,IAAI,UAAU,CAAC,KAAK,KAAK,QAAQ,CAAC,KAAK,EAAE,CAAC;YACzC,MAAM,aAAa,GAAG,KAAK,GAAG,UAAU,CAAC,SAAS,CAAC,MAAM,CAAA;YACzD,MAAM,WAAW,GAAG,GAAG,GAAG,UAAU,CAAC,SAAS,CAAC,MAAM,CAAA;YACrD,OAAO,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,aAAa,EAAE,WAAW,CAAU,CAAA;QACtE,CAAC;QAED,8DAA8D;QAC9D,MAAM,WAAW,GAAG,IAAI,CAAC,sBAAsB,EAAE,CAAA;QACjD,MAAM,MAAM,GAAG,GAAG,GAAG,KAAK,CAAA;QAC1B,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,MAAM,CAAC,CAAA;QAEtC,IAAI,WAAW,GAAG,CAAC,CAAA;QACnB,IAAI,iBAAiB,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,CAAA;QAE9D,OAAO,iBAAiB,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YAChD,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAE,CAAA;YACrD,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,iBAAiB,CAAE,CAAA;YAExD,MAAM,SAAS,GAAG,YAAY,KAAK,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;YAClF,MAAM,OAAO,GAAG,YAAY,KAAK,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAA;YAE9F,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,WAAW,CAAC,CAAA;YAClE,WAAW,IAAI,OAAO,GAAG,SAAS,CAAA;YAElC,IAAI,YAAY,KAAK,QAAQ,CAAC,KAAK;gBAAE,MAAK;YAC1C,iBAAiB,EAAE,CAAA;QACpB,CAAC;QAED,OAAO,MAAe,CAAA;IACvB,CAAC;IAED;;OAEG;IACI,IAAI;QACV,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAA;IACvB,CAAC;IAED;;;;OAIG;IACI,KAAK;IACX;;;;OAIG;IACH,QAAgB,CAAC;IACjB;;;;OAIG;IACH,MAAc,IAAI,CAAC,mBAAmB;QAEtC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACf,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC,CAAA;QACtD,CAAC;QAED,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;YACb,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,mBAAmB,GAAG,GAAG,CAAC,CAAA;QAClD,CAAC;QAED,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;IACjC,CAAC;CACD"}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
import { CharacterSequenceInput } from "./CharacterSequence.js";
|
|
7
|
+
import { AsyncDataResource, ByteRange } from "./shared.js";
|
|
8
|
+
/**
|
|
9
|
+
* Options for the delimited chunk reader.
|
|
10
|
+
*/
|
|
11
|
+
export interface DeliminatedChunkReaderInit {
|
|
12
|
+
/**
|
|
13
|
+
* The desired number of chunks. This is a target, not a guarantee.
|
|
14
|
+
*
|
|
15
|
+
* This will never be less than 1, nor greater than the number of delimiters in the file, nor
|
|
16
|
+
* greater than the byte length of the file.
|
|
17
|
+
*/
|
|
18
|
+
chunks?: number;
|
|
19
|
+
/**
|
|
20
|
+
* The character to delimit by. Typically a newline or comma.
|
|
21
|
+
*/
|
|
22
|
+
delimiter?: CharacterSequenceInput;
|
|
23
|
+
/**
|
|
24
|
+
* Whether to close the file handle after reading.
|
|
25
|
+
*/
|
|
26
|
+
autoClose?: boolean;
|
|
27
|
+
/**
|
|
28
|
+
* The limit of bytes to read from the file.
|
|
29
|
+
*/
|
|
30
|
+
limit?: number;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Given a file handle containing delimited data and a desired slice count, returns an array of
|
|
34
|
+
* slices of the buffer between delimiters.
|
|
35
|
+
*
|
|
36
|
+
* This is an advanced function so an analogy is provided:
|
|
37
|
+
*
|
|
38
|
+
* Suppose you had to manually search through a very large book page by page to find where each
|
|
39
|
+
* chapter begins and ends. For a book with 1,000,000 pages, a single person would take a long time
|
|
40
|
+
* to go through it all.
|
|
41
|
+
*
|
|
42
|
+
* You could add more people to the task by laying out all the pages, measuring the length and
|
|
43
|
+
* assigning each person a length of pages to traverse.
|
|
44
|
+
*
|
|
45
|
+
* There's a few ways to go about this:
|
|
46
|
+
*
|
|
47
|
+
* We could approach this in serial -- having the first worker start from page 1 and scanning until
|
|
48
|
+
* they find the beginning of the next chapter, handing off the range to the next worker. This is
|
|
49
|
+
* how `AsyncSlidingWindow` approaches the problem.
|
|
50
|
+
*
|
|
51
|
+
* However this is inefficient because no matter how many workers we have, they must wait for the
|
|
52
|
+
* previous worker to finish before they can start. Ideally, a desired number of workers would be
|
|
53
|
+
* able to scan their own _length_ of pages simultaneously, and settle up on the boundaries of the
|
|
54
|
+
* chapters they find.
|
|
55
|
+
*
|
|
56
|
+
* `DeliminatedChunkReader` is like the second approach. Given a desired number of chunks
|
|
57
|
+
*
|
|
58
|
+
* @returns A contiguous array of slices of the buffer separated by a delimiter.
|
|
59
|
+
*/
|
|
60
|
+
export declare class DelimitedChunkReader {
|
|
61
|
+
static fromAsync(
|
|
62
|
+
/**
|
|
63
|
+
* The buffer containing newline-delimited data.
|
|
64
|
+
*/
|
|
65
|
+
source: AsyncDataResource, init?: DeliminatedChunkReaderInit): Promise<ByteRange[]>;
|
|
66
|
+
}
|
|
67
|
+
//# sourceMappingURL=DelimitedChunkReader.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"DelimitedChunkReader.d.ts","sourceRoot":"","sources":["../../lib/DelimitedChunkReader.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAqB,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AAClF,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAwC,MAAM,aAAa,CAAA;AAGhG;;GAEG;AACH,MAAM,WAAW,0BAA0B;IAC1C;;;;;OAKG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf;;OAEG;IACH,SAAS,CAAC,EAAE,sBAAsB,CAAA;IAElC;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAA;IAEnB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;CACd;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,qBAAa,oBAAoB;WACnB,SAAS;IACrB;;OAEG;IACH,MAAM,EAAE,iBAAiB,EACzB,IAAI,GAAE,0BAA+B,GACnC,OAAO,CAAC,SAAS,EAAE,CAAC;CAyFvB"}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
import { CharacterSequence } from "./CharacterSequence.js";
|
|
7
|
+
import { isFileResourceLike } from "./shared.js";
|
|
8
|
+
import { AsyncSlidingWindow } from "./SlidingWindow.js";
|
|
9
|
+
/**
|
|
10
|
+
* Given a file handle containing delimited data and a desired slice count, returns an array of
|
|
11
|
+
* slices of the buffer between delimiters.
|
|
12
|
+
*
|
|
13
|
+
* This is an advanced function so an analogy is provided:
|
|
14
|
+
*
|
|
15
|
+
* Suppose you had to manually search through a very large book page by page to find where each
|
|
16
|
+
* chapter begins and ends. For a book with 1,000,000 pages, a single person would take a long time
|
|
17
|
+
* to go through it all.
|
|
18
|
+
*
|
|
19
|
+
* You could add more people to the task by laying out all the pages, measuring the length and
|
|
20
|
+
* assigning each person a length of pages to traverse.
|
|
21
|
+
*
|
|
22
|
+
* There's a few ways to go about this:
|
|
23
|
+
*
|
|
24
|
+
* We could approach this in serial -- having the first worker start from page 1 and scanning until
|
|
25
|
+
* they find the beginning of the next chapter, handing off the range to the next worker. This is
|
|
26
|
+
* how `AsyncSlidingWindow` approaches the problem.
|
|
27
|
+
*
|
|
28
|
+
* However this is inefficient because no matter how many workers we have, they must wait for the
|
|
29
|
+
* previous worker to finish before they can start. Ideally, a desired number of workers would be
|
|
30
|
+
* able to scan their own _length_ of pages simultaneously, and settle up on the boundaries of the
|
|
31
|
+
* chapters they find.
|
|
32
|
+
*
|
|
33
|
+
* `DeliminatedChunkReader` is like the second approach. Given a desired number of chunks
|
|
34
|
+
*
|
|
35
|
+
* @returns A contiguous array of slices of the buffer separated by a delimiter.
|
|
36
|
+
*/
|
|
37
|
+
export class DelimitedChunkReader {
|
|
38
|
+
static async fromAsync(
|
|
39
|
+
/**
|
|
40
|
+
* The buffer containing newline-delimited data.
|
|
41
|
+
*/
|
|
42
|
+
source, init = {}) {
|
|
43
|
+
let file;
|
|
44
|
+
if (isFileResourceLike(source)) {
|
|
45
|
+
file = source;
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
const { NodeFileResource } = await import("spliterator/node/fs");
|
|
49
|
+
file = await NodeFileResource.open(source);
|
|
50
|
+
}
|
|
51
|
+
const byteLimit = init.limit ?? file.size;
|
|
52
|
+
// const byteLimit = init.limit ?? (await fileHandle.stat()).size
|
|
53
|
+
const delimiter = new CharacterSequence(init.delimiter);
|
|
54
|
+
const delimiterLength = delimiter.length;
|
|
55
|
+
const desiredSlices = Math.min(Math.max(1, init.chunks ?? 2), byteLimit / delimiterLength, byteLimit);
|
|
56
|
+
const fallback = [[0, byteLimit]];
|
|
57
|
+
if (desiredSlices === 1) {
|
|
58
|
+
return fallback;
|
|
59
|
+
}
|
|
60
|
+
const ranges = [];
|
|
61
|
+
const chunkSize = Math.floor(byteLimit / desiredSlices);
|
|
62
|
+
for (let i = 0; i < desiredSlices; i++) {
|
|
63
|
+
const previousSlice = ranges[i - 1] ?? [0, 0];
|
|
64
|
+
const chunkEnd = Math.min(i * chunkSize, byteLimit);
|
|
65
|
+
const searchStart = Math.max(chunkEnd - delimiterLength * 2, previousSlice[1]);
|
|
66
|
+
const searchEnd = Math.min(chunkEnd + delimiterLength * 2, byteLimit);
|
|
67
|
+
const reverse = new AsyncSlidingWindow(file, {
|
|
68
|
+
delimiter,
|
|
69
|
+
position: searchStart,
|
|
70
|
+
byteLength: searchEnd,
|
|
71
|
+
});
|
|
72
|
+
const forward = new AsyncSlidingWindow(file, {
|
|
73
|
+
delimiter,
|
|
74
|
+
position: searchStart,
|
|
75
|
+
byteLength: searchEnd,
|
|
76
|
+
});
|
|
77
|
+
const [previousRange, nextRange] = await Promise.all([
|
|
78
|
+
// We look backward to find the previous delimiter at a midpoint in the chunk...
|
|
79
|
+
reverse.previous(),
|
|
80
|
+
// And forward to find the next delimiter.
|
|
81
|
+
forward.next(),
|
|
82
|
+
]);
|
|
83
|
+
if (i === 0) {
|
|
84
|
+
// The first slice is always from the beginning of the file to the first delimiter.
|
|
85
|
+
ranges.push(nextRange.value);
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
if (previousRange.done || nextRange.done)
|
|
89
|
+
break;
|
|
90
|
+
if (i === desiredSlices - 1) {
|
|
91
|
+
const range = [previousRange.value[0] + delimiterLength, byteLimit];
|
|
92
|
+
// We need to fix the previous slice to end at the previous delimiter.
|
|
93
|
+
previousSlice[1] = previousRange.value[0] - delimiterLength;
|
|
94
|
+
// The last slice is always from the last delimiter to the end of the file.
|
|
95
|
+
ranges.push(range);
|
|
96
|
+
break;
|
|
97
|
+
}
|
|
98
|
+
// We need to fix the previous slice to end at the previous delimiter.
|
|
99
|
+
previousSlice[1] = previousRange.value[0] - delimiterLength;
|
|
100
|
+
// And we need to fix the current slice to start at the previous delimiter.
|
|
101
|
+
nextRange.value[0] = previousRange.value[0] + delimiterLength;
|
|
102
|
+
// We can now add the current slice to the list.
|
|
103
|
+
ranges.push(nextRange.value);
|
|
104
|
+
}
|
|
105
|
+
if (!ranges.length)
|
|
106
|
+
return fallback;
|
|
107
|
+
if (init.autoClose) {
|
|
108
|
+
await file[Symbol.asyncDispose]?.();
|
|
109
|
+
}
|
|
110
|
+
return ranges;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
//# sourceMappingURL=DelimitedChunkReader.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"DelimitedChunkReader.js","sourceRoot":"","sources":["../../lib/DelimitedChunkReader.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,iBAAiB,EAA0B,MAAM,wBAAwB,CAAA;AAClF,OAAO,EAAkD,kBAAkB,EAAE,MAAM,aAAa,CAAA;AAChG,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAA;AA8BvD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAM,OAAO,oBAAoB;IAChC,MAAM,CAAC,KAAK,CAAC,SAAS;IACrB;;OAEG;IACH,MAAyB,EACzB,OAAmC,EAAE;QAErC,IAAI,IAAsB,CAAA;QAE1B,IAAI,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC;YAChC,IAAI,GAAG,MAAM,CAAA;QACd,CAAC;aAAM,CAAC;YACP,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAA;YAChE,IAAI,GAAG,MAAM,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAC3C,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,CAAA;QACzC,iEAAiE;QAEjE,MAAM,SAAS,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QACvD,MAAM,eAAe,GAAG,SAAS,CAAC,MAAM,CAAA;QAExC,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,SAAS,GAAG,eAAe,EAAE,SAAS,CAAC,CAAA;QACrG,MAAM,QAAQ,GAAgB,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAA;QAE9C,IAAI,aAAa,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,QAAQ,CAAA;QAChB,CAAC;QAED,MAAM,MAAM,GAAgB,EAAE,CAAA;QAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,aAAa,CAAC,CAAA;QAEvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,aAAa,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;YAE7C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,EAAE,SAAS,CAAC,CAAA;YACnD,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,eAAe,GAAG,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,CAAA;YAC9E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,eAAe,GAAG,CAAC,EAAE,SAAS,CAAC,CAAA;YAErE,MAAM,OAAO,GAAG,IAAI,kBAAkB,CAAC,IAAI,EAAE;gBAC5C,SAAS;gBACT,QAAQ,EAAE,WAAW;gBACrB,UAAU,EAAE,SAAS;aACrB,CAAC,CAAA;YACF,MAAM,OAAO,GAAG,IAAI,kBAAkB,CAAC,IAAI,EAAE;gBAC5C,SAAS;gBACT,QAAQ,EAAE,WAAW;gBACrB,UAAU,EAAE,SAAS;aACrB,CAAC,CAAA;YAEF,MAAM,CAAC,aAAa,EAAE,SAAS,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;gBACpD,gFAAgF;gBAChF,OAAO,CAAC,QAAQ,EAAE;gBAClB,0CAA0C;gBAC1C,OAAO,CAAC,IAAI,EAAE;aACd,CAAC,CAAA;YAEF,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBACb,mFAAmF;gBACnF,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;gBAE5B,SAAQ;YACT,CAAC;YAED,IAAI,aAAa,CAAC,IAAI,IAAI,SAAS,CAAC,IAAI;gBAAE,MAAK;YAE/C,IAAI,CAAC,KAAK,aAAa,GAAG,CAAC,EAAE,CAAC;gBAC7B,MAAM,KAAK,GAAc,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,eAAe,EAAE,SAAS,CAAC,CAAA;gBAC9E,sEAAsE;gBACtE,aAAa,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,eAAe,CAAA;gBAE3D,2EAA2E;gBAC3E,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;gBAElB,MAAK;YACN,CAAC;YAED,sEAAsE;YACtE,aAAa,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,eAAe,CAAA;YAE3D,2EAA2E;YAC3E,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,eAAe,CAAA;YAE7D,gDAAgD;YAChD,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;QAC7B,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,MAAM;YAAE,OAAO,QAAQ,CAAA;QAEnC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE,CAAA;QACpC,CAAC;QAED,OAAO,MAAM,CAAA;IACd,CAAC;CACD"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
import { CharacterSequenceInput } from "./CharacterSequence.js";
|
|
7
|
+
import { TypedArray } from "./shared.js";
|
|
8
|
+
/**
|
|
9
|
+
* Options for the `DelimiterTransformer` class.
|
|
10
|
+
*/
|
|
11
|
+
export interface DelimiterTransformerOptions {
|
|
12
|
+
/**
|
|
13
|
+
* The delimiter to use when splitting fields.
|
|
14
|
+
*
|
|
15
|
+
* @default `Delimiter.Comma`
|
|
16
|
+
*/
|
|
17
|
+
delimiter?: CharacterSequenceInput;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* A transform stream that splits incoming text into fields using a delimiter.
|
|
21
|
+
*
|
|
22
|
+
* This is useful for parsing CSV files, for example.
|
|
23
|
+
*/
|
|
24
|
+
export declare class DelimiterTransformer<T extends TypedArray = TypedArray> extends TransformStream<T, T[]> {
|
|
25
|
+
constructor(options?: DelimiterTransformerOptions);
|
|
26
|
+
}
|
|
27
|
+
//# sourceMappingURL=DelimiterTransformer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"DelimiterTransformer.d.ts","sourceRoot":"","sources":["../../lib/DelimiterTransformer.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAqB,sBAAsB,EAA6B,MAAM,wBAAwB,CAAA;AAC7G,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAExC;;GAEG;AACH,MAAM,WAAW,2BAA2B;IAC3C;;;;OAIG;IACH,SAAS,CAAC,EAAE,sBAAsB,CAAA;CAClC;AAED;;;;GAIG;AACH,qBAAa,oBAAoB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,CAAE,SAAQ,eAAe,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;gBACvF,OAAO,GAAE,2BAAgC;CAerD"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
import { CharacterSequence, Delimiters, takeDelimited } from "./CharacterSequence.js";
|
|
7
|
+
/**
|
|
8
|
+
* A transform stream that splits incoming text into fields using a delimiter.
|
|
9
|
+
*
|
|
10
|
+
* This is useful for parsing CSV files, for example.
|
|
11
|
+
*/
|
|
12
|
+
export class DelimiterTransformer extends TransformStream {
|
|
13
|
+
constructor(options = {}) {
|
|
14
|
+
const delimiter = new CharacterSequence(options.delimiter ?? Delimiters.Comma);
|
|
15
|
+
super({
|
|
16
|
+
transform: (line, controller) => {
|
|
17
|
+
if (line.length === 0) {
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
20
|
+
const columns = Array.from(takeDelimited(line, delimiter));
|
|
21
|
+
controller.enqueue(columns);
|
|
22
|
+
},
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
//# sourceMappingURL=DelimiterTransformer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"DelimiterTransformer.js","sourceRoot":"","sources":["../../lib/DelimiterTransformer.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,iBAAiB,EAA0B,UAAU,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAA;AAe7G;;;;GAIG;AACH,MAAM,OAAO,oBAAwD,SAAQ,eAAuB;IACnG,YAAY,UAAuC,EAAE;QACpD,MAAM,SAAS,GAAG,IAAI,iBAAiB,CAAC,OAAO,CAAC,SAAS,IAAI,UAAU,CAAC,KAAK,CAAC,CAAA;QAE9E,KAAK,CAAC;YACL,SAAS,EAAE,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE;gBAC/B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACvB,OAAM;gBACP,CAAC;gBAED,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAA;gBAE1D,UAAU,CAAC,OAAO,CAAC,OAAyB,CAAC,CAAA;YAC9C,CAAC;SACD,CAAC,CAAA;IACH,CAAC;CACD"}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
import type { ByteRange } from "./shared.js";
|
|
7
|
+
/**
|
|
8
|
+
* A first-in, first-out queue for marking and dequeuing index tuples.
|
|
9
|
+
*/
|
|
10
|
+
export declare class IndexQueue implements IterableIterator<ByteRange> {
|
|
11
|
+
#private;
|
|
12
|
+
/**
|
|
13
|
+
* The number of tuples in the queue.
|
|
14
|
+
*/
|
|
15
|
+
get size(): number;
|
|
16
|
+
/**
|
|
17
|
+
* The total byte length spanned by the tuples in the queue.
|
|
18
|
+
*
|
|
19
|
+
* This is a cached value and may not be accurate if the queue is modified externally.
|
|
20
|
+
*/
|
|
21
|
+
get byteLength(): number;
|
|
22
|
+
/**
|
|
23
|
+
* Append a new tuple to the queue.
|
|
24
|
+
*/
|
|
25
|
+
enqueue(tuple: ByteRange): void;
|
|
26
|
+
/**
|
|
27
|
+
* Get the next tuple in the queue, removing it.
|
|
28
|
+
*/
|
|
29
|
+
dequeue(): ByteRange | undefined;
|
|
30
|
+
/**
|
|
31
|
+
* Peek at the next tuple in the queue without removing it.
|
|
32
|
+
*/
|
|
33
|
+
peek(): ByteRange | undefined;
|
|
34
|
+
/**
|
|
35
|
+
* Peek at the last tuple in the queue without removing it.
|
|
36
|
+
*/
|
|
37
|
+
peekLast(): ByteRange | undefined;
|
|
38
|
+
/**
|
|
39
|
+
* Clear the queue.
|
|
40
|
+
*/
|
|
41
|
+
clear(): void;
|
|
42
|
+
/**
|
|
43
|
+
* Alias for `dequeue`, conforming to the iterator protocol.
|
|
44
|
+
*/
|
|
45
|
+
next(): IteratorResult<ByteRange>;
|
|
46
|
+
[Symbol.iterator](): IndexQueue;
|
|
47
|
+
}
|
|
48
|
+
//# sourceMappingURL=IndexQueue.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"IndexQueue.d.ts","sourceRoot":"","sources":["../../lib/IndexQueue.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAE5C;;GAEG;AACH,qBAAa,UAAW,YAAW,gBAAgB,CAAC,SAAS,CAAC;;IAG7D;;OAEG;IACH,IAAW,IAAI,IAAI,MAAM,CAExB;IAID;;;;OAIG;IACH,IAAW,UAAU,IAAI,MAAM,CAE9B;IAED;;OAEG;IACI,OAAO,CAAC,KAAK,EAAE,SAAS,GAAG,IAAI;IAMtC;;OAEG;IACI,OAAO,IAAI,SAAS,GAAG,SAAS;IAUvC;;OAEG;IACI,IAAI,IAAI,SAAS,GAAG,SAAS;IAMpC;;OAEG;IACI,QAAQ,IAAI,SAAS,GAAG,SAAS;IAMxC;;OAEG;IACI,KAAK,IAAI,IAAI;IAKpB;;OAEG;IACI,IAAI,IAAI,cAAc,CAAC,SAAS,CAAC;IAUjC,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,UAAU;CAGtC"}
|