spliterator 5.0.0 → 6.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -0
- package/out/index.d.ts +7 -1
- package/out/index.d.ts.map +1 -1
- package/out/index.js +7 -1
- package/out/index.js.map +1 -1
- package/out/lib/AsyncSequence.d.ts +180 -0
- package/out/lib/AsyncSequence.d.ts.map +1 -0
- package/out/lib/AsyncSequence.js +432 -0
- package/out/lib/AsyncSequence.js.map +1 -0
- package/out/lib/AsyncSpliterator.d.ts +4 -2
- package/out/lib/AsyncSpliterator.d.ts.map +1 -1
- package/out/lib/AsyncSpliterator.js.map +1 -1
- package/out/lib/CSVSpliterator.d.ts +6 -5
- package/out/lib/CSVSpliterator.d.ts.map +1 -1
- package/out/lib/CSVSpliterator.js +46 -37
- package/out/lib/CSVSpliterator.js.map +1 -1
- package/out/lib/JSONSpliterator.d.ts +23 -11
- package/out/lib/JSONSpliterator.d.ts.map +1 -1
- package/out/lib/JSONSpliterator.js +28 -19
- package/out/lib/JSONSpliterator.js.map +1 -1
- package/out/lib/TextSpliterator.d.ts +4 -2
- package/out/lib/TextSpliterator.d.ts.map +1 -1
- package/out/lib/TextSpliterator.js +8 -9
- package/out/lib/TextSpliterator.js.map +1 -1
- package/out/lib/adaptive-source.d.ts +39 -0
- package/out/lib/adaptive-source.d.ts.map +1 -0
- package/out/lib/adaptive-source.js +95 -0
- package/out/lib/adaptive-source.js.map +1 -0
- package/out/lib/chunks.d.ts +20 -0
- package/out/lib/chunks.d.ts.map +1 -0
- package/out/lib/chunks.js +36 -0
- package/out/lib/chunks.js.map +1 -0
- package/out/lib/parallel-map-workers.d.ts +57 -0
- package/out/lib/parallel-map-workers.d.ts.map +1 -0
- package/out/lib/parallel-map-workers.js +82 -0
- package/out/lib/parallel-map-workers.js.map +1 -0
- package/out/lib/parallel-map.d.ts +17 -42
- package/out/lib/parallel-map.d.ts.map +1 -1
- package/out/lib/parallel-map.js +18 -66
- package/out/lib/parallel-map.js.map +1 -1
- package/out/lib/pivot.d.ts +30 -0
- package/out/lib/pivot.d.ts.map +1 -0
- package/out/lib/pivot.js +47 -0
- package/out/lib/pivot.js.map +1 -0
- package/out/lib/shared.d.ts +0 -27
- package/out/lib/shared.d.ts.map +1 -1
- package/out/lib/shared.js +0 -53
- package/out/lib/shared.js.map +1 -1
- package/out/lib/take.d.ts +16 -0
- package/out/lib/take.d.ts.map +1 -1
- package/out/lib/take.js +28 -0
- package/out/lib/take.js.map +1 -1
- package/out/lib/utils.d.ts +37 -0
- package/out/lib/utils.d.ts.map +1 -0
- package/out/lib/utils.js +39 -0
- package/out/lib/utils.js.map +1 -0
- package/out/lib/zip.d.ts +33 -0
- package/out/lib/zip.d.ts.map +1 -0
- package/out/lib/zip.js +59 -0
- package/out/lib/zip.js.map +1 -0
- package/package.json +6 -6
|
@@ -3,10 +3,12 @@
|
|
|
3
3
|
* @license MIT
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
|
+
import { openDelimitedRows } from "./adaptive-source.js";
|
|
7
|
+
import { AsyncSequence } from "./AsyncSequence.js";
|
|
6
8
|
import { normalizeColumnNames } from "./casing.js";
|
|
7
9
|
import { CharacterSequence, Delimiters } from "./CharacterSequence.js";
|
|
8
|
-
import { zipSync } from "./shared.js";
|
|
9
10
|
import { Spliterator } from "./Spliterator.js";
|
|
11
|
+
import { zipSync } from "./zip.js";
|
|
10
12
|
const identity = (value) => value;
|
|
11
13
|
const doubleQuoteSequence = new CharacterSequence('"');
|
|
12
14
|
/**
|
|
@@ -154,7 +156,8 @@ export class CSVSpliterator {
|
|
|
154
156
|
*
|
|
155
157
|
* @yields Each row, shaped according to the `mode` option.
|
|
156
158
|
*/
|
|
157
|
-
static
|
|
159
|
+
static fromAsync(source, init = {}) {
|
|
160
|
+
const defaultColumnDelimiter = this.ColumnDelimiter;
|
|
158
161
|
const {
|
|
159
162
|
// ---
|
|
160
163
|
header = true, mode = "array", transformers: transformersInput = [], normalizeKeys = mode !== "array", columnDelimiter: columnDelimiterInput, enableQuoteHandling = false,
|
|
@@ -162,45 +165,51 @@ export class CSVSpliterator {
|
|
|
162
165
|
// never carries a stray `\r` on Windows-lineage sources.
|
|
163
166
|
crlf = true, take = Infinity, drop = 0, ...rowInit } = init;
|
|
164
167
|
const emitter = CSVSpliteratorEmitters[mode];
|
|
165
|
-
|
|
166
|
-
let yieldCount = 0;
|
|
167
|
-
const yieldLimit = take + drop;
|
|
168
|
-
const columnDelimiter = new CharacterSequence(columnDelimiterInput ?? this.ColumnDelimiter);
|
|
168
|
+
const columnDelimiter = new CharacterSequence(columnDelimiterInput ?? defaultColumnDelimiter);
|
|
169
169
|
const decoder = new TextDecoder();
|
|
170
|
-
//
|
|
171
|
-
//
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
170
|
+
// Populated by the header pass below before the first row op runs, since the source thunk resolves on the first
|
|
171
|
+
// pull and the ops only run against what it returns.
|
|
172
|
+
let transformers = [];
|
|
173
|
+
const openRows = async () => {
|
|
174
|
+
// Quote handling applies at both levels: rows must not split on newlines inside quotes,
|
|
175
|
+
// columns must not split on quoted column delimiters.
|
|
176
|
+
const rows = await openDelimitedRows(source, { ...rowInit, crlf, enableQuoteHandling });
|
|
177
|
+
if (header) {
|
|
178
|
+
// Both engines return `this` from their iterator method, so consuming the header row here advances the very
|
|
179
|
+
// cursor the row ops will go on to read — returning `rows` afterwards resumes at row two, not row one.
|
|
180
|
+
const iterator = Symbol.asyncIterator in rows ? rows[Symbol.asyncIterator]() : rows[Symbol.iterator]();
|
|
181
|
+
const result = await iterator.next();
|
|
182
|
+
if (result.done)
|
|
183
|
+
return rows;
|
|
184
|
+
const columns = splitRowColumns(result.value, columnDelimiter, decoder, enableQuoteHandling);
|
|
185
|
+
const headers = normalizeKeys ? normalizeColumnNames(columns) : columns;
|
|
186
|
+
// oxlint-disable-next-line unicorn/prefer-ternary
|
|
187
|
+
if (Array.isArray(transformersInput)) {
|
|
188
|
+
transformers = Array.from(zipSync(headers, transformersInput), ([columnName, transformer]) => [
|
|
189
|
+
columnName,
|
|
190
|
+
transformer ?? identity,
|
|
191
|
+
]);
|
|
192
|
+
}
|
|
193
|
+
else {
|
|
194
|
+
transformers = headers.map((columnName) => {
|
|
195
|
+
const transform = transformersInput[columnName] || identity;
|
|
196
|
+
return [columnName, transform];
|
|
197
|
+
});
|
|
198
|
+
}
|
|
197
199
|
}
|
|
198
|
-
|
|
199
|
-
|
|
200
|
+
return rows;
|
|
201
|
+
};
|
|
202
|
+
let sequence = AsyncSequence.from(openRows).map((row) => {
|
|
200
203
|
const columns = splitRowColumns(row, columnDelimiter, decoder, enableQuoteHandling);
|
|
201
|
-
|
|
202
|
-
|
|
204
|
+
return emitter ? emitter(columns, transformers) : columns;
|
|
205
|
+
});
|
|
206
|
+
if (drop > 0) {
|
|
207
|
+
sequence = sequence.drop(drop);
|
|
208
|
+
}
|
|
209
|
+
if (Number.isFinite(take)) {
|
|
210
|
+
sequence = sequence.take(take);
|
|
203
211
|
}
|
|
212
|
+
return sequence;
|
|
204
213
|
}
|
|
205
214
|
}
|
|
206
215
|
//# sourceMappingURL=CSVSpliterator.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CSVSpliterator.js","sourceRoot":"","sources":["../../lib/CSVSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAA;AAClD,OAAO,EAAE,iBAAiB,EAA+B,UAAU,EAAE,MAAM,wBAAwB,CAAA;
|
|
1
|
+
{"version":3,"file":"CSVSpliterator.js","sourceRoot":"","sources":["../../lib/CSVSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAA2B,iBAAiB,EAAE,MAAM,sBAAsB,CAAA;AACjF,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAClD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAA;AAClD,OAAO,EAAE,iBAAiB,EAA+B,UAAU,EAAE,MAAM,wBAAwB,CAAA;AAEnG,OAAO,EAA6B,WAAW,EAAwB,MAAM,kBAAkB,CAAA;AAC/F,OAAO,EAAE,OAAO,EAAE,MAAM,UAAU,CAAA;AAelC,MAAM,QAAQ,GAA2B,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAA;AAEzD,MAAM,mBAAmB,GAAG,IAAI,iBAAiB,CAAC,GAAG,CAAC,CAAA;AAEtD;;;GAGG;AACH,SAAS,YAAY,CAAC,KAAiB,EAAE,OAAoB,EAAE,mBAA4B;IAC1F,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;IAEnC,IAAI,mBAAmB,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC9F,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;IAChD,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;;;;GAMG;AACH,SAAS,eAAe,CACvB,GAAe,EACf,eAAkC,EAClC,OAAoB,EACpB,mBAA4B;IAE5B,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC1B,OAAO,eAAe,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAA;IACtG,CAAC;IAED,MAAM,OAAO,GAAa,EAAE,CAAA;IAC5B,IAAI,UAAU,GAAG,CAAC,CAAA;IAClB,IAAI,YAAY,GAAG,KAAK,CAAA;IAExB,KAAK,MAAM,KAAK,IAAI,eAAe,CAAC,aAAa,CAAC,GAAG,EAAE,mBAAmB,CAAC,EAAE,CAAC;QAC7E,IAAI,KAAK,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;YAC3B,YAAY,GAAG,CAAC,YAAY,CAAA;YAE5B,SAAQ;QACT,CAAC;QAED,IAAI,YAAY;YAAE,SAAQ;QAE1B,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAC,CAAA;QAChG,UAAU,GAAG,KAAK,CAAC,MAAM,GAAG,eAAe,CAAC,MAAM,CAAA;IACnD,CAAC;IAED,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAC,CAAA;IAElF,OAAO,OAAO,CAAA;AACf,CAAC;AAaD,MAAM,CAAC,MAAM,sBAAsB,GAAG;IACrC,KAAK,EAAE,IAAI;IAEX,OAAO,CAAC,OAAyB,EAAE,aAAa,GAAkC,EAAE;QACnF,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE;YAChF,MAAM,GAAG,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,CAAA;YAC5B,MAAM,SAAS,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAA;YAE9C,OAAO,CAAC,GAAG,IAAI,UAAU,GAAG,EAAE,EAAE,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,CAAA;QAC7D,CAAC,CAAC,CAAA;IACH,CAAC;IACD,MAAM,CACL,OAAyB,EACzB,aAAa,GAAkC,EAAE;QAEjD,MAAM,MAAM,GAAyC,EAAE,CAAA;QAEvD,KAAK,MAAM,CAAC,WAAW,EAAE,KAAK,EAAE,GAAG,CAAC,IAAI,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,EAAE,CAAC;YACzE,MAAM,GAAG,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,UAAU,GAAG,EAAE,CAAA;YAC/C,MAAM,SAAS,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAA;YAE9C,MAAM,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;QACrC,CAAC;QAED,OAAO,MAAM,CAAA;IACd,CAAC;CAC2D,CAAA;AAmD7D;;;;;;;;;;GAUG;AACH,MAAM,OAAgB,cAAc;IACnC;;;;OAIG;IACI,MAAM,CAAC,eAAe,GAAW,UAAU,CAAC,KAAK,CAAA;IAExD;QACC,MAAM,IAAI,SAAS,CAAC,0EAA0E,CAAC,CAAA;IAChG,CAAC;IAuBD;;;;OAIG;IACH,MAAM,CAAC,CAAC,IAAI,CAAC,MAA8B,EAAE,IAAI,GAAuB,EAAE;QACzE,MAAM;QACL,MAAM;QACN,MAAM,GAAG,IAAI;QACb,uFAAuF;QACvF,IAAI,GAAG,OAAO,EACd,YAAY,EAAE,iBAAiB,GAAG,EAAE;QACpC,wFAAwF;QACxF,8FAA8F;QAC9F,aAAa,GAAG,IAAI,KAAK,OAAO,EAChC,eAAe,EAAE,oBAAoB,GAAG,IAAI,CAAC,eAAe,EAC5D,mBAAmB,GAAG,KAAK;QAC3B,qFAAqF;QACrF,yDAAyD;QACzD,IAAI,GAAG,IAAI,EACX,IAAI,GAAG,QAAQ,EACf,IAAI,GAAG,CAAC,EACR,GAAG,OAAO,EACV,GAAG,IAAI,CAAA;QAER,MAAM,OAAO,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAA;QAC5C,IAAI,YAAY,GAA0B,EAAE,CAAA;QAC5C,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,MAAM,UAAU,GAAG,IAAI,GAAG,IAAI,CAAA;QAE9B,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QACjC,MAAM,eAAe,GAAG,IAAI,iBAAiB,CAAC,oBAAoB,IAAI,IAAI,CAAC,eAAe,CAAC,CAAA;QAE3F,wFAAwF;QACxF,sDAAsD;QACtD,MAAM,IAAI,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,mBAAmB,EAAE,CAAC,CAAA;QAEpF,IAAI,MAAM,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;YAE1B,IAAI,MAAM,CAAC,IAAI;gBAAE,OAAM;YAEvB,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,CAAC,KAAK,EAAE,eAAe,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAA;YAC5F,MAAM,OAAO,GAAG,aAAa,CAAC,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;YAEvE,kDAAkD;YAClD,IAAI,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,CAAC;gBACtC,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,iBAAiB,CAAC,EAAE,CAAC,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC;oBAC7F,UAAW;oBACX,WAAW,IAAI,QAAQ;iBACvB,CAAC,CAAA;YACH,CAAC;iBAAM,CAAC;gBACP,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;oBACzC,MAAM,SAAS,GAAI,iBAA0C,CAAC,UAAU,CAAC,IAAI,QAAQ,CAAA;oBAErF,OAAO,CAAC,UAAU,EAAE,SAAS,CAAC,CAAA;gBAC/B,CAAC,CAAC,CAAA;YACH,CAAC;QACF,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,IAAI,UAAU,GAAG,IAAI,EAAE,CAAC;gBACvB,UAAU,EAAE,CAAA;gBAEZ,SAAQ;YACT,CAAC;YAED,IAAI,UAAU,IAAI,UAAU;gBAAE,MAAK;YAEnC,MAAM,OAAO,GAAG,eAAe,CAAC,GAAG,EAAE,eAAe,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAA;YAEnF,MAAM,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;YAExD,UAAU,EAAE,CAAA;QACb,CAAC;IACF,CAAC;IAqCD;;;;;;;;OAQG;IACH,MAAM,CAAC,SAAS,CACf,MAA8C,EAC9C,IAAI,GAA4C,EAAE;QAElD,MAAM,sBAAsB,GAAG,IAAI,CAAC,eAAe,CAAA;QAEnD,MAAM;QACL,MAAM;QACN,MAAM,GAAG,IAAI,EACb,IAAI,GAAG,OAAO,EACd,YAAY,EAAE,iBAAiB,GAAG,EAAE,EACpC,aAAa,GAAG,IAAI,KAAK,OAAO,EAChC,eAAe,EAAE,oBAAoB,EACrC,mBAAmB,GAAG,KAAK;QAC3B,qFAAqF;QACrF,yDAAyD;QACzD,IAAI,GAAG,IAAI,EACX,IAAI,GAAG,QAAQ,EACf,IAAI,GAAG,CAAC,EACR,GAAG,OAAO,EACV,GAAG,IAAI,CAAA;QAER,MAAM,OAAO,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAA;QAC5C,MAAM,eAAe,GAAG,IAAI,iBAAiB,CAAC,oBAAoB,IAAI,sBAAsB,CAAC,CAAA;QAC7F,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QAEjC,gHAAgH;QAChH,qDAAqD;QACrD,IAAI,YAAY,GAA0B,EAAE,CAAA;QAE5C,MAAM,QAAQ,GAAG,KAAK,IAA+D,EAAE;YACtF,wFAAwF;YACxF,sDAAsD;YACtD,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,mBAAmB,EAAE,CAAC,CAAA;YAEvF,IAAI,MAAM,EAAE,CAAC;gBACZ,4GAA4G;gBAC5G,uGAAuG;gBACvG,MAAM,QAAQ,GAAG,MAAM,CAAC,aAAa,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAA;gBACtG,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;gBAEpC,IAAI,MAAM,CAAC,IAAI;oBAAE,OAAO,IAAI,CAAA;gBAE5B,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,CAAC,KAAK,EAAE,eAAe,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAA;gBAC5F,MAAM,OAAO,GAAG,aAAa,CAAC,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;gBAEvE,kDAAkD;gBAClD,IAAI,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,CAAC;oBACtC,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,iBAAiB,CAAC,EAAE,CAAC,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC;wBAC7F,UAAW;wBACX,WAAW,IAAI,QAAQ;qBACvB,CAAC,CAAA;gBACH,CAAC;qBAAM,CAAC;oBACP,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;wBACzC,MAAM,SAAS,GAAI,iBAA0C,CAAC,UAAU,CAAC,IAAI,QAAQ,CAAA;wBAErF,OAAO,CAAC,UAAU,EAAE,SAAS,CAAC,CAAA;oBAC/B,CAAC,CAAC,CAAA;gBACH,CAAC;YACF,CAAC;YAED,OAAO,IAAI,CAAA;QACZ,CAAC,CAAA;QAED,IAAI,QAAQ,GAA2B,aAAa,CAAC,IAAI,CAAa,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YAC3F,MAAM,OAAO,GAAG,eAAe,CAAC,GAAG,EAAE,eAAe,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAA;YAEnF,OAAO,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;QAC1D,CAAC,CAAC,CAAA;QAEF,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;YACd,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC/B,CAAC;QAED,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3B,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC/B,CAAC;QAED,OAAO,QAAQ,CAAA;IAChB,CAAC;CACD"}
|
|
@@ -3,20 +3,32 @@
|
|
|
3
3
|
* @license MIT
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
|
+
import { type AdaptiveSourceInit } from "./adaptive-source.js";
|
|
7
|
+
import { AsyncSequence } from "./AsyncSequence.js";
|
|
6
8
|
import type { CharacterSequenceInput } from "./CharacterSequence.js";
|
|
7
9
|
import type { AsyncDataResource } from "./shared.js";
|
|
8
|
-
import { type
|
|
10
|
+
import { type SpliteratorInit } from "./Spliterator.js";
|
|
9
11
|
/**
|
|
10
|
-
* Stream a delimited byte source and `JSON.parse` each row — one parsed value per line, for JSONL / NDJSON.
|
|
11
|
-
* delimiter via `options.delimiter` (
|
|
12
|
+
* Stream a delimited byte source and `JSON.parse` each row — one parsed value per line, for JSONL / NDJSON. The row
|
|
13
|
+
* delimiter defaults to a line feed; override it via `options.delimiter`. `skipEmpty` (on by default) drops blank
|
|
14
|
+
* rows.
|
|
12
15
|
*
|
|
13
|
-
* **Performance caveat — this path is `JSON.parse`-bound, not scan-bound.**
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
16
|
+
* **Performance caveat — this path is `JSON.parse`-bound, not scan-bound.** Measured over 500k rows (88MB, ~177B per
|
|
17
|
+
* row, Node 26): the delimiter scan is ~140ms and decoding ~46ms, while `JSON.parse` and the per-row async machinery
|
|
18
|
+
* account for the rest of ~1280ms. Against Node's `readline` + `JSON.parse` at ~630ms, this runs **roughly half the
|
|
19
|
+
* speed — a net loss.** Use it for API convenience, or when streaming to bound memory; do NOT swap a working `readline`
|
|
20
|
+
* loop to it expecting a speedup on parse-heavy JSONL.
|
|
21
|
+
*
|
|
22
|
+
* The scan advantage only shows when you _don't_ fully parse every row. Filtering on the raw text and parsing only what
|
|
23
|
+
* survives is the shape that wins — parsing 20% of rows measured ~2.5× faster than parsing all of them:
|
|
24
|
+
*
|
|
25
|
+
* ```ts
|
|
26
|
+
* TextSpliterator.fromAsync(path, { delimiter: "\n" })
|
|
27
|
+
* .filter((line) => line.includes('"category":"Novelty"'))
|
|
28
|
+
* .map((line) => JSON.parse(line))
|
|
29
|
+
* ```
|
|
30
|
+
*
|
|
31
|
+
* For segmentation or counting, use {@link Spliterator} (raw byte ranges) or {@link TextSpliterator}. When unsure,
|
|
20
32
|
* benchmark.
|
|
21
33
|
*/
|
|
22
34
|
export declare abstract class JSONSpliterator {
|
|
@@ -27,6 +39,6 @@ export declare abstract class JSONSpliterator {
|
|
|
27
39
|
*
|
|
28
40
|
* @yields Each row as an array of columns.
|
|
29
41
|
*/
|
|
30
|
-
static fromAsync<T = unknown>(source: AsyncDataResource, options?:
|
|
42
|
+
static fromAsync<T = unknown>(source: AsyncDataResource, options?: AdaptiveSourceInit): AsyncSequence<T>;
|
|
31
43
|
}
|
|
32
44
|
//# sourceMappingURL=JSONSpliterator.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"JSONSpliterator.d.ts","sourceRoot":"","sources":["../../lib/JSONSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AACpE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AACpD,OAAO,
|
|
1
|
+
{"version":3,"file":"JSONSpliterator.d.ts","sourceRoot":"","sources":["../../lib/JSONSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,KAAK,kBAAkB,EAAqB,MAAM,sBAAsB,CAAA;AACjF,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAClD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AACpE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AACpD,OAAO,EAA0C,KAAK,eAAe,EAAE,MAAM,kBAAkB,CAAA;AAE/F;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,8BAAsB,eAAe;IACpC,cAEC;IAED,MAAM,CAAE,IAAI,CAAC,CAAC,GAAG,OAAO,EAAE,MAAM,EAAE,sBAAsB,EAAE,OAAO,GAAE,eAAoB,GAAG,SAAS,CAAC,CAAC,CAAC,CAwBrG;IAED;;;;OAIG;IACH,MAAM,CAAC,SAAS,CAAC,CAAC,GAAG,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,OAAO,GAAE,kBAAuB,GAAG,aAAa,CAAC,CAAC,CAAC,CAgB3G;CACD"}
|
|
@@ -3,18 +3,30 @@
|
|
|
3
3
|
* @license MIT
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
|
+
import { openDelimitedRows } from "./adaptive-source.js";
|
|
7
|
+
import { AsyncSequence } from "./AsyncSequence.js";
|
|
6
8
|
import { Spliterator } from "./Spliterator.js";
|
|
7
9
|
/**
|
|
8
|
-
* Stream a delimited byte source and `JSON.parse` each row — one parsed value per line, for JSONL / NDJSON.
|
|
9
|
-
* delimiter via `options.delimiter` (
|
|
10
|
+
* Stream a delimited byte source and `JSON.parse` each row — one parsed value per line, for JSONL / NDJSON. The row
|
|
11
|
+
* delimiter defaults to a line feed; override it via `options.delimiter`. `skipEmpty` (on by default) drops blank
|
|
12
|
+
* rows.
|
|
10
13
|
*
|
|
11
|
-
* **Performance caveat — this path is `JSON.parse`-bound, not scan-bound.**
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
14
|
+
* **Performance caveat — this path is `JSON.parse`-bound, not scan-bound.** Measured over 500k rows (88MB, ~177B per
|
|
15
|
+
* row, Node 26): the delimiter scan is ~140ms and decoding ~46ms, while `JSON.parse` and the per-row async machinery
|
|
16
|
+
* account for the rest of ~1280ms. Against Node's `readline` + `JSON.parse` at ~630ms, this runs **roughly half the
|
|
17
|
+
* speed — a net loss.** Use it for API convenience, or when streaming to bound memory; do NOT swap a working `readline`
|
|
18
|
+
* loop to it expecting a speedup on parse-heavy JSONL.
|
|
19
|
+
*
|
|
20
|
+
* The scan advantage only shows when you _don't_ fully parse every row. Filtering on the raw text and parsing only what
|
|
21
|
+
* survives is the shape that wins — parsing 20% of rows measured ~2.5× faster than parsing all of them:
|
|
22
|
+
*
|
|
23
|
+
* ```ts
|
|
24
|
+
* TextSpliterator.fromAsync(path, { delimiter: "\n" })
|
|
25
|
+
* .filter((line) => line.includes('"category":"Novelty"'))
|
|
26
|
+
* .map((line) => JSON.parse(line))
|
|
27
|
+
* ```
|
|
28
|
+
*
|
|
29
|
+
* For segmentation or counting, use {@link Spliterator} (raw byte ranges) or {@link TextSpliterator}. When unsure,
|
|
18
30
|
* benchmark.
|
|
19
31
|
*/
|
|
20
32
|
export class JSONSpliterator {
|
|
@@ -45,24 +57,21 @@ export class JSONSpliterator {
|
|
|
45
57
|
*
|
|
46
58
|
* @yields Each row as an array of columns.
|
|
47
59
|
*/
|
|
48
|
-
static
|
|
60
|
+
static fromAsync(source, options = {}) {
|
|
49
61
|
const decoder = new TextDecoder();
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
62
|
+
// Parsing is an op on the sequence, not a generator wrapped inside one. An allocating row body makes the extra
|
|
63
|
+
// async frame a wrapping generator adds disproportionately expensive: 1460ms against 1278ms over 500k rows, where
|
|
64
|
+
// the same layer costs a third as much when the body only decodes.
|
|
65
|
+
return AsyncSequence.from(() => openDelimitedRows(source, options)).map((row, rowCursor) => {
|
|
54
66
|
try {
|
|
55
|
-
|
|
56
|
-
parsed = JSON.parse(content);
|
|
67
|
+
return JSON.parse(decoder.decode(row));
|
|
57
68
|
}
|
|
58
69
|
catch (parsedError) {
|
|
59
70
|
const error = new SyntaxError(`Failed to parse JSON at row ${rowCursor}`);
|
|
60
71
|
error.cause = parsedError;
|
|
61
72
|
throw error;
|
|
62
73
|
}
|
|
63
|
-
|
|
64
|
-
rowCursor++;
|
|
65
|
-
}
|
|
74
|
+
});
|
|
66
75
|
}
|
|
67
76
|
}
|
|
68
77
|
//# sourceMappingURL=JSONSpliterator.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"JSONSpliterator.js","sourceRoot":"","sources":["../../lib/JSONSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;
|
|
1
|
+
{"version":3,"file":"JSONSpliterator.js","sourceRoot":"","sources":["../../lib/JSONSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAA2B,iBAAiB,EAAE,MAAM,sBAAsB,CAAA;AACjF,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAGlD,OAAO,EAA6B,WAAW,EAAwB,MAAM,kBAAkB,CAAA;AAE/F;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,OAAgB,eAAe;IACpC;QACC,MAAM,IAAI,SAAS,CAAC,2EAA2E,CAAC,CAAA;IACjG,CAAC;IAED,MAAM,CAAC,CAAC,IAAI,CAAc,MAA8B,EAAE,OAAO,GAAoB,EAAE;QACtF,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QACjC,IAAI,SAAS,GAAG,CAAC,CAAA;QAEjB,MAAM,WAAW,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAEzD,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YAC/B,IAAI,MAAS,CAAA;YAEb,IAAI,CAAC;gBACJ,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;gBAEnC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAM,CAAA;YAClC,CAAC;YAAC,OAAO,WAAW,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,IAAI,WAAW,CAAC,+BAA+B,SAAS,EAAE,CAAC,CAAA;gBACzE,KAAK,CAAC,KAAK,GAAG,WAAW,CAAA;gBAEzB,MAAM,KAAK,CAAA;YACZ,CAAC;YAED,MAAM,MAAM,CAAA;YAEZ,SAAS,EAAE,CAAA;QACZ,CAAC;IACF,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,SAAS,CAAc,MAAyB,EAAE,OAAO,GAAuB,EAAE;QACxF,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QAEjC,+GAA+G;QAC/G,kHAAkH;QAClH,mEAAmE;QACnE,OAAO,aAAa,CAAC,IAAI,CAAa,GAAG,EAAE,CAAC,iBAAiB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE;YACtG,IAAI,CAAC;gBACJ,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAM,CAAA;YAC5C,CAAC;YAAC,OAAO,WAAW,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,IAAI,WAAW,CAAC,+BAA+B,SAAS,EAAE,CAAC,CAAA;gBACzE,KAAK,CAAC,KAAK,GAAG,WAAW,CAAA;gBAEzB,MAAM,KAAK,CAAA;YACZ,CAAC;QACF,CAAC,CAAC,CAAA;IACH,CAAC;CACD"}
|
|
@@ -3,9 +3,11 @@
|
|
|
3
3
|
* @license MIT
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
|
+
import { type AdaptiveSourceInit } from "./adaptive-source.js";
|
|
7
|
+
import { AsyncSequence } from "./AsyncSequence.js";
|
|
6
8
|
import type { CharacterSequenceInput } from "./CharacterSequence.js";
|
|
7
9
|
import type { AsyncDataResource } from "./shared.js";
|
|
8
|
-
import { type
|
|
10
|
+
import { type SpliteratorInit } from "./Spliterator.js";
|
|
9
11
|
export interface TextSpliteratorInit {
|
|
10
12
|
/**
|
|
11
13
|
* The encoding to use when decoding the data.
|
|
@@ -46,6 +48,6 @@ export declare abstract class TextSpliterator {
|
|
|
46
48
|
* @see {@linkcode TextSpliterator.from} for synchronous iteration with decoding.
|
|
47
49
|
* @see {@linkcode Spliterator.fromAsync} for asynchronous iteration without decoding.
|
|
48
50
|
*/
|
|
49
|
-
static fromAsync(source: AsyncDataResource, { encoding, fatal, ignoreBOM, ...options }?: TextSpliteratorInit &
|
|
51
|
+
static fromAsync(source: AsyncDataResource, { encoding, fatal, ignoreBOM, ...options }?: TextSpliteratorInit & AdaptiveSourceInit): AsyncSequence<string>;
|
|
50
52
|
}
|
|
51
53
|
//# sourceMappingURL=TextSpliterator.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"TextSpliterator.d.ts","sourceRoot":"","sources":["../../lib/TextSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AACpE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AACpD,OAAO,
|
|
1
|
+
{"version":3,"file":"TextSpliterator.d.ts","sourceRoot":"","sources":["../../lib/TextSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,KAAK,kBAAkB,EAAqB,MAAM,sBAAsB,CAAA;AACjF,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAClD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AACpE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AACpD,OAAO,EAA0C,KAAK,eAAe,EAAE,MAAM,kBAAkB,CAAA;AAE/F,MAAM,WAAW,mBAAmB;IACnC;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,CAAA;IAEf;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAA;CACnB;AAED,8BAAsB,eAAe;IACpC,cAEC;IAED;;;;;;;;;OASG;IACH,MAAM,CAAE,IAAI,CACX,MAAM,EAAE,sBAAsB,EAC9B,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,OAAO,EAAE,GAAE,mBAAmB,GAAG,eAAoB,GACpF,SAAS,CAAC,MAAM,CAAC,CAsBnB;IAED;;;;;;;;;OASG;IACH,MAAM,CAAC,SAAS,CACf,MAAM,EAAE,iBAAiB,EACzB,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,OAAO,EAAE,GAAE,mBAAmB,GAAG,kBAAuB,GACvF,aAAa,CAAC,MAAM,CAAC,CAevB;CACD"}
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
* @license MIT
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
|
+
import { openDelimitedRows } from "./adaptive-source.js";
|
|
7
|
+
import { AsyncSequence } from "./AsyncSequence.js";
|
|
6
8
|
import { Spliterator } from "./Spliterator.js";
|
|
7
9
|
export class TextSpliterator {
|
|
8
10
|
constructor() {
|
|
@@ -46,23 +48,20 @@ export class TextSpliterator {
|
|
|
46
48
|
* @see {@linkcode TextSpliterator.from} for synchronous iteration with decoding.
|
|
47
49
|
* @see {@linkcode Spliterator.fromAsync} for asynchronous iteration without decoding.
|
|
48
50
|
*/
|
|
49
|
-
static
|
|
51
|
+
static fromAsync(source, { encoding, fatal, ignoreBOM, ...options } = {}) {
|
|
50
52
|
const decoder = new TextDecoder(encoding, { fatal, ignoreBOM });
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
let decoded;
|
|
53
|
+
// Decoding is an op on the sequence, not a generator wrapped inside one. A wrapping generator adds an async frame
|
|
54
|
+
// per row on top of the sequence's own, which measured 297ms against 279ms over 500k rows.
|
|
55
|
+
return AsyncSequence.from(() => openDelimitedRows(source, options)).map((row, rowCursor) => {
|
|
55
56
|
try {
|
|
56
|
-
|
|
57
|
+
return decoder.decode(row);
|
|
57
58
|
}
|
|
58
59
|
catch (parsedError) {
|
|
59
60
|
const error = new SyntaxError(`Failed to decode data at row ${rowCursor}`);
|
|
60
61
|
error.cause = parsedError;
|
|
61
62
|
throw error;
|
|
62
63
|
}
|
|
63
|
-
|
|
64
|
-
rowCursor++;
|
|
65
|
-
}
|
|
64
|
+
});
|
|
66
65
|
}
|
|
67
66
|
}
|
|
68
67
|
//# sourceMappingURL=TextSpliterator.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"TextSpliterator.js","sourceRoot":"","sources":["../../lib/TextSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;
|
|
1
|
+
{"version":3,"file":"TextSpliterator.js","sourceRoot":"","sources":["../../lib/TextSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAA2B,iBAAiB,EAAE,MAAM,sBAAsB,CAAA;AACjF,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAGlD,OAAO,EAA6B,WAAW,EAAwB,MAAM,kBAAkB,CAAA;AAsB/F,MAAM,OAAgB,eAAe;IACpC;QACC,MAAM,IAAI,SAAS,CAAC,2EAA2E,CAAC,CAAA;IACjG,CAAC;IAED;;;;;;;;;OASG;IACH,MAAM,CAAC,CAAC,IAAI,CACX,MAA8B,EAC9B,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,OAAO,EAAE,GAA0C,EAAE;QAEtF,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAA;QAC/D,IAAI,SAAS,GAAG,CAAC,CAAA;QAEjB,MAAM,WAAW,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAEzD,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YAC/B,IAAI,OAAe,CAAA;YAEnB,IAAI,CAAC;gBACJ,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YAC9B,CAAC;YAAC,OAAO,WAAW,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,IAAI,WAAW,CAAC,gCAAgC,SAAS,EAAE,CAAC,CAAA;gBAC1E,KAAK,CAAC,KAAK,GAAG,WAAW,CAAA;gBAEzB,MAAM,KAAK,CAAA;YACZ,CAAC;YAED,MAAM,OAAO,CAAA;YAEb,SAAS,EAAE,CAAA;QACZ,CAAC;IACF,CAAC;IAED;;;;;;;;;OASG;IACH,MAAM,CAAC,SAAS,CACf,MAAyB,EACzB,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,OAAO,EAAE,GAA6C,EAAE;QAEzF,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAA;QAE/D,kHAAkH;QAClH,2FAA2F;QAC3F,OAAO,aAAa,CAAC,IAAI,CAAa,GAAG,EAAE,CAAC,iBAAiB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE;YACtG,IAAI,CAAC;gBACJ,OAAO,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YAC3B,CAAC;YAAC,OAAO,WAAW,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,IAAI,WAAW,CAAC,gCAAgC,SAAS,EAAE,CAAC,CAAA;gBAC1E,KAAK,CAAC,KAAK,GAAG,WAAW,CAAA;gBAEzB,MAAM,KAAK,CAAA;YACZ,CAAC;QACF,CAAC,CAAC,CAAA;IACH,CAAC;CACD"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license MIT
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
import type { AsyncChunkIterator, AsyncDataResource } from "./shared.js";
|
|
7
|
+
import { type AsyncSpliteratorInit } from "./Spliterator.js";
|
|
8
|
+
/**
|
|
9
|
+
* Byte length at or below which a source is read whole and parsed synchronously.
|
|
10
|
+
*
|
|
11
|
+
* What this buys is **fixed setup cost** — opening a handle and standing up a read stream, ~100µs — not throughput, so
|
|
12
|
+
* the win decays as the source grows and the threshold marks where it stops being measurable. End-to-end against the
|
|
13
|
+
* streaming path (Node 26, min of 200 runs, two independent sweeps): ~1.85× at 635B, ~1.45× at 6.5KB, ~1.4× at 125KiB.
|
|
14
|
+
* At 253KiB and above the two runs disagreed on which was faster, so there is nothing reliable left to win.
|
|
15
|
+
*
|
|
16
|
+
* Raising this does not recover the much larger gap a raw synchronous parse shows (~1.6× even at 1GiB); that gap is
|
|
17
|
+
* eaten by the per-row cost of the sequence itself, which both paths pay. It only trades resident memory — a 1GiB
|
|
18
|
+
* source costs ~105MB streamed against ~1121MB read whole — for a difference that no longer measures.
|
|
19
|
+
*/
|
|
20
|
+
export declare const DEFAULT_BULK_THRESHOLD: number;
|
|
21
|
+
export interface AdaptiveSourceInit extends AsyncSpliteratorInit {
|
|
22
|
+
/**
|
|
23
|
+
* Byte length at or below which the whole source is read into memory and parsed synchronously, trading resident
|
|
24
|
+
* memory for speed.
|
|
25
|
+
*
|
|
26
|
+
* Raising it buys nothing measurable and costs memory linearly — the advantage is gone by ~256KiB. Set `0` to always
|
|
27
|
+
* stream, which is what you want when a bounded footprint is the reason you reached for this library.
|
|
28
|
+
*
|
|
29
|
+
* @default 128 KiB ({@linkcode DEFAULT_BULK_THRESHOLD})
|
|
30
|
+
*/
|
|
31
|
+
bulkThreshold?: number;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Take the delimited rows of `source`, reading it whole when it is small enough to be worth the memory and streaming it
|
|
35
|
+
* otherwise. Returns a sync iterable in the first case and an async one in the second; both satisfy
|
|
36
|
+
* {@linkcode AsyncSequence}.
|
|
37
|
+
*/
|
|
38
|
+
export declare function openDelimitedRows(source: AsyncDataResource | AsyncChunkIterator, init?: AdaptiveSourceInit): Promise<AsyncIterable<Uint8Array> | Iterable<Uint8Array>>;
|
|
39
|
+
//# sourceMappingURL=adaptive-source.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"adaptive-source.d.ts","sourceRoot":"","sources":["../../lib/adaptive-source.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AACxE,OAAO,EAAE,KAAK,oBAAoB,EAAe,MAAM,kBAAkB,CAAA;AAEzE;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,sBAAsB,QAAa,CAAA;AAEhD,MAAM,WAAW,kBAAmB,SAAQ,oBAAoB;IAC/D;;;;;;;;OAQG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;CACtB;AAuBD;;;;GAIG;AACH,wBAAsB,iBAAiB,CACtC,MAAM,EAAE,iBAAiB,GAAG,kBAAkB,EAC9C,IAAI,GAAE,kBAAuB,GAC3B,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC,CAuB3D"}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license MIT
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
import { CharacterSequence } from "./CharacterSequence.js";
|
|
7
|
+
import { Spliterator } from "./Spliterator.js";
|
|
8
|
+
/**
|
|
9
|
+
* Byte length at or below which a source is read whole and parsed synchronously.
|
|
10
|
+
*
|
|
11
|
+
* What this buys is **fixed setup cost** — opening a handle and standing up a read stream, ~100µs — not throughput, so
|
|
12
|
+
* the win decays as the source grows and the threshold marks where it stops being measurable. End-to-end against the
|
|
13
|
+
* streaming path (Node 26, min of 200 runs, two independent sweeps): ~1.85× at 635B, ~1.45× at 6.5KB, ~1.4× at 125KiB.
|
|
14
|
+
* At 253KiB and above the two runs disagreed on which was faster, so there is nothing reliable left to win.
|
|
15
|
+
*
|
|
16
|
+
* Raising this does not recover the much larger gap a raw synchronous parse shows (~1.6× even at 1GiB); that gap is
|
|
17
|
+
* eaten by the per-row cost of the sequence itself, which both paths pay. It only trades resident memory — a 1GiB
|
|
18
|
+
* source costs ~105MB streamed against ~1121MB read whole — for a difference that no longer measures.
|
|
19
|
+
*/
|
|
20
|
+
export const DEFAULT_BULK_THRESHOLD = 128 * 1024;
|
|
21
|
+
function isChunkIterator(source) {
|
|
22
|
+
return typeof source === "object" && source !== null && Symbol.asyncIterator in source;
|
|
23
|
+
}
|
|
24
|
+
function toBytes(chunk) {
|
|
25
|
+
return typeof chunk === "string" ? new TextEncoder().encode(chunk) : chunk;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Parse in-memory bytes with the synchronous engine.
|
|
29
|
+
*
|
|
30
|
+
* Awaiting {@linkcode CharacterSequence.whenReady} first is the point: the WASM scanner loads asynchronously, so a
|
|
31
|
+
* synchronous caller normally completes before it is available and silently uses the JS scanner. Reaching the sync
|
|
32
|
+
* engine through an async path is the one place that can be avoided — worth ~26% on a large source.
|
|
33
|
+
*/
|
|
34
|
+
async function bulk(bytes, init) {
|
|
35
|
+
await CharacterSequence.whenReady();
|
|
36
|
+
return Spliterator.fromSync(bytes, init);
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Take the delimited rows of `source`, reading it whole when it is small enough to be worth the memory and streaming it
|
|
40
|
+
* otherwise. Returns a sync iterable in the first case and an async one in the second; both satisfy
|
|
41
|
+
* {@linkcode AsyncSequence}.
|
|
42
|
+
*/
|
|
43
|
+
export async function openDelimitedRows(source, init = {}) {
|
|
44
|
+
const threshold = init.bulkThreshold ?? DEFAULT_BULK_THRESHOLD;
|
|
45
|
+
if (threshold <= 0)
|
|
46
|
+
return Spliterator.fromAsync(source, init);
|
|
47
|
+
if (isChunkIterator(source))
|
|
48
|
+
return openChunkIterator(source, threshold, init);
|
|
49
|
+
let size;
|
|
50
|
+
try {
|
|
51
|
+
const { readFileSize } = await import("spliterator/node/fs");
|
|
52
|
+
size = await readFileSize(source);
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
// Unsized or unreachable by the Node adapter — the streaming path owns reporting why.
|
|
56
|
+
return Spliterator.fromAsync(source, init);
|
|
57
|
+
}
|
|
58
|
+
if (size > threshold)
|
|
59
|
+
return Spliterator.fromAsync(source, init);
|
|
60
|
+
const { readBytes } = await import("spliterator/node/fs");
|
|
61
|
+
return bulk(await readBytes(source, 0, size), init);
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* A stream cannot be measured before it is read, so the size test becomes an end-of-input test: pull one chunk, and if
|
|
65
|
+
* the stream is already exhausted the whole input is in hand and no threshold policy is needed.
|
|
66
|
+
*
|
|
67
|
+
* When it is not exhausted the pulled chunks cannot be discarded, so the stream is re-headed with them in front.
|
|
68
|
+
*/
|
|
69
|
+
async function openChunkIterator(source, threshold, init) {
|
|
70
|
+
const iterator = source[Symbol.asyncIterator]();
|
|
71
|
+
const first = await iterator.next();
|
|
72
|
+
if (first.done)
|
|
73
|
+
return bulk(new Uint8Array(0), init);
|
|
74
|
+
const second = await iterator.next();
|
|
75
|
+
const head = toBytes(first.value);
|
|
76
|
+
if (second.done && head.byteLength <= threshold)
|
|
77
|
+
return bulk(head, init);
|
|
78
|
+
const pulled = second.done ? [first.value] : [first.value, second.value];
|
|
79
|
+
const reheaded = {
|
|
80
|
+
async *[Symbol.asyncIterator]() {
|
|
81
|
+
yield* pulled;
|
|
82
|
+
for (;;) {
|
|
83
|
+
const next = await iterator.next();
|
|
84
|
+
if (next.done)
|
|
85
|
+
return;
|
|
86
|
+
yield next.value;
|
|
87
|
+
}
|
|
88
|
+
},
|
|
89
|
+
};
|
|
90
|
+
if (Symbol.asyncDispose in source) {
|
|
91
|
+
reheaded[Symbol.asyncDispose] = () => source[Symbol.asyncDispose]();
|
|
92
|
+
}
|
|
93
|
+
return Spliterator.fromAsync(reheaded, init);
|
|
94
|
+
}
|
|
95
|
+
//# sourceMappingURL=adaptive-source.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"adaptive-source.js","sourceRoot":"","sources":["../../lib/adaptive-source.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAE1D,OAAO,EAA6B,WAAW,EAAE,MAAM,kBAAkB,CAAA;AAEzE;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,GAAG,IAAI,CAAA;AAehD,SAAS,eAAe,CAAC,MAAe;IACvC,OAAO,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,CAAC,aAAa,IAAI,MAAM,CAAA;AACvF,CAAC;AAED,SAAS,OAAO,CAAC,KAA0B;IAC1C,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAA;AAC3E,CAAC;AAED;;;;;;GAMG;AACH,KAAK,UAAU,IAAI,CAAC,KAAiB,EAAE,IAAwB;IAC9D,MAAM,iBAAiB,CAAC,SAAS,EAAE,CAAA;IAEnC,OAAO,WAAW,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;AACzC,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACtC,MAA8C,EAC9C,IAAI,GAAuB,EAAE;IAE7B,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,IAAI,sBAAsB,CAAA;IAE9D,IAAI,SAAS,IAAI,CAAC;QAAE,OAAO,WAAW,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;IAE9D,IAAI,eAAe,CAAC,MAAM,CAAC;QAAE,OAAO,iBAAiB,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,CAAA;IAE9E,IAAI,IAAY,CAAA;IAEhB,IAAI,CAAC;QACJ,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAA;QAE5D,IAAI,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC,CAAA;IAClC,CAAC;IAAC,MAAM,CAAC;QACR,sFAAsF;QACtF,OAAO,WAAW,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;IAC3C,CAAC;IAED,IAAI,IAAI,GAAG,SAAS;QAAE,OAAO,WAAW,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;IAEhE,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAA;IAEzD,OAAO,IAAI,CAAC,MAAM,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,CAAA;AACpD,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,iBAAiB,CAC/B,MAA0B,EAC1B,SAAiB,EACjB,IAAwB;IAExB,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAA;IAC/C,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;IAEnC,IAAI,KAAK,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAA;IAEpD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;IACpC,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IAEjC,IAAI,MAAM,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,IAAI,SAAS;QAAE,OAAO,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;IAExE,MAAM,MAAM,GAA+B,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAA;IAEpG,MAAM,QAAQ,GAAuB;QACpC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;YAC5B,KAAK,CAAC,CAAC,MAAM,CAAA;YAEb,SAAS,CAAC;gBACT,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;gBAElC,IAAI,IAAI,CAAC,IAAI;oBAAE,OAAM;gBAErB,MAAM,IAAI,CAAC,KAAK,CAAA;YACjB,CAAC;QACF,CAAC;KACD,CAAA;IAED,IAAI,MAAM,CAAC,YAAY,IAAI,MAAM,EAAE,CAAC;QACnC,QAAQ,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAE,EAAE,CAAA;IACrE,CAAC;IAED,OAAO,WAAW,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAA;AAC7C,CAAC"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license MIT
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* Group an iterable's values into arrays of `size`, with a shorter final batch if the collection does not divide
|
|
8
|
+
* evenly.
|
|
9
|
+
*
|
|
10
|
+
* Distinct from `Iterator.prototype.take`, which yields the first `size` _values_. Named for the [iterator chunking
|
|
11
|
+
* proposal](https://github.com/tc39/proposal-iterator-chunking).
|
|
12
|
+
*
|
|
13
|
+
* @param collection The collection to batch.
|
|
14
|
+
* @param size The size of each batch.
|
|
15
|
+
*
|
|
16
|
+
* @yields Each batch of values.
|
|
17
|
+
* @see {@linkcode AsyncSequence.chunks} for the asynchronous form.
|
|
18
|
+
*/
|
|
19
|
+
export declare function chunks<T>(collection: Iterable<T>, size: number): Generator<T[]>;
|
|
20
|
+
//# sourceMappingURL=chunks.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunks.d.ts","sourceRoot":"","sources":["../../lib/chunks.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;;;;;;;;;;;GAYG;AACH,wBAAiB,MAAM,CAAC,CAAC,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,EAAE,CAAC,CAsBhF"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license MIT
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* Group an iterable's values into arrays of `size`, with a shorter final batch if the collection does not divide
|
|
8
|
+
* evenly.
|
|
9
|
+
*
|
|
10
|
+
* Distinct from `Iterator.prototype.take`, which yields the first `size` _values_. Named for the [iterator chunking
|
|
11
|
+
* proposal](https://github.com/tc39/proposal-iterator-chunking).
|
|
12
|
+
*
|
|
13
|
+
* @param collection The collection to batch.
|
|
14
|
+
* @param size The size of each batch.
|
|
15
|
+
*
|
|
16
|
+
* @yields Each batch of values.
|
|
17
|
+
* @see {@linkcode AsyncSequence.chunks} for the asynchronous form.
|
|
18
|
+
*/
|
|
19
|
+
export function* chunks(collection, size) {
|
|
20
|
+
const normalized = Math.trunc(size);
|
|
21
|
+
if (!Number.isFinite(normalized) || normalized < 1) {
|
|
22
|
+
throw new RangeError(`chunks(${size}): size must be a positive finite number`);
|
|
23
|
+
}
|
|
24
|
+
let batch = [];
|
|
25
|
+
for (const item of collection) {
|
|
26
|
+
batch.push(item);
|
|
27
|
+
if (batch.length === normalized) {
|
|
28
|
+
yield batch;
|
|
29
|
+
batch = [];
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
if (batch.length) {
|
|
33
|
+
yield batch;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
//# sourceMappingURL=chunks.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunks.js","sourceRoot":"","sources":["../../lib/chunks.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;;;;;;;;;;;GAYG;AACH,MAAM,SAAS,CAAC,CAAC,MAAM,CAAI,UAAuB,EAAE,IAAY;IAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAEnC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QACpD,MAAM,IAAI,UAAU,CAAC,UAAU,IAAI,0CAA0C,CAAC,CAAA;IAC/E,CAAC;IAED,IAAI,KAAK,GAAQ,EAAE,CAAA;IAEnB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAEhB,IAAI,KAAK,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YACjC,MAAM,KAAK,CAAA;YAEX,KAAK,GAAG,EAAE,CAAA;QACX,CAAC;IACF,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QAClB,MAAM,KAAK,CAAA;IACZ,CAAC;AACF,CAAC"}
|