spliterator 5.0.0 → 6.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/README.md +56 -0
  2. package/out/index.d.ts +7 -1
  3. package/out/index.d.ts.map +1 -1
  4. package/out/index.js +7 -1
  5. package/out/index.js.map +1 -1
  6. package/out/lib/AsyncSequence.d.ts +180 -0
  7. package/out/lib/AsyncSequence.d.ts.map +1 -0
  8. package/out/lib/AsyncSequence.js +432 -0
  9. package/out/lib/AsyncSequence.js.map +1 -0
  10. package/out/lib/AsyncSpliterator.d.ts +4 -2
  11. package/out/lib/AsyncSpliterator.d.ts.map +1 -1
  12. package/out/lib/AsyncSpliterator.js.map +1 -1
  13. package/out/lib/CSVSpliterator.d.ts +6 -5
  14. package/out/lib/CSVSpliterator.d.ts.map +1 -1
  15. package/out/lib/CSVSpliterator.js +46 -37
  16. package/out/lib/CSVSpliterator.js.map +1 -1
  17. package/out/lib/JSONSpliterator.d.ts +23 -11
  18. package/out/lib/JSONSpliterator.d.ts.map +1 -1
  19. package/out/lib/JSONSpliterator.js +28 -19
  20. package/out/lib/JSONSpliterator.js.map +1 -1
  21. package/out/lib/TextSpliterator.d.ts +4 -2
  22. package/out/lib/TextSpliterator.d.ts.map +1 -1
  23. package/out/lib/TextSpliterator.js +8 -9
  24. package/out/lib/TextSpliterator.js.map +1 -1
  25. package/out/lib/adaptive-source.d.ts +39 -0
  26. package/out/lib/adaptive-source.d.ts.map +1 -0
  27. package/out/lib/adaptive-source.js +95 -0
  28. package/out/lib/adaptive-source.js.map +1 -0
  29. package/out/lib/chunks.d.ts +20 -0
  30. package/out/lib/chunks.d.ts.map +1 -0
  31. package/out/lib/chunks.js +36 -0
  32. package/out/lib/chunks.js.map +1 -0
  33. package/out/lib/parallel-map-workers.d.ts +57 -0
  34. package/out/lib/parallel-map-workers.d.ts.map +1 -0
  35. package/out/lib/parallel-map-workers.js +82 -0
  36. package/out/lib/parallel-map-workers.js.map +1 -0
  37. package/out/lib/parallel-map.d.ts +17 -42
  38. package/out/lib/parallel-map.d.ts.map +1 -1
  39. package/out/lib/parallel-map.js +18 -66
  40. package/out/lib/parallel-map.js.map +1 -1
  41. package/out/lib/pivot.d.ts +30 -0
  42. package/out/lib/pivot.d.ts.map +1 -0
  43. package/out/lib/pivot.js +47 -0
  44. package/out/lib/pivot.js.map +1 -0
  45. package/out/lib/shared.d.ts +0 -27
  46. package/out/lib/shared.d.ts.map +1 -1
  47. package/out/lib/shared.js +0 -53
  48. package/out/lib/shared.js.map +1 -1
  49. package/out/lib/take.d.ts +16 -0
  50. package/out/lib/take.d.ts.map +1 -1
  51. package/out/lib/take.js +28 -0
  52. package/out/lib/take.js.map +1 -1
  53. package/out/lib/utils.d.ts +37 -0
  54. package/out/lib/utils.d.ts.map +1 -0
  55. package/out/lib/utils.js +39 -0
  56. package/out/lib/utils.js.map +1 -0
  57. package/out/lib/zip.d.ts +33 -0
  58. package/out/lib/zip.d.ts.map +1 -0
  59. package/out/lib/zip.js +59 -0
  60. package/out/lib/zip.js.map +1 -0
  61. package/package.json +6 -6
@@ -3,10 +3,12 @@
3
3
  * @license MIT
4
4
  * @author Teffen Ellis, et al.
5
5
  */
6
+ import { openDelimitedRows } from "./adaptive-source.js";
7
+ import { AsyncSequence } from "./AsyncSequence.js";
6
8
  import { normalizeColumnNames } from "./casing.js";
7
9
  import { CharacterSequence, Delimiters } from "./CharacterSequence.js";
8
- import { zipSync } from "./shared.js";
9
10
  import { Spliterator } from "./Spliterator.js";
11
+ import { zipSync } from "./zip.js";
10
12
  const identity = (value) => value;
11
13
  const doubleQuoteSequence = new CharacterSequence('"');
12
14
  /**
@@ -154,7 +156,8 @@ export class CSVSpliterator {
154
156
  *
155
157
  * @yields Each row, shaped according to the `mode` option.
156
158
  */
157
- static async *fromAsync(source, init = {}) {
159
+ static fromAsync(source, init = {}) {
160
+ const defaultColumnDelimiter = this.ColumnDelimiter;
158
161
  const {
159
162
  // ---
160
163
  header = true, mode = "array", transformers: transformersInput = [], normalizeKeys = mode !== "array", columnDelimiter: columnDelimiterInput, enableQuoteHandling = false,
@@ -162,45 +165,51 @@ export class CSVSpliterator {
162
165
  // never carries a stray `\r` on Windows-lineage sources.
163
166
  crlf = true, take = Infinity, drop = 0, ...rowInit } = init;
164
167
  const emitter = CSVSpliteratorEmitters[mode];
165
- let transformers = [];
166
- let yieldCount = 0;
167
- const yieldLimit = take + drop;
168
- const columnDelimiter = new CharacterSequence(columnDelimiterInput ?? this.ColumnDelimiter);
168
+ const columnDelimiter = new CharacterSequence(columnDelimiterInput ?? defaultColumnDelimiter);
169
169
  const decoder = new TextDecoder();
170
- // Quote handling applies at both levels: rows must not split on newlines inside quotes,
171
- // columns must not split on quoted column delimiters.
172
- const rows = await Spliterator.from(source, { ...rowInit, crlf, enableQuoteHandling });
173
- if (header) {
174
- const result = await rows.next();
175
- if (result.done)
176
- return;
177
- const columns = splitRowColumns(result.value, columnDelimiter, decoder, enableQuoteHandling);
178
- const headers = normalizeKeys ? normalizeColumnNames(columns) : columns;
179
- // oxlint-disable-next-line unicorn/prefer-ternary
180
- if (Array.isArray(transformersInput)) {
181
- transformers = Array.from(zipSync(headers, transformersInput), ([columnName, transformer]) => [
182
- columnName,
183
- transformer ?? identity,
184
- ]);
185
- }
186
- else {
187
- transformers = headers.map((columnName) => {
188
- const transform = transformersInput[columnName] || identity;
189
- return [columnName, transform];
190
- });
191
- }
192
- }
193
- for await (const row of rows) {
194
- if (yieldCount < drop) {
195
- yieldCount++;
196
- continue;
170
+ // Populated by the header pass below before the first row op runs, since the source thunk resolves on the first
171
+ // pull and the ops only run against what it returns.
172
+ let transformers = [];
173
+ const openRows = async () => {
174
+ // Quote handling applies at both levels: rows must not split on newlines inside quotes,
175
+ // columns must not split on quoted column delimiters.
176
+ const rows = await openDelimitedRows(source, { ...rowInit, crlf, enableQuoteHandling });
177
+ if (header) {
178
+ // Both engines return `this` from their iterator method, so consuming the header row here advances the very
179
+ // cursor the row ops will go on to read — returning `rows` afterwards resumes at row two, not row one.
180
+ const iterator = Symbol.asyncIterator in rows ? rows[Symbol.asyncIterator]() : rows[Symbol.iterator]();
181
+ const result = await iterator.next();
182
+ if (result.done)
183
+ return rows;
184
+ const columns = splitRowColumns(result.value, columnDelimiter, decoder, enableQuoteHandling);
185
+ const headers = normalizeKeys ? normalizeColumnNames(columns) : columns;
186
+ // oxlint-disable-next-line unicorn/prefer-ternary
187
+ if (Array.isArray(transformersInput)) {
188
+ transformers = Array.from(zipSync(headers, transformersInput), ([columnName, transformer]) => [
189
+ columnName,
190
+ transformer ?? identity,
191
+ ]);
192
+ }
193
+ else {
194
+ transformers = headers.map((columnName) => {
195
+ const transform = transformersInput[columnName] || identity;
196
+ return [columnName, transform];
197
+ });
198
+ }
197
199
  }
198
- if (yieldCount >= yieldLimit)
199
- break;
200
+ return rows;
201
+ };
202
+ let sequence = AsyncSequence.from(openRows).map((row) => {
200
203
  const columns = splitRowColumns(row, columnDelimiter, decoder, enableQuoteHandling);
201
- yield emitter ? emitter(columns, transformers) : columns;
202
- yieldCount++;
204
+ return emitter ? emitter(columns, transformers) : columns;
205
+ });
206
+ if (drop > 0) {
207
+ sequence = sequence.drop(drop);
208
+ }
209
+ if (Number.isFinite(take)) {
210
+ sequence = sequence.take(take);
203
211
  }
212
+ return sequence;
204
213
  }
205
214
  }
206
215
  //# sourceMappingURL=CSVSpliterator.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"CSVSpliterator.js","sourceRoot":"","sources":["../../lib/CSVSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAA;AAClD,OAAO,EAAE,iBAAiB,EAA+B,UAAU,EAAE,MAAM,wBAAwB,CAAA;AACnG,OAAO,EAAmD,OAAO,EAAE,MAAM,aAAa,CAAA;AACtF,OAAO,EAA6B,WAAW,EAAwB,MAAM,kBAAkB,CAAA;AAe/F,MAAM,QAAQ,GAA2B,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAA;AAEzD,MAAM,mBAAmB,GAAG,IAAI,iBAAiB,CAAC,GAAG,CAAC,CAAA;AAEtD;;;GAGG;AACH,SAAS,YAAY,CAAC,KAAiB,EAAE,OAAoB,EAAE,mBAA4B;IAC1F,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;IAEnC,IAAI,mBAAmB,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC9F,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;IAChD,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;;;;GAMG;AACH,SAAS,eAAe,CACvB,GAAe,EACf,eAAkC,EAClC,OAAoB,EACpB,mBAA4B;IAE5B,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC1B,OAAO,eAAe,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAA;IACtG,CAAC;IAED,MAAM,OAAO,GAAa,EAAE,CAAA;IAC5B,IAAI,UAAU,GAAG,CAAC,CAAA;IAClB,IAAI,YAAY,GAAG,KAAK,CAAA;IAExB,KAAK,MAAM,KAAK,IAAI,eAAe,CAAC,aAAa,CAAC,GAAG,EAAE,mBAAmB,CAAC,EAAE,CAAC;QAC7E,IAAI,KAAK,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;YAC3B,YAAY,GAAG,CAAC,YAAY,CAAA;YAE5B,SAAQ;QACT,CAAC;QAED,IAAI,YAAY;YAAE,SAAQ;QAE1B,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAC,CAAA;QAChG,UAAU,GAAG,KAAK,CAAC,MAAM,GAAG,eAAe,CAAC,MAAM,CAAA;IACnD,CAAC;IAED,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAC,CAAA;IAElF,OAAO,OAAO,CAAA;AACf,CAAC;AAaD,MAAM,CAAC,MAAM,sBAAsB,GAAG;IACrC,KAAK,EAAE,IAAI;IAEX,OAAO,CAAC,OAAyB,EAAE,aAAa,GAAkC,EAAE;QACnF,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE;YAChF,MAAM,GAAG,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,CAAA;YAC5B,MAAM,SAAS,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAA;YAE9C,OAAO,CAAC,GAAG,IAAI,UAAU,GAAG,EAAE,EAAE,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,CAAA;QAC7D,CAAC,CAAC,CAAA;IACH,CAAC;IACD,MAAM,CACL,OAAyB,EACzB,aAAa,GAAkC,EAAE;QAEjD,MAAM,MAAM,GAAyC,EAAE,CAAA;QAEvD,KAAK,MAAM,CAAC,WAAW,EAAE,KAAK,EAAE,GAAG,CAAC,IAAI,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,EAAE,CAAC;YACzE,MAAM,GAAG,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,UAAU,GAAG,EAAE,CAAA;YAC/C,MAAM,SAAS,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAA;YAE9C,MAAM,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;QACrC,CAAC;QAED,OAAO,MAAM,CAAA;IACd,CAAC;CAC2D,CAAA;AAmD7D;;;;;;;;;;GAUG;AACH,MAAM,OAAgB,cAAc;IACnC;;;;OAIG;IACI,MAAM,CAAC,eAAe,GAAW,UAAU,CAAC,KAAK,CAAA;IAExD;QACC,MAAM,IAAI,SAAS,CAAC,0EAA0E,CAAC,CAAA;IAChG,CAAC;IAuBD;;;;OAIG;IACH,MAAM,CAAC,CAAC,IAAI,CAAC,MAA8B,EAAE,IAAI,GAAuB,EAAE;QACzE,MAAM;QACL,MAAM;QACN,MAAM,GAAG,IAAI;QACb,uFAAuF;QACvF,IAAI,GAAG,OAAO,EACd,YAAY,EAAE,iBAAiB,GAAG,EAAE;QACpC,wFAAwF;QACxF,8FAA8F;QAC9F,aAAa,GAAG,IAAI,KAAK,OAAO,EAChC,eAAe,EAAE,oBAAoB,GAAG,IAAI,CAAC,eAAe,EAC5D,mBAAmB,GAAG,KAAK;QAC3B,qFAAqF;QACrF,yDAAyD;QACzD,IAAI,GAAG,IAAI,EACX,IAAI,GAAG,QAAQ,EACf,IAAI,GAAG,CAAC,EACR,GAAG,OAAO,EACV,GAAG,IAAI,CAAA;QAER,MAAM,OAAO,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAA;QAC5C,IAAI,YAAY,GAA0B,EAAE,CAAA;QAC5C,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,MAAM,UAAU,GAAG,IAAI,GAAG,IAAI,CAAA;QAE9B,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QACjC,MAAM,eAAe,GAAG,IAAI,iBAAiB,CAAC,oBAAoB,IAAI,IAAI,CAAC,eAAe,CAAC,CAAA;QAE3F,wFAAwF;QACxF,sDAAsD;QACtD,MAAM,IAAI,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,mBAAmB,EAAE,CAAC,CAAA;QAEpF,IAAI,MAAM,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;YAE1B,IAAI,MAAM,CAAC,IAAI;gBAAE,OAAM;YAEvB,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,CAAC,KAAK,EAAE,eAAe,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAA;YAC5F,MAAM,OAAO,GAAG,aAAa,CAAC,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;YAEvE,kDAAkD;YAClD,IAAI,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,CAAC;gBACtC,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,iBAAiB,CAAC,EAAE,CAAC,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC;oBAC7F,UAAW;oBACX,WAAW,IAAI,QAAQ;iBACvB,CAAC,CAAA;YACH,CAAC;iBAAM,CAAC;gBACP,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;oBACzC,MAAM,SAAS,GAAI,iBAA0C,CAAC,UAAU,CAAC,IAAI,QAAQ,CAAA;oBAErF,OAAO,CAAC,UAAU,EAAE,SAAS,CAAC,CAAA;gBAC/B,CAAC,CAAC,CAAA;YACH,CAAC;QACF,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,IAAI,UAAU,GAAG,IAAI,EAAE,CAAC;gBACvB,UAAU,EAAE,CAAA;gBAEZ,SAAQ;YACT,CAAC;YAED,IAAI,UAAU,IAAI,UAAU;gBAAE,MAAK;YAEnC,MAAM,OAAO,GAAG,eAAe,CAAC,GAAG,EAAE,eAAe,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAA;YAEnF,MAAM,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;YAExD,UAAU,EAAE,CAAA;QACb,CAAC;IACF,CAAC;IAqCD;;;;;;;;OAQG;IACH,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CACtB,MAA8C,EAC9C,IAAI,GAA8C,EAAE;QAEpD,MAAM;QACL,MAAM;QACN,MAAM,GAAG,IAAI,EACb,IAAI,GAAG,OAAO,EACd,YAAY,EAAE,iBAAiB,GAAG,EAAE,EACpC,aAAa,GAAG,IAAI,KAAK,OAAO,EAChC,eAAe,EAAE,oBAAoB,EACrC,mBAAmB,GAAG,KAAK;QAC3B,qFAAqF;QACrF,yDAAyD;QACzD,IAAI,GAAG,IAAI,EACX,IAAI,GAAG,QAAQ,EACf,IAAI,GAAG,CAAC,EACR,GAAG,OAAO,EACV,GAAG,IAAI,CAAA;QAER,MAAM,OAAO,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAA;QAC5C,IAAI,YAAY,GAA0B,EAAE,CAAA;QAC5C,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,MAAM,UAAU,GAAG,IAAI,GAAG,IAAI,CAAA;QAE9B,MAAM,eAAe,GAAG,IAAI,iBAAiB,CAAC,oBAAoB,IAAI,IAAI,CAAC,eAAe,CAAC,CAAA;QAE3F,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QAEjC,wFAAwF;QACxF,sDAAsD;QACtD,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,mBAAmB,EAAE,CAAC,CAAA;QAEtF,IAAI,MAAM,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAA;YAEhC,IAAI,MAAM,CAAC,IAAI;gBAAE,OAAM;YAEvB,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,CAAC,KAAK,EAAE,eAAe,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAA;YAC5F,MAAM,OAAO,GAAG,aAAa,CAAC,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;YAEvE,kDAAkD;YAClD,IAAI,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,CAAC;gBACtC,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,iBAAiB,CAAC,EAAE,CAAC,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC;oBAC7F,UAAW;oBACX,WAAW,IAAI,QAAQ;iBACvB,CAAC,CAAA;YACH,CAAC;iBAAM,CAAC;gBACP,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;oBACzC,MAAM,SAAS,GAAI,iBAA0C,CAAC,UAAU,CAAC,IAAI,QAAQ,CAAA;oBAErF,OAAO,CAAC,UAAU,EAAE,SAAS,CAAC,CAAA;gBAC/B,CAAC,CAAC,CAAA;YACH,CAAC;QACF,CAAC;QAED,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YAC9B,IAAI,UAAU,GAAG,IAAI,EAAE,CAAC;gBACvB,UAAU,EAAE,CAAA;gBAEZ,SAAQ;YACT,CAAC;YAED,IAAI,UAAU,IAAI,UAAU;gBAAE,MAAK;YAEnC,MAAM,OAAO,GAAG,eAAe,CAAC,GAAG,EAAE,eAAe,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAA;YAEnF,MAAM,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;YAExD,UAAU,EAAE,CAAA;QACb,CAAC;IACF,CAAC;CACD"}
1
+ {"version":3,"file":"CSVSpliterator.js","sourceRoot":"","sources":["../../lib/CSVSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAA2B,iBAAiB,EAAE,MAAM,sBAAsB,CAAA;AACjF,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAClD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAA;AAClD,OAAO,EAAE,iBAAiB,EAA+B,UAAU,EAAE,MAAM,wBAAwB,CAAA;AAEnG,OAAO,EAA6B,WAAW,EAAwB,MAAM,kBAAkB,CAAA;AAC/F,OAAO,EAAE,OAAO,EAAE,MAAM,UAAU,CAAA;AAelC,MAAM,QAAQ,GAA2B,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAA;AAEzD,MAAM,mBAAmB,GAAG,IAAI,iBAAiB,CAAC,GAAG,CAAC,CAAA;AAEtD;;;GAGG;AACH,SAAS,YAAY,CAAC,KAAiB,EAAE,OAAoB,EAAE,mBAA4B;IAC1F,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;IAEnC,IAAI,mBAAmB,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC9F,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;IAChD,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;;;;GAMG;AACH,SAAS,eAAe,CACvB,GAAe,EACf,eAAkC,EAClC,OAAoB,EACpB,mBAA4B;IAE5B,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC1B,OAAO,eAAe,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAA;IACtG,CAAC;IAED,MAAM,OAAO,GAAa,EAAE,CAAA;IAC5B,IAAI,UAAU,GAAG,CAAC,CAAA;IAClB,IAAI,YAAY,GAAG,KAAK,CAAA;IAExB,KAAK,MAAM,KAAK,IAAI,eAAe,CAAC,aAAa,CAAC,GAAG,EAAE,mBAAmB,CAAC,EAAE,CAAC;QAC7E,IAAI,KAAK,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;YAC3B,YAAY,GAAG,CAAC,YAAY,CAAA;YAE5B,SAAQ;QACT,CAAC;QAED,IAAI,YAAY;YAAE,SAAQ;QAE1B,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAC,CAAA;QAChG,UAAU,GAAG,KAAK,CAAC,MAAM,GAAG,eAAe,CAAC,MAAM,CAAA;IACnD,CAAC;IAED,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAC,CAAA;IAElF,OAAO,OAAO,CAAA;AACf,CAAC;AAaD,MAAM,CAAC,MAAM,sBAAsB,GAAG;IACrC,KAAK,EAAE,IAAI;IAEX,OAAO,CAAC,OAAyB,EAAE,aAAa,GAAkC,EAAE;QACnF,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE;YAChF,MAAM,GAAG,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,CAAA;YAC5B,MAAM,SAAS,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAA;YAE9C,OAAO,CAAC,GAAG,IAAI,UAAU,GAAG,EAAE,EAAE,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,CAAA;QAC7D,CAAC,CAAC,CAAA;IACH,CAAC;IACD,MAAM,CACL,OAAyB,EACzB,aAAa,GAAkC,EAAE;QAEjD,MAAM,MAAM,GAAyC,EAAE,CAAA;QAEvD,KAAK,MAAM,CAAC,WAAW,EAAE,KAAK,EAAE,GAAG,CAAC,IAAI,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,EAAE,CAAC;YACzE,MAAM,GAAG,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,UAAU,GAAG,EAAE,CAAA;YAC/C,MAAM,SAAS,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAA;YAE9C,MAAM,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;QACrC,CAAC;QAED,OAAO,MAAM,CAAA;IACd,CAAC;CAC2D,CAAA;AAmD7D;;;;;;;;;;GAUG;AACH,MAAM,OAAgB,cAAc;IACnC;;;;OAIG;IACI,MAAM,CAAC,eAAe,GAAW,UAAU,CAAC,KAAK,CAAA;IAExD;QACC,MAAM,IAAI,SAAS,CAAC,0EAA0E,CAAC,CAAA;IAChG,CAAC;IAuBD;;;;OAIG;IACH,MAAM,CAAC,CAAC,IAAI,CAAC,MAA8B,EAAE,IAAI,GAAuB,EAAE;QACzE,MAAM;QACL,MAAM;QACN,MAAM,GAAG,IAAI;QACb,uFAAuF;QACvF,IAAI,GAAG,OAAO,EACd,YAAY,EAAE,iBAAiB,GAAG,EAAE;QACpC,wFAAwF;QACxF,8FAA8F;QAC9F,aAAa,GAAG,IAAI,KAAK,OAAO,EAChC,eAAe,EAAE,oBAAoB,GAAG,IAAI,CAAC,eAAe,EAC5D,mBAAmB,GAAG,KAAK;QAC3B,qFAAqF;QACrF,yDAAyD;QACzD,IAAI,GAAG,IAAI,EACX,IAAI,GAAG,QAAQ,EACf,IAAI,GAAG,CAAC,EACR,GAAG,OAAO,EACV,GAAG,IAAI,CAAA;QAER,MAAM,OAAO,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAA;QAC5C,IAAI,YAAY,GAA0B,EAAE,CAAA;QAC5C,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,MAAM,UAAU,GAAG,IAAI,GAAG,IAAI,CAAA;QAE9B,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QACjC,MAAM,eAAe,GAAG,IAAI,iBAAiB,CAAC,oBAAoB,IAAI,IAAI,CAAC,eAAe,CAAC,CAAA;QAE3F,wFAAwF;QACxF,sDAAsD;QACtD,MAAM,IAAI,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,mBAAmB,EAAE,CAAC,CAAA;QAEpF,IAAI,MAAM,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;YAE1B,IAAI,MAAM,CAAC,IAAI;gBAAE,OAAM;YAEvB,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,CAAC,KAAK,EAAE,eAAe,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAA;YAC5F,MAAM,OAAO,GAAG,aAAa,CAAC,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;YAEvE,kDAAkD;YAClD,IAAI,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,CAAC;gBACtC,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,iBAAiB,CAAC,EAAE,CAAC,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC;oBAC7F,UAAW;oBACX,WAAW,IAAI,QAAQ;iBACvB,CAAC,CAAA;YACH,CAAC;iBAAM,CAAC;gBACP,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;oBACzC,MAAM,SAAS,GAAI,iBAA0C,CAAC,UAAU,CAAC,IAAI,QAAQ,CAAA;oBAErF,OAAO,CAAC,UAAU,EAAE,SAAS,CAAC,CAAA;gBAC/B,CAAC,CAAC,CAAA;YACH,CAAC;QACF,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,IAAI,UAAU,GAAG,IAAI,EAAE,CAAC;gBACvB,UAAU,EAAE,CAAA;gBAEZ,SAAQ;YACT,CAAC;YAED,IAAI,UAAU,IAAI,UAAU;gBAAE,MAAK;YAEnC,MAAM,OAAO,GAAG,eAAe,CAAC,GAAG,EAAE,eAAe,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAA;YAEnF,MAAM,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;YAExD,UAAU,EAAE,CAAA;QACb,CAAC;IACF,CAAC;IAqCD;;;;;;;;OAQG;IACH,MAAM,CAAC,SAAS,CACf,MAA8C,EAC9C,IAAI,GAA4C,EAAE;QAElD,MAAM,sBAAsB,GAAG,IAAI,CAAC,eAAe,CAAA;QAEnD,MAAM;QACL,MAAM;QACN,MAAM,GAAG,IAAI,EACb,IAAI,GAAG,OAAO,EACd,YAAY,EAAE,iBAAiB,GAAG,EAAE,EACpC,aAAa,GAAG,IAAI,KAAK,OAAO,EAChC,eAAe,EAAE,oBAAoB,EACrC,mBAAmB,GAAG,KAAK;QAC3B,qFAAqF;QACrF,yDAAyD;QACzD,IAAI,GAAG,IAAI,EACX,IAAI,GAAG,QAAQ,EACf,IAAI,GAAG,CAAC,EACR,GAAG,OAAO,EACV,GAAG,IAAI,CAAA;QAER,MAAM,OAAO,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAA;QAC5C,MAAM,eAAe,GAAG,IAAI,iBAAiB,CAAC,oBAAoB,IAAI,sBAAsB,CAAC,CAAA;QAC7F,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QAEjC,gHAAgH;QAChH,qDAAqD;QACrD,IAAI,YAAY,GAA0B,EAAE,CAAA;QAE5C,MAAM,QAAQ,GAAG,KAAK,IAA+D,EAAE;YACtF,wFAAwF;YACxF,sDAAsD;YACtD,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,mBAAmB,EAAE,CAAC,CAAA;YAEvF,IAAI,MAAM,EAAE,CAAC;gBACZ,4GAA4G;gBAC5G,uGAAuG;gBACvG,MAAM,QAAQ,GAAG,MAAM,CAAC,aAAa,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAA;gBACtG,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;gBAEpC,IAAI,MAAM,CAAC,IAAI;oBAAE,OAAO,IAAI,CAAA;gBAE5B,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,CAAC,KAAK,EAAE,eAAe,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAA;gBAC5F,MAAM,OAAO,GAAG,aAAa,CAAC,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;gBAEvE,kDAAkD;gBAClD,IAAI,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,CAAC;oBACtC,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,iBAAiB,CAAC,EAAE,CAAC,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC;wBAC7F,UAAW;wBACX,WAAW,IAAI,QAAQ;qBACvB,CAAC,CAAA;gBACH,CAAC;qBAAM,CAAC;oBACP,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;wBACzC,MAAM,SAAS,GAAI,iBAA0C,CAAC,UAAU,CAAC,IAAI,QAAQ,CAAA;wBAErF,OAAO,CAAC,UAAU,EAAE,SAAS,CAAC,CAAA;oBAC/B,CAAC,CAAC,CAAA;gBACH,CAAC;YACF,CAAC;YAED,OAAO,IAAI,CAAA;QACZ,CAAC,CAAA;QAED,IAAI,QAAQ,GAA2B,aAAa,CAAC,IAAI,CAAa,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YAC3F,MAAM,OAAO,GAAG,eAAe,CAAC,GAAG,EAAE,eAAe,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAA;YAEnF,OAAO,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;QAC1D,CAAC,CAAC,CAAA;QAEF,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;YACd,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC/B,CAAC;QAED,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3B,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC/B,CAAC;QAED,OAAO,QAAQ,CAAA;IAChB,CAAC;CACD"}
@@ -3,20 +3,32 @@
3
3
  * @license MIT
4
4
  * @author Teffen Ellis, et al.
5
5
  */
6
+ import { type AdaptiveSourceInit } from "./adaptive-source.js";
7
+ import { AsyncSequence } from "./AsyncSequence.js";
6
8
  import type { CharacterSequenceInput } from "./CharacterSequence.js";
7
9
  import type { AsyncDataResource } from "./shared.js";
8
- import { type AsyncSpliteratorInit, type SpliteratorInit } from "./Spliterator.js";
10
+ import { type SpliteratorInit } from "./Spliterator.js";
9
11
  /**
10
- * Stream a delimited byte source and `JSON.parse` each row — one parsed value per line, for JSONL / NDJSON. Set the row
11
- * delimiter via `options.delimiter` (there is no implicit default); `skipEmpty` drops blank rows.
12
+ * Stream a delimited byte source and `JSON.parse` each row — one parsed value per line, for JSONL / NDJSON. The row
13
+ * delimiter defaults to a line feed; override it via `options.delimiter`. `skipEmpty` (on by default) drops blank
14
+ * rows.
12
15
  *
13
- * **Performance caveat — this path is `JSON.parse`-bound, not scan-bound.** The underlying SIMD newline scan is
14
- * genuinely fast (scan-only measures ~1.75× a JS line-splitter on large files), but a full `JSON.parse` per row dwarfs
15
- * the scan, so end-to-end this typically runs **~0.75× of Node's `readline` + `JSON.parse` a net loss** because the
16
- * per-row decode + parse dominates and `readline`'s line emission is already well-tuned. Use this for API convenience,
17
- * or when streaming to bound memory; do NOT swap a working `readline` loop to it expecting a speedup on parse-heavy
18
- * JSONL. The scan advantage only shows when you _don't_ fully parse every row (segmentation, counting, extracting a
19
- * couple of fields) use {@link Spliterator} (raw byte ranges) or {@link TextSpliterator} for those. When unsure,
16
+ * **Performance caveat — this path is `JSON.parse`-bound, not scan-bound.** Measured over 500k rows (88MB, ~177B per
17
+ * row, Node 26): the delimiter scan is ~140ms and decoding ~46ms, while `JSON.parse` and the per-row async machinery
18
+ * account for the rest of ~1280ms. Against Node's `readline` + `JSON.parse` at ~630ms, this runs **roughly half the
19
+ * speed a net loss.** Use it for API convenience, or when streaming to bound memory; do NOT swap a working `readline`
20
+ * loop to it expecting a speedup on parse-heavy JSONL.
21
+ *
22
+ * The scan advantage only shows when you _don't_ fully parse every row. Filtering on the raw text and parsing only what
23
+ * survives is the shape that wins — parsing 20% of rows measured ~2.5× faster than parsing all of them:
24
+ *
25
+ * ```ts
26
+ * TextSpliterator.fromAsync(path, { delimiter: "\n" })
27
+ * .filter((line) => line.includes('"category":"Novelty"'))
28
+ * .map((line) => JSON.parse(line))
29
+ * ```
30
+ *
31
+ * For segmentation or counting, use {@link Spliterator} (raw byte ranges) or {@link TextSpliterator}. When unsure,
20
32
  * benchmark.
21
33
  */
22
34
  export declare abstract class JSONSpliterator {
@@ -27,6 +39,6 @@ export declare abstract class JSONSpliterator {
27
39
  *
28
40
  * @yields Each row as an array of columns.
29
41
  */
30
- static fromAsync<T = unknown>(source: AsyncDataResource, options?: AsyncSpliteratorInit): AsyncGenerator<Awaited<T>, void, unknown>;
42
+ static fromAsync<T = unknown>(source: AsyncDataResource, options?: AdaptiveSourceInit): AsyncSequence<T>;
31
43
  }
32
44
  //# sourceMappingURL=JSONSpliterator.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"JSONSpliterator.d.ts","sourceRoot":"","sources":["../../lib/JSONSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AACpE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AACpD,OAAO,EAAE,KAAK,oBAAoB,EAAe,KAAK,eAAe,EAAE,MAAM,kBAAkB,CAAA;AAE/F;;;;;;;;;;;;GAYG;AACH,8BAAsB,eAAe;IACpC,cAEC;IAED,MAAM,CAAE,IAAI,CAAC,CAAC,GAAG,OAAO,EAAE,MAAM,EAAE,sBAAsB,EAAE,OAAO,GAAE,eAAoB,GAAG,SAAS,CAAC,CAAC,CAAC,CAwBrG;IAED;;;;OAIG;IACH,OAAc,SAAS,CAAC,CAAC,GAAG,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,OAAO,GAAE,oBAAyB,6CAuBjG;CACD"}
1
+ {"version":3,"file":"JSONSpliterator.d.ts","sourceRoot":"","sources":["../../lib/JSONSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,KAAK,kBAAkB,EAAqB,MAAM,sBAAsB,CAAA;AACjF,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAClD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AACpE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AACpD,OAAO,EAA0C,KAAK,eAAe,EAAE,MAAM,kBAAkB,CAAA;AAE/F;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,8BAAsB,eAAe;IACpC,cAEC;IAED,MAAM,CAAE,IAAI,CAAC,CAAC,GAAG,OAAO,EAAE,MAAM,EAAE,sBAAsB,EAAE,OAAO,GAAE,eAAoB,GAAG,SAAS,CAAC,CAAC,CAAC,CAwBrG;IAED;;;;OAIG;IACH,MAAM,CAAC,SAAS,CAAC,CAAC,GAAG,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,OAAO,GAAE,kBAAuB,GAAG,aAAa,CAAC,CAAC,CAAC,CAgB3G;CACD"}
@@ -3,18 +3,30 @@
3
3
  * @license MIT
4
4
  * @author Teffen Ellis, et al.
5
5
  */
6
+ import { openDelimitedRows } from "./adaptive-source.js";
7
+ import { AsyncSequence } from "./AsyncSequence.js";
6
8
  import { Spliterator } from "./Spliterator.js";
7
9
  /**
8
- * Stream a delimited byte source and `JSON.parse` each row — one parsed value per line, for JSONL / NDJSON. Set the row
9
- * delimiter via `options.delimiter` (there is no implicit default); `skipEmpty` drops blank rows.
10
+ * Stream a delimited byte source and `JSON.parse` each row — one parsed value per line, for JSONL / NDJSON. The row
11
+ * delimiter defaults to a line feed; override it via `options.delimiter`. `skipEmpty` (on by default) drops blank
12
+ * rows.
10
13
  *
11
- * **Performance caveat — this path is `JSON.parse`-bound, not scan-bound.** The underlying SIMD newline scan is
12
- * genuinely fast (scan-only measures ~1.75× a JS line-splitter on large files), but a full `JSON.parse` per row dwarfs
13
- * the scan, so end-to-end this typically runs **~0.75× of Node's `readline` + `JSON.parse` a net loss** because the
14
- * per-row decode + parse dominates and `readline`'s line emission is already well-tuned. Use this for API convenience,
15
- * or when streaming to bound memory; do NOT swap a working `readline` loop to it expecting a speedup on parse-heavy
16
- * JSONL. The scan advantage only shows when you _don't_ fully parse every row (segmentation, counting, extracting a
17
- * couple of fields) use {@link Spliterator} (raw byte ranges) or {@link TextSpliterator} for those. When unsure,
14
+ * **Performance caveat — this path is `JSON.parse`-bound, not scan-bound.** Measured over 500k rows (88MB, ~177B per
15
+ * row, Node 26): the delimiter scan is ~140ms and decoding ~46ms, while `JSON.parse` and the per-row async machinery
16
+ * account for the rest of ~1280ms. Against Node's `readline` + `JSON.parse` at ~630ms, this runs **roughly half the
17
+ * speed a net loss.** Use it for API convenience, or when streaming to bound memory; do NOT swap a working `readline`
18
+ * loop to it expecting a speedup on parse-heavy JSONL.
19
+ *
20
+ * The scan advantage only shows when you _don't_ fully parse every row. Filtering on the raw text and parsing only what
21
+ * survives is the shape that wins — parsing 20% of rows measured ~2.5× faster than parsing all of them:
22
+ *
23
+ * ```ts
24
+ * TextSpliterator.fromAsync(path, { delimiter: "\n" })
25
+ * .filter((line) => line.includes('"category":"Novelty"'))
26
+ * .map((line) => JSON.parse(line))
27
+ * ```
28
+ *
29
+ * For segmentation or counting, use {@link Spliterator} (raw byte ranges) or {@link TextSpliterator}. When unsure,
18
30
  * benchmark.
19
31
  */
20
32
  export class JSONSpliterator {
@@ -45,24 +57,21 @@ export class JSONSpliterator {
45
57
  *
46
58
  * @yields Each row as an array of columns.
47
59
  */
48
- static async *fromAsync(source, options = {}) {
60
+ static fromAsync(source, options = {}) {
49
61
  const decoder = new TextDecoder();
50
- let rowCursor = 0;
51
- const spliterator = await Spliterator.from(source, options);
52
- for await (const row of spliterator) {
53
- let parsed;
62
+ // Parsing is an op on the sequence, not a generator wrapped inside one. An allocating row body makes the extra
63
+ // async frame a wrapping generator adds disproportionately expensive: 1460ms against 1278ms over 500k rows, where
64
+ // the same layer costs a third as much when the body only decodes.
65
+ return AsyncSequence.from(() => openDelimitedRows(source, options)).map((row, rowCursor) => {
54
66
  try {
55
- const content = decoder.decode(row);
56
- parsed = JSON.parse(content);
67
+ return JSON.parse(decoder.decode(row));
57
68
  }
58
69
  catch (parsedError) {
59
70
  const error = new SyntaxError(`Failed to parse JSON at row ${rowCursor}`);
60
71
  error.cause = parsedError;
61
72
  throw error;
62
73
  }
63
- yield parsed;
64
- rowCursor++;
65
- }
74
+ });
66
75
  }
67
76
  }
68
77
  //# sourceMappingURL=JSONSpliterator.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"JSONSpliterator.js","sourceRoot":"","sources":["../../lib/JSONSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,OAAO,EAA6B,WAAW,EAAwB,MAAM,kBAAkB,CAAA;AAE/F;;;;;;;;;;;;GAYG;AACH,MAAM,OAAgB,eAAe;IACpC;QACC,MAAM,IAAI,SAAS,CAAC,2EAA2E,CAAC,CAAA;IACjG,CAAC;IAED,MAAM,CAAC,CAAC,IAAI,CAAc,MAA8B,EAAE,OAAO,GAAoB,EAAE;QACtF,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QACjC,IAAI,SAAS,GAAG,CAAC,CAAA;QAEjB,MAAM,WAAW,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAEzD,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YAC/B,IAAI,MAAS,CAAA;YAEb,IAAI,CAAC;gBACJ,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;gBAEnC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAM,CAAA;YAClC,CAAC;YAAC,OAAO,WAAW,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,IAAI,WAAW,CAAC,+BAA+B,SAAS,EAAE,CAAC,CAAA;gBACzE,KAAK,CAAC,KAAK,GAAG,WAAW,CAAA;gBAEzB,MAAM,KAAK,CAAA;YACZ,CAAC;YAED,MAAM,MAAM,CAAA;YAEZ,SAAS,EAAE,CAAA;QACZ,CAAC;IACF,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CAAc,MAAyB,EAAE,OAAO,GAAyB,EAAE;QACjG,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QACjC,IAAI,SAAS,GAAG,CAAC,CAAA;QACjB,MAAM,WAAW,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAE3D,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YACrC,IAAI,MAAS,CAAA;YAEb,IAAI,CAAC;gBACJ,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;gBAEnC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAM,CAAA;YAClC,CAAC;YAAC,OAAO,WAAW,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,IAAI,WAAW,CAAC,+BAA+B,SAAS,EAAE,CAAC,CAAA;gBACzE,KAAK,CAAC,KAAK,GAAG,WAAW,CAAA;gBAEzB,MAAM,KAAK,CAAA;YACZ,CAAC;YAED,MAAM,MAAM,CAAA;YAEZ,SAAS,EAAE,CAAA;QACZ,CAAC;IACF,CAAC;CACD"}
1
+ {"version":3,"file":"JSONSpliterator.js","sourceRoot":"","sources":["../../lib/JSONSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAA2B,iBAAiB,EAAE,MAAM,sBAAsB,CAAA;AACjF,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAGlD,OAAO,EAA6B,WAAW,EAAwB,MAAM,kBAAkB,CAAA;AAE/F;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,OAAgB,eAAe;IACpC;QACC,MAAM,IAAI,SAAS,CAAC,2EAA2E,CAAC,CAAA;IACjG,CAAC;IAED,MAAM,CAAC,CAAC,IAAI,CAAc,MAA8B,EAAE,OAAO,GAAoB,EAAE;QACtF,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QACjC,IAAI,SAAS,GAAG,CAAC,CAAA;QAEjB,MAAM,WAAW,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAEzD,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YAC/B,IAAI,MAAS,CAAA;YAEb,IAAI,CAAC;gBACJ,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;gBAEnC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAM,CAAA;YAClC,CAAC;YAAC,OAAO,WAAW,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,IAAI,WAAW,CAAC,+BAA+B,SAAS,EAAE,CAAC,CAAA;gBACzE,KAAK,CAAC,KAAK,GAAG,WAAW,CAAA;gBAEzB,MAAM,KAAK,CAAA;YACZ,CAAC;YAED,MAAM,MAAM,CAAA;YAEZ,SAAS,EAAE,CAAA;QACZ,CAAC;IACF,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,SAAS,CAAc,MAAyB,EAAE,OAAO,GAAuB,EAAE;QACxF,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QAEjC,+GAA+G;QAC/G,kHAAkH;QAClH,mEAAmE;QACnE,OAAO,aAAa,CAAC,IAAI,CAAa,GAAG,EAAE,CAAC,iBAAiB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE;YACtG,IAAI,CAAC;gBACJ,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAM,CAAA;YAC5C,CAAC;YAAC,OAAO,WAAW,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,IAAI,WAAW,CAAC,+BAA+B,SAAS,EAAE,CAAC,CAAA;gBACzE,KAAK,CAAC,KAAK,GAAG,WAAW,CAAA;gBAEzB,MAAM,KAAK,CAAA;YACZ,CAAC;QACF,CAAC,CAAC,CAAA;IACH,CAAC;CACD"}
@@ -3,9 +3,11 @@
3
3
  * @license MIT
4
4
  * @author Teffen Ellis, et al.
5
5
  */
6
+ import { type AdaptiveSourceInit } from "./adaptive-source.js";
7
+ import { AsyncSequence } from "./AsyncSequence.js";
6
8
  import type { CharacterSequenceInput } from "./CharacterSequence.js";
7
9
  import type { AsyncDataResource } from "./shared.js";
8
- import { type AsyncSpliteratorInit, type SpliteratorInit } from "./Spliterator.js";
10
+ import { type SpliteratorInit } from "./Spliterator.js";
9
11
  export interface TextSpliteratorInit {
10
12
  /**
11
13
  * The encoding to use when decoding the data.
@@ -46,6 +48,6 @@ export declare abstract class TextSpliterator {
46
48
  * @see {@linkcode TextSpliterator.from} for synchronous iteration with decoding.
47
49
  * @see {@linkcode Spliterator.fromAsync} for asynchronous iteration without decoding.
48
50
  */
49
- static fromAsync(source: AsyncDataResource, { encoding, fatal, ignoreBOM, ...options }?: TextSpliteratorInit & AsyncSpliteratorInit): AsyncGenerator<string>;
51
+ static fromAsync(source: AsyncDataResource, { encoding, fatal, ignoreBOM, ...options }?: TextSpliteratorInit & AdaptiveSourceInit): AsyncSequence<string>;
50
52
  }
51
53
  //# sourceMappingURL=TextSpliterator.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"TextSpliterator.d.ts","sourceRoot":"","sources":["../../lib/TextSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AACpE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AACpD,OAAO,EAAE,KAAK,oBAAoB,EAAe,KAAK,eAAe,EAAE,MAAM,kBAAkB,CAAA;AAE/F,MAAM,WAAW,mBAAmB;IACnC;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,CAAA;IAEf;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAA;CACnB;AAED,8BAAsB,eAAe;IACpC,cAEC;IAED;;;;;;;;;OASG;IACH,MAAM,CAAE,IAAI,CACX,MAAM,EAAE,sBAAsB,EAC9B,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,OAAO,EAAE,GAAE,mBAAmB,GAAG,eAAoB,GACpF,SAAS,CAAC,MAAM,CAAC,CAsBnB;IAED;;;;;;;;;OASG;IACH,OAAc,SAAS,CACtB,MAAM,EAAE,iBAAiB,EACzB,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,OAAO,EAAE,GAAE,mBAAmB,GAAG,oBAAyB,GACzF,cAAc,CAAC,MAAM,CAAC,CAqBxB;CACD"}
1
+ {"version":3,"file":"TextSpliterator.d.ts","sourceRoot":"","sources":["../../lib/TextSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,KAAK,kBAAkB,EAAqB,MAAM,sBAAsB,CAAA;AACjF,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAClD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AACpE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AACpD,OAAO,EAA0C,KAAK,eAAe,EAAE,MAAM,kBAAkB,CAAA;AAE/F,MAAM,WAAW,mBAAmB;IACnC;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,CAAA;IAEf;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAA;CACnB;AAED,8BAAsB,eAAe;IACpC,cAEC;IAED;;;;;;;;;OASG;IACH,MAAM,CAAE,IAAI,CACX,MAAM,EAAE,sBAAsB,EAC9B,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,OAAO,EAAE,GAAE,mBAAmB,GAAG,eAAoB,GACpF,SAAS,CAAC,MAAM,CAAC,CAsBnB;IAED;;;;;;;;;OASG;IACH,MAAM,CAAC,SAAS,CACf,MAAM,EAAE,iBAAiB,EACzB,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,OAAO,EAAE,GAAE,mBAAmB,GAAG,kBAAuB,GACvF,aAAa,CAAC,MAAM,CAAC,CAevB;CACD"}
@@ -3,6 +3,8 @@
3
3
  * @license MIT
4
4
  * @author Teffen Ellis, et al.
5
5
  */
6
+ import { openDelimitedRows } from "./adaptive-source.js";
7
+ import { AsyncSequence } from "./AsyncSequence.js";
6
8
  import { Spliterator } from "./Spliterator.js";
7
9
  export class TextSpliterator {
8
10
  constructor() {
@@ -46,23 +48,20 @@ export class TextSpliterator {
46
48
  * @see {@linkcode TextSpliterator.from} for synchronous iteration with decoding.
47
49
  * @see {@linkcode Spliterator.fromAsync} for asynchronous iteration without decoding.
48
50
  */
49
- static async *fromAsync(source, { encoding, fatal, ignoreBOM, ...options } = {}) {
51
+ static fromAsync(source, { encoding, fatal, ignoreBOM, ...options } = {}) {
50
52
  const decoder = new TextDecoder(encoding, { fatal, ignoreBOM });
51
- let rowCursor = 0;
52
- const spliterator = await Spliterator.fromAsync(source, options);
53
- for await (const row of spliterator) {
54
- let decoded;
53
+ // Decoding is an op on the sequence, not a generator wrapped inside one. A wrapping generator adds an async frame
54
+ // per row on top of the sequence's own, which measured 297ms against 279ms over 500k rows.
55
+ return AsyncSequence.from(() => openDelimitedRows(source, options)).map((row, rowCursor) => {
55
56
  try {
56
- decoded = decoder.decode(row);
57
+ return decoder.decode(row);
57
58
  }
58
59
  catch (parsedError) {
59
60
  const error = new SyntaxError(`Failed to decode data at row ${rowCursor}`);
60
61
  error.cause = parsedError;
61
62
  throw error;
62
63
  }
63
- yield decoded;
64
- rowCursor++;
65
- }
64
+ });
66
65
  }
67
66
  }
68
67
  //# sourceMappingURL=TextSpliterator.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"TextSpliterator.js","sourceRoot":"","sources":["../../lib/TextSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,OAAO,EAA6B,WAAW,EAAwB,MAAM,kBAAkB,CAAA;AAsB/F,MAAM,OAAgB,eAAe;IACpC;QACC,MAAM,IAAI,SAAS,CAAC,2EAA2E,CAAC,CAAA;IACjG,CAAC;IAED;;;;;;;;;OASG;IACH,MAAM,CAAC,CAAC,IAAI,CACX,MAA8B,EAC9B,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,OAAO,EAAE,GAA0C,EAAE;QAEtF,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAA;QAC/D,IAAI,SAAS,GAAG,CAAC,CAAA;QAEjB,MAAM,WAAW,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAEzD,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YAC/B,IAAI,OAAe,CAAA;YAEnB,IAAI,CAAC;gBACJ,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YAC9B,CAAC;YAAC,OAAO,WAAW,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,IAAI,WAAW,CAAC,gCAAgC,SAAS,EAAE,CAAC,CAAA;gBAC1E,KAAK,CAAC,KAAK,GAAG,WAAW,CAAA;gBAEzB,MAAM,KAAK,CAAA;YACZ,CAAC;YAED,MAAM,OAAO,CAAA;YAEb,SAAS,EAAE,CAAA;QACZ,CAAC;IACF,CAAC;IAED;;;;;;;;;OASG;IACH,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CACtB,MAAyB,EACzB,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,OAAO,EAAE,GAA+C,EAAE;QAE3F,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAA;QAC/D,IAAI,SAAS,GAAG,CAAC,CAAA;QACjB,MAAM,WAAW,GAAG,MAAM,WAAW,CAAC,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAEhE,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YACrC,IAAI,OAAe,CAAA;YAEnB,IAAI,CAAC;gBACJ,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YAC9B,CAAC;YAAC,OAAO,WAAW,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,IAAI,WAAW,CAAC,gCAAgC,SAAS,EAAE,CAAC,CAAA;gBAC1E,KAAK,CAAC,KAAK,GAAG,WAAW,CAAA;gBAEzB,MAAM,KAAK,CAAA;YACZ,CAAC;YAED,MAAM,OAAO,CAAA;YAEb,SAAS,EAAE,CAAA;QACZ,CAAC;IACF,CAAC;CACD"}
1
+ {"version":3,"file":"TextSpliterator.js","sourceRoot":"","sources":["../../lib/TextSpliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAA2B,iBAAiB,EAAE,MAAM,sBAAsB,CAAA;AACjF,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAGlD,OAAO,EAA6B,WAAW,EAAwB,MAAM,kBAAkB,CAAA;AAsB/F,MAAM,OAAgB,eAAe;IACpC;QACC,MAAM,IAAI,SAAS,CAAC,2EAA2E,CAAC,CAAA;IACjG,CAAC;IAED;;;;;;;;;OASG;IACH,MAAM,CAAC,CAAC,IAAI,CACX,MAA8B,EAC9B,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,OAAO,EAAE,GAA0C,EAAE;QAEtF,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAA;QAC/D,IAAI,SAAS,GAAG,CAAC,CAAA;QAEjB,MAAM,WAAW,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAEzD,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YAC/B,IAAI,OAAe,CAAA;YAEnB,IAAI,CAAC;gBACJ,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YAC9B,CAAC;YAAC,OAAO,WAAW,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,IAAI,WAAW,CAAC,gCAAgC,SAAS,EAAE,CAAC,CAAA;gBAC1E,KAAK,CAAC,KAAK,GAAG,WAAW,CAAA;gBAEzB,MAAM,KAAK,CAAA;YACZ,CAAC;YAED,MAAM,OAAO,CAAA;YAEb,SAAS,EAAE,CAAA;QACZ,CAAC;IACF,CAAC;IAED;;;;;;;;;OASG;IACH,MAAM,CAAC,SAAS,CACf,MAAyB,EACzB,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,OAAO,EAAE,GAA6C,EAAE;QAEzF,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAA;QAE/D,kHAAkH;QAClH,2FAA2F;QAC3F,OAAO,aAAa,CAAC,IAAI,CAAa,GAAG,EAAE,CAAC,iBAAiB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE;YACtG,IAAI,CAAC;gBACJ,OAAO,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YAC3B,CAAC;YAAC,OAAO,WAAW,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,IAAI,WAAW,CAAC,gCAAgC,SAAS,EAAE,CAAC,CAAA;gBAC1E,KAAK,CAAC,KAAK,GAAG,WAAW,CAAA;gBAEzB,MAAM,KAAK,CAAA;YACZ,CAAC;QACF,CAAC,CAAC,CAAA;IACH,CAAC;CACD"}
@@ -0,0 +1,39 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license MIT
4
+ * @author Teffen Ellis, et al.
5
+ */
6
+ import type { AsyncChunkIterator, AsyncDataResource } from "./shared.js";
7
+ import { type AsyncSpliteratorInit } from "./Spliterator.js";
8
+ /**
9
+ * Byte length at or below which a source is read whole and parsed synchronously.
10
+ *
11
+ * What this buys is **fixed setup cost** — opening a handle and standing up a read stream, ~100µs — not throughput, so
12
+ * the win decays as the source grows and the threshold marks where it stops being measurable. End-to-end against the
13
+ * streaming path (Node 26, min of 200 runs, two independent sweeps): ~1.85× at 635B, ~1.45× at 6.5KB, ~1.4× at 125KiB.
14
+ * At 253KiB and above the two runs disagreed on which was faster, so there is nothing reliable left to win.
15
+ *
16
+ * Raising this does not recover the much larger gap a raw synchronous parse shows (~1.6× even at 1GiB); that gap is
17
+ * eaten by the per-row cost of the sequence itself, which both paths pay. It only trades resident memory — a 1GiB
18
+ * source costs ~105MB streamed against ~1121MB read whole — for a difference that no longer measures.
19
+ */
20
+ export declare const DEFAULT_BULK_THRESHOLD: number;
21
+ export interface AdaptiveSourceInit extends AsyncSpliteratorInit {
22
+ /**
23
+ * Byte length at or below which the whole source is read into memory and parsed synchronously, trading resident
24
+ * memory for speed.
25
+ *
26
+ * Raising it buys nothing measurable and costs memory linearly — the advantage is gone by ~256KiB. Set `0` to always
27
+ * stream, which is what you want when a bounded footprint is the reason you reached for this library.
28
+ *
29
+ * @default 128 KiB ({@linkcode DEFAULT_BULK_THRESHOLD})
30
+ */
31
+ bulkThreshold?: number;
32
+ }
33
+ /**
34
+ * Take the delimited rows of `source`, reading it whole when it is small enough to be worth the memory and streaming it
35
+ * otherwise. Returns a sync iterable in the first case and an async one in the second; both satisfy
36
+ * {@linkcode AsyncSequence}.
37
+ */
38
+ export declare function openDelimitedRows(source: AsyncDataResource | AsyncChunkIterator, init?: AdaptiveSourceInit): Promise<AsyncIterable<Uint8Array> | Iterable<Uint8Array>>;
39
+ //# sourceMappingURL=adaptive-source.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"adaptive-source.d.ts","sourceRoot":"","sources":["../../lib/adaptive-source.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AACxE,OAAO,EAAE,KAAK,oBAAoB,EAAe,MAAM,kBAAkB,CAAA;AAEzE;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,sBAAsB,QAAa,CAAA;AAEhD,MAAM,WAAW,kBAAmB,SAAQ,oBAAoB;IAC/D;;;;;;;;OAQG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;CACtB;AAuBD;;;;GAIG;AACH,wBAAsB,iBAAiB,CACtC,MAAM,EAAE,iBAAiB,GAAG,kBAAkB,EAC9C,IAAI,GAAE,kBAAuB,GAC3B,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC,CAuB3D"}
@@ -0,0 +1,95 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license MIT
4
+ * @author Teffen Ellis, et al.
5
+ */
6
+ import { CharacterSequence } from "./CharacterSequence.js";
7
+ import { Spliterator } from "./Spliterator.js";
8
+ /**
9
+ * Byte length at or below which a source is read whole and parsed synchronously.
10
+ *
11
+ * What this buys is **fixed setup cost** — opening a handle and standing up a read stream, ~100µs — not throughput, so
12
+ * the win decays as the source grows and the threshold marks where it stops being measurable. End-to-end against the
13
+ * streaming path (Node 26, min of 200 runs, two independent sweeps): ~1.85× at 635B, ~1.45× at 6.5KB, ~1.4× at 125KiB.
14
+ * At 253KiB and above the two runs disagreed on which was faster, so there is nothing reliable left to win.
15
+ *
16
+ * Raising this does not recover the much larger gap a raw synchronous parse shows (~1.6× even at 1GiB); that gap is
17
+ * eaten by the per-row cost of the sequence itself, which both paths pay. It only trades resident memory — a 1GiB
18
+ * source costs ~105MB streamed against ~1121MB read whole — for a difference that no longer measures.
19
+ */
20
+ export const DEFAULT_BULK_THRESHOLD = 128 * 1024;
21
+ function isChunkIterator(source) {
22
+ return typeof source === "object" && source !== null && Symbol.asyncIterator in source;
23
+ }
24
+ function toBytes(chunk) {
25
+ return typeof chunk === "string" ? new TextEncoder().encode(chunk) : chunk;
26
+ }
27
+ /**
28
+ * Parse in-memory bytes with the synchronous engine.
29
+ *
30
+ * Awaiting {@linkcode CharacterSequence.whenReady} first is the point: the WASM scanner loads asynchronously, so a
31
+ * synchronous caller normally completes before it is available and silently uses the JS scanner. Reaching the sync
32
+ * engine through an async path is the one place that can be avoided — worth ~26% on a large source.
33
+ */
34
+ async function bulk(bytes, init) {
35
+ await CharacterSequence.whenReady();
36
+ return Spliterator.fromSync(bytes, init);
37
+ }
38
+ /**
39
+ * Take the delimited rows of `source`, reading it whole when it is small enough to be worth the memory and streaming it
40
+ * otherwise. Returns a sync iterable in the first case and an async one in the second; both satisfy
41
+ * {@linkcode AsyncSequence}.
42
+ */
43
+ export async function openDelimitedRows(source, init = {}) {
44
+ const threshold = init.bulkThreshold ?? DEFAULT_BULK_THRESHOLD;
45
+ if (threshold <= 0)
46
+ return Spliterator.fromAsync(source, init);
47
+ if (isChunkIterator(source))
48
+ return openChunkIterator(source, threshold, init);
49
+ let size;
50
+ try {
51
+ const { readFileSize } = await import("spliterator/node/fs");
52
+ size = await readFileSize(source);
53
+ }
54
+ catch {
55
+ // Unsized or unreachable by the Node adapter — the streaming path owns reporting why.
56
+ return Spliterator.fromAsync(source, init);
57
+ }
58
+ if (size > threshold)
59
+ return Spliterator.fromAsync(source, init);
60
+ const { readBytes } = await import("spliterator/node/fs");
61
+ return bulk(await readBytes(source, 0, size), init);
62
+ }
63
+ /**
64
+ * A stream cannot be measured before it is read, so the size test becomes an end-of-input test: pull one chunk, and if
65
+ * the stream is already exhausted the whole input is in hand and no threshold policy is needed.
66
+ *
67
+ * When it is not exhausted the pulled chunks cannot be discarded, so the stream is re-headed with them in front.
68
+ */
69
+ async function openChunkIterator(source, threshold, init) {
70
+ const iterator = source[Symbol.asyncIterator]();
71
+ const first = await iterator.next();
72
+ if (first.done)
73
+ return bulk(new Uint8Array(0), init);
74
+ const second = await iterator.next();
75
+ const head = toBytes(first.value);
76
+ if (second.done && head.byteLength <= threshold)
77
+ return bulk(head, init);
78
+ const pulled = second.done ? [first.value] : [first.value, second.value];
79
+ const reheaded = {
80
+ async *[Symbol.asyncIterator]() {
81
+ yield* pulled;
82
+ for (;;) {
83
+ const next = await iterator.next();
84
+ if (next.done)
85
+ return;
86
+ yield next.value;
87
+ }
88
+ },
89
+ };
90
+ if (Symbol.asyncDispose in source) {
91
+ reheaded[Symbol.asyncDispose] = () => source[Symbol.asyncDispose]();
92
+ }
93
+ return Spliterator.fromAsync(reheaded, init);
94
+ }
95
+ //# sourceMappingURL=adaptive-source.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"adaptive-source.js","sourceRoot":"","sources":["../../lib/adaptive-source.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAE1D,OAAO,EAA6B,WAAW,EAAE,MAAM,kBAAkB,CAAA;AAEzE;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,GAAG,IAAI,CAAA;AAehD,SAAS,eAAe,CAAC,MAAe;IACvC,OAAO,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,CAAC,aAAa,IAAI,MAAM,CAAA;AACvF,CAAC;AAED,SAAS,OAAO,CAAC,KAA0B;IAC1C,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAA;AAC3E,CAAC;AAED;;;;;;GAMG;AACH,KAAK,UAAU,IAAI,CAAC,KAAiB,EAAE,IAAwB;IAC9D,MAAM,iBAAiB,CAAC,SAAS,EAAE,CAAA;IAEnC,OAAO,WAAW,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;AACzC,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACtC,MAA8C,EAC9C,IAAI,GAAuB,EAAE;IAE7B,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,IAAI,sBAAsB,CAAA;IAE9D,IAAI,SAAS,IAAI,CAAC;QAAE,OAAO,WAAW,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;IAE9D,IAAI,eAAe,CAAC,MAAM,CAAC;QAAE,OAAO,iBAAiB,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,CAAA;IAE9E,IAAI,IAAY,CAAA;IAEhB,IAAI,CAAC;QACJ,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAA;QAE5D,IAAI,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC,CAAA;IAClC,CAAC;IAAC,MAAM,CAAC;QACR,sFAAsF;QACtF,OAAO,WAAW,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;IAC3C,CAAC;IAED,IAAI,IAAI,GAAG,SAAS;QAAE,OAAO,WAAW,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;IAEhE,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAA;IAEzD,OAAO,IAAI,CAAC,MAAM,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,CAAA;AACpD,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,iBAAiB,CAC/B,MAA0B,EAC1B,SAAiB,EACjB,IAAwB;IAExB,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAA;IAC/C,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;IAEnC,IAAI,KAAK,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAA;IAEpD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;IACpC,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IAEjC,IAAI,MAAM,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,IAAI,SAAS;QAAE,OAAO,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;IAExE,MAAM,MAAM,GAA+B,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAA;IAEpG,MAAM,QAAQ,GAAuB;QACpC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;YAC5B,KAAK,CAAC,CAAC,MAAM,CAAA;YAEb,SAAS,CAAC;gBACT,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;gBAElC,IAAI,IAAI,CAAC,IAAI;oBAAE,OAAM;gBAErB,MAAM,IAAI,CAAC,KAAK,CAAA;YACjB,CAAC;QACF,CAAC;KACD,CAAA;IAED,IAAI,MAAM,CAAC,YAAY,IAAI,MAAM,EAAE,CAAC;QACnC,QAAQ,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAE,EAAE,CAAA;IACrE,CAAC;IAED,OAAO,WAAW,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAA;AAC7C,CAAC"}
@@ -0,0 +1,20 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license MIT
4
+ * @author Teffen Ellis, et al.
5
+ */
6
+ /**
7
+ * Group an iterable's values into arrays of `size`, with a shorter final batch if the collection does not divide
8
+ * evenly.
9
+ *
10
+ * Distinct from `Iterator.prototype.take`, which yields the first `size` _values_. Named for the [iterator chunking
11
+ * proposal](https://github.com/tc39/proposal-iterator-chunking).
12
+ *
13
+ * @param collection The collection to batch.
14
+ * @param size The size of each batch.
15
+ *
16
+ * @yields Each batch of values.
17
+ * @see {@linkcode AsyncSequence.chunks} for the asynchronous form.
18
+ */
19
+ export declare function chunks<T>(collection: Iterable<T>, size: number): Generator<T[]>;
20
+ //# sourceMappingURL=chunks.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"chunks.d.ts","sourceRoot":"","sources":["../../lib/chunks.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;;;;;;;;;;;GAYG;AACH,wBAAiB,MAAM,CAAC,CAAC,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,EAAE,CAAC,CAsBhF"}
@@ -0,0 +1,36 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license MIT
4
+ * @author Teffen Ellis, et al.
5
+ */
6
+ /**
7
+ * Group an iterable's values into arrays of `size`, with a shorter final batch if the collection does not divide
8
+ * evenly.
9
+ *
10
+ * Distinct from `Iterator.prototype.take`, which yields the first `size` _values_. Named for the [iterator chunking
11
+ * proposal](https://github.com/tc39/proposal-iterator-chunking).
12
+ *
13
+ * @param collection The collection to batch.
14
+ * @param size The size of each batch.
15
+ *
16
+ * @yields Each batch of values.
17
+ * @see {@linkcode AsyncSequence.chunks} for the asynchronous form.
18
+ */
19
+ export function* chunks(collection, size) {
20
+ const normalized = Math.trunc(size);
21
+ if (!Number.isFinite(normalized) || normalized < 1) {
22
+ throw new RangeError(`chunks(${size}): size must be a positive finite number`);
23
+ }
24
+ let batch = [];
25
+ for (const item of collection) {
26
+ batch.push(item);
27
+ if (batch.length === normalized) {
28
+ yield batch;
29
+ batch = [];
30
+ }
31
+ }
32
+ if (batch.length) {
33
+ yield batch;
34
+ }
35
+ }
36
+ //# sourceMappingURL=chunks.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"chunks.js","sourceRoot":"","sources":["../../lib/chunks.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;;;;;;;;;;;GAYG;AACH,MAAM,SAAS,CAAC,CAAC,MAAM,CAAI,UAAuB,EAAE,IAAY;IAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAEnC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QACpD,MAAM,IAAI,UAAU,CAAC,UAAU,IAAI,0CAA0C,CAAC,CAAA;IAC/E,CAAC;IAED,IAAI,KAAK,GAAQ,EAAE,CAAA;IAEnB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAEhB,IAAI,KAAK,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YACjC,MAAM,KAAK,CAAA;YAEX,KAAK,GAAG,EAAE,CAAA;QACX,CAAC;IACF,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QAClB,MAAM,KAAK,CAAA;IACZ,CAAC;AACF,CAAC"}