spliterator 3.2.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/lib/AsyncSpliterator.d.ts.map +1 -1
- package/out/lib/AsyncSpliterator.js +6 -4
- package/out/lib/AsyncSpliterator.js.map +1 -1
- package/out/lib/BufferController.d.ts.map +1 -1
- package/out/lib/BufferController.js.map +1 -1
- package/out/lib/CSVSpliterator.d.ts +8 -6
- package/out/lib/CSVSpliterator.d.ts.map +1 -1
- package/out/lib/CSVSpliterator.js +8 -1
- package/out/lib/CSVSpliterator.js.map +1 -1
- package/out/lib/CharacterSequence.d.ts +10 -1
- package/out/lib/CharacterSequence.d.ts.map +1 -1
- package/out/lib/CharacterSequence.js +76 -26
- package/out/lib/CharacterSequence.js.map +1 -1
- package/out/lib/CompositeDataView.d.ts.map +1 -1
- package/out/lib/CompositeDataView.js.map +1 -1
- package/out/lib/IndexQueue.d.ts.map +1 -1
- package/out/lib/IndexQueue.js +3 -1
- package/out/lib/IndexQueue.js.map +1 -1
- package/out/lib/JSONSpliterator.d.ts.map +1 -1
- package/out/lib/JSONSpliterator.js +2 -2
- package/out/lib/JSONSpliterator.js.map +1 -1
- package/out/lib/PSVSpliterator.js.map +1 -1
- package/out/lib/SlidingWindow.d.ts.map +1 -1
- package/out/lib/SlidingWindow.js.map +1 -1
- package/out/lib/Spliterator.d.ts.map +1 -1
- package/out/lib/Spliterator.js +6 -4
- package/out/lib/Spliterator.js.map +1 -1
- package/out/lib/TSVSpliterator.js.map +1 -1
- package/out/lib/TextSpliterator.d.ts +2 -2
- package/out/lib/TextSpliterator.d.ts.map +1 -1
- package/out/lib/TextSpliterator.js +2 -4
- package/out/lib/TextSpliterator.js.map +1 -1
- package/out/lib/casing.js +3 -3
- package/out/lib/casing.js.map +1 -1
- package/out/lib/merge-async-iterators.d.ts.map +1 -1
- package/out/lib/merge-async-iterators.js +3 -2
- package/out/lib/merge-async-iterators.js.map +1 -1
- package/out/lib/parallel-map-runtime.d.ts +3 -1
- package/out/lib/parallel-map-runtime.d.ts.map +1 -1
- package/out/lib/parallel-map-runtime.js +1 -1
- package/out/lib/parallel-map-runtime.js.map +1 -1
- package/out/lib/parallel-map-worker-entry.js +3 -2
- package/out/lib/parallel-map-worker-entry.js.map +1 -1
- package/out/lib/parallel-map.d.ts +12 -4
- package/out/lib/parallel-map.d.ts.map +1 -1
- package/out/lib/parallel-map.js +7 -3
- package/out/lib/parallel-map.js.map +1 -1
- package/out/lib/segment-runtime.d.ts.map +1 -1
- package/out/lib/segment-runtime.js +7 -4
- package/out/lib/segment-runtime.js.map +1 -1
- package/out/lib/segment-worker-entry.js +20 -17
- package/out/lib/segment-worker-entry.js.map +1 -1
- package/out/lib/segment-workers.d.ts +22 -9
- package/out/lib/segment-workers.d.ts.map +1 -1
- package/out/lib/segment-workers.js +8 -4
- package/out/lib/segment-workers.js.map +1 -1
- package/out/lib/segments.d.ts +9 -3
- package/out/lib/segments.d.ts.map +1 -1
- package/out/lib/segments.js +6 -4
- package/out/lib/segments.js.map +1 -1
- package/out/lib/shared.d.ts +3 -2
- package/out/lib/shared.d.ts.map +1 -1
- package/out/lib/shared.js +0 -2
- package/out/lib/shared.js.map +1 -1
- package/out/lib/take.d.ts.map +1 -1
- package/out/lib/take.js +3 -1
- package/out/lib/take.js.map +1 -1
- package/out/lib/wasm_module.d.ts +9 -3
- package/out/lib/wasm_module.d.ts.map +1 -1
- package/out/lib/wasm_module.js.map +1 -1
- package/out/lib/writer.d.ts +16 -5
- package/out/lib/writer.d.ts.map +1 -1
- package/out/lib/writer.js +20 -4
- package/out/lib/writer.js.map +1 -1
- package/out/node/cli/commands/csv.d.ts +3 -38
- package/out/node/cli/commands/csv.d.ts.map +1 -1
- package/out/node/cli/commands/csv.js +55 -47
- package/out/node/cli/commands/csv.js.map +1 -1
- package/out/node/cli/commands/iterate.d.ts +3 -31
- package/out/node/cli/commands/iterate.d.ts.map +1 -1
- package/out/node/cli/commands/iterate.js +45 -24
- package/out/node/cli/commands/iterate.js.map +1 -1
- package/out/node/cli/index.js +36 -22
- package/out/node/cli/index.js.map +1 -1
- package/out/node/cli/parse-args.d.ts +120 -0
- package/out/node/cli/parse-args.d.ts.map +1 -0
- package/out/node/cli/parse-args.js +118 -0
- package/out/node/cli/parse-args.js.map +1 -0
- package/out/node/cli/utils.d.ts +79 -28
- package/out/node/cli/utils.d.ts.map +1 -1
- package/out/node/cli/utils.js +63 -63
- package/out/node/cli/utils.js.map +1 -1
- package/out/node/fs/index.d.ts.map +1 -1
- package/out/node/fs/index.js +7 -3
- package/out/node/fs/index.js.map +1 -1
- package/package.json +13 -17
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
import { Worker } from "node:worker_threads";
|
|
7
7
|
import { mergeAsyncIterators } from "./merge-async-iterators.js";
|
|
8
8
|
import { computeSegments } from "./segments.js";
|
|
9
|
-
export { mergeAsyncIterators };
|
|
9
|
+
export { mergeAsyncIterators } from "./merge-async-iterators.js";
|
|
10
10
|
/**
|
|
11
11
|
* Drain a worker's batched messages into an async iterator. Listeners attach **eagerly** (messages posted before
|
|
12
12
|
* iteration starts are buffered, not lost) and draining uses a `batches[] + head` pointer (no `Array.shift()`).
|
|
@@ -25,10 +25,12 @@ export function workerToIterable(worker, onBatchConsumed) {
|
|
|
25
25
|
};
|
|
26
26
|
worker.on("message", (msg) => {
|
|
27
27
|
const m = msg;
|
|
28
|
-
if (m.type === "batch")
|
|
28
|
+
if (m.type === "batch") {
|
|
29
29
|
batches.push(m.records);
|
|
30
|
-
|
|
30
|
+
}
|
|
31
|
+
else if (m.type === "done") {
|
|
31
32
|
done = true;
|
|
33
|
+
}
|
|
32
34
|
else if (m.type === "error") {
|
|
33
35
|
error = new Error(m.message);
|
|
34
36
|
done = true;
|
|
@@ -44,8 +46,9 @@ export function workerToIterable(worker, onBatchConsumed) {
|
|
|
44
46
|
for (;;) {
|
|
45
47
|
if (head < batches.length) {
|
|
46
48
|
const batch = batches[head++];
|
|
47
|
-
for (const record of batch)
|
|
49
|
+
for (const record of batch) {
|
|
48
50
|
yield record;
|
|
51
|
+
}
|
|
49
52
|
onBatchConsumed();
|
|
50
53
|
continue;
|
|
51
54
|
}
|
|
@@ -53,6 +56,7 @@ export function workerToIterable(worker, onBatchConsumed) {
|
|
|
53
56
|
throw error;
|
|
54
57
|
if (done)
|
|
55
58
|
return;
|
|
59
|
+
// oxlint-disable-next-line no-loop-func no-promise-executor-return
|
|
56
60
|
await new Promise((resolve) => (wake = resolve));
|
|
57
61
|
}
|
|
58
62
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"segment-workers.js","sourceRoot":"","sources":["../../lib/segment-workers.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAG5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAG/C,OAAO,EAAE,mBAAmB,EAAE,CAAA;
|
|
1
|
+
{"version":3,"file":"segment-workers.js","sourceRoot":"","sources":["../../lib/segment-workers.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAG5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAG/C,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AAShE;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAI,MAAqB,EAAE,eAA2B;IACrF,MAAM,OAAO,GAAU,EAAE,CAAA;IACzB,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,IAAI,GAAG,KAAK,CAAA;IAChB,IAAI,KAAwB,CAAA;IAC5B,IAAI,IAA8B,CAAA;IAElC,MAAM,MAAM,GAAG,GAAG,EAAE;QACnB,IAAI,EAAE,EAAE,CAAA;QACR,IAAI,GAAG,SAAS,CAAA;IACjB,CAAC,CAAA;IAED,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,EAAE;QAC5B,MAAM,CAAC,GAAG,GAAuB,CAAA;QAEjC,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAA;QACxB,CAAC;aAAM,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC9B,IAAI,GAAG,IAAI,CAAA;QACZ,CAAC;aAAM,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC/B,KAAK,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAA;YAC5B,IAAI,GAAG,IAAI,CAAA;QACZ,CAAC;QAED,MAAM,EAAE,CAAA;IACT,CAAC,CAAC,CAAA;IAEF,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;QAC1B,KAAK,GAAG,GAAG,CAAA;QACX,IAAI,GAAG,IAAI,CAAA;QACX,MAAM,EAAE,CAAA;IACT,CAAC,CAAC,CAAA;IAEF,KAAK,SAAS,CAAC,CAAC,KAAK;QACpB,SAAS,CAAC;YACT,IAAI,IAAI,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC3B,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,EAAE,CAAE,CAAA;gBAE9B,KAAK,MAAM,MAAM,IAAI,KAAK,EAAE,CAAC;oBAC5B,MAAM,MAAM,CAAA;gBACb,CAAC;gBAED,eAAe,EAAE,CAAA;gBAEjB,SAAQ;YACT,CAAC;YAED,IAAI,KAAK;gBAAE,MAAM,KAAK,CAAA;YAEtB,IAAI,IAAI;gBAAE,OAAM;YAEhB,mEAAmE;YACnE,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,IAAI,GAAG,OAAO,CAAC,CAAC,CAAA;QACvD,CAAC;IACF,CAAC;IAED,OAAO,KAAK,EAAE,CAAA;AACf,CAAC;AAiCD;;;;GAIG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,iBAAiB,CACvC,MAAyB,EACzB,OAA6B;IAE7B,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,YAAY,GAAG,CAAC,EAAE,CAAC;QAC5D,MAAM,IAAI,SAAS,CAAC,gFAAgF,CAAC,CAAA;IACtG,CAAC;IAED,MAAM,UAAU,GACf,OAAO,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,CAAA;IAE/G,MAAM,UAAU,GAAG,MAAM,YAAY,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAA;IAE/D,MAAM,QAAQ,GAAgB,MAAM,eAAe,CAAC,MAAM,EAAE;QAC3D,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,SAAS,EAAE,OAAO,CAAC,SAAS;KAC5B,CAAC,CAAA;IAEF,MAAM,OAAO,GAAa,EAAE,CAAA;IAC5B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,2BAA2B,EAAE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAA;IAEtE,IAAI,CAAC;QACJ,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,YAAY,EAAE,EAAE;YAC7D,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,QAAQ,EAAE;gBACnC,UAAU,EAAE;oBACX,MAAM,EAAE,UAAU;oBAClB,UAAU;oBACV,KAAK;oBACL,GAAG;oBACH,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,IAAI;oBACpC,YAAY;oBACZ,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,GAAG;oBACnC,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,CAAC;oBACrC,QAAQ,EAAE,OAAO,CAAC,UAAU;iBAC5B;aACD,CAAC,CAAA;YAEF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAEpB,OAAO,gBAAgB,CAAI,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAA;QACpE,CAAC,CAAC,CAAA;QAEF,gGAAgG;QAChG,6FAA6F;QAC7F,mDAAmD;QACnD,KAAK,CAAC,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAA;IACtC,CAAC;YAAS,CAAC;QACV,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAA;IACrD,CAAC;AACF,CAAC"}
|
package/out/lib/segments.d.ts
CHANGED
|
@@ -6,11 +6,17 @@
|
|
|
6
6
|
import { type CharacterSequenceInput } from "./CharacterSequence.js";
|
|
7
7
|
import type { AsyncDataResource, ByteRange } from "./shared.js";
|
|
8
8
|
export interface SegmentOptions {
|
|
9
|
-
/**
|
|
9
|
+
/**
|
|
10
|
+
* The record delimiter. @default LineFeed
|
|
11
|
+
*/
|
|
10
12
|
delimiter?: CharacterSequenceInput;
|
|
11
|
-
/**
|
|
13
|
+
/**
|
|
14
|
+
* Desired number of segments. Clamped to ≥ 1; the result may be fewer.
|
|
15
|
+
*/
|
|
12
16
|
concurrency: number;
|
|
13
|
-
/**
|
|
17
|
+
/**
|
|
18
|
+
* Bytes read at each ideal boundary to find the next delimiter. @default 65536
|
|
19
|
+
*/
|
|
14
20
|
probeSize?: number;
|
|
15
21
|
}
|
|
16
22
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"segments.d.ts","sourceRoot":"","sources":["../../lib/segments.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAqB,KAAK,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AACvF,OAAO,KAAK,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAE/D,MAAM,WAAW,cAAc;IAC9B
|
|
1
|
+
{"version":3,"file":"segments.d.ts","sourceRoot":"","sources":["../../lib/segments.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAqB,KAAK,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AACvF,OAAO,KAAK,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAE/D,MAAM,WAAW,cAAc;IAC9B;;OAEG;IACH,SAAS,CAAC,EAAE,sBAAsB,CAAA;IAClC;;OAEG;IACH,WAAW,EAAE,MAAM,CAAA;IACnB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;CAClB;AAED;;;;;;;GAOG;AACH,wBAAsB,eAAe,CAAC,MAAM,EAAE,iBAAiB,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAgD9G"}
|
package/out/lib/segments.js
CHANGED
|
@@ -15,7 +15,7 @@ import { CharacterSequence } from "./CharacterSequence.js";
|
|
|
15
15
|
export async function computeSegments(source, options) {
|
|
16
16
|
const { readFileSize, readBytes } = await import("spliterator/node/fs");
|
|
17
17
|
const needle = new CharacterSequence(options.delimiter);
|
|
18
|
-
const probeSize = options.probeSize ??
|
|
18
|
+
const probeSize = options.probeSize ?? 65_536;
|
|
19
19
|
const concurrency = Math.max(1, Math.floor(options.concurrency));
|
|
20
20
|
const fileSize = await readFileSize(source);
|
|
21
21
|
if (fileSize === 0)
|
|
@@ -34,16 +34,18 @@ export async function computeSegments(source, options) {
|
|
|
34
34
|
}));
|
|
35
35
|
const boundaries = new Set([0, fileSize]);
|
|
36
36
|
for (const cut of alignedCuts) {
|
|
37
|
-
if (cut !== null && cut > 0 && cut < fileSize)
|
|
37
|
+
if (cut !== null && cut > 0 && cut < fileSize) {
|
|
38
38
|
boundaries.add(cut);
|
|
39
|
+
}
|
|
39
40
|
}
|
|
40
|
-
const sorted = Array.from(boundaries).
|
|
41
|
+
const sorted = Array.from(boundaries).toSorted((a, b) => a - b);
|
|
41
42
|
const segments = [];
|
|
42
43
|
for (let i = 1; i < sorted.length; i++) {
|
|
43
44
|
const start = sorted[i - 1];
|
|
44
45
|
const end = sorted[i];
|
|
45
|
-
if (end > start)
|
|
46
|
+
if (end > start) {
|
|
46
47
|
segments.push([start, end]);
|
|
48
|
+
}
|
|
47
49
|
}
|
|
48
50
|
return segments;
|
|
49
51
|
}
|
package/out/lib/segments.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"segments.js","sourceRoot":"","sources":["../../lib/segments.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,iBAAiB,EAA+B,MAAM,wBAAwB,CAAA;
|
|
1
|
+
{"version":3,"file":"segments.js","sourceRoot":"","sources":["../../lib/segments.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,iBAAiB,EAA+B,MAAM,wBAAwB,CAAA;AAkBvF;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAyB,EAAE,OAAuB;IACvF,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAA;IACvE,MAAM,MAAM,GAAG,IAAI,iBAAiB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;IACvD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,MAAM,CAAA;IAC7C,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAA;IAEhE,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC,CAAA;IAE3C,IAAI,QAAQ,KAAK,CAAC;QAAE,OAAO,EAAE,CAAA;IAE7B,IAAI,WAAW,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAA;IAE7C,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,GAAG,WAAW,CAAC,CAAC,CAAA;IAEnH,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;QAC3B,IAAI,GAAG,IAAI,CAAC,IAAI,GAAG,IAAI,QAAQ;YAAE,OAAO,IAAI,CAAA;QAE5C,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,GAAG,EAAE,SAAS,CAAC,CAAA;QACtD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;QAErD,IAAI,KAAK,KAAK,CAAC,CAAC;YAAE,OAAO,IAAI,CAAA;QAE7B,OAAO,GAAG,GAAG,KAAK,GAAG,MAAM,CAAC,MAAM,CAAA;IACnC,CAAC,CAAC,CACF,CAAA;IAED,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAA;IAEjD,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QAC/B,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,GAAG,CAAC,IAAI,GAAG,GAAG,QAAQ,EAAE,CAAC;YAC/C,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACpB,CAAC;IACF,CAAC;IAED,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IAC/D,MAAM,QAAQ,GAAgB,EAAE,CAAA;IAEhC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,CAAA;QAC5B,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;QAEtB,IAAI,GAAG,GAAG,KAAK,EAAE,CAAC;YACjB,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAA;QAC5B,CAAC;IACF,CAAC;IAED,OAAO,QAAQ,CAAA;AAChB,CAAC"}
|
package/out/lib/shared.d.ts
CHANGED
|
@@ -3,8 +3,9 @@
|
|
|
3
3
|
* @license MIT
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
|
-
import {
|
|
6
|
+
import type { Stats } from "node:fs";
|
|
7
7
|
import type { FileHandle } from "node:fs/promises";
|
|
8
|
+
import type { PathBuilderLike } from "path-ts";
|
|
8
9
|
/**
|
|
9
10
|
* A trimmed-down version of the Node.js `Stats` interface.
|
|
10
11
|
*/
|
|
@@ -137,7 +138,7 @@ export interface AsyncChunkIterator extends AsyncIterable<Uint8Array | string> {
|
|
|
137
138
|
* qualifies: a Node `Readable` (child-process stdout, a gunzip pipe), a web `ReadableStream` wrapped via
|
|
138
139
|
* `Symbol.asyncIterator`, or a hand-rolled generator.
|
|
139
140
|
*/
|
|
140
|
-
export type AsyncDataResource = string | URL | FileHandleLike | AsyncChunkIterator;
|
|
141
|
+
export type AsyncDataResource = string | URL | FileHandleLike | AsyncChunkIterator | PathBuilderLike;
|
|
141
142
|
/**
|
|
142
143
|
* A resource to a delimited byte stream, i.e., a file buffer, handle, or path.
|
|
143
144
|
*
|
package/out/lib/shared.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../lib/shared.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAE,KAAK,
|
|
1
|
+
{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../lib/shared.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,SAAS,CAAA;AAEpC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAElD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAE9C;;GAEG;AACH,MAAM,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,SAAS,CAAC,CAAA;AAEnE;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,IAAI,CAChC,UAAU,EACV,IAAI,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,GAAG,mBAAmB,GAAG,kBAAkB,CAC3E,CAAA;AAED;;GAEG;AACH,MAAM,MAAM,SAAS,GAAG;IACvB;;OAEG;IACH,KAAK,EAAE,MAAM;IACb;;OAEG;IACH,GAAG,EAAE,MAAM;CACX,CAAA;AAED;;GAEG;AACH,MAAM,MAAM,UAAU,GACnB,UAAU,GACV,iBAAiB,GACjB,WAAW,GACX,WAAW,GACX,SAAS,GACT,UAAU,GACV,UAAU,GACV,cAAc,GACd,aAAa,GACb,YAAY,GACZ,YAAY,CAAA;AAEf;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,cAAc,GAAG,UAAU,CAAA;AAExD;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,cAAc,CAExE;AAED;;;;GAIG;AACH,MAAM,MAAM,kBAAkB,CAAC,CAAC,IAAI,CAAC,SAAS,UAAU,GAAG,CAAC,GAAG,UAAU,CAAA;AAEzE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;;;;;OAMG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;CACjB;AAED;;;;;GAKG;AACH,MAAM,WAAW,eAAgB,SAAQ,aAAa,CAAC,UAAU,CAAC;IACjE;;OAEG;IACH,IAAI,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;IACrD,IAAI,CAAC,CAAC,SAAS,UAAU,EAAE,OAAO,EAAE,gBAAgB,GAAG;QAAE,MAAM,EAAE,CAAC,CAAA;KAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;CACjF;AAED;;;;GAIG;AACH,MAAM,WAAW,gBAAiB,SAAQ,IAAI;IAC7C;;;;;OAKG;IACH,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,gBAAgB,CAAA;IAEnD;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAElB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,CAAA;CAC3C;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,gBAAgB,CAE5E;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,CAAC,SAAS,gBAAgB,EAAE,IAAI,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,GAAG,eAAe,CAsB5G;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,kBAAmB,SAAQ,aAAa,CAAC,UAAU,GAAG,MAAM,CAAC;IAC7E;;;;OAIG;IACH,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,CAAA;CAC3C;AAED;;;;;;;;;GASG;AACH,MAAM,MAAM,iBAAiB,GAAG,MAAM,GAAG,GAAG,GAAG,cAAc,GAAG,kBAAkB,GAAG,eAAe,CAAA;AAEpG;;;;;GAKG;AACH,MAAM,MAAM,YAAY,GAAG,iBAAiB,GAAG,UAAU,CAAA;AAEzD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B,MAAM,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAAA;CACjC;AAED,MAAM,MAAM,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,GAAG,EAAE,MAAM,CAAC,CAAA;AAE5E,MAAM,MAAM,aAAa,CAAC,CAAC,IAAI,CAAC,SAAS,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAA;AAElF,wBAAgB,aAAa,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAErF;AAED;;;;;;;;;;GAUG;AACH,wBAAiB,OAAO,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAkBtF;AAED;;;;;;;;;;GAUG;AACH,wBAAuB,QAAQ,CAAC,CAAC,EAAE,CAAC,EACnC,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,EACjC,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,GAC/B,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,CAkBnE"}
|
package/out/lib/shared.js
CHANGED
package/out/lib/shared.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../lib/shared.ts"],"names":[],"mappings":"AAAA;;;;GAIG;
|
|
1
|
+
{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../lib/shared.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAyDH;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAc;IAC9C,OAAO,OAAO,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,IAAI,IAAI,KAAK,CAAC,CAAA;AACpE,CAAC;AAuED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAAc;IAChD,OAAO,OAAO,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,IAAI,KAAK,CAAC,CAAA;AACvE,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAA6B,IAAO;IACtE,IAAI,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;QACvD,OAAM;IACP,CAAC;IAED,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE;QACnB,KAAK,CAAC,IAAI,CAAuB,OAAO,GAAsC,EAAE;YAC/E,MAAM,EAAE,QAAQ,EAAE,KAAK,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,CAAC,EAAE,GAAG,OAAO,CAAA;YACvE,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC,CAAA;YAElD,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,CAAC,KAAK,EAAE,CAA0B,CAAA;YAE/D,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBACpB,MAAM,WAAW,GAAG,OAAO,CAAC,MAA+B,CAAA;gBAC3D,WAAW,CAAC,GAAG,CAAC,KAAY,EAAE,MAAM,CAAC,CAAA;gBAErC,OAAO,WAAW,CAAA;YACnB,CAAC;YAED,OAAO,KAAK,CAAA;QACb,CAAC;KACD,CAAC,CAAA;AACH,CAAC;AAkDD,MAAM,UAAU,aAAa,CAAO,MAAoB;IACvD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAgC,CAAA;AACzD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,SAAS,CAAC,CAAC,OAAO,CAAO,CAAc,EAAE,CAAc;IAC5D,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAA;IACtC,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAA;IAEtC,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,OAAO,IAAI,EAAE,CAAC;QACb,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC,IAAI,EAAE,CAAA;QACvD,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC,IAAI,EAAE,CAAA;QAEvD,IAAI,KAAK,IAAI,KAAK,EAAE,CAAC;YACpB,MAAK;QACN,CAAC;QAED,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAA;QAE7B,KAAK,EAAE,CAAA;IACR,CAAC;AACF,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,QAAQ,CAC9B,CAAiC,EACjC,CAAiC;IAEjC,MAAM,SAAS,GAAG,MAAM,CAAC,aAAa,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAA;IAC9F,MAAM,SAAS,GAAG,MAAM,CAAC,aAAa,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAA;IAE9F,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,OAAO,IAAI,EAAE,CAAC;QACb,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAA;QAC7D,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAA;QAE7D,IAAI,KAAK,IAAI,KAAK,EAAE,CAAC;YACpB,MAAK;QACN,CAAC;QAED,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAA;QAE7B,KAAK,EAAE,CAAA;IACR,CAAC;AACF,CAAC"}
|
package/out/lib/take.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"take.d.ts","sourceRoot":"","sources":["../../lib/take.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;;;;;;;;;GAUG;AACH,wBAAuB,qBAAqB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,EACvF,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,EAC1C,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,CAAC,EACX,WAAW,CAAC,EAAE,WAAW,GACvB,aAAa,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,
|
|
1
|
+
{"version":3,"file":"take.d.ts","sourceRoot":"","sources":["../../lib/take.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;;;;;;;;;GAUG;AACH,wBAAuB,qBAAqB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,EACvF,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,EAC1C,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,CAAC,EACX,WAAW,CAAC,EAAE,WAAW,GACvB,aAAa,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAqDvC;AAED;;;;;;GAMG;AACH,wBAAuB,SAAS,CAAC,CAAC,EAAE,UAAU,EAAE,aAAa,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,GAAG,aAAa,CAAC,CAAC,EAAE,CAAC,CAgBvG"}
|
package/out/lib/take.js
CHANGED
|
@@ -19,6 +19,7 @@ export async function* asyncParallelIterator(collection, batchSize, callback, ab
|
|
|
19
19
|
const runningTasks = new Map();
|
|
20
20
|
const results = new Map();
|
|
21
21
|
let iterationResult = await iterator.next();
|
|
22
|
+
// oxlint-disable-next-line no-unmodified-loop-condition
|
|
22
23
|
while ((!iterationResult.done || results.size) && !abortSignal?.aborted) {
|
|
23
24
|
for (const [key, result] of results) {
|
|
24
25
|
yield result;
|
|
@@ -36,6 +37,7 @@ export async function* asyncParallelIterator(collection, batchSize, callback, ab
|
|
|
36
37
|
const futureResult = callback(entry).then((result) => {
|
|
37
38
|
runningTasks.delete(entry);
|
|
38
39
|
results.set(entry, result);
|
|
40
|
+
return void 0;
|
|
39
41
|
});
|
|
40
42
|
runningTasks.set(entry, futureResult);
|
|
41
43
|
iterationResult = await iterator.next();
|
|
@@ -64,7 +66,7 @@ export async function* takeAsync(collection, batchSize) {
|
|
|
64
66
|
buffer = [];
|
|
65
67
|
}
|
|
66
68
|
}
|
|
67
|
-
if (buffer.length
|
|
69
|
+
if (buffer.length) {
|
|
68
70
|
yield buffer;
|
|
69
71
|
}
|
|
70
72
|
}
|
package/out/lib/take.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"take.js","sourceRoot":"","sources":["../../lib/take.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,qBAAqB,CAC3C,UAA0C,EAC1C,SAAiB,EACjB,QAAW,EACX,WAAyB;IAEzB,MAAM,QAAQ,GACb,MAAM,CAAC,aAAa,IAAI,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAA;IAExG,MAAM,YAAY,GAAG,IAAI,GAAG,EAAuB,CAAA;IACnD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAoB,CAAA;IAE3C,IAAI,eAAe,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;IAE3C,OAAO,CAAC,CAAC,eAAe,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,OAAO,EAAE,CAAC;QACzE,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACrC,MAAM,MAAM,CAAA;YAEZ,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QACpB,CAAC;QAED,IAAI,YAAY,CAAC,IAAI,IAAI,SAAS,EAAE,CAAC;YACpC,MAAM,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAA;
|
|
1
|
+
{"version":3,"file":"take.js","sourceRoot":"","sources":["../../lib/take.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,qBAAqB,CAC3C,UAA0C,EAC1C,SAAiB,EACjB,QAAW,EACX,WAAyB;IAEzB,MAAM,QAAQ,GACb,MAAM,CAAC,aAAa,IAAI,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAA;IAExG,MAAM,YAAY,GAAG,IAAI,GAAG,EAAuB,CAAA;IACnD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAoB,CAAA;IAE3C,IAAI,eAAe,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;IAE3C,wDAAwD;IACxD,OAAO,CAAC,CAAC,eAAe,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,OAAO,EAAE,CAAC;QACzE,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACrC,MAAM,MAAM,CAAA;YAEZ,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QACpB,CAAC;QAED,IAAI,YAAY,CAAC,IAAI,IAAI,SAAS,EAAE,CAAC;YACpC,MAAM,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAA;YAEzC,SAAQ;QACT,CAAC;QAED,IAAI,eAAe,CAAC,IAAI,EAAE,CAAC;YAC1B,MAAM,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAA;YAExC,MAAK;QACN,CAAC;QAED,MAAM,KAAK,GAAG,eAAe,CAAC,KAAK,CAAA;QAEnC,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;YACpD,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YAE1B,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,MAAuB,CAAC,CAAA;YAE3C,OAAO,KAAK,CAAC,CAAA;QACd,CAAC,CAAC,CAAA;QAEF,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,YAAY,CAAC,CAAA;QAErC,eAAe,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;IACxC,CAAC;IAED,IAAI,CAAC,WAAW,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QAC1C,MAAM,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAA;QAExC,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACrC,MAAM,MAAM,CAAA;YAEZ,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QACpB,CAAC;IACF,CAAC;AACF,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,SAAS,CAAI,UAA4B,EAAE,SAAiB;IAClF,IAAI,MAAM,GAAQ,EAAE,CAAA;IAEpB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QACrC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAEjB,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YACjC,MAAM,MAAM,CAAA;YAEZ,MAAM,GAAG,EAAE,CAAA;QACZ,CAAC;IACF,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,MAAM,CAAA;IACb,CAAC;AACF,CAAC"}
|
package/out/lib/wasm_module.d.ts
CHANGED
|
@@ -17,11 +17,17 @@ export interface WasmDelimiterScanner {
|
|
|
17
17
|
findAllDelimiters: WasmFindAllDelimiters;
|
|
18
18
|
findAllMatches: WasmFindAllMatches;
|
|
19
19
|
}
|
|
20
|
-
/**
|
|
20
|
+
/**
|
|
21
|
+
* Match result from find_all_matches: [offset, pattern_id].
|
|
22
|
+
*/
|
|
21
23
|
export interface MatchResult {
|
|
22
|
-
/**
|
|
24
|
+
/**
|
|
25
|
+
* Byte offset of the match within the haystack.
|
|
26
|
+
*/
|
|
23
27
|
offset: number;
|
|
24
|
-
/**
|
|
28
|
+
/**
|
|
29
|
+
* 0 = pattern 1 (delimiter), 1 = pattern 2 (quote).
|
|
30
|
+
*/
|
|
25
31
|
patternId: number;
|
|
26
32
|
}
|
|
27
33
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"wasm_module.d.ts","sourceRoot":"","sources":["../../lib/wasm_module.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAiBH,wBAAsB,cAAc,IAAI,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CAkB3E;AAED,KAAK,iBAAiB,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,KAAK,MAAM,CAAA;AAEnF,KAAK,qBAAqB,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,KAAK,MAAM,CAAA;AAE/G,KAAK,kBAAkB,GAAG,CACzB,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,EACV,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,MAAM,EACX,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,KACN,MAAM,CAAA;AAEX,MAAM,WAAW,UAAU;IAC1B,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAA;IAC5B,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAA;CAC3B;AAED,MAAM,WAAW,oBAAoB;IACpC,MAAM,EAAE,UAAU,CAAA;IAClB,aAAa,EAAE,iBAAiB,CAAA;IAChC,iBAAiB,EAAE,qBAAqB,CAAA;IACxC,cAAc,EAAE,kBAAkB,CAAA;CAClC;AAED
|
|
1
|
+
{"version":3,"file":"wasm_module.d.ts","sourceRoot":"","sources":["../../lib/wasm_module.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAiBH,wBAAsB,cAAc,IAAI,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CAkB3E;AAED,KAAK,iBAAiB,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,KAAK,MAAM,CAAA;AAEnF,KAAK,qBAAqB,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,KAAK,MAAM,CAAA;AAE/G,KAAK,kBAAkB,GAAG,CACzB,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,EACV,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,MAAM,EACX,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,KACN,MAAM,CAAA;AAEX,MAAM,WAAW,UAAU;IAC1B,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAA;IAC5B,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAA;CAC3B;AAED,MAAM,WAAW,oBAAoB;IACpC,MAAM,EAAE,UAAU,CAAA;IAClB,aAAa,EAAE,iBAAiB,CAAA;IAChC,iBAAiB,EAAE,qBAAqB,CAAA;IACxC,cAAc,EAAE,kBAAkB,CAAA;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC3B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;CACjB;AAED;;;;;GAKG;AACH,eAAO,MAAM,cAAc,MAAM,CAAA;AACjC,eAAO,MAAM,gBAAgB,OAAO,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"wasm_module.js","sourceRoot":"","sources":["../../lib/wasm_module.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAA;AAe9C,MAAM,CAAC,KAAK,UAAU,cAAc;IACnC,IAAI,CAAC,WAAW;QAAE,OAAO,IAAI,CAAA;IAE7B,IAAI,CAAC;QACJ,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAA;QACxE,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAC/C,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;QAC1D,MAAM,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAA;QAE1B,MAAM,MAAM,GAAG,CAAC,CAAC,MAAoB,CAAA;QACrC,MAAM,aAAa,GAAG,CAAC,CAAC,cAAmC,CAAA;QAC3D,MAAM,iBAAiB,GAAG,CAAC,CAAC,mBAA4C,CAAA;QACxE,MAAM,cAAc,GAAG,CAAC,CAAC,gBAAsC,CAAA;QAE/D,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,iBAAiB,EAAE,cAAc,EAAE,CAAA;IACpE,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAA;IACZ,CAAC;AACF,CAAC;
|
|
1
|
+
{"version":3,"file":"wasm_module.js","sourceRoot":"","sources":["../../lib/wasm_module.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAA;AAe9C,MAAM,CAAC,KAAK,UAAU,cAAc;IACnC,IAAI,CAAC,WAAW;QAAE,OAAO,IAAI,CAAA;IAE7B,IAAI,CAAC;QACJ,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAA;QACxE,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAC/C,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;QAC1D,MAAM,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAA;QAE1B,MAAM,MAAM,GAAG,CAAC,CAAC,MAAoB,CAAA;QACrC,MAAM,aAAa,GAAG,CAAC,CAAC,cAAmC,CAAA;QAC3D,MAAM,iBAAiB,GAAG,CAAC,CAAC,mBAA4C,CAAA;QACxE,MAAM,cAAc,GAAG,CAAC,CAAC,gBAAsC,CAAA;QAE/D,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,iBAAiB,EAAE,cAAc,EAAE,CAAA;IACpE,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAA;IACZ,CAAC;AACF,CAAC;AA0CD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,GAAG,CAAA;AACjC,MAAM,CAAC,MAAM,gBAAgB,GAAG,IAAI,CAAA"}
|
package/out/lib/writer.d.ts
CHANGED
|
@@ -3,25 +3,27 @@
|
|
|
3
3
|
* @license MIT
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
|
-
import { type
|
|
6
|
+
import { type WriteStream, type PathLike } from "node:fs";
|
|
7
7
|
/**
|
|
8
8
|
* Callback for writing a line to a newline-delimited file.
|
|
9
|
+
*
|
|
10
|
+
* The trailing newline is appended for you — pass the content alone.
|
|
9
11
|
*/
|
|
10
12
|
export interface WriteLineCallback {
|
|
11
13
|
/**
|
|
12
|
-
* Write a line to the file. The
|
|
14
|
+
* Write a line to the file, terminated with a newline. The content should not contain one itself.
|
|
13
15
|
*/
|
|
14
16
|
(content: string): Promise<void>;
|
|
15
17
|
/**
|
|
16
|
-
* Write a chunk to the file. The chunk should not contain
|
|
18
|
+
* Write a chunk to the file, terminated with a newline. The chunk should not contain one itself.
|
|
17
19
|
*/
|
|
18
20
|
(chunk: any, encoding: BufferEncoding): Promise<void>;
|
|
19
21
|
}
|
|
20
22
|
export interface NewlineWriter extends AsyncDisposable {
|
|
21
23
|
/**
|
|
22
|
-
* Write a line to the file.
|
|
24
|
+
* Write a line to the file, terminated with a newline.
|
|
23
25
|
*
|
|
24
|
-
* The content should not contain a newline character.
|
|
26
|
+
* The content should not contain a newline character — the writer supplies the delimiter.
|
|
25
27
|
*/
|
|
26
28
|
write: WriteLineCallback;
|
|
27
29
|
dispose: () => Promise<void>;
|
|
@@ -29,6 +31,15 @@ export interface NewlineWriter extends AsyncDisposable {
|
|
|
29
31
|
}
|
|
30
32
|
/**
|
|
31
33
|
* Creates a writer for newline-delimited files.
|
|
34
|
+
*
|
|
35
|
+
* Every {@linkcode NewlineWriter.write} terminates its content with `\n`, so the file is delimited (and round-trips
|
|
36
|
+
* through {@linkcode TextSpliterator}) without the caller appending anything:
|
|
37
|
+
*
|
|
38
|
+
* ```ts
|
|
39
|
+
* await using out = createNewlineWriter("rows.jsonl")
|
|
40
|
+
*
|
|
41
|
+
* for (const row of rows) await out.write(JSON.stringify(row))
|
|
42
|
+
* ```
|
|
32
43
|
*/
|
|
33
44
|
export declare function createNewlineWriter(filePath: PathLike): NewlineWriter;
|
|
34
45
|
//# sourceMappingURL=writer.d.ts.map
|
package/out/lib/writer.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../lib/writer.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAqB,KAAK,QAAQ,EAAE,
|
|
1
|
+
{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../lib/writer.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,KAAK,WAAW,EAAqB,KAAK,QAAQ,EAAE,MAAM,SAAS,CAAA;AAE5E;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IACjC;;OAEG;IACH,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAChC;;OAEG;IACH,CAAC,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CACrD;AAED,MAAM,WAAW,aAAc,SAAQ,eAAe;IACrD;;;;OAIG;IACH,KAAK,EAAE,iBAAiB,CAAA;IACxB,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;IAC5B,MAAM,EAAE,WAAW,CAAA;CACnB;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,GAAG,aAAa,CA6CrE"}
|
package/out/lib/writer.js
CHANGED
|
@@ -3,21 +3,37 @@
|
|
|
3
3
|
* @license MIT
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
|
-
import { createWriteStream
|
|
6
|
+
import { createWriteStream } from "node:fs";
|
|
7
7
|
/**
|
|
8
8
|
* Creates a writer for newline-delimited files.
|
|
9
|
+
*
|
|
10
|
+
* Every {@linkcode NewlineWriter.write} terminates its content with `\n`, so the file is delimited (and round-trips
|
|
11
|
+
* through {@linkcode TextSpliterator}) without the caller appending anything:
|
|
12
|
+
*
|
|
13
|
+
* ```ts
|
|
14
|
+
* await using out = createNewlineWriter("rows.jsonl")
|
|
15
|
+
*
|
|
16
|
+
* for (const row of rows) await out.write(JSON.stringify(row))
|
|
17
|
+
* ```
|
|
9
18
|
*/
|
|
10
19
|
export function createNewlineWriter(filePath) {
|
|
11
20
|
const writer = createWriteStream(filePath);
|
|
12
|
-
const write = (content) => {
|
|
21
|
+
const write = (content, encoding) => {
|
|
13
22
|
return new Promise((resolve, reject) => {
|
|
14
|
-
|
|
23
|
+
const settle = (error) => {
|
|
15
24
|
if (error) {
|
|
16
25
|
reject(error);
|
|
17
26
|
return;
|
|
18
27
|
}
|
|
19
28
|
resolve();
|
|
20
|
-
}
|
|
29
|
+
};
|
|
30
|
+
// The delimiter goes out as its own chunk rather than concatenated onto `content`: the
|
|
31
|
+
// two-argument overload accepts a Buffer, and `buffer + "\n"` would stringify it via
|
|
32
|
+
// `toString()` and corrupt the bytes. Stream writes are ordered, so the newline always
|
|
33
|
+
// follows its content, and settling on the delimiter's callback resolves once the whole
|
|
34
|
+
// line has flushed.
|
|
35
|
+
writer.write(content, encoding ?? "utf8");
|
|
36
|
+
writer.write("\n", "utf8", settle);
|
|
21
37
|
});
|
|
22
38
|
};
|
|
23
39
|
const dispose = () => {
|
package/out/lib/writer.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"writer.js","sourceRoot":"","sources":["../../lib/writer.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,
|
|
1
|
+
{"version":3,"file":"writer.js","sourceRoot":"","sources":["../../lib/writer.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAoB,iBAAiB,EAAiB,MAAM,SAAS,CAAA;AA6B5E;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,mBAAmB,CAAC,QAAkB;IACrD,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAA;IAE1C,MAAM,KAAK,GAAsB,CAAC,OAAY,EAAE,QAAyB,EAAE,EAAE;QAC5E,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACtC,MAAM,MAAM,GAAG,CAAC,KAA+B,EAAQ,EAAE;gBACxD,IAAI,KAAK,EAAE,CAAC;oBACX,MAAM,CAAC,KAAK,CAAC,CAAA;oBAEb,OAAM;gBACP,CAAC;gBAED,OAAO,EAAE,CAAA;YACV,CAAC,CAAA;YAED,uFAAuF;YACvF,qFAAqF;YACrF,uFAAuF;YACvF,wFAAwF;YACxF,oBAAoB;YACpB,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,QAAQ,IAAI,MAAM,CAAC,CAAA;YACzC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAA;QACnC,CAAC,CAAC,CAAA;IACH,CAAC,CAAA;IAED,MAAM,OAAO,GAAG,GAAG,EAAE;QACpB,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC5C,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gBACtB,IAAI,KAAK,EAAE,CAAC;oBACX,MAAM,CAAC,KAAK,CAAC,CAAA;oBAEb,OAAM;gBACP,CAAC;gBAED,OAAO,EAAE,CAAA;YACV,CAAC,CAAC,CAAA;QACH,CAAC,CAAC,CAAA;IACH,CAAC,CAAA;IAED,OAAO;QACN,KAAK;QACL,MAAM;QACN,OAAO;QACP,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,OAAO;KAC9B,CAAA;AACF,CAAC"}
|
|
@@ -2,43 +2,8 @@
|
|
|
2
2
|
* @copyright Sister Software
|
|
3
3
|
* @license MIT
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
|
-
* @file
|
|
5
|
+
* @file Command to split a CSV file into JSONL format.
|
|
6
6
|
*/
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
export declare const command = "csv [source] [destination]";
|
|
10
|
-
export declare const describe = false;
|
|
11
|
-
export declare const builder: (argv: Argv) => Argv<{
|
|
12
|
-
split: string;
|
|
13
|
-
} & {
|
|
14
|
-
join: string;
|
|
15
|
-
} & {
|
|
16
|
-
"skip-empty": boolean;
|
|
17
|
-
} & {
|
|
18
|
-
take: number | undefined;
|
|
19
|
-
} & {
|
|
20
|
-
drop: number;
|
|
21
|
-
} & {
|
|
22
|
-
debug: boolean;
|
|
23
|
-
} & {
|
|
24
|
-
"reader-high-water-mark": number;
|
|
25
|
-
} & {
|
|
26
|
-
"writer-high-water-mark": number;
|
|
27
|
-
} & {
|
|
28
|
-
filter: string | undefined;
|
|
29
|
-
} & {
|
|
30
|
-
source: unknown;
|
|
31
|
-
} & {
|
|
32
|
-
destination: unknown;
|
|
33
|
-
} & {
|
|
34
|
-
header: boolean;
|
|
35
|
-
} & {
|
|
36
|
-
transformers: string | undefined;
|
|
37
|
-
} & {
|
|
38
|
-
"column-delimiter": string;
|
|
39
|
-
} & {
|
|
40
|
-
mode: string;
|
|
41
|
-
}>;
|
|
42
|
-
export type CSVCommandArgs = PluckArgv<typeof builder>;
|
|
43
|
-
export declare const handler: (argv: ArgumentsCamelCase<CSVCommandArgs>) => Promise<void>;
|
|
7
|
+
export declare const help: string;
|
|
8
|
+
export declare function run(args: string[]): Promise<void>;
|
|
44
9
|
//# sourceMappingURL=csv.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"csv.d.ts","sourceRoot":"","sources":["../../../../node/cli/commands/csv.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;
|
|
1
|
+
{"version":3,"file":"csv.d.ts","sourceRoot":"","sources":["../../../../node/cli/commands/csv.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAmBH,eAAO,MAAM,IAAI,QAWL,CAAA;AAEZ,wBAAsB,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CA4EvD"}
|
|
@@ -2,70 +2,78 @@
|
|
|
2
2
|
* @copyright Sister Software
|
|
3
3
|
* @license MIT
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
|
-
* @file
|
|
5
|
+
* @file Command to split a CSV file into JSONL format.
|
|
6
6
|
*/
|
|
7
7
|
import { resolve as resolvePath } from "node:path";
|
|
8
|
+
import { parseArgs } from "node:util";
|
|
8
9
|
import { CharacterSequence, CSVSpliterator, } from "spliterator";
|
|
9
10
|
import { createFileWritableStream, createReadStream } from "spliterator/node/fs";
|
|
10
|
-
import {} from "
|
|
11
|
-
|
|
12
|
-
export const
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
choices: ["object", "array"],
|
|
37
|
-
default: "object",
|
|
38
|
-
string: true,
|
|
11
|
+
import { commonOptions, commonOptionsHelp, resolveIO, toNumber, usageError } from "../utils.js";
|
|
12
|
+
const MODES = ["object", "array"];
|
|
13
|
+
export const help = [
|
|
14
|
+
"Split a CSV file into JSONL format.",
|
|
15
|
+
"",
|
|
16
|
+
"Usage: spliterator csv [source] [destination] [options]",
|
|
17
|
+
"",
|
|
18
|
+
"Options:",
|
|
19
|
+
commonOptionsHelp,
|
|
20
|
+
" -H, --header Whether the CSV file has a header (default: true, disable with --no-header)",
|
|
21
|
+
" -T, --transformers <path> Path to JS file exporting transformer functions",
|
|
22
|
+
" -c, --column-delimiter <str> Delimiter to split columns on (default: ,)",
|
|
23
|
+
` -m, --mode <mode> Output mode: ${MODES.join(" | ")} (default: object)`,
|
|
24
|
+
].join("\n");
|
|
25
|
+
export async function run(args) {
|
|
26
|
+
const { values, positionals } = parseArgs({
|
|
27
|
+
args,
|
|
28
|
+
allowPositionals: true,
|
|
29
|
+
allowNegative: true,
|
|
30
|
+
options: {
|
|
31
|
+
...commonOptions,
|
|
32
|
+
header: { type: "boolean", short: "H", default: true },
|
|
33
|
+
transformers: { type: "string", short: "T" },
|
|
34
|
+
"column-delimiter": { type: "string", short: "c", default: "," },
|
|
35
|
+
mode: { type: "string", short: "m", default: "object" },
|
|
36
|
+
},
|
|
39
37
|
});
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
38
|
+
if (values.help) {
|
|
39
|
+
console.log(help);
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
const [source, destination] = resolveIO(positionals, values);
|
|
43
|
+
if (!MODES.includes(values.mode)) {
|
|
44
|
+
usageError(`Option --mode expects one of: ${MODES.join(", ")}.`);
|
|
45
|
+
}
|
|
46
|
+
const take = toNumber("take", values.take);
|
|
47
|
+
const drop = toNumber("drop", values.drop);
|
|
48
|
+
const readerHighWaterMark = toNumber("reader-high-water-mark", values["reader-high-water-mark"]);
|
|
49
|
+
const writerHighWaterMark = toNumber("writer-high-water-mark", values["writer-high-water-mark"]);
|
|
50
|
+
const joinDelimiter = new CharacterSequence(values.join).decode();
|
|
43
51
|
let transformers = {};
|
|
44
|
-
if (
|
|
45
|
-
transformers = await import(resolvePath(
|
|
52
|
+
if (values.transformers) {
|
|
53
|
+
transformers = await import(resolvePath(values.transformers));
|
|
46
54
|
}
|
|
47
55
|
let filter = () => true;
|
|
48
|
-
if (
|
|
49
|
-
const module = await import(resolvePath(
|
|
56
|
+
if (values.filter) {
|
|
57
|
+
const module = await import(resolvePath(values.filter));
|
|
50
58
|
filter = module.default;
|
|
51
59
|
}
|
|
52
60
|
const [readStream, writeStream] = await Promise.all([
|
|
53
|
-
createReadStream(
|
|
54
|
-
createFileWritableStream(
|
|
61
|
+
createReadStream(source, readerHighWaterMark),
|
|
62
|
+
createFileWritableStream(destination, {
|
|
55
63
|
encoding: "utf8",
|
|
56
|
-
highWaterMark:
|
|
64
|
+
highWaterMark: writerHighWaterMark,
|
|
57
65
|
}),
|
|
58
66
|
]);
|
|
59
67
|
const writer = writeStream.getWriter();
|
|
60
68
|
const spliterator = CSVSpliterator.fromAsync(readStream, {
|
|
61
|
-
mode:
|
|
62
|
-
delimiter:
|
|
69
|
+
mode: values.mode,
|
|
70
|
+
delimiter: values.split,
|
|
63
71
|
autoDispose: true,
|
|
64
|
-
header:
|
|
72
|
+
header: values.header,
|
|
65
73
|
transformers,
|
|
66
|
-
columnDelimiter:
|
|
67
|
-
take
|
|
68
|
-
drop
|
|
74
|
+
columnDelimiter: values["column-delimiter"],
|
|
75
|
+
take,
|
|
76
|
+
drop,
|
|
69
77
|
});
|
|
70
78
|
for await (const row of spliterator) {
|
|
71
79
|
const emit = await filter(row);
|
|
@@ -75,5 +83,5 @@ export const handler = async (argv) => {
|
|
|
75
83
|
await writer.write(joinDelimiter);
|
|
76
84
|
}
|
|
77
85
|
await writer.close();
|
|
78
|
-
}
|
|
86
|
+
}
|
|
79
87
|
//# sourceMappingURL=csv.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"csv.js","sourceRoot":"","sources":["../../../../node/cli/commands/csv.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,OAAO,IAAI,WAAW,EAAE,MAAM,WAAW,CAAA;
|
|
1
|
+
{"version":3,"file":"csv.js","sourceRoot":"","sources":["../../../../node/cli/commands/csv.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,OAAO,IAAI,WAAW,EAAE,MAAM,WAAW,CAAA;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AAErC,OAAO,EACN,iBAAiB,EACjB,cAAc,GAKd,MAAM,aAAa,CAAA;AACpB,OAAO,EAAE,wBAAwB,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAA;AAEhF,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAA0B,MAAM,aAAa,CAAA;AAEvH,MAAM,KAAK,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAoC,CAAA;AAEpE,MAAM,CAAC,MAAM,IAAI,GAAG;IACnB,qCAAqC;IACrC,EAAE;IACF,yDAAyD;IACzD,EAAE;IACF,UAAU;IACV,iBAAiB;IACjB,6GAA6G;IAC7G,iFAAiF;IACjF,4EAA4E;IAC5E,gDAAgD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,oBAAoB;CACrF,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AAEZ,MAAM,CAAC,KAAK,UAAU,GAAG,CAAC,IAAc;IACvC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,SAAS,CAAC;QACzC,IAAI;QACJ,gBAAgB,EAAE,IAAI;QACtB,aAAa,EAAE,IAAI;QACnB,OAAO,EAAE;YACR,GAAG,aAAa;YAChB,MAAM,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE;YACtD,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE;YAC5C,kBAAkB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE;YAChE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE;SACvD;KACD,CAAC,CAAA;IAEF,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QACjB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAEjB,OAAM;IACP,CAAC;IAED,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,SAAS,CAAC,WAAW,EAAE,MAAM,CAAC,CAAA;IAE5D,IAAI,CAAE,KAA2B,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QACzD,UAAU,CAAC,iCAAiC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACjE,CAAC;IAED,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAA;IAC1C,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAA;IAC1C,MAAM,mBAAmB,GAAG,QAAQ,CAAC,wBAAwB,EAAE,MAAM,CAAC,wBAAwB,CAAC,CAAE,CAAA;IACjG,MAAM,mBAAmB,GAAG,QAAQ,CAAC,wBAAwB,EAAE,MAAM,CAAC,wBAAwB,CAAC,CAAC,CAAA;IAEhG,MAAM,aAAa,GAAG,IAAI,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAA;IACjE,IAAI,YAAY,GAAyB,EAAE,CAAA;IAE3C,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACzB,YAAY,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAA;IAC9D,CAAC;IAED,IAAI,MAAM,GAAsB,GAAG,EAAE,CAAC,IAAI,CAAA;IAE1C,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAA;QACvD,MAAM,GAAG,MAAM,CAAC,OAAO,CAAA;IACxB,CAAC;IAED,MAAM,CAAC,UAAU,EAAE,WAAW,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACnD,gBAAgB,CAAC,MAAM,EAAE,mBAAmB,CAAC;QAC7C,wBAAwB,CAAC,WAAW,EAAE;YACrC,QAAQ,EAAE,MAAM;YAChB,aAAa,EAAE,mBAAmB;SAClC,CAAC;KACF,CAAC,CAAA;IAEF,MAAM,MAAM,GAAG,WAAW,CAAC,SAAS,EAAE,CAAA;IAEtC,MAAM,WAAW,GAAG,cAAc,CAAC,SAAS,CAAC,UAAU,EAAE;QACxD,IAAI,EAAE,MAAM,CAAC,IAAqB;QAClC,SAAS,EAAE,MAAM,CAAC,KAAK;QACvB,WAAW,EAAE,IAAI;QACjB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,YAAY;QACZ,eAAe,EAAE,MAAM,CAAC,kBAAkB,CAAC;QAC3C,IAAI;QACJ,IAAI;KACgD,CAAC,CAAA;IAEtD,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,CAAA;QAE9B,IAAI,CAAC,IAAI;YAAE,SAAQ;QAEnB,MAAM,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAA;QACvC,MAAM,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAA;IAClC,CAAC;IAED,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;AACrB,CAAC"}
|