spliterator 6.1.0 → 6.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -0
- package/out/index.d.ts +26 -25
- package/out/index.d.ts.map +1 -1
- package/out/index.js +26 -25
- package/out/index.js.map +1 -1
- package/out/lib/AsyncSpliterator.d.ts.map +1 -1
- package/out/lib/AsyncSpliterator.js +28 -0
- package/out/lib/AsyncSpliterator.js.map +1 -1
- package/out/lib/CharacterSequence.d.ts +16 -1
- package/out/lib/CharacterSequence.d.ts.map +1 -1
- package/out/lib/CharacterSequence.js +37 -0
- package/out/lib/CharacterSequence.js.map +1 -1
- package/out/lib/core/AsyncSpliterator.d.ts +227 -0
- package/out/lib/core/AsyncSpliterator.d.ts.map +1 -0
- package/out/lib/core/AsyncSpliterator.js +510 -0
- package/out/lib/core/AsyncSpliterator.js.map +1 -0
- package/out/lib/core/BufferController.d.ts +77 -0
- package/out/lib/core/BufferController.d.ts.map +1 -0
- package/out/lib/core/BufferController.js +118 -0
- package/out/lib/core/BufferController.js.map +1 -0
- package/out/lib/core/CharacterSequence.d.ts +72 -0
- package/out/lib/core/CharacterSequence.d.ts.map +1 -0
- package/out/lib/core/CharacterSequence.js +347 -0
- package/out/lib/core/CharacterSequence.js.map +1 -0
- package/out/lib/core/CompositeDataView.d.ts +111 -0
- package/out/lib/core/CompositeDataView.d.ts.map +1 -0
- package/out/lib/core/CompositeDataView.js +241 -0
- package/out/lib/core/CompositeDataView.js.map +1 -0
- package/out/lib/core/IndexQueue.d.ts +48 -0
- package/out/lib/core/IndexQueue.d.ts.map +1 -0
- package/out/lib/core/IndexQueue.js +111 -0
- package/out/lib/core/IndexQueue.js.map +1 -0
- package/out/lib/core/SlidingWindow.d.ts +47 -0
- package/out/lib/core/SlidingWindow.d.ts.map +1 -0
- package/out/lib/core/SlidingWindow.js +63 -0
- package/out/lib/core/SlidingWindow.js.map +1 -0
- package/out/lib/core/Spliterator.d.ts +96 -0
- package/out/lib/core/Spliterator.d.ts.map +1 -0
- package/out/lib/core/Spliterator.js +330 -0
- package/out/lib/core/Spliterator.js.map +1 -0
- package/out/lib/core/wasm_base64.d.ts +19 -0
- package/out/lib/core/wasm_base64.d.ts.map +1 -0
- package/out/lib/core/wasm_base64.js +19 -0
- package/out/lib/core/wasm_base64.js.map +1 -0
- package/out/lib/core/wasm_module.d.ts +57 -0
- package/out/lib/core/wasm_module.d.ts.map +1 -0
- package/out/lib/core/wasm_module.js +34 -0
- package/out/lib/core/wasm_module.js.map +1 -0
- package/out/lib/formats/CSVSpliterator.d.ts +131 -0
- package/out/lib/formats/CSVSpliterator.d.ts.map +1 -0
- package/out/lib/formats/CSVSpliterator.js +172 -0
- package/out/lib/formats/CSVSpliterator.js.map +1 -0
- package/out/lib/formats/JSONSpliterator.d.ts +44 -0
- package/out/lib/formats/JSONSpliterator.d.ts.map +1 -0
- package/out/lib/formats/JSONSpliterator.js +77 -0
- package/out/lib/formats/JSONSpliterator.js.map +1 -0
- package/out/lib/formats/PSVSpliterator.d.ts +21 -0
- package/out/lib/formats/PSVSpliterator.d.ts.map +1 -0
- package/out/lib/formats/PSVSpliterator.js +22 -0
- package/out/lib/formats/PSVSpliterator.js.map +1 -0
- package/out/lib/formats/TSVSpliterator.d.ts +21 -0
- package/out/lib/formats/TSVSpliterator.d.ts.map +1 -0
- package/out/lib/formats/TSVSpliterator.js +22 -0
- package/out/lib/formats/TSVSpliterator.js.map +1 -0
- package/out/lib/formats/TextSpliterator.d.ts +53 -0
- package/out/lib/formats/TextSpliterator.d.ts.map +1 -0
- package/out/lib/formats/TextSpliterator.js +67 -0
- package/out/lib/formats/TextSpliterator.js.map +1 -0
- package/out/lib/formats/XLSXSpliterator.d.ts +143 -0
- package/out/lib/formats/XLSXSpliterator.d.ts.map +1 -0
- package/out/lib/formats/XLSXSpliterator.js +161 -0
- package/out/lib/formats/XLSXSpliterator.js.map +1 -0
- package/out/lib/formats/casing.d.ts +27 -0
- package/out/lib/formats/casing.d.ts.map +1 -0
- package/out/lib/formats/casing.js +68 -0
- package/out/lib/formats/casing.js.map +1 -0
- package/out/lib/formats/row-emitters.d.ts +35 -0
- package/out/lib/formats/row-emitters.d.ts.map +1 -0
- package/out/lib/formats/row-emitters.js +50 -0
- package/out/lib/formats/row-emitters.js.map +1 -0
- package/out/lib/internal/shared.d.ts +155 -0
- package/out/lib/internal/shared.d.ts.map +1 -0
- package/out/lib/internal/shared.js +41 -0
- package/out/lib/internal/shared.js.map +1 -0
- package/out/lib/internal/utils.d.ts +37 -0
- package/out/lib/internal/utils.d.ts.map +1 -0
- package/out/lib/internal/utils.js +39 -0
- package/out/lib/internal/utils.js.map +1 -0
- package/out/lib/io/adaptive-source.d.ts +39 -0
- package/out/lib/io/adaptive-source.d.ts.map +1 -0
- package/out/lib/io/adaptive-source.js +95 -0
- package/out/lib/io/adaptive-source.js.map +1 -0
- package/out/lib/io/writer.d.ts +45 -0
- package/out/lib/io/writer.d.ts.map +1 -0
- package/out/lib/io/writer.js +57 -0
- package/out/lib/io/writer.js.map +1 -0
- package/out/lib/iterators/AsyncSequence.d.ts +180 -0
- package/out/lib/iterators/AsyncSequence.d.ts.map +1 -0
- package/out/lib/iterators/AsyncSequence.js +432 -0
- package/out/lib/iterators/AsyncSequence.js.map +1 -0
- package/out/lib/iterators/chunks.d.ts +20 -0
- package/out/lib/iterators/chunks.d.ts.map +1 -0
- package/out/lib/iterators/chunks.js +36 -0
- package/out/lib/iterators/chunks.js.map +1 -0
- package/out/lib/iterators/pivot.d.ts +30 -0
- package/out/lib/iterators/pivot.d.ts.map +1 -0
- package/out/lib/iterators/pivot.js +47 -0
- package/out/lib/iterators/pivot.js.map +1 -0
- package/out/lib/iterators/zip.d.ts +33 -0
- package/out/lib/iterators/zip.d.ts.map +1 -0
- package/out/lib/iterators/zip.js +59 -0
- package/out/lib/iterators/zip.js.map +1 -0
- package/out/lib/parallel/merge-async-iterators.d.ts +12 -0
- package/out/lib/parallel/merge-async-iterators.d.ts.map +1 -0
- package/out/lib/parallel/merge-async-iterators.js +39 -0
- package/out/lib/parallel/merge-async-iterators.js.map +1 -0
- package/out/lib/parallel/parallel-map-runtime.d.ts +22 -0
- package/out/lib/parallel/parallel-map-runtime.d.ts.map +1 -0
- package/out/lib/parallel/parallel-map-runtime.js +60 -0
- package/out/lib/parallel/parallel-map-runtime.js.map +1 -0
- package/out/lib/parallel/parallel-map-worker-entry.d.ts +11 -0
- package/out/lib/parallel/parallel-map-worker-entry.d.ts.map +1 -0
- package/out/lib/parallel/parallel-map-worker-entry.js +41 -0
- package/out/lib/parallel/parallel-map-worker-entry.js.map +1 -0
- package/out/lib/parallel/parallel-map-workers.d.ts +57 -0
- package/out/lib/parallel/parallel-map-workers.d.ts.map +1 -0
- package/out/lib/parallel/parallel-map-workers.js +82 -0
- package/out/lib/parallel/parallel-map-workers.js.map +1 -0
- package/out/lib/parallel/parallel-map.d.ts +28 -0
- package/out/lib/parallel/parallel-map.d.ts.map +1 -0
- package/out/lib/parallel/parallel-map.js +30 -0
- package/out/lib/parallel/parallel-map.js.map +1 -0
- package/out/lib/parallel/segment-runtime.d.ts +27 -0
- package/out/lib/parallel/segment-runtime.d.ts.map +1 -0
- package/out/lib/parallel/segment-runtime.js +41 -0
- package/out/lib/parallel/segment-runtime.js.map +1 -0
- package/out/lib/parallel/segment-worker-entry.d.ts +10 -0
- package/out/lib/parallel/segment-worker-entry.d.ts.map +1 -0
- package/out/lib/parallel/segment-worker-entry.js +59 -0
- package/out/lib/parallel/segment-worker-entry.js.map +1 -0
- package/out/lib/parallel/segment-workers.d.ts +56 -0
- package/out/lib/parallel/segment-workers.d.ts.map +1 -0
- package/out/lib/parallel/segment-workers.js +110 -0
- package/out/lib/parallel/segment-workers.js.map +1 -0
- package/out/lib/parallel/segments.d.ts +31 -0
- package/out/lib/parallel/segments.d.ts.map +1 -0
- package/out/lib/parallel/segments.js +52 -0
- package/out/lib/parallel/segments.js.map +1 -0
- package/out/lib/wasm_base64.d.ts +4 -1
- package/out/lib/wasm_base64.d.ts.map +1 -1
- package/out/lib/wasm_base64.js +4 -1
- package/out/lib/wasm_base64.js.map +1 -1
- package/out/lib/wasm_module.d.ts +15 -0
- package/out/lib/wasm_module.d.ts.map +1 -1
- package/out/lib/wasm_module.js +2 -1
- package/out/lib/wasm_module.js.map +1 -1
- package/out/node/cli/commands/csv.d.ts +113 -0
- package/out/node/cli/commands/csv.d.ts.map +1 -1
- package/out/node/cli/commands/csv.js +34 -36
- package/out/node/cli/commands/csv.js.map +1 -1
- package/out/node/cli/commands/iterate.d.ts +92 -0
- package/out/node/cli/commands/iterate.d.ts.map +1 -1
- package/out/node/cli/commands/iterate.js +26 -28
- package/out/node/cli/commands/iterate.js.map +1 -1
- package/out/node/cli/commands/parallel.d.ts +102 -0
- package/out/node/cli/commands/parallel.d.ts.map +1 -0
- package/out/node/cli/commands/parallel.js +426 -0
- package/out/node/cli/commands/parallel.js.map +1 -0
- package/out/node/cli/index.js +5 -0
- package/out/node/cli/index.js.map +1 -1
- package/out/node/cli/spec.d.ts +51 -0
- package/out/node/cli/spec.d.ts.map +1 -0
- package/out/node/cli/spec.js +78 -0
- package/out/node/cli/spec.js.map +1 -0
- package/out/node/cli/utils.d.ts +36 -18
- package/out/node/cli/utils.d.ts.map +1 -1
- package/out/node/cli/utils.js +48 -39
- package/out/node/cli/utils.js.map +1 -1
- package/out/node/fs/index.d.ts +1 -1
- package/out/node/fs/index.d.ts.map +1 -1
- package/out/node/fs/index.js +1 -1
- package/out/node/fs/index.js.map +1 -1
- package/package.json +6 -6
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license MIT
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
import { AsyncSequence } from "../iterators/AsyncSequence.js";
|
|
7
|
+
/**
|
|
8
|
+
* Map an iterable through a callback with up to `concurrency` calls in flight, yielding in **completion order** — not
|
|
9
|
+
* input order.
|
|
10
|
+
*
|
|
11
|
+
* The callback is a closure, so every call runs on the caller's thread. This overlaps _latency_ — file reads, network
|
|
12
|
+
* round-trips, anything that spends its time waiting — and does nothing for CPU-bound work, which still occupies the
|
|
13
|
+
* one thread it always did.
|
|
14
|
+
*
|
|
15
|
+
* **Concurrency is not core count.** Work that contends for a shared resource (one disk, one socket pool, one on-disk
|
|
16
|
+
* database) peaks _low_ — often ~2–3 — and then degrades. Sweep it.
|
|
17
|
+
*
|
|
18
|
+
* @param source The collection to map.
|
|
19
|
+
* @param fn The callback, receiving `(value, counter)`.
|
|
20
|
+
* @param options Concurrency ceiling and optional abort signal.
|
|
21
|
+
*
|
|
22
|
+
* @yields Each result, in completion order.
|
|
23
|
+
* @see {@linkcode parallelMapWorkers} to run the handler on worker threads instead — it takes a module path rather than
|
|
24
|
+
* a closure, because a closure cannot cross a thread boundary.
|
|
25
|
+
* @see The "Choosing a primitive" table in the README, which keys the decision off per-row work.
|
|
26
|
+
*/
|
|
27
|
+
export function parallelMap(source, fn, options) {
|
|
28
|
+
return AsyncSequence.from(source).parallelMap(fn, options);
|
|
29
|
+
}
|
|
30
|
+
//# sourceMappingURL=parallel-map.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parallel-map.js","sourceRoot":"","sources":["../../../lib/parallel/parallel-map.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,aAAa,EAAmC,MAAM,+BAA+B,CAAA;AAE9F;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,UAAU,WAAW,CAC1B,MAAsC,EACtC,EAAqD,EACrD,OAAmC;IAEnC,OAAO,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,WAAW,CAAC,EAAE,EAAE,OAAO,CAAC,CAAA;AAC3D,CAAC"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license MIT
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
export type SegmentHandler = (record: Uint8Array, ctx: {
|
|
7
|
+
index: number;
|
|
8
|
+
segmentIndex: number;
|
|
9
|
+
}) => unknown | Promise<unknown>;
|
|
10
|
+
export interface RunSegmentIO {
|
|
11
|
+
records: AsyncIterable<Uint8Array>;
|
|
12
|
+
handleRecord: SegmentHandler;
|
|
13
|
+
segmentIndex: number;
|
|
14
|
+
batchSize: number;
|
|
15
|
+
maxInFlight: number;
|
|
16
|
+
post: (batch: unknown[], transfer: ArrayBuffer[]) => void;
|
|
17
|
+
waitForAck: () => Promise<void>;
|
|
18
|
+
inFlight: () => number;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Drive one segment: read records, run the handler, accumulate results into `batchSize` batches, transfer `Uint8Array`
|
|
22
|
+
* result buffers zero-copy, and respect a bounded in-flight window by awaiting an ack before exceeding `maxInFlight`
|
|
23
|
+
* outstanding batches. Results of `undefined` are skipped (the filter case). Transport-agnostic so it unit-tests
|
|
24
|
+
* without `worker_threads`.
|
|
25
|
+
*/
|
|
26
|
+
export declare function runSegment(io: RunSegmentIO): Promise<void>;
|
|
27
|
+
//# sourceMappingURL=segment-runtime.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"segment-runtime.d.ts","sourceRoot":"","sources":["../../../lib/parallel/segment-runtime.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,MAAM,cAAc,GAAG,CAC5B,MAAM,EAAE,UAAU,EAClB,GAAG,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,KACxC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;AAE/B,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;IAClC,YAAY,EAAE,cAAc,CAAA;IAC5B,YAAY,EAAE,MAAM,CAAA;IACpB,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;IACnB,IAAI,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,IAAI,CAAA;IACzD,UAAU,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;IAC/B,QAAQ,EAAE,MAAM,MAAM,CAAA;CACtB;AAED;;;;;GAKG;AACH,wBAAsB,UAAU,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAoChE"}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license MIT
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* Drive one segment: read records, run the handler, accumulate results into `batchSize` batches, transfer `Uint8Array`
|
|
8
|
+
* result buffers zero-copy, and respect a bounded in-flight window by awaiting an ack before exceeding `maxInFlight`
|
|
9
|
+
* outstanding batches. Results of `undefined` are skipped (the filter case). Transport-agnostic so it unit-tests
|
|
10
|
+
* without `worker_threads`.
|
|
11
|
+
*/
|
|
12
|
+
export async function runSegment(io) {
|
|
13
|
+
let batch = [];
|
|
14
|
+
let transfer = [];
|
|
15
|
+
let index = 0;
|
|
16
|
+
const flush = async () => {
|
|
17
|
+
if (!batch.length)
|
|
18
|
+
return;
|
|
19
|
+
while (io.inFlight() >= io.maxInFlight) {
|
|
20
|
+
await io.waitForAck();
|
|
21
|
+
}
|
|
22
|
+
io.post(batch, transfer);
|
|
23
|
+
batch = [];
|
|
24
|
+
transfer = [];
|
|
25
|
+
};
|
|
26
|
+
for await (const record of io.records) {
|
|
27
|
+
const result = await io.handleRecord(record, { index, segmentIndex: io.segmentIndex });
|
|
28
|
+
index++;
|
|
29
|
+
if (result === undefined)
|
|
30
|
+
continue;
|
|
31
|
+
batch.push(result);
|
|
32
|
+
if (result instanceof Uint8Array) {
|
|
33
|
+
transfer.push(result.buffer);
|
|
34
|
+
}
|
|
35
|
+
if (batch.length >= io.batchSize) {
|
|
36
|
+
await flush();
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
await flush();
|
|
40
|
+
}
|
|
41
|
+
//# sourceMappingURL=segment-runtime.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"segment-runtime.js","sourceRoot":"","sources":["../../../lib/parallel/segment-runtime.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAkBH;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,EAAgB;IAChD,IAAI,KAAK,GAAc,EAAE,CAAA;IACzB,IAAI,QAAQ,GAAkB,EAAE,CAAA;IAChC,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,MAAM,KAAK,GAAG,KAAK,IAAmB,EAAE;QACvC,IAAI,CAAC,KAAK,CAAC,MAAM;YAAE,OAAM;QAEzB,OAAO,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YACxC,MAAM,EAAE,CAAC,UAAU,EAAE,CAAA;QACtB,CAAC;QAED,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;QACxB,KAAK,GAAG,EAAE,CAAA;QACV,QAAQ,GAAG,EAAE,CAAA;IACd,CAAC,CAAA;IAED,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,YAAY,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,CAAC,YAAY,EAAE,CAAC,CAAA;QAEtF,KAAK,EAAE,CAAA;QAEP,IAAI,MAAM,KAAK,SAAS;YAAE,SAAQ;QAElC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAElB,IAAI,MAAM,YAAY,UAAU,EAAE,CAAC;YAClC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,MAAqB,CAAC,CAAA;QAC5C,CAAC;QAED,IAAI,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC,SAAS,EAAE,CAAC;YAClC,MAAM,KAAK,EAAE,CAAA;QACd,CAAC;IACF,CAAC;IAED,MAAM,KAAK,EAAE,CAAA;AACd,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license MIT
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Runs inside a worker thread (spawned by runSegmentWorkers). Reads its segment via its own handle,
|
|
7
|
+
* runs the user handler per record, and posts batched results to the parent with ack backpressure.
|
|
8
|
+
*/
|
|
9
|
+
export {};
|
|
10
|
+
//# sourceMappingURL=segment-worker-entry.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"segment-worker-entry.d.ts","sourceRoot":"","sources":["../../../lib/parallel/segment-worker-entry.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG"}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license MIT
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Runs inside a worker thread (spawned by runSegmentWorkers). Reads its segment via its own handle,
|
|
7
|
+
* runs the user handler per record, and posts batched results to the parent with ack backpressure.
|
|
8
|
+
*/
|
|
9
|
+
import { parentPort, workerData } from "node:worker_threads";
|
|
10
|
+
import { AsyncSpliterator } from "../core/AsyncSpliterator.js";
|
|
11
|
+
import { runSegment } from "./segment-runtime.js";
|
|
12
|
+
async function main() {
|
|
13
|
+
const data = workerData;
|
|
14
|
+
const port = parentPort;
|
|
15
|
+
// Ack backpressure: the parent posts `"ack"` per consumed batch; we track outstanding batches.
|
|
16
|
+
let acked = 0;
|
|
17
|
+
let posted = 0;
|
|
18
|
+
let wakeAck;
|
|
19
|
+
port.on("message", (msg) => {
|
|
20
|
+
if (msg === "ack") {
|
|
21
|
+
acked++;
|
|
22
|
+
wakeAck?.();
|
|
23
|
+
wakeAck = undefined;
|
|
24
|
+
}
|
|
25
|
+
});
|
|
26
|
+
const mod = (await import(data.handlerUrl));
|
|
27
|
+
const handleRecord = mod.handleRecord ?? mod.default;
|
|
28
|
+
if (typeof handleRecord !== "function") {
|
|
29
|
+
port.postMessage({ type: "error", message: `Worker module ${data.handlerUrl} has no handleRecord export.` });
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
const { createChunkIterator } = await import("spliterator/node/fs");
|
|
33
|
+
const chunkIterator = await createChunkIterator(data.source, { start: data.start, end: data.end - 1 });
|
|
34
|
+
const records = new AsyncSpliterator(chunkIterator, { delimiter: data.delimiter, autoDispose: true });
|
|
35
|
+
return runSegment({
|
|
36
|
+
records,
|
|
37
|
+
handleRecord,
|
|
38
|
+
segmentIndex: data.segmentIndex,
|
|
39
|
+
batchSize: data.batchSize,
|
|
40
|
+
maxInFlight: data.maxInFlight,
|
|
41
|
+
post: (batch, transfer) => {
|
|
42
|
+
posted++;
|
|
43
|
+
port.postMessage({ type: "batch", records: batch }, transfer);
|
|
44
|
+
},
|
|
45
|
+
waitForAck: () => new Promise((resolve) => {
|
|
46
|
+
wakeAck = resolve;
|
|
47
|
+
}),
|
|
48
|
+
inFlight: () => posted - acked,
|
|
49
|
+
})
|
|
50
|
+
.then(() => {
|
|
51
|
+
port.postMessage({ type: "done" });
|
|
52
|
+
return void 0;
|
|
53
|
+
})
|
|
54
|
+
.catch((error) => {
|
|
55
|
+
port.postMessage({ type: "error", message: error instanceof Error ? error.message : String(error) });
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
void main();
|
|
59
|
+
//# sourceMappingURL=segment-worker-entry.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"segment-worker-entry.js","sourceRoot":"","sources":["../../../lib/parallel/segment-worker-entry.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AAE5D,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAA;AAC9D,OAAO,EAAE,UAAU,EAAuB,MAAM,sBAAsB,CAAA;AActE,KAAK,UAAU,IAAI;IAClB,MAAM,IAAI,GAAG,UAAwB,CAAA;IACrC,MAAM,IAAI,GAAG,UAAW,CAAA;IAExB,+FAA+F;IAC/F,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,OAAiC,CAAA;IAErC,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAY,EAAE,EAAE;QACnC,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;YACnB,KAAK,EAAE,CAAA;YACP,OAAO,EAAE,EAAE,CAAA;YACX,OAAO,GAAG,SAAS,CAAA;QACpB,CAAC;IACF,CAAC,CAAC,CAAA;IAEF,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAgE,CAAA;IAC1G,MAAM,YAAY,GAAG,GAAG,CAAC,YAAY,IAAI,GAAG,CAAC,OAAO,CAAA;IAEpD,IAAI,OAAO,YAAY,KAAK,UAAU,EAAE,CAAC;QACxC,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,iBAAiB,IAAI,CAAC,UAAU,8BAA8B,EAAE,CAAC,CAAA;QAE5G,OAAM;IACP,CAAC;IAED,MAAM,EAAE,mBAAmB,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAA;IACnE,MAAM,aAAa,GAAG,MAAM,mBAAmB,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAA;IACtG,MAAM,OAAO,GAAG,IAAI,gBAAgB,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,SAAkB,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAA;IAE9G,OAAO,UAAU,CAAC;QACjB,OAAO;QACP,YAAY;QACZ,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,IAAI,EAAE,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;YACzB,MAAM,EAAE,CAAA;YACR,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,CAAC,CAAA;QAC9D,CAAC;QACD,UAAU,EAAE,GAAG,EAAE,CAChB,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAC7B,OAAO,GAAG,OAAO,CAAA;QAClB,CAAC,CAAC;QACH,QAAQ,EAAE,GAAG,EAAE,CAAC,MAAM,GAAG,KAAK;KAC9B,CAAC;SACA,IAAI,CAAC,GAAG,EAAE;QACV,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAA;QAElC,OAAO,KAAK,CAAC,CAAA;IACd,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QAChB,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IACrG,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,KAAK,IAAI,EAAE,CAAA"}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license MIT
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
import type { CharacterSequenceInput } from "../core/CharacterSequence.js";
|
|
7
|
+
import type { AsyncDataResource } from "../internal/shared.js";
|
|
8
|
+
export { mergeAsyncIterators } from "./merge-async-iterators.js";
|
|
9
|
+
export interface MinimalWorker {
|
|
10
|
+
on(event: "message", cb: (msg: unknown) => void): void;
|
|
11
|
+
on(event: "error", cb: (err: Error) => void): void;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Drain a worker's batched messages into an async iterator. Listeners attach **eagerly** (messages posted before
|
|
15
|
+
* iteration starts are buffered, not lost) and draining uses a `batches[] + head` pointer (no `Array.shift()`).
|
|
16
|
+
* `onBatchConsumed` fires once per batch after its records are yielded — the ack hook for backpressure. An `error`
|
|
17
|
+
* message or worker `error` rejects the iterator.
|
|
18
|
+
*/
|
|
19
|
+
export declare function workerToIterable<R>(worker: MinimalWorker, onBatchConsumed: () => void): AsyncIterableIterator<R>;
|
|
20
|
+
export interface AsManyWorkersOptions {
|
|
21
|
+
/**
|
|
22
|
+
* Module path or URL exporting `handleRecord(bytes, ctx)`. Runs once per worker at import.
|
|
23
|
+
*/
|
|
24
|
+
worker: string | URL;
|
|
25
|
+
/**
|
|
26
|
+
* The record delimiter. @default LineFeed
|
|
27
|
+
*/
|
|
28
|
+
delimiter?: CharacterSequenceInput;
|
|
29
|
+
/**
|
|
30
|
+
* Desired number of segments/workers. Clamped to ≥ 1; fewer may run.
|
|
31
|
+
*/
|
|
32
|
+
concurrency: number;
|
|
33
|
+
/**
|
|
34
|
+
* Bytes read at each ideal boundary to find the next delimiter. @default 65536
|
|
35
|
+
*/
|
|
36
|
+
probeSize?: number;
|
|
37
|
+
/**
|
|
38
|
+
* Results per message. @default 256
|
|
39
|
+
*/
|
|
40
|
+
batchSize?: number;
|
|
41
|
+
/**
|
|
42
|
+
* Unacked batches per worker before it pauses (bounds memory). @default 8
|
|
43
|
+
*/
|
|
44
|
+
maxInFlight?: number;
|
|
45
|
+
/**
|
|
46
|
+
* Forwarded to every worker via `workerData.userData`.
|
|
47
|
+
*/
|
|
48
|
+
workerData?: unknown;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Spawn one worker per delimiter-aligned segment, each running the `worker` handler module over its own handle, and
|
|
52
|
+
* merge their results into a single async iterator. Results interleave across segments. Sends an `ack` per consumed
|
|
53
|
+
* batch (backpressure); terminates all workers on completion, error, or early return.
|
|
54
|
+
*/
|
|
55
|
+
export declare function runSegmentWorkers<R>(source: AsyncDataResource, options: AsManyWorkersOptions): AsyncIterableIterator<R>;
|
|
56
|
+
//# sourceMappingURL=segment-workers.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"segment-workers.d.ts","sourceRoot":"","sources":["../../../lib/parallel/segment-workers.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,8BAA8B,CAAA;AAC1E,OAAO,KAAK,EAAE,iBAAiB,EAAa,MAAM,uBAAuB,CAAA;AAIzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AAEhE,MAAM,WAAW,aAAa;IAC7B,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,GAAG,EAAE,OAAO,KAAK,IAAI,GAAG,IAAI,CAAA;IACtD,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI,CAAA;CAClD;AAID;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,IAAI,GAAG,qBAAqB,CAAC,CAAC,CAAC,CAyDhH;AAED,MAAM,WAAW,oBAAoB;IACpC;;OAEG;IACH,MAAM,EAAE,MAAM,GAAG,GAAG,CAAA;IACpB;;OAEG;IACH,SAAS,CAAC,EAAE,sBAAsB,CAAA;IAClC;;OAEG;IACH,WAAW,EAAE,MAAM,CAAA;IACnB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,UAAU,CAAC,EAAE,OAAO,CAAA;CACpB;AAED;;;;GAIG;AACH,wBAAuB,iBAAiB,CAAC,CAAC,EACzC,MAAM,EAAE,iBAAiB,EACzB,OAAO,EAAE,oBAAoB,GAC3B,qBAAqB,CAAC,CAAC,CAAC,CA+C1B"}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license MIT
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
import { Worker } from "node:worker_threads";
|
|
7
|
+
import { mergeAsyncIterators } from "./merge-async-iterators.js";
|
|
8
|
+
import { computeSegments } from "./segments.js";
|
|
9
|
+
export { mergeAsyncIterators } from "./merge-async-iterators.js";
|
|
10
|
+
/**
|
|
11
|
+
* Drain a worker's batched messages into an async iterator. Listeners attach **eagerly** (messages posted before
|
|
12
|
+
* iteration starts are buffered, not lost) and draining uses a `batches[] + head` pointer (no `Array.shift()`).
|
|
13
|
+
* `onBatchConsumed` fires once per batch after its records are yielded — the ack hook for backpressure. An `error`
|
|
14
|
+
* message or worker `error` rejects the iterator.
|
|
15
|
+
*/
|
|
16
|
+
export function workerToIterable(worker, onBatchConsumed) {
|
|
17
|
+
const batches = [];
|
|
18
|
+
let head = 0;
|
|
19
|
+
let done = false;
|
|
20
|
+
let error;
|
|
21
|
+
let wake;
|
|
22
|
+
const signal = () => {
|
|
23
|
+
wake?.();
|
|
24
|
+
wake = undefined;
|
|
25
|
+
};
|
|
26
|
+
worker.on("message", (msg) => {
|
|
27
|
+
const m = msg;
|
|
28
|
+
if (m.type === "batch") {
|
|
29
|
+
batches.push(m.records);
|
|
30
|
+
}
|
|
31
|
+
else if (m.type === "done") {
|
|
32
|
+
done = true;
|
|
33
|
+
}
|
|
34
|
+
else if (m.type === "error") {
|
|
35
|
+
error = new Error(m.message);
|
|
36
|
+
done = true;
|
|
37
|
+
}
|
|
38
|
+
signal();
|
|
39
|
+
});
|
|
40
|
+
worker.on("error", (err) => {
|
|
41
|
+
error = err;
|
|
42
|
+
done = true;
|
|
43
|
+
signal();
|
|
44
|
+
});
|
|
45
|
+
async function* drain() {
|
|
46
|
+
for (;;) {
|
|
47
|
+
if (head < batches.length) {
|
|
48
|
+
const batch = batches[head++];
|
|
49
|
+
for (const record of batch) {
|
|
50
|
+
yield record;
|
|
51
|
+
}
|
|
52
|
+
onBatchConsumed();
|
|
53
|
+
continue;
|
|
54
|
+
}
|
|
55
|
+
if (error)
|
|
56
|
+
throw error;
|
|
57
|
+
if (done)
|
|
58
|
+
return;
|
|
59
|
+
// oxlint-disable-next-line no-loop-func no-promise-executor-return
|
|
60
|
+
await new Promise((resolve) => (wake = resolve));
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return drain();
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Spawn one worker per delimiter-aligned segment, each running the `worker` handler module over its own handle, and
|
|
67
|
+
* merge their results into a single async iterator. Results interleave across segments. Sends an `ack` per consumed
|
|
68
|
+
* batch (backpressure); terminates all workers on completion, error, or early return.
|
|
69
|
+
*/
|
|
70
|
+
export async function* runSegmentWorkers(source, options) {
|
|
71
|
+
if (typeof source !== "string" && !(source instanceof URL)) {
|
|
72
|
+
throw new TypeError("asManyWorkers requires a file path or URL — file handles cannot cross threads.");
|
|
73
|
+
}
|
|
74
|
+
const handlerUrl = options.worker instanceof URL ? options.worker.href : new URL(options.worker, `file://${process.cwd()}/`).href;
|
|
75
|
+
const sourcePath = source instanceof URL ? source.href : source;
|
|
76
|
+
const segments = await computeSegments(source, {
|
|
77
|
+
delimiter: options.delimiter,
|
|
78
|
+
concurrency: options.concurrency,
|
|
79
|
+
probeSize: options.probeSize,
|
|
80
|
+
});
|
|
81
|
+
const workers = [];
|
|
82
|
+
const entryUrl = new URL("./segment-worker-entry.js", import.meta.url);
|
|
83
|
+
try {
|
|
84
|
+
const iterables = segments.map(([start, end], segmentIndex) => {
|
|
85
|
+
const worker = new Worker(entryUrl, {
|
|
86
|
+
workerData: {
|
|
87
|
+
source: sourcePath,
|
|
88
|
+
handlerUrl,
|
|
89
|
+
start,
|
|
90
|
+
end,
|
|
91
|
+
delimiter: options.delimiter ?? null,
|
|
92
|
+
segmentIndex,
|
|
93
|
+
batchSize: options.batchSize ?? 256,
|
|
94
|
+
maxInFlight: options.maxInFlight ?? 8,
|
|
95
|
+
userData: options.workerData,
|
|
96
|
+
},
|
|
97
|
+
});
|
|
98
|
+
workers.push(worker);
|
|
99
|
+
return workerToIterable(worker, () => worker.postMessage("ack"));
|
|
100
|
+
});
|
|
101
|
+
// Drain all workers concurrently so they run in parallel — a sequential drain would consume one
|
|
102
|
+
// to completion while the rest stall at their in-flight window. Order across segments is not
|
|
103
|
+
// guaranteed (results arrive in completion order).
|
|
104
|
+
yield* mergeAsyncIterators(iterables);
|
|
105
|
+
}
|
|
106
|
+
finally {
|
|
107
|
+
await Promise.all(workers.map((w) => w.terminate()));
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
//# sourceMappingURL=segment-workers.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"segment-workers.js","sourceRoot":"","sources":["../../../lib/parallel/segment-workers.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAI5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAE/C,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AAShE;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAI,MAAqB,EAAE,eAA2B;IACrF,MAAM,OAAO,GAAU,EAAE,CAAA;IACzB,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,IAAI,GAAG,KAAK,CAAA;IAChB,IAAI,KAAwB,CAAA;IAC5B,IAAI,IAA8B,CAAA;IAElC,MAAM,MAAM,GAAG,GAAG,EAAE;QACnB,IAAI,EAAE,EAAE,CAAA;QACR,IAAI,GAAG,SAAS,CAAA;IACjB,CAAC,CAAA;IAED,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,EAAE;QAC5B,MAAM,CAAC,GAAG,GAAuB,CAAA;QAEjC,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAA;QACxB,CAAC;aAAM,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC9B,IAAI,GAAG,IAAI,CAAA;QACZ,CAAC;aAAM,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC/B,KAAK,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAA;YAC5B,IAAI,GAAG,IAAI,CAAA;QACZ,CAAC;QAED,MAAM,EAAE,CAAA;IACT,CAAC,CAAC,CAAA;IAEF,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;QAC1B,KAAK,GAAG,GAAG,CAAA;QACX,IAAI,GAAG,IAAI,CAAA;QACX,MAAM,EAAE,CAAA;IACT,CAAC,CAAC,CAAA;IAEF,KAAK,SAAS,CAAC,CAAC,KAAK;QACpB,SAAS,CAAC;YACT,IAAI,IAAI,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC3B,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,EAAE,CAAE,CAAA;gBAE9B,KAAK,MAAM,MAAM,IAAI,KAAK,EAAE,CAAC;oBAC5B,MAAM,MAAM,CAAA;gBACb,CAAC;gBAED,eAAe,EAAE,CAAA;gBAEjB,SAAQ;YACT,CAAC;YAED,IAAI,KAAK;gBAAE,MAAM,KAAK,CAAA;YAEtB,IAAI,IAAI;gBAAE,OAAM;YAEhB,mEAAmE;YACnE,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,IAAI,GAAG,OAAO,CAAC,CAAC,CAAA;QACvD,CAAC;IACF,CAAC;IAED,OAAO,KAAK,EAAE,CAAA;AACf,CAAC;AAiCD;;;;GAIG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,iBAAiB,CACvC,MAAyB,EACzB,OAA6B;IAE7B,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,YAAY,GAAG,CAAC,EAAE,CAAC;QAC5D,MAAM,IAAI,SAAS,CAAC,gFAAgF,CAAC,CAAA;IACtG,CAAC;IAED,MAAM,UAAU,GACf,OAAO,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,CAAA;IAE/G,MAAM,UAAU,GAAG,MAAM,YAAY,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAA;IAE/D,MAAM,QAAQ,GAAgB,MAAM,eAAe,CAAC,MAAM,EAAE;QAC3D,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,SAAS,EAAE,OAAO,CAAC,SAAS;KAC5B,CAAC,CAAA;IAEF,MAAM,OAAO,GAAa,EAAE,CAAA;IAC5B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,2BAA2B,EAAE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAA;IAEtE,IAAI,CAAC;QACJ,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,YAAY,EAAE,EAAE;YAC7D,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,QAAQ,EAAE;gBACnC,UAAU,EAAE;oBACX,MAAM,EAAE,UAAU;oBAClB,UAAU;oBACV,KAAK;oBACL,GAAG;oBACH,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,IAAI;oBACpC,YAAY;oBACZ,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,GAAG;oBACnC,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,CAAC;oBACrC,QAAQ,EAAE,OAAO,CAAC,UAAU;iBAC5B;aACD,CAAC,CAAA;YAEF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAEpB,OAAO,gBAAgB,CAAI,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAA;QACpE,CAAC,CAAC,CAAA;QAEF,gGAAgG;QAChG,6FAA6F;QAC7F,mDAAmD;QACnD,KAAK,CAAC,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAA;IACtC,CAAC;YAAS,CAAC;QACV,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAA;IACrD,CAAC;AACF,CAAC"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license MIT
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
import { type CharacterSequenceInput } from "../core/CharacterSequence.js";
|
|
7
|
+
import type { AsyncDataResource, ByteRange } from "../internal/shared.js";
|
|
8
|
+
export interface SegmentOptions {
|
|
9
|
+
/**
|
|
10
|
+
* The record delimiter. @default LineFeed
|
|
11
|
+
*/
|
|
12
|
+
delimiter?: CharacterSequenceInput;
|
|
13
|
+
/**
|
|
14
|
+
* Desired number of segments. Clamped to ≥ 1; the result may be fewer.
|
|
15
|
+
*/
|
|
16
|
+
concurrency: number;
|
|
17
|
+
/**
|
|
18
|
+
* Bytes read at each ideal boundary to find the next delimiter. @default 65536
|
|
19
|
+
*/
|
|
20
|
+
probeSize?: number;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Divide `source` into up to `concurrency` delimiter-aligned byte ranges. Each internal boundary is placed immediately
|
|
24
|
+
* after the first delimiter at or past the ideal cut, so concatenating each segment's records reproduces the file
|
|
25
|
+
* exactly — no record is split or duplicated.
|
|
26
|
+
*
|
|
27
|
+
* A probe window with no delimiter (a record longer than `probeSize`) collapses that boundary, so the result may have
|
|
28
|
+
* fewer than `concurrency` segments. An empty file yields no segments.
|
|
29
|
+
*/
|
|
30
|
+
export declare function computeSegments(source: AsyncDataResource, options: SegmentOptions): Promise<ByteRange[]>;
|
|
31
|
+
//# sourceMappingURL=segments.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"segments.d.ts","sourceRoot":"","sources":["../../../lib/parallel/segments.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAqB,KAAK,sBAAsB,EAAE,MAAM,8BAA8B,CAAA;AAC7F,OAAO,KAAK,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAA;AAEzE,MAAM,WAAW,cAAc;IAC9B;;OAEG;IACH,SAAS,CAAC,EAAE,sBAAsB,CAAA;IAClC;;OAEG;IACH,WAAW,EAAE,MAAM,CAAA;IACnB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;CAClB;AAED;;;;;;;GAOG;AACH,wBAAsB,eAAe,CAAC,MAAM,EAAE,iBAAiB,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAgD9G"}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license MIT
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
import { CharacterSequence } from "../core/CharacterSequence.js";
|
|
7
|
+
/**
|
|
8
|
+
* Divide `source` into up to `concurrency` delimiter-aligned byte ranges. Each internal boundary is placed immediately
|
|
9
|
+
* after the first delimiter at or past the ideal cut, so concatenating each segment's records reproduces the file
|
|
10
|
+
* exactly — no record is split or duplicated.
|
|
11
|
+
*
|
|
12
|
+
* A probe window with no delimiter (a record longer than `probeSize`) collapses that boundary, so the result may have
|
|
13
|
+
* fewer than `concurrency` segments. An empty file yields no segments.
|
|
14
|
+
*/
|
|
15
|
+
export async function computeSegments(source, options) {
|
|
16
|
+
const { readFileSize, readBytes } = await import("spliterator/node/fs");
|
|
17
|
+
const needle = new CharacterSequence(options.delimiter);
|
|
18
|
+
const probeSize = options.probeSize ?? 65_536;
|
|
19
|
+
const concurrency = Math.max(1, Math.floor(options.concurrency));
|
|
20
|
+
const fileSize = await readFileSize(source);
|
|
21
|
+
if (fileSize === 0)
|
|
22
|
+
return [];
|
|
23
|
+
if (concurrency === 1)
|
|
24
|
+
return [[0, fileSize]];
|
|
25
|
+
const idealCuts = Array.from({ length: concurrency - 1 }, (_, i) => Math.round(((i + 1) * fileSize) / concurrency));
|
|
26
|
+
const alignedCuts = await Promise.all(idealCuts.map(async (cut) => {
|
|
27
|
+
if (cut <= 0 || cut >= fileSize)
|
|
28
|
+
return null;
|
|
29
|
+
const window = await readBytes(source, cut, probeSize);
|
|
30
|
+
const index = needle.search(window, 0, window.length);
|
|
31
|
+
if (index === -1)
|
|
32
|
+
return null;
|
|
33
|
+
return cut + index + needle.length;
|
|
34
|
+
}));
|
|
35
|
+
const boundaries = new Set([0, fileSize]);
|
|
36
|
+
for (const cut of alignedCuts) {
|
|
37
|
+
if (cut !== null && cut > 0 && cut < fileSize) {
|
|
38
|
+
boundaries.add(cut);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
const sorted = Array.from(boundaries).toSorted((a, b) => a - b);
|
|
42
|
+
const segments = [];
|
|
43
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
44
|
+
const start = sorted[i - 1];
|
|
45
|
+
const end = sorted[i];
|
|
46
|
+
if (end > start) {
|
|
47
|
+
segments.push([start, end]);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return segments;
|
|
51
|
+
}
|
|
52
|
+
//# sourceMappingURL=segments.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"segments.js","sourceRoot":"","sources":["../../../lib/parallel/segments.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,iBAAiB,EAA+B,MAAM,8BAA8B,CAAA;AAkB7F;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAyB,EAAE,OAAuB;IACvF,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAA;IACvE,MAAM,MAAM,GAAG,IAAI,iBAAiB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;IACvD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,MAAM,CAAA;IAC7C,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAA;IAEhE,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC,CAAA;IAE3C,IAAI,QAAQ,KAAK,CAAC;QAAE,OAAO,EAAE,CAAA;IAE7B,IAAI,WAAW,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAA;IAE7C,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,GAAG,WAAW,CAAC,CAAC,CAAA;IAEnH,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;QAC3B,IAAI,GAAG,IAAI,CAAC,IAAI,GAAG,IAAI,QAAQ;YAAE,OAAO,IAAI,CAAA;QAE5C,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,GAAG,EAAE,SAAS,CAAC,CAAA;QACtD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;QAErD,IAAI,KAAK,KAAK,CAAC,CAAC;YAAE,OAAO,IAAI,CAAA;QAE7B,OAAO,GAAG,GAAG,KAAK,GAAG,MAAM,CAAC,MAAM,CAAA;IACnC,CAAC,CAAC,CACF,CAAA;IAED,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAA;IAEjD,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QAC/B,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,GAAG,CAAC,IAAI,GAAG,GAAG,QAAQ,EAAE,CAAC;YAC/C,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACpB,CAAC;IACF,CAAC;IAED,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IAC/D,MAAM,QAAQ,GAAgB,EAAE,CAAA;IAEhC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,CAAA;QAC5B,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;QAEtB,IAAI,GAAG,GAAG,KAAK,EAAE,CAAC;YACjB,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAA;QAC5B,CAAC;IACF,CAAC;IAED,OAAO,QAAQ,CAAA;AAChB,CAAC"}
|
package/out/lib/wasm_base64.d.ts
CHANGED
|
@@ -11,6 +11,9 @@
|
|
|
11
11
|
* - find_all_delimiters(haystack_offset, haystack_len, pattern_offset, pattern_len,
|
|
12
12
|
* results_offset, max_results) → usize
|
|
13
13
|
* Writes (start, end) i32 pairs to results_offset, returns count.
|
|
14
|
+
* - scan_delimited_ranges(haystack_offset, haystack_len, scan_start, pending_slice_start,
|
|
15
|
+
* delimiter, quote, inside_quotes, results_offset, max_ranges) → usize
|
|
16
|
+
* Writes resumable scan state followed by bounded (start, end) i32 pairs.
|
|
14
17
|
*/
|
|
15
|
-
export declare const WASM_BASE64 = "
|
|
18
|
+
export declare const WASM_BASE64 = "AGFzbQEAAAABKwRgBH9/f38Bf2AGf39/f39/AX9gB39/f39/f38Bf2AJf39/f39/f39/AX8DBgUAAQIAAwUDAQABB1wFBm1lbW9yeQIAE2ZpbmRfYWxsX2RlbGltaXRlcnMAARBmaW5kX2FsbF9tYXRjaGVzAAIOZmluZF9kZWxpbWl0ZXIAAxVzY2FuX2RlbGltaXRlZF9yYW5nZXMABArfDwXEAgINfwF7IAItAAAhCyABIANBAWsiDWsiBEEAIAEgBE8bIglBEE8EQCAL/Q8hESAAIQpBECEEA0AgBSEMIAQhBSAAIAxq/QAAACAR/SP9ZCIGBEADQCABIAZoIgQgDGoiDiADak8EQCAEIApqIQcgAyEEIAIhCANAIARFBEAgDg8LIARBAWshBCAILQAAIActAAAgCEEBaiEIIAdBAWohB0YNAAsLIAZBAWsgBnEiBg0ACwsgCkEQaiEKIAVBEGoiBCAJTQ0ACwsgBSAJSQRAIAJBAWohASAAIAVqQQFqIQYDQCANIQQgASEHIAYhCAJAIAAgBWotAAAgC0cNAANAIAQEQCAEQQFrIQQgBy0AACAILQAAIAdBAWohByAIQQFqIQhGDQEMAgsLIAUPCyAGQQFqIQYgBUEBaiIFIAlJDQALC0F/C5MCAgl/AXsCQCAFRSABRSADRXJyRQRAIAEgA0kNASADQQFHIQ0DQCAAIAlqIQogASAJayELAn8gDUUEQCACLQAAIg79DyEPQQAhBgJAA0AgBkEQaiIHIAtLDQEgBiAKaiAHIQb9AAAAIA/9I/1kIgdFDQALIAdoIAZBEGtqDAILA0AgBiALTw0FIAYgCmogBkEBaiEGLQAAIA5HDQALIAZBAWsMAQsgCiALIAIgAxAACyIGQQBIDQIgBCAIQQN0aiIMIAYgCWoiBzYCBCAMIAk2AgAgASADIAdqIgkgA2pPIAhBAWoiCCAFSSIHcQ0ACyAHDQELIAgPCyAEIAhBA3RqIgAgATYCBCAAIAk2AgAgCEEBaguDBgILfwN7AkAgBkUgAyAEckUgAUVycg0AIAIgA2ohDiADQQFGIhAgBEEBRnFFBEADQCABIAxrIQsCfwJAAkAgEEUEQCADDQFB/////wcMAwsgACAMaiEKIAItAAAiDf0PIRJBACEHAkADQCAHQRBqIgkgC0sNASAHIApqIAkhB/0AAAAgEv0j/WQiCUUNAAsgCWggB0EQa2oMAwsDQCAHIAtJBEAgByAKaiEJIAdBAWohByANIAktAABHDQEMAwsLQX8MAgsgACAMaiALIAIgAxAADAELIAdBAWsLIgkCfwJAAkAgBEEBRgRAIAAgDGohDSAOLQAAIg/9DyESQQAhBwJAA0AgB0EQaiIKIAtLDQEgByANaiAKIQf9AAAAIBL9I/1kIgpFDQALIApoIAdBEGtqDAQLA0AgByALSQRAIAcgDWohCiAHQQFqIQcgDyAKLQAARw0BDAMLC0F/DAMLIAQNAUH/////BwwCCyAHQQFrDAELIAAgDGogCyAOIAQQAAsiB3FBAEgNAiAFIAhBA3RqIgsgByAJSCAHQQBOcSAJQQBIciIKNgIEIAsgByAJIAobIAxqIgc2AgAgCEEBaiEIIAcgBCADIAobaiIMIAFPDQIgBiAISw0ACwwBCyAOLQAAIQogAi0AACEDAkAgAUEQSQ0AIAr9DyESIAP9DyETQRAhB0EAIQIDQCAHIQkCQCAAIAJq/QAAACIUIBP9I/1kIgQgFCAS/SP9ZHIiB0UNACAEQX9zIQwgBSAIQQN0aiEEA0AgBCAHaCILIAJqNgIAIARBBGogDCALdkEBcTYCACAIQQFqIQggB0EBayAHcSIHRQ0BIARBCGohBCAGIAhLDQALCyAJQRBqIgcgAUsNASAJIQIgBiAISw0ACwsgASAJTSAGIAhNcg0AA0ACQCADIAAgCWotAAAiAkYEf0EABSACIApHDQFBAQshByAFIAhBA3RqIgIgBzYCBCACIAk2AgAgCEEBaiEICyAJQQFqIgkgAU8NASAGIAhLDQALCyAIC5IBAgN/AXtBfyEFAkAgA0EBayABTw0AIANBAUYEQCACLQAAIgL9DyEHQQAhAwNAIAEgA0EQaiIESQRAA0AgASADTQ0EIAAgA2ogA0EBaiEDLQAAIAJHDQALIANBAWsPCyAAIANqIAQhA/0AAAAgB/0j/WQiBEUNAAsgBGggA0EQa2oPCyAAIAEgAiADEAAhBQsgBQvoAwIIfwN7IAEgAyABIANJGyEKIAZBAEchBiAHAn8CQCABIAEgAiABIAJJGyICQRBqIgNPBEAgBf0PIREgBP0PIRIgBUEATiENA0AgAiELIAMhAiAAIAtq/QAAACITIBH9I/1kQQAgDRsiDiATIBL9I/1kIg9yIgMEQANAIANoIQwCQAJAAkACQCAFQQBIDQBBASAMdCIQIA9xDQAgDiAQcQ0BCyAGQQFxRQ0BQQEhBgwCCyAGQQFzIQYMAQsgCyAMaiEGIAggCUsEQCAHIAlBA3RqIgxBEGogBjYCACAMQQxqIAo2AgAgBkEBaiEKIAlBAWohCUEAIQYMAQsgByAKNgIEIAcgBjYCAAwFCyADQQFrIANxIgMNAAsLIAJBEGoiAyABTQ0ACwsgASACSwRAIARB/wFxIQMgBUEATiEEA0ACQCADIAAgAmotAAAiC0cEQCAGIAQgCyAFQf8BcUZxcyEGDAELIAZBAXEEQEEBIQYMAQsgCCAJSwRAIAcgCUEDdGoiBkEQaiACNgIAIAZBDGogCjYCACACQQFqIQogCUEBaiEJQQAhBgwBCyAHIAo2AgQgByACNgIADAMLIAEgAkEBaiICRw0ACwsgByAKNgIEIAcgATYCACAGQQFxDAELQQALNgIIIAkL";
|
|
16
19
|
//# sourceMappingURL=wasm_base64.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"wasm_base64.d.ts","sourceRoot":"","sources":["../../lib/wasm_base64.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"wasm_base64.d.ts","sourceRoot":"","sources":["../../lib/wasm_base64.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AACH,eAAO,MAAM,WAAW,61FACm0F,CAAA"}
|
package/out/lib/wasm_base64.js
CHANGED
|
@@ -11,6 +11,9 @@
|
|
|
11
11
|
* - find_all_delimiters(haystack_offset, haystack_len, pattern_offset, pattern_len,
|
|
12
12
|
* results_offset, max_results) → usize
|
|
13
13
|
* Writes (start, end) i32 pairs to results_offset, returns count.
|
|
14
|
+
* - scan_delimited_ranges(haystack_offset, haystack_len, scan_start, pending_slice_start,
|
|
15
|
+
* delimiter, quote, inside_quotes, results_offset, max_ranges) → usize
|
|
16
|
+
* Writes resumable scan state followed by bounded (start, end) i32 pairs.
|
|
14
17
|
*/
|
|
15
|
-
export const WASM_BASE64 = "
|
|
18
|
+
export const WASM_BASE64 = "AGFzbQEAAAABKwRgBH9/f38Bf2AGf39/f39/AX9gB39/f39/f38Bf2AJf39/f39/f39/AX8DBgUAAQIAAwUDAQABB1wFBm1lbW9yeQIAE2ZpbmRfYWxsX2RlbGltaXRlcnMAARBmaW5kX2FsbF9tYXRjaGVzAAIOZmluZF9kZWxpbWl0ZXIAAxVzY2FuX2RlbGltaXRlZF9yYW5nZXMABArfDwXEAgINfwF7IAItAAAhCyABIANBAWsiDWsiBEEAIAEgBE8bIglBEE8EQCAL/Q8hESAAIQpBECEEA0AgBSEMIAQhBSAAIAxq/QAAACAR/SP9ZCIGBEADQCABIAZoIgQgDGoiDiADak8EQCAEIApqIQcgAyEEIAIhCANAIARFBEAgDg8LIARBAWshBCAILQAAIActAAAgCEEBaiEIIAdBAWohB0YNAAsLIAZBAWsgBnEiBg0ACwsgCkEQaiEKIAVBEGoiBCAJTQ0ACwsgBSAJSQRAIAJBAWohASAAIAVqQQFqIQYDQCANIQQgASEHIAYhCAJAIAAgBWotAAAgC0cNAANAIAQEQCAEQQFrIQQgBy0AACAILQAAIAdBAWohByAIQQFqIQhGDQEMAgsLIAUPCyAGQQFqIQYgBUEBaiIFIAlJDQALC0F/C5MCAgl/AXsCQCAFRSABRSADRXJyRQRAIAEgA0kNASADQQFHIQ0DQCAAIAlqIQogASAJayELAn8gDUUEQCACLQAAIg79DyEPQQAhBgJAA0AgBkEQaiIHIAtLDQEgBiAKaiAHIQb9AAAAIA/9I/1kIgdFDQALIAdoIAZBEGtqDAILA0AgBiALTw0FIAYgCmogBkEBaiEGLQAAIA5HDQALIAZBAWsMAQsgCiALIAIgAxAACyIGQQBIDQIgBCAIQQN0aiIMIAYgCWoiBzYCBCAMIAk2AgAgASADIAdqIgkgA2pPIAhBAWoiCCAFSSIHcQ0ACyAHDQELIAgPCyAEIAhBA3RqIgAgATYCBCAAIAk2AgAgCEEBaguDBgILfwN7AkAgBkUgAyAEckUgAUVycg0AIAIgA2ohDiADQQFGIhAgBEEBRnFFBEADQCABIAxrIQsCfwJAAkAgEEUEQCADDQFB/////wcMAwsgACAMaiEKIAItAAAiDf0PIRJBACEHAkADQCAHQRBqIgkgC0sNASAHIApqIAkhB/0AAAAgEv0j/WQiCUUNAAsgCWggB0EQa2oMAwsDQCAHIAtJBEAgByAKaiEJIAdBAWohByANIAktAABHDQEMAwsLQX8MAgsgACAMaiALIAIgAxAADAELIAdBAWsLIgkCfwJAAkAgBEEBRgRAIAAgDGohDSAOLQAAIg/9DyESQQAhBwJAA0AgB0EQaiIKIAtLDQEgByANaiAKIQf9AAAAIBL9I/1kIgpFDQALIApoIAdBEGtqDAQLA0AgByALSQRAIAcgDWohCiAHQQFqIQcgDyAKLQAARw0BDAMLC0F/DAMLIAQNAUH/////BwwCCyAHQQFrDAELIAAgDGogCyAOIAQQAAsiB3FBAEgNAiAFIAhBA3RqIgsgByAJSCAHQQBOcSAJQQBIciIKNgIEIAsgByAJIAobIAxqIgc2AgAgCEEBaiEIIAcgBCADIAobaiIMIAFPDQIgBiAISw0ACwwBCyAOLQAAIQogAi0AACEDAkAgAUEQSQ0AIAr9DyESIAP9DyETQRAhB0EAIQIDQCAHIQkCQCAAIAJq/QAAACIUIBP9I/1kIgQgFCAS/SP9ZHIiB0UNACAEQX9zIQwgBSAIQQN0aiEEA0AgBCAHaCILIAJqNgIAIARBBGogDCALdkEBcTYCACAIQQFqIQggB0EBayAHcSIHRQ0BIARBCGohBCAGIAhLDQALCyAJQRBqIgcgAUsNASAJIQIgBiAISw0ACwsgASAJTSAGIAhNcg0AA0ACQCADIAAgCWotAAAiAkYEf0EABSACIApHDQFBAQshByAFIAhBA3RqIgIgBzYCBCACIAk2AgAgCEEBaiEICyAJQQFqIgkgAU8NASAGIAhLDQALCyAIC5IBAgN/AXtBfyEFAkAgA0EBayABTw0AIANBAUYEQCACLQAAIgL9DyEHQQAhAwNAIAEgA0EQaiIESQRAA0AgASADTQ0EIAAgA2ogA0EBaiEDLQAAIAJHDQALIANBAWsPCyAAIANqIAQhA/0AAAAgB/0j/WQiBEUNAAsgBGggA0EQa2oPCyAAIAEgAiADEAAhBQsgBQvoAwIIfwN7IAEgAyABIANJGyEKIAZBAEchBiAHAn8CQCABIAEgAiABIAJJGyICQRBqIgNPBEAgBf0PIREgBP0PIRIgBUEATiENA0AgAiELIAMhAiAAIAtq/QAAACITIBH9I/1kQQAgDRsiDiATIBL9I/1kIg9yIgMEQANAIANoIQwCQAJAAkACQCAFQQBIDQBBASAMdCIQIA9xDQAgDiAQcQ0BCyAGQQFxRQ0BQQEhBgwCCyAGQQFzIQYMAQsgCyAMaiEGIAggCUsEQCAHIAlBA3RqIgxBEGogBjYCACAMQQxqIAo2AgAgBkEBaiEKIAlBAWohCUEAIQYMAQsgByAKNgIEIAcgBjYCAAwFCyADQQFrIANxIgMNAAsLIAJBEGoiAyABTQ0ACwsgASACSwRAIARB/wFxIQMgBUEATiEEA0ACQCADIAAgAmotAAAiC0cEQCAGIAQgCyAFQf8BcUZxcyEGDAELIAZBAXEEQEEBIQYMAQsgCCAJSwRAIAcgCUEDdGoiBkEQaiACNgIAIAZBDGogCjYCACACQQFqIQogCUEBaiEJQQAhBgwBCyAHIAo2AgQgByACNgIADAMLIAEgAkEBaiICRw0ACwsgByAKNgIEIAcgATYCACAGQQFxDAELQQALNgIIIAkL";
|
|
16
19
|
//# sourceMappingURL=wasm_base64.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"wasm_base64.js","sourceRoot":"","sources":["../../lib/wasm_base64.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"wasm_base64.js","sourceRoot":"","sources":["../../lib/wasm_base64.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,MAAM,WAAW,GACvB,01FAA01F,CAAA"}
|
package/out/lib/wasm_module.d.ts
CHANGED
|
@@ -7,6 +7,7 @@ export declare function loadWasmModule(): Promise<WasmDelimiterScanner | null>;
|
|
|
7
7
|
type WasmFindDelimiter = (ho: number, hl: number, po: number, pl: number) => number;
|
|
8
8
|
type WasmFindAllDelimiters = (ho: number, hl: number, po: number, pl: number, ro: number, mr: number) => number;
|
|
9
9
|
type WasmFindAllMatches = (ho: number, hl: number, p1o: number, p1l: number, p2l: number, ro: number, mr: number) => number;
|
|
10
|
+
type WasmScanDelimitedRanges = (ho: number, hl: number, ss: number, pss: number, delimiter: number, quote: number, insideQuotes: number, ro: number, mr: number) => number;
|
|
10
11
|
export interface WasmMemory {
|
|
11
12
|
readonly buffer: ArrayBuffer;
|
|
12
13
|
grow(pages: number): number;
|
|
@@ -16,6 +17,7 @@ export interface WasmDelimiterScanner {
|
|
|
16
17
|
findDelimiter: WasmFindDelimiter;
|
|
17
18
|
findAllDelimiters: WasmFindAllDelimiters;
|
|
18
19
|
findAllMatches: WasmFindAllMatches;
|
|
20
|
+
scanDelimitedRanges: WasmScanDelimitedRanges;
|
|
19
21
|
}
|
|
20
22
|
/**
|
|
21
23
|
* Match result from find_all_matches: [offset, pattern_id].
|
|
@@ -38,5 +40,18 @@ export interface MatchResult {
|
|
|
38
40
|
*/
|
|
39
41
|
export declare const WASM_THRESHOLD = 512;
|
|
40
42
|
export declare const WASM_MAX_RESULTS = 4096;
|
|
43
|
+
/**
|
|
44
|
+
* State returned by the bounded, resumable range scanner.
|
|
45
|
+
*/
|
|
46
|
+
export interface WasmRangeScanResult {
|
|
47
|
+
/**
|
|
48
|
+
* A shared-memory view of `[start, end]` pairs. Consume it before the next WASM scan.
|
|
49
|
+
*/
|
|
50
|
+
ranges: Int32Array;
|
|
51
|
+
count: number;
|
|
52
|
+
scanCursor: number;
|
|
53
|
+
pendingSliceStart: number;
|
|
54
|
+
insideQuotes: boolean;
|
|
55
|
+
}
|
|
41
56
|
export {};
|
|
42
57
|
//# sourceMappingURL=wasm_module.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"wasm_module.d.ts","sourceRoot":"","sources":["../../lib/wasm_module.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAiBH,wBAAsB,cAAc,IAAI,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"wasm_module.d.ts","sourceRoot":"","sources":["../../lib/wasm_module.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAiBH,wBAAsB,cAAc,IAAI,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CAmB3E;AAED,KAAK,iBAAiB,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,KAAK,MAAM,CAAA;AAEnF,KAAK,qBAAqB,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,KAAK,MAAM,CAAA;AAE/G,KAAK,kBAAkB,GAAG,CACzB,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,EACV,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,MAAM,EACX,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,KACN,MAAM,CAAA;AAGX,KAAK,uBAAuB,GAAG,CAC9B,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,EACV,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,EACb,YAAY,EAAE,MAAM,EACpB,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,KACN,MAAM,CAAA;AAEX,MAAM,WAAW,UAAU;IAC1B,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAA;IAC5B,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAA;CAC3B;AAED,MAAM,WAAW,oBAAoB;IACpC,MAAM,EAAE,UAAU,CAAA;IAClB,aAAa,EAAE,iBAAiB,CAAA;IAChC,iBAAiB,EAAE,qBAAqB,CAAA;IACxC,cAAc,EAAE,kBAAkB,CAAA;IAClC,mBAAmB,EAAE,uBAAuB,CAAA;CAC5C;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC3B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;CACjB;AAED;;;;;GAKG;AACH,eAAO,MAAM,cAAc,MAAM,CAAA;AACjC,eAAO,MAAM,gBAAgB,OAAO,CAAA;AAEpC;;GAEG;AACH,MAAM,WAAW,mBAAmB;IACnC;;OAEG;IACH,MAAM,EAAE,UAAU,CAAA;IAClB,KAAK,EAAE,MAAM,CAAA;IACb,UAAU,EAAE,MAAM,CAAA;IAClB,iBAAiB,EAAE,MAAM,CAAA;IACzB,YAAY,EAAE,OAAO,CAAA;CACrB"}
|
package/out/lib/wasm_module.js
CHANGED
|
@@ -16,7 +16,8 @@ export async function loadWasmModule() {
|
|
|
16
16
|
const findDelimiter = e.find_delimiter;
|
|
17
17
|
const findAllDelimiters = e.find_all_delimiters;
|
|
18
18
|
const findAllMatches = e.find_all_matches;
|
|
19
|
-
|
|
19
|
+
const scanDelimitedRanges = e.scan_delimited_ranges;
|
|
20
|
+
return { memory, findDelimiter, findAllDelimiters, findAllMatches, scanDelimitedRanges };
|
|
20
21
|
}
|
|
21
22
|
catch {
|
|
22
23
|
return null;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"wasm_module.js","sourceRoot":"","sources":["../../lib/wasm_module.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAA;AAe9C,MAAM,CAAC,KAAK,UAAU,cAAc;IACnC,IAAI,CAAC,WAAW;QAAE,OAAO,IAAI,CAAA;IAE7B,IAAI,CAAC;QACJ,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAA;QACxE,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAC/C,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;QAC1D,MAAM,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAA;QAE1B,MAAM,MAAM,GAAG,CAAC,CAAC,MAAoB,CAAA;QACrC,MAAM,aAAa,GAAG,CAAC,CAAC,cAAmC,CAAA;QAC3D,MAAM,iBAAiB,GAAG,CAAC,CAAC,mBAA4C,CAAA;QACxE,MAAM,cAAc,GAAG,CAAC,CAAC,gBAAsC,CAAA;
|
|
1
|
+
{"version":3,"file":"wasm_module.js","sourceRoot":"","sources":["../../lib/wasm_module.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAA;AAe9C,MAAM,CAAC,KAAK,UAAU,cAAc;IACnC,IAAI,CAAC,WAAW;QAAE,OAAO,IAAI,CAAA;IAE7B,IAAI,CAAC;QACJ,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAA;QACxE,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAC/C,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;QAC1D,MAAM,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAA;QAE1B,MAAM,MAAM,GAAG,CAAC,CAAC,MAAoB,CAAA;QACrC,MAAM,aAAa,GAAG,CAAC,CAAC,cAAmC,CAAA;QAC3D,MAAM,iBAAiB,GAAG,CAAC,CAAC,mBAA4C,CAAA;QACxE,MAAM,cAAc,GAAG,CAAC,CAAC,gBAAsC,CAAA;QAC/D,MAAM,mBAAmB,GAAG,CAAC,CAAC,qBAAgD,CAAA;QAE9E,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,iBAAiB,EAAE,cAAc,EAAE,mBAAmB,EAAE,CAAA;IACzF,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAA;IACZ,CAAC;AACF,CAAC;AAwDD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,GAAG,CAAA;AACjC,MAAM,CAAC,MAAM,gBAAgB,GAAG,IAAI,CAAA"}
|