spliterator 2.2.2 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +21 -650
- package/README.md +42 -0
- package/out/index.d.ts +2 -1
- package/out/index.d.ts.map +1 -1
- package/out/index.js +2 -1
- package/out/index.js.map +1 -1
- package/out/lib/AsyncSpliterator.d.ts +33 -28
- package/out/lib/AsyncSpliterator.d.ts.map +1 -1
- package/out/lib/AsyncSpliterator.js +84 -70
- package/out/lib/AsyncSpliterator.js.map +1 -1
- package/out/lib/BufferController.d.ts +1 -1
- package/out/lib/BufferController.js +1 -1
- package/out/lib/CSVSpliterator.d.ts +1 -1
- package/out/lib/CSVSpliterator.d.ts.map +1 -1
- package/out/lib/CSVSpliterator.js +5 -3
- package/out/lib/CSVSpliterator.js.map +1 -1
- package/out/lib/CharacterSequence.d.ts +15 -64
- package/out/lib/CharacterSequence.d.ts.map +1 -1
- package/out/lib/CharacterSequence.js +194 -89
- package/out/lib/CharacterSequence.js.map +1 -1
- package/out/lib/CompositeDataView.d.ts +1 -1
- package/out/lib/CompositeDataView.js +1 -1
- package/out/lib/IndexQueue.d.ts +1 -1
- package/out/lib/IndexQueue.d.ts.map +1 -1
- package/out/lib/IndexQueue.js +40 -11
- package/out/lib/IndexQueue.js.map +1 -1
- package/out/lib/JSONSpliterator.d.ts +1 -1
- package/out/lib/JSONSpliterator.js +1 -1
- package/out/lib/PSVSpliterator.d.ts +1 -1
- package/out/lib/PSVSpliterator.js +1 -1
- package/out/lib/SlidingWindow.d.ts +1 -1
- package/out/lib/SlidingWindow.d.ts.map +1 -1
- package/out/lib/SlidingWindow.js +11 -10
- package/out/lib/SlidingWindow.js.map +1 -1
- package/out/lib/Spliterator.d.ts +1 -1
- package/out/lib/Spliterator.d.ts.map +1 -1
- package/out/lib/Spliterator.js +99 -110
- package/out/lib/Spliterator.js.map +1 -1
- package/out/lib/TSVSpliterator.d.ts +1 -1
- package/out/lib/TSVSpliterator.js +1 -1
- package/out/lib/TextSpliterator.d.ts +1 -1
- package/out/lib/TextSpliterator.js +1 -1
- package/out/lib/casing.d.ts +1 -1
- package/out/lib/casing.js +1 -1
- package/out/lib/segment-runtime.d.ts +27 -0
- package/out/lib/segment-runtime.d.ts.map +1 -0
- package/out/lib/segment-runtime.js +38 -0
- package/out/lib/segment-runtime.js.map +1 -0
- package/out/lib/segment-worker-entry.d.ts +10 -0
- package/out/lib/segment-worker-entry.d.ts.map +1 -0
- package/out/lib/segment-worker-entry.js +56 -0
- package/out/lib/segment-worker-entry.js.map +1 -0
- package/out/lib/segment-workers.d.ts +41 -0
- package/out/lib/segment-workers.d.ts.map +1 -0
- package/out/lib/segment-workers.js +105 -0
- package/out/lib/segment-workers.js.map +1 -0
- package/out/lib/segments.d.ts +25 -0
- package/out/lib/segments.d.ts.map +1 -0
- package/out/lib/segments.js +50 -0
- package/out/lib/segments.js.map +1 -0
- package/out/lib/shared.d.ts +1 -1
- package/out/lib/shared.js +1 -1
- package/out/lib/take.d.ts +1 -1
- package/out/lib/take.js +1 -1
- package/out/lib/wasm_base64.d.ts +16 -0
- package/out/lib/wasm_base64.d.ts.map +1 -0
- package/out/lib/wasm_base64.js +16 -0
- package/out/lib/wasm_base64.js.map +1 -0
- package/out/lib/wasm_module.d.ts +36 -0
- package/out/lib/wasm_module.d.ts.map +1 -0
- package/out/lib/wasm_module.js +33 -0
- package/out/lib/wasm_module.js.map +1 -0
- package/out/lib/writer.d.ts +1 -1
- package/out/lib/writer.js +1 -1
- package/out/node/cli/commands/csv.d.ts +1 -1
- package/out/node/cli/commands/csv.js +1 -1
- package/out/node/cli/commands/iterate.d.ts +1 -1
- package/out/node/cli/commands/iterate.js +1 -1
- package/out/node/cli/index.d.ts +1 -1
- package/out/node/cli/index.js +1 -1
- package/out/node/cli/utils.d.ts +1 -1
- package/out/node/cli/utils.js +1 -1
- package/out/node/fs/index.d.ts +14 -2
- package/out/node/fs/index.d.ts.map +1 -1
- package/out/node/fs/index.js +26 -2
- package/out/node/fs/index.js.map +1 -1
- package/package.json +5 -3
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SlidingWindow.d.ts","sourceRoot":"","sources":["../../lib/SlidingWindow.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAqB,KAAK,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AACvF,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAExD,MAAM,WAAW,iBAAiB;IACjC;;OAEG;IACH,SAAS,CAAC,EAAE,sBAAsB,CAAA;IAElC;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;CACnB;AAED;;;;GAIG;AACH,qBAAa,aAAa,CAAC,CAAC,SAAS,UAAU,CAAE,YAAW,gBAAgB,CAAC,SAAS,CAAC;;IACtF,MAAM,EAAE,CAAC,CAAA;IAKT;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IAEd;;;;OAIG;;IAEF;;OAEG;IACH,MAAM,EAAE,CAAC,EACT,IAAI,GAAE,iBAAsB;IAQtB,IAAI,IAAI,cAAc,CAAC,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"SlidingWindow.d.ts","sourceRoot":"","sources":["../../lib/SlidingWindow.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAqB,KAAK,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AACvF,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAExD,MAAM,WAAW,iBAAiB;IACjC;;OAEG;IACH,SAAS,CAAC,EAAE,sBAAsB,CAAA;IAElC;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;CACnB;AAED;;;;GAIG;AACH,qBAAa,aAAa,CAAC,CAAC,SAAS,UAAU,CAAE,YAAW,gBAAgB,CAAC,SAAS,CAAC;;IACtF,MAAM,EAAE,CAAC,CAAA;IAKT;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IAEd;;;;OAIG;;IAEF;;OAEG;IACH,MAAM,EAAE,CAAC,EACT,IAAI,GAAE,iBAAsB;IAQtB,IAAI,IAAI,cAAc,CAAC,SAAS,CAAC;IA+BjC,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,gBAAgB,CAAC,SAAS,CAAC;CAGvD"}
|
package/out/lib/SlidingWindow.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @copyright Sister Software
|
|
3
|
-
* @license
|
|
3
|
+
* @license MIT
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
import { CharacterSequence } from "./CharacterSequence.js";
|
|
@@ -34,15 +34,16 @@ export class SlidingWindow {
|
|
|
34
34
|
this.#byteLength = Math.min(init.byteLength ?? buffer.byteLength, buffer.byteLength);
|
|
35
35
|
}
|
|
36
36
|
next() {
|
|
37
|
-
|
|
38
|
-
//
|
|
39
|
-
|
|
40
|
-
//
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
37
|
+
if (this.cursor < this.#byteLength) {
|
|
38
|
+
// Delegate scanning to CharacterSequence.search, which uses native indexOf for
|
|
39
|
+
// single-byte delimiters and the SIMD/BMH paths otherwise — far faster than a
|
|
40
|
+
// per-byte `.every` callback across the buffer.
|
|
41
|
+
const delimiterIndex = this.#delimiter.search(this.buffer, this.cursor, this.#byteLength);
|
|
42
|
+
if (delimiterIndex !== -1) {
|
|
43
|
+
const range = [this.cursor, delimiterIndex];
|
|
44
|
+
this.cursor = delimiterIndex + this.#delimiter.length;
|
|
45
|
+
return { value: range, done: false };
|
|
46
|
+
}
|
|
46
47
|
}
|
|
47
48
|
if (this.cursor <= this.#byteLength && !this.#done) {
|
|
48
49
|
const range = [this.cursor, this.#byteLength];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SlidingWindow.js","sourceRoot":"","sources":["../../lib/SlidingWindow.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,iBAAiB,EAA+B,MAAM,wBAAwB,CAAA;AAoBvF;;;;GAIG;AACH,MAAM,OAAO,aAAa;IACzB,MAAM,CAAG;IACT,UAAU,CAAmB;IAC7B,WAAW,CAAQ;IACnB,KAAK,GAAG,KAAK,CAAA;IAEb;;OAEG;IACH,MAAM,CAAQ;IAEd;;;;OAIG;IACH;IACC;;OAEG;IACH,MAAS,EACT,OAA0B,EAAE;QAE5B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;QACpB,IAAI,CAAC,UAAU,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QACvD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAA;QAChC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,CAAA;IACrF,CAAC;IAEM,IAAI;QACV,
|
|
1
|
+
{"version":3,"file":"SlidingWindow.js","sourceRoot":"","sources":["../../lib/SlidingWindow.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,iBAAiB,EAA+B,MAAM,wBAAwB,CAAA;AAoBvF;;;;GAIG;AACH,MAAM,OAAO,aAAa;IACzB,MAAM,CAAG;IACT,UAAU,CAAmB;IAC7B,WAAW,CAAQ;IACnB,KAAK,GAAG,KAAK,CAAA;IAEb;;OAEG;IACH,MAAM,CAAQ;IAEd;;;;OAIG;IACH;IACC;;OAEG;IACH,MAAS,EACT,OAA0B,EAAE;QAE5B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;QACpB,IAAI,CAAC,UAAU,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QACvD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAA;QAChC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,CAAA;IACrF,CAAC;IAEM,IAAI;QACV,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;YACpC,+EAA+E;YAC/E,8EAA8E;YAC9E,gDAAgD;YAChD,MAAM,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,MAA+B,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;YAElH,IAAI,cAAc,KAAK,CAAC,CAAC,EAAE,CAAC;gBAC3B,MAAM,KAAK,GAAc,CAAC,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;gBAEtD,IAAI,CAAC,MAAM,GAAG,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAA;gBAErD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAA;YACrC,CAAC;QACF,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,WAAW,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACpD,MAAM,KAAK,GAAc,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;YAExD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,WAAW,CAAA;YAC9B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;YAEjB,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAA;QACrC,CAAC;QAED,OAAO;YACN,KAAK,EAAE,SAAS;YAChB,IAAI,EAAE,IAAI;SACV,CAAA;IACF,CAAC;IAEM,CAAC,MAAM,CAAC,QAAQ,CAAC;QACvB,OAAO,IAAI,CAAA;IACZ,CAAC;CACD"}
|
package/out/lib/Spliterator.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Spliterator.d.ts","sourceRoot":"","sources":["../../lib/Spliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,gBAAgB,EAAE,KAAK,oBAAoB,EAAE,KAAK,eAAe,EAAE,MAAM,uBAAuB,CAAA;AACzG,OAAO,EAIN,KAAK,sBAAsB,EAC3B,MAAM,wBAAwB,CAAA;AAE/B,OAAO,EAAoB,KAAK,kBAAkB,EAAE,KAAK,iBAAiB,EAAkB,MAAM,aAAa,CAAA;AAE/G,OAAO,EAAE,gBAAgB,EAAE,CAAA;AAE3B,YAAY,EAAE,oBAAoB,EAAE,eAAe,EAAE,CAAA;AAErD;;GAEG;AACH,qBAAa,WAAW,CAAC,CAAC,SAAS,UAAU,GAAG,QAAQ,GAAG,WAAW,GAAG,UAAU,CAClF,YAAW,gBAAgB,CAAC,CAAC,CAAC,EAAE,UAAU;;IAI1C;;;;;;;;OAQG;IACH,MAAM,CAAC,SAAS,+BAAwB;IAExC;;;;;OAKG;IACH,MAAM,CAAC,QAAQ,CAAC,CAAC,SAAS,sBAAsB,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,GAAE,eAAoB,GAAG,WAAW;IAIrG;;;;;;OAMG;IACH,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,GAAG,WAAW,GAAG,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,EAAE,eAAe,GAAG,WAAW;IAC5G;;;;;;;OAOG;IACH,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,kBAAkB,EAAE,IAAI,CAAC,EAAE,oBAAoB,GAAG,gBAAgB;IACtF;;;;;;;;OAQG;IACH,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,iBAAiB,EAAE,IAAI,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAC9F;;;;;;;;OAQG;IACH,MAAM,CAAC,IAAI,CACV,MAAM,EAAE,sBAAsB,GAAG,iBAAiB,GAAG,kBAAkB,EACvE,IAAI,CAAC,EAAE,eAAe,GAAG,oBAAoB,GAC3C,WAAW,GAAG,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAsC7D;;OAEG;WACW,iBAAiB,CAAC,IAAI,EAAE,eAAe,GAAG,eAAe,CAAC,UAAU,EAAE,UAAU,EAAE,CAAC;IAUjG;;OAEG;IACI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI;gBAInB,MAAM,EAAE,UAAU,EAAE,IAAI,GAAE,eAAoB;
|
|
1
|
+
{"version":3,"file":"Spliterator.d.ts","sourceRoot":"","sources":["../../lib/Spliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,gBAAgB,EAAE,KAAK,oBAAoB,EAAE,KAAK,eAAe,EAAE,MAAM,uBAAuB,CAAA;AACzG,OAAO,EAIN,KAAK,sBAAsB,EAC3B,MAAM,wBAAwB,CAAA;AAE/B,OAAO,EAAoB,KAAK,kBAAkB,EAAE,KAAK,iBAAiB,EAAkB,MAAM,aAAa,CAAA;AAE/G,OAAO,EAAE,gBAAgB,EAAE,CAAA;AAE3B,YAAY,EAAE,oBAAoB,EAAE,eAAe,EAAE,CAAA;AAErD;;GAEG;AACH,qBAAa,WAAW,CAAC,CAAC,SAAS,UAAU,GAAG,QAAQ,GAAG,WAAW,GAAG,UAAU,CAClF,YAAW,gBAAgB,CAAC,CAAC,CAAC,EAAE,UAAU;;IAI1C;;;;;;;;OAQG;IACH,MAAM,CAAC,SAAS,+BAAwB;IAExC;;;;;OAKG;IACH,MAAM,CAAC,QAAQ,CAAC,CAAC,SAAS,sBAAsB,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,GAAE,eAAoB,GAAG,WAAW;IAIrG;;;;;;OAMG;IACH,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,GAAG,WAAW,GAAG,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,EAAE,eAAe,GAAG,WAAW;IAC5G;;;;;;;OAOG;IACH,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,kBAAkB,EAAE,IAAI,CAAC,EAAE,oBAAoB,GAAG,gBAAgB;IACtF;;;;;;;;OAQG;IACH,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,iBAAiB,EAAE,IAAI,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAC9F;;;;;;;;OAQG;IACH,MAAM,CAAC,IAAI,CACV,MAAM,EAAE,sBAAsB,GAAG,iBAAiB,GAAG,kBAAkB,EACvE,IAAI,CAAC,EAAE,eAAe,GAAG,oBAAoB,GAC3C,WAAW,GAAG,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAsC7D;;OAEG;WACW,iBAAiB,CAAC,IAAI,EAAE,eAAe,GAAG,eAAe,CAAC,UAAU,EAAE,UAAU,EAAE,CAAC;IAUjG;;OAEG;IACI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI;gBAInB,MAAM,EAAE,UAAU,EAAE,IAAI,GAAE,eAAoB;IAyP1D;;OAEG;IACI,IAAI,IAAI,cAAc,CAAC,CAAC,CAAC;IA4ChC;;;;;OAKG;IACI,OAAO,IAAI,CAAC,EAAE;IAIrB;;OAEG;IACI,cAAc,CAAC,OAAO,kCAAoB,GAAG,MAAM,EAAE;IAIrD,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,gBAAgB,CAAC,CAAC,CAAC;CAG/C"}
|
package/out/lib/Spliterator.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @copyright Sister Software
|
|
3
|
-
* @license
|
|
3
|
+
* @license MIT
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
import { AsyncSpliterator } from "./AsyncSpliterator.js";
|
|
@@ -83,6 +83,7 @@ export class Spliterator {
|
|
|
83
83
|
this.#yieldStopCount = Math.max(init.take ?? Infinity, 0) + this.#yieldDropCount;
|
|
84
84
|
this.#yieldByteEstimate = this.#source.byteLength - this.#readPosition;
|
|
85
85
|
this.#skipEmpty = init.skipEmpty ?? true;
|
|
86
|
+
this.#enableQuoteHandling = init.enableQuoteHandling ?? false;
|
|
86
87
|
this.#debug = init.debug ?? false;
|
|
87
88
|
this.#log = this.#debug ? console.debug.bind(console) : () => void 0;
|
|
88
89
|
}
|
|
@@ -108,6 +109,7 @@ export class Spliterator {
|
|
|
108
109
|
* The byte sequence for a double quote.
|
|
109
110
|
*/
|
|
110
111
|
#doubleQuoteSequence = new CharacterSequence('"');
|
|
112
|
+
#enableQuoteHandling;
|
|
111
113
|
/**
|
|
112
114
|
* How many yields to skip.
|
|
113
115
|
*/
|
|
@@ -144,10 +146,6 @@ export class Spliterator {
|
|
|
144
146
|
* The current byte index to perform read operations from.
|
|
145
147
|
*/
|
|
146
148
|
#readPosition;
|
|
147
|
-
/**
|
|
148
|
-
* A bit of a hack to keep track of double quotes.
|
|
149
|
-
*/
|
|
150
|
-
#doubleQuoteStartIndex = -1;
|
|
151
149
|
/**
|
|
152
150
|
* The previous byte range seen while draining the buffer.
|
|
153
151
|
*/
|
|
@@ -191,26 +189,24 @@ export class Spliterator {
|
|
|
191
189
|
this.#log("Reached end of file. Preparing to finalize.");
|
|
192
190
|
// There's a few special cases we could get this far and not have drained the buffer.
|
|
193
191
|
const lastByteRange = this.#previousByteRange;
|
|
194
|
-
if (
|
|
195
|
-
|
|
196
|
-
// There's a special case where the last delimiter is at the end of the *file*.
|
|
197
|
-
// We need to determine if we're at the end of the *file* and there's no delimiter.
|
|
198
|
-
const possibleDelimiterIndex = this.#needle.search(this.#source, sourceByteLength - this.#needle.length);
|
|
199
|
-
if (possibleDelimiterIndex !== -1) {
|
|
200
|
-
this.#log("Inserted byte range at the last delimiter", [possibleDelimiterIndex, sourceByteLength]);
|
|
201
|
-
// We found a delimiter at the end of the file, so we enqueue the byte range.
|
|
202
|
-
this.#indices.enqueue([possibleDelimiterIndex + this.#needle.length, sourceByteLength]);
|
|
203
|
-
}
|
|
204
|
-
else {
|
|
205
|
-
this.#log("Inserted byte range at the last byte", [lastByteIndex, sourceByteLength]);
|
|
206
|
-
// The file didn't end with a delimiter, so we just enqueue the last byte range.
|
|
207
|
-
this.#indices.enqueue([lastByteIndex + this.#needle.length, sourceByteLength]);
|
|
208
|
-
}
|
|
192
|
+
if (this.#readPosition < sourceByteLength) {
|
|
193
|
+
this.#indices.enqueue([this.#readPosition, sourceByteLength]);
|
|
209
194
|
}
|
|
210
|
-
else if (this.#yieldCount === 0) {
|
|
211
|
-
//
|
|
195
|
+
else if (this.#yieldCount === 0 && lastByteRange === undefined) {
|
|
196
|
+
// Nothing was ever found or yielded (e.g. an empty source) — emit the whole buffer as
|
|
197
|
+
// a single field. A run of all-empty fields that `skipEmpty` dropped has a defined
|
|
198
|
+
// `lastByteRange`, so it falls through to the trailing-delimiter case instead of
|
|
199
|
+
// resurfacing the entire delimiter run as one (non-empty) row.
|
|
212
200
|
this.#indices.enqueue([0, sourceByteLength]);
|
|
213
201
|
}
|
|
202
|
+
else if (!this.#enableQuoteHandling) {
|
|
203
|
+
// Simple path: emit empty field for trailing delimiter (match String.split).
|
|
204
|
+
// Quote-aware path handles this inside #fill() via searchMatches.
|
|
205
|
+
const trailingDelimPos = sourceByteLength - this.#needle.length;
|
|
206
|
+
if (trailingDelimPos >= 0 && this.#needle.search(this.#source, trailingDelimPos) !== -1) {
|
|
207
|
+
this.#indices.enqueue([sourceByteLength, sourceByteLength]);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
214
210
|
this.#done = true;
|
|
215
211
|
}
|
|
216
212
|
/**
|
|
@@ -218,75 +214,64 @@ export class Spliterator {
|
|
|
218
214
|
*/
|
|
219
215
|
#fill() {
|
|
220
216
|
const sourceByteLength = this.#source.byteLength;
|
|
217
|
+
if (!this.#enableQuoteHandling) {
|
|
218
|
+
while (this.#readPosition < sourceByteLength && this.#indices.byteLength < this.#highWaterMark) {
|
|
219
|
+
const delimiterIndex = this.#needle.search(this.#source, this.#readPosition);
|
|
220
|
+
if (delimiterIndex === -1)
|
|
221
|
+
return;
|
|
222
|
+
this.#indices.enqueue([this.#readPosition, delimiterIndex]);
|
|
223
|
+
this.#readPosition = delimiterIndex + this.#needle.length;
|
|
224
|
+
}
|
|
225
|
+
return;
|
|
226
|
+
}
|
|
221
227
|
while (this.#readPosition < sourceByteLength && this.#indices.byteLength < this.#highWaterMark) {
|
|
222
|
-
const
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
let sliceEnd;
|
|
228
|
+
const matches = this.#needle.searchMatches(this.#source, this.#doubleQuoteSequence, this.#readPosition, sourceByteLength);
|
|
229
|
+
if (matches.length === 0)
|
|
230
|
+
return;
|
|
226
231
|
let sliceStart = this.#readPosition;
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
// this.#doubleQuoteStartIndex = -1
|
|
240
|
-
// } else {
|
|
241
|
-
// // console.log("Searching for double quote at read position", this.#readPosition)
|
|
242
|
-
// nextDoubleQuoteIndex = this.#doubleQuoteSequence.search(this.#source, this.#readPosition)
|
|
243
|
-
// if (nextDoubleQuoteIndex !== -1) {
|
|
244
|
-
// console.log("Found double quote at", nextDoubleQuoteIndex)
|
|
245
|
-
// this.#doubleQuoteStartIndex = nextDoubleQuoteIndex
|
|
246
|
-
// sliceStart = doubleQuoteStartIndex + 1
|
|
247
|
-
// delimiterSearchStart = nextDoubleQuoteIndex + 1
|
|
248
|
-
// } else {
|
|
249
|
-
// // console.log("No double quote found at read position")
|
|
250
|
-
// delimiterSearchStart = this.#readPosition
|
|
251
|
-
// }
|
|
252
|
-
// }
|
|
253
|
-
let delimiterIndex;
|
|
254
|
-
while (true) {
|
|
255
|
-
delimiterIndex = this.#needle.search(this.#source, delimiterSearchStart);
|
|
256
|
-
if (delimiterIndex === -1) {
|
|
257
|
-
// this.#log(`No delimiters left. Breaking.`)
|
|
258
|
-
return;
|
|
259
|
-
}
|
|
260
|
-
nextDoubleQuoteIndex = this.#doubleQuoteSequence.search(this.#source, delimiterSearchStart, delimiterIndex);
|
|
261
|
-
// Didn't find a double quote before the delimiter.
|
|
262
|
-
if (nextDoubleQuoteIndex === -1 && this.#doubleQuoteStartIndex === -1) {
|
|
263
|
-
// And we haven't found a double quote before.
|
|
264
|
-
sliceEnd = delimiterIndex;
|
|
265
|
-
this.#readPosition = sliceEnd + this.#needle.length;
|
|
266
|
-
break;
|
|
267
|
-
}
|
|
268
|
-
if (nextDoubleQuoteIndex !== -1) {
|
|
269
|
-
// We found a double quote before the delimiter...
|
|
270
|
-
if (this.#doubleQuoteStartIndex === -1) {
|
|
271
|
-
// But this is the first double quote we've found.
|
|
272
|
-
this.#doubleQuoteStartIndex = nextDoubleQuoteIndex;
|
|
273
|
-
delimiterSearchStart = delimiterIndex + 1;
|
|
274
|
-
continue;
|
|
232
|
+
let insideQuotes = false;
|
|
233
|
+
let quoteStart = -1;
|
|
234
|
+
for (const match of matches) {
|
|
235
|
+
if (match.patternId === 1) {
|
|
236
|
+
// Quote found
|
|
237
|
+
if (!insideQuotes) {
|
|
238
|
+
// Emit normal field before the opening quote
|
|
239
|
+
if (match.offset > sliceStart) {
|
|
240
|
+
this.#indices.enqueue([sliceStart, match.offset]);
|
|
241
|
+
}
|
|
242
|
+
insideQuotes = true;
|
|
243
|
+
quoteStart = match.offset;
|
|
275
244
|
}
|
|
276
245
|
else {
|
|
277
|
-
|
|
246
|
+
// Closing quote: emit quoted field, advance past quote
|
|
247
|
+
this.#indices.enqueue([quoteStart + 1, match.offset]);
|
|
248
|
+
insideQuotes = false;
|
|
249
|
+
sliceStart = match.offset + 1;
|
|
278
250
|
}
|
|
279
251
|
}
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
252
|
+
else {
|
|
253
|
+
// Delimiter found; one inside quotes is part of the field, so skip it.
|
|
254
|
+
if (insideQuotes)
|
|
255
|
+
continue;
|
|
256
|
+
// Emit content before this delimiter (empty if delimiter follows closing quote)
|
|
257
|
+
if (sliceStart < match.offset) {
|
|
258
|
+
this.#indices.enqueue([sliceStart, match.offset]);
|
|
259
|
+
}
|
|
260
|
+
sliceStart = match.offset + this.#needle.length;
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
const lastMatch = matches[matches.length - 1];
|
|
264
|
+
this.#readPosition = lastMatch.offset + (lastMatch.patternId === 0 ? this.#needle.length : 1);
|
|
265
|
+
// Emit trailing content after the last match
|
|
266
|
+
// Case 1: Content after last delimiter (final field, no trailing delimiter)
|
|
267
|
+
if (sliceStart < this.#readPosition) {
|
|
268
|
+
this.#indices.enqueue([sliceStart, sourceByteLength]);
|
|
269
|
+
this.#readPosition = sourceByteLength;
|
|
270
|
+
// Case 2: Trailing delimiter (emit empty field to match String.split)
|
|
271
|
+
}
|
|
272
|
+
else if (sliceStart === sourceByteLength && this.#readPosition === sourceByteLength) {
|
|
273
|
+
this.#indices.enqueue([sourceByteLength, sourceByteLength]);
|
|
285
274
|
}
|
|
286
|
-
const byteRange = [sliceStart, sliceEnd];
|
|
287
|
-
// this.#log("Found byte ranges", byteRange)
|
|
288
|
-
this.#log(`${byteRange} --> (${sliceStart - delimiterSearchStart})`, debugAsVisibleCharacters(this.#source.subarray(...byteRange)));
|
|
289
|
-
this.#indices.enqueue(byteRange);
|
|
290
275
|
}
|
|
291
276
|
}
|
|
292
277
|
//#endregion
|
|
@@ -295,33 +280,37 @@ export class Spliterator {
|
|
|
295
280
|
* Read the next byte range from the source.
|
|
296
281
|
*/
|
|
297
282
|
next() {
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
this.#
|
|
302
|
-
|
|
303
|
-
this.#
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
283
|
+
// Loop rather than recurse: a long run of skipped (empty or dropped) ranges would
|
|
284
|
+
// otherwise grow the call stack one frame per skip and overflow.
|
|
285
|
+
while (true) {
|
|
286
|
+
if (this.#done || this.#yieldCount >= this.#yieldStopCount)
|
|
287
|
+
return this.#finalize();
|
|
288
|
+
if (!this.#indices.size)
|
|
289
|
+
this.#fill();
|
|
290
|
+
if (!this.#indices.size) {
|
|
291
|
+
this.#drain();
|
|
292
|
+
}
|
|
293
|
+
const currentByteRange = this.#indices.dequeue();
|
|
294
|
+
this.#previousByteRange = currentByteRange;
|
|
295
|
+
if (!currentByteRange) {
|
|
296
|
+
this.#done = true;
|
|
297
|
+
return this.#finalize();
|
|
298
|
+
}
|
|
299
|
+
const [start, end] = currentByteRange;
|
|
300
|
+
const slice = this.#source.subarray(start, end);
|
|
301
|
+
if (slice.length === 0 && this.#skipEmpty) {
|
|
302
|
+
continue;
|
|
303
|
+
}
|
|
304
|
+
this.#yieldCount++;
|
|
305
|
+
if (this.#yieldCount <= this.#yieldDropCount) {
|
|
306
|
+
continue;
|
|
307
|
+
}
|
|
308
|
+
this.#yieldedByteLength += end - start;
|
|
309
|
+
return {
|
|
310
|
+
value: slice,
|
|
311
|
+
done: false,
|
|
312
|
+
};
|
|
319
313
|
}
|
|
320
|
-
this.#yieldedByteLength += end - start;
|
|
321
|
-
return {
|
|
322
|
-
value: slice,
|
|
323
|
-
done: false,
|
|
324
|
-
};
|
|
325
314
|
}
|
|
326
315
|
/**
|
|
327
316
|
* Collect all the byte ranges from the file.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Spliterator.js","sourceRoot":"","sources":["../../lib/Spliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,gBAAgB,EAAmD,MAAM,uBAAuB,CAAA;AACzG,OAAO,EACN,iBAAiB,EACjB,wBAAwB,EACxB,uBAAuB,GAEvB,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EAAE,gBAAgB,EAAmE,MAAM,aAAa,CAAA;AAE/G,OAAO,EAAE,gBAAgB,EAAE,CAAA;AAI3B;;GAEG;AACH,MAAM,OAAO,WAAW;IAGvB,mBAAmB;IAEnB;;;;;;;;OAQG;IACH,MAAM,CAAC,SAAS,GAAG,gBAAgB,CAAC,IAAI,CAAA;IAExC;;;;;OAKG;IACH,MAAM,CAAC,QAAQ,CAAmC,MAAS,EAAE,OAAwB,EAAE;QACtF,OAAO,IAAI,WAAW,CAAC,uBAAuB,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,CAAA;IAC9D,CAAC;IA0CD,MAAM,CAAC,IAAI,CACV,MAAuE,EACvE,OAA+C,EAAE;QAEjD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,aAAa,IAAI,MAAM,EAAE,CAAC;YAClE,OAAO,IAAI,gBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QAC1C,CAAC;QAED,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,YAAY,GAAG,IAAI,gBAAgB,CAAC,MAAM,CAAC,EAAE,CAAC;YACrF,OAAO,MAAM,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,mBAAmB,EAAE,EAAE,EAAE;gBAC3E,IAAI,aAAiC,CAAA;gBAErC,IAAI,CAAC;oBACJ,aAAa,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE;wBACjD,aAAa,EAAE,IAAI,CAAC,aAAa;qBACjC,CAAC,CAAA;gBACH,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,YAAY,GAAG,EAAE,CAAC;wBACzD,MAAM,OAAO,GAAG,IAAI,KAAK,CACxB,gHAAgH,CAChH,CAAA;wBAED,OAAO,CAAC,KAAK,GAAG,KAAK,CAAA;wBAErB,MAAM,OAAO,CAAA;oBACd,CAAC;oBAED,MAAM,KAAK,CAAA;gBACZ,CAAC;gBAED,OAAO,IAAI,gBAAgB,CAAC,aAAa,EAAE,IAAI,CAAC,CAAA;YACjD,CAAC,CAAC,CAAA;QACH,CAAC;QAED,OAAO,IAAI,WAAW,CAAC,uBAAuB,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,CAAA;IAC9D,CAAC;IAED;;OAEG;IACI,MAAM,CAAC,iBAAiB,CAAC,IAAqB;QACpD,OAAO,IAAI,eAAe,CAA2B;YACpD,SAAS,CAAC,KAAK,EAAE,UAAU;gBAC1B,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;gBAEhD,UAAU,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAA;YAC1C,CAAC;SACD,CAAC,CAAA;IACH,CAAC;IAED;;OAEG;IACI,CAAC,MAAM,CAAC,OAAO,CAAC;QACtB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;IACtB,CAAC;IAED,YAAY,MAAkB,EAAE,OAAwB,EAAE;QACzD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QAErB,IAAI,CAAC,OAAO,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QAEpD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAA;QAEvC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,CAAA;QAE7D,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAA;QAClD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,eAAe,CAAA;QAChF,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,aAAa,CAAA;QAEtE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAA;
|
|
1
|
+
{"version":3,"file":"Spliterator.js","sourceRoot":"","sources":["../../lib/Spliterator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,gBAAgB,EAAmD,MAAM,uBAAuB,CAAA;AACzG,OAAO,EACN,iBAAiB,EACjB,wBAAwB,EACxB,uBAAuB,GAEvB,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EAAE,gBAAgB,EAAmE,MAAM,aAAa,CAAA;AAE/G,OAAO,EAAE,gBAAgB,EAAE,CAAA;AAI3B;;GAEG;AACH,MAAM,OAAO,WAAW;IAGvB,mBAAmB;IAEnB;;;;;;;;OAQG;IACH,MAAM,CAAC,SAAS,GAAG,gBAAgB,CAAC,IAAI,CAAA;IAExC;;;;;OAKG;IACH,MAAM,CAAC,QAAQ,CAAmC,MAAS,EAAE,OAAwB,EAAE;QACtF,OAAO,IAAI,WAAW,CAAC,uBAAuB,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,CAAA;IAC9D,CAAC;IA0CD,MAAM,CAAC,IAAI,CACV,MAAuE,EACvE,OAA+C,EAAE;QAEjD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,aAAa,IAAI,MAAM,EAAE,CAAC;YAClE,OAAO,IAAI,gBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QAC1C,CAAC;QAED,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,YAAY,GAAG,IAAI,gBAAgB,CAAC,MAAM,CAAC,EAAE,CAAC;YACrF,OAAO,MAAM,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,mBAAmB,EAAE,EAAE,EAAE;gBAC3E,IAAI,aAAiC,CAAA;gBAErC,IAAI,CAAC;oBACJ,aAAa,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE;wBACjD,aAAa,EAAE,IAAI,CAAC,aAAa;qBACjC,CAAC,CAAA;gBACH,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,YAAY,GAAG,EAAE,CAAC;wBACzD,MAAM,OAAO,GAAG,IAAI,KAAK,CACxB,gHAAgH,CAChH,CAAA;wBAED,OAAO,CAAC,KAAK,GAAG,KAAK,CAAA;wBAErB,MAAM,OAAO,CAAA;oBACd,CAAC;oBAED,MAAM,KAAK,CAAA;gBACZ,CAAC;gBAED,OAAO,IAAI,gBAAgB,CAAC,aAAa,EAAE,IAAI,CAAC,CAAA;YACjD,CAAC,CAAC,CAAA;QACH,CAAC;QAED,OAAO,IAAI,WAAW,CAAC,uBAAuB,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,CAAA;IAC9D,CAAC;IAED;;OAEG;IACI,MAAM,CAAC,iBAAiB,CAAC,IAAqB;QACpD,OAAO,IAAI,eAAe,CAA2B;YACpD,SAAS,CAAC,KAAK,EAAE,UAAU;gBAC1B,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;gBAEhD,UAAU,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAA;YAC1C,CAAC;SACD,CAAC,CAAA;IACH,CAAC;IAED;;OAEG;IACI,CAAC,MAAM,CAAC,OAAO,CAAC;QACtB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;IACtB,CAAC;IAED,YAAY,MAAkB,EAAE,OAAwB,EAAE;QACzD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QAErB,IAAI,CAAC,OAAO,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QAEpD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAA;QAEvC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,CAAA;QAE7D,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAA;QAClD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,eAAe,CAAA;QAChF,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,aAAa,CAAA;QAEtE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAA;QACxC,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,mBAAmB,IAAI,KAAK,CAAA;QAE7D,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,IAAI,KAAK,CAAA;QACjC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAA;IACrE,CAAC;IAED,YAAY;IAEZ,4BAA4B;IAE5B;;OAEG;IACM,OAAO,CAAY;IAE5B;;;;;;OAMG;IACM,QAAQ,GAAG,IAAI,UAAU,EAAE,CAAA;IAEpC;;OAEG;IACM,OAAO,CAAmB;IAEnC;;OAEG;IACM,oBAAoB,GAAsB,IAAI,iBAAiB,CAAC,GAAG,CAAC,CAAA;IACpE,oBAAoB,CAAS;IAEtC;;OAEG;IACM,eAAe,CAAQ;IAEhC;;OAEG;IACM,eAAe,CAAQ;IAEhC;;;;OAIG;IACM,kBAAkB,CAAQ;IAEnC;;OAEG;IACM,UAAU,CAAS;IAE5B;;;;OAIG;IACM,cAAc,CAAQ;IAE/B;;OAEG;IACH,kBAAkB,GAAG,CAAC,CAAA;IAEtB;;OAEG;IACH,WAAW,GAAG,CAAC,CAAA;IAEf;;OAEG;IACH,aAAa,CAAQ;IAErB;;OAEG;IACH,kBAAkB,CAAuB;IAEzC;;OAEG;IACH,MAAM,CAAS;IAEf;;OAEG;IACH,KAAK,GAAG,KAAK,CAAA;IAEb,YAAY;IAEZ,yBAAyB;IAEzB,IAAI,CAA0B;IAE9B,SAAS;QACR,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB;;;eAGG;YACH,MAAM,6BAA6B,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;YAClF,MAAM,YAAY,GAAG,IAAI,CAAC,kBAAkB,GAAG,CAAC,IAAI,CAAC,kBAAkB,GAAG,6BAA6B,CAAC,CAAA;YAExG,IAAI,CAAC,IAAI,CAAC;gBACT,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU;gBACtC,YAAY,EAAE,IAAI,CAAC,aAAa;gBAChC,iBAAiB,EAAE,IAAI,CAAC,kBAAkB;gBAC1C,iBAAiB,EAAE,IAAI,CAAC,kBAAkB;gBAC1C,UAAU,EAAE,IAAI,CAAC,WAAW;gBAC5B,6BAA6B;gBAC7B,YAAY;aACZ,CAAC,CAAA;QACH,CAAC;QAED,OAAO;YACN,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,SAAS;SAChB,CAAA;IACF,CAAC;IAED,MAAM;QACL,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAA;QAEhD,IAAI,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAA;QAExD,qFAAqF;QACrF,MAAM,aAAa,GAAG,IAAI,CAAC,kBAAkB,CAAA;QAE7C,IAAI,IAAI,CAAC,aAAa,GAAG,gBAAgB,EAAE,CAAC;YAC3C,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,aAAa,EAAE,gBAAgB,CAAC,CAAC,CAAA;QAC9D,CAAC;aAAM,IAAI,IAAI,CAAC,WAAW,KAAK,CAAC,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;YAClE,sFAAsF;YACtF,mFAAmF;YACnF,iFAAiF;YACjF,+DAA+D;YAC/D,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC,CAAA;QAC7C,CAAC;aAAM,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;YACvC,6EAA6E;YAC7E,kEAAkE;YAClE,MAAM,gBAAgB,GAAG,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;YAE/D,IAAI,gBAAgB,IAAI,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gBACzF,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,gBAAgB,EAAE,gBAAgB,CAAC,CAAC,CAAA;YAC5D,CAAC;QACF,CAAC;QAED,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;IAClB,CAAC;IAED;;OAEG;IACH,KAAK;QACJ,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAA;QAEhD,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAChC,OAAO,IAAI,CAAC,aAAa,GAAG,gBAAgB,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;gBAChG,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,aAAa,CAAC,CAAA;gBAE5E,IAAI,cAAc,KAAK,CAAC,CAAC;oBAAE,OAAM;gBAEjC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC,CAAA;gBAC3D,IAAI,CAAC,aAAa,GAAG,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;YAC1D,CAAC;YAED,OAAM;QACP,CAAC;QAED,OAAO,IAAI,CAAC,aAAa,GAAG,gBAAgB,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;YAChG,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CACzC,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,oBAAoB,EACzB,IAAI,CAAC,aAAa,EAClB,gBAAgB,CAChB,CAAA;YAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAM;YAEhC,IAAI,UAAU,GAAG,IAAI,CAAC,aAAa,CAAA;YACnC,IAAI,YAAY,GAAG,KAAK,CAAA;YACxB,IAAI,UAAU,GAAG,CAAC,CAAC,CAAA;YAEnB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC7B,IAAI,KAAK,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;oBAC3B,cAAc;oBACd,IAAI,CAAC,YAAY,EAAE,CAAC;wBACnB,6CAA6C;wBAC7C,IAAI,KAAK,CAAC,MAAM,GAAG,UAAU,EAAE,CAAC;4BAC/B,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;wBAClD,CAAC;wBACD,YAAY,GAAG,IAAI,CAAA;wBACnB,UAAU,GAAG,KAAK,CAAC,MAAM,CAAA;oBAC1B,CAAC;yBAAM,CAAC;wBACP,uDAAuD;wBACvD,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,UAAU,GAAG,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;wBACrD,YAAY,GAAG,KAAK,CAAA;wBACpB,UAAU,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAA;oBAC9B,CAAC;gBACF,CAAC;qBAAM,CAAC;oBACP,uEAAuE;oBACvE,IAAI,YAAY;wBAAE,SAAQ;oBAE1B,gFAAgF;oBAChF,IAAI,UAAU,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;wBAC/B,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;oBAClD,CAAC;oBACD,UAAU,GAAG,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;gBAChD,CAAC;YACF,CAAC;YAED,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAE,CAAA;YAC9C,IAAI,CAAC,aAAa,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,SAAS,CAAC,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;YAE7F,6CAA6C;YAC7C,4EAA4E;YAC5E,IAAI,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;gBACrC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAC,CAAA;gBACrD,IAAI,CAAC,aAAa,GAAG,gBAAgB,CAAA;gBACrC,sEAAsE;YACvE,CAAC;iBAAM,IAAI,UAAU,KAAK,gBAAgB,IAAI,IAAI,CAAC,aAAa,KAAK,gBAAgB,EAAE,CAAC;gBACvF,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,gBAAgB,EAAE,gBAAgB,CAAC,CAAC,CAAA;YAC5D,CAAC;QACF,CAAC;IACF,CAAC;IAED,YAAY;IAEZ,0BAA0B;IAE1B;;OAEG;IACI,IAAI;QACV,kFAAkF;QAClF,iEAAiE;QACjE,OAAO,IAAI,EAAE,CAAC;YACb,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,eAAe;gBAAE,OAAO,IAAI,CAAC,SAAS,EAAE,CAAA;YAEnF,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI;gBAAE,IAAI,CAAC,KAAK,EAAE,CAAA;YAErC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACzB,IAAI,CAAC,MAAM,EAAE,CAAA;YACd,CAAC;YAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAA;YAChD,IAAI,CAAC,kBAAkB,GAAG,gBAAgB,CAAA;YAE1C,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACvB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;gBAEjB,OAAO,IAAI,CAAC,SAAS,EAAE,CAAA;YACxB,CAAC;YAED,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,gBAAgB,CAAA;YAErC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;YAE/C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC3C,SAAQ;YACT,CAAC;YAED,IAAI,CAAC,WAAW,EAAE,CAAA;YAElB,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;gBAC9C,SAAQ;YACT,CAAC;YAED,IAAI,CAAC,kBAAkB,IAAI,GAAG,GAAG,KAAK,CAAA;YAEtC,OAAO;gBACN,KAAK,EAAE,KAAqB;gBAC5B,IAAI,EAAE,KAAK;aACX,CAAA;QACF,CAAC;IACF,CAAC;IAED;;;;;OAKG;IACI,OAAO;QACb,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACxB,CAAC;IAED;;OAEG;IACI,cAAc,CAAC,OAAO,GAAG,IAAI,WAAW,EAAE;QAChD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;IAC1D,CAAC;IAEM,CAAC,MAAM,CAAC,QAAQ,CAAC;QACvB,OAAO,IAAI,CAAA;IACZ,CAAC"}
|
package/out/lib/casing.d.ts
CHANGED
package/out/lib/casing.js
CHANGED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license MIT
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
export type SegmentHandler = (record: Uint8Array, ctx: {
|
|
7
|
+
index: number;
|
|
8
|
+
segmentIndex: number;
|
|
9
|
+
}) => unknown | Promise<unknown>;
|
|
10
|
+
export interface RunSegmentIO {
|
|
11
|
+
records: AsyncIterable<Uint8Array>;
|
|
12
|
+
handleRecord: SegmentHandler;
|
|
13
|
+
segmentIndex: number;
|
|
14
|
+
batchSize: number;
|
|
15
|
+
maxInFlight: number;
|
|
16
|
+
post: (batch: unknown[], transfer: ArrayBuffer[]) => void;
|
|
17
|
+
waitForAck: () => Promise<void>;
|
|
18
|
+
inFlight: () => number;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Drive one segment: read records, run the handler, accumulate results into `batchSize` batches, transfer `Uint8Array`
|
|
22
|
+
* result buffers zero-copy, and respect a bounded in-flight window by awaiting an ack before exceeding `maxInFlight`
|
|
23
|
+
* outstanding batches. Results of `undefined` are skipped (the filter case). Transport-agnostic so it unit-tests
|
|
24
|
+
* without `worker_threads`.
|
|
25
|
+
*/
|
|
26
|
+
export declare function runSegment(io: RunSegmentIO): Promise<void>;
|
|
27
|
+
//# sourceMappingURL=segment-runtime.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"segment-runtime.d.ts","sourceRoot":"","sources":["../../lib/segment-runtime.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,MAAM,cAAc,GAAG,CAC5B,MAAM,EAAE,UAAU,EAClB,GAAG,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,KACxC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;AAE/B,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;IAClC,YAAY,EAAE,cAAc,CAAA;IAC5B,YAAY,EAAE,MAAM,CAAA;IACpB,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;IACnB,IAAI,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,IAAI,CAAA;IACzD,UAAU,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;IAC/B,QAAQ,EAAE,MAAM,MAAM,CAAA;CACtB;AAED;;;;;GAKG;AACH,wBAAsB,UAAU,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CA6BhE"}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license MIT
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* Drive one segment: read records, run the handler, accumulate results into `batchSize` batches, transfer `Uint8Array`
|
|
8
|
+
* result buffers zero-copy, and respect a bounded in-flight window by awaiting an ack before exceeding `maxInFlight`
|
|
9
|
+
* outstanding batches. Results of `undefined` are skipped (the filter case). Transport-agnostic so it unit-tests
|
|
10
|
+
* without `worker_threads`.
|
|
11
|
+
*/
|
|
12
|
+
export async function runSegment(io) {
|
|
13
|
+
let batch = [];
|
|
14
|
+
let transfer = [];
|
|
15
|
+
let index = 0;
|
|
16
|
+
const flush = async () => {
|
|
17
|
+
if (batch.length === 0)
|
|
18
|
+
return;
|
|
19
|
+
while (io.inFlight() >= io.maxInFlight)
|
|
20
|
+
await io.waitForAck();
|
|
21
|
+
io.post(batch, transfer);
|
|
22
|
+
batch = [];
|
|
23
|
+
transfer = [];
|
|
24
|
+
};
|
|
25
|
+
for await (const record of io.records) {
|
|
26
|
+
const result = await io.handleRecord(record, { index, segmentIndex: io.segmentIndex });
|
|
27
|
+
index++;
|
|
28
|
+
if (result === undefined)
|
|
29
|
+
continue;
|
|
30
|
+
batch.push(result);
|
|
31
|
+
if (result instanceof Uint8Array)
|
|
32
|
+
transfer.push(result.buffer);
|
|
33
|
+
if (batch.length >= io.batchSize)
|
|
34
|
+
await flush();
|
|
35
|
+
}
|
|
36
|
+
await flush();
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=segment-runtime.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"segment-runtime.js","sourceRoot":"","sources":["../../lib/segment-runtime.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAkBH;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,EAAgB;IAChD,IAAI,KAAK,GAAc,EAAE,CAAA;IACzB,IAAI,QAAQ,GAAkB,EAAE,CAAA;IAChC,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,MAAM,KAAK,GAAG,KAAK,IAAmB,EAAE;QACvC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAM;QAE9B,OAAO,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,WAAW;YAAE,MAAM,EAAE,CAAC,UAAU,EAAE,CAAA;QAE7D,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;QACxB,KAAK,GAAG,EAAE,CAAA;QACV,QAAQ,GAAG,EAAE,CAAA;IACd,CAAC,CAAA;IAED,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,YAAY,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,CAAC,YAAY,EAAE,CAAC,CAAA;QACtF,KAAK,EAAE,CAAA;QAEP,IAAI,MAAM,KAAK,SAAS;YAAE,SAAQ;QAElC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAElB,IAAI,MAAM,YAAY,UAAU;YAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,MAAqB,CAAC,CAAA;QAE7E,IAAI,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC,SAAS;YAAE,MAAM,KAAK,EAAE,CAAA;IAChD,CAAC;IAED,MAAM,KAAK,EAAE,CAAA;AACd,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license MIT
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Runs inside a worker thread (spawned by runSegmentWorkers). Reads its segment via its own handle,
|
|
7
|
+
* runs the user handler per record, and posts batched results to the parent with ack backpressure.
|
|
8
|
+
*/
|
|
9
|
+
export {};
|
|
10
|
+
//# sourceMappingURL=segment-worker-entry.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"segment-worker-entry.d.ts","sourceRoot":"","sources":["../../lib/segment-worker-entry.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG"}
|