spliterator 4.0.0 → 6.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/README.md +56 -0
  2. package/out/index.d.ts +7 -1
  3. package/out/index.d.ts.map +1 -1
  4. package/out/index.js +7 -1
  5. package/out/index.js.map +1 -1
  6. package/out/lib/AsyncSequence.d.ts +180 -0
  7. package/out/lib/AsyncSequence.d.ts.map +1 -0
  8. package/out/lib/AsyncSequence.js +432 -0
  9. package/out/lib/AsyncSequence.js.map +1 -0
  10. package/out/lib/AsyncSpliterator.d.ts.map +1 -1
  11. package/out/lib/AsyncSpliterator.js +6 -4
  12. package/out/lib/AsyncSpliterator.js.map +1 -1
  13. package/out/lib/BufferController.d.ts.map +1 -1
  14. package/out/lib/BufferController.js.map +1 -1
  15. package/out/lib/CSVSpliterator.d.ts +14 -11
  16. package/out/lib/CSVSpliterator.d.ts.map +1 -1
  17. package/out/lib/CSVSpliterator.js +53 -37
  18. package/out/lib/CSVSpliterator.js.map +1 -1
  19. package/out/lib/CharacterSequence.d.ts +10 -1
  20. package/out/lib/CharacterSequence.d.ts.map +1 -1
  21. package/out/lib/CharacterSequence.js +76 -26
  22. package/out/lib/CharacterSequence.js.map +1 -1
  23. package/out/lib/CompositeDataView.d.ts.map +1 -1
  24. package/out/lib/CompositeDataView.js.map +1 -1
  25. package/out/lib/IndexQueue.d.ts.map +1 -1
  26. package/out/lib/IndexQueue.js +3 -1
  27. package/out/lib/IndexQueue.js.map +1 -1
  28. package/out/lib/JSONSpliterator.d.ts +20 -9
  29. package/out/lib/JSONSpliterator.d.ts.map +1 -1
  30. package/out/lib/JSONSpliterator.js +27 -19
  31. package/out/lib/JSONSpliterator.js.map +1 -1
  32. package/out/lib/PSVSpliterator.js.map +1 -1
  33. package/out/lib/SlidingWindow.d.ts.map +1 -1
  34. package/out/lib/SlidingWindow.js.map +1 -1
  35. package/out/lib/Spliterator.d.ts.map +1 -1
  36. package/out/lib/Spliterator.js +6 -4
  37. package/out/lib/Spliterator.js.map +1 -1
  38. package/out/lib/TSVSpliterator.js.map +1 -1
  39. package/out/lib/TextSpliterator.d.ts +6 -4
  40. package/out/lib/TextSpliterator.d.ts.map +1 -1
  41. package/out/lib/TextSpliterator.js +10 -13
  42. package/out/lib/TextSpliterator.js.map +1 -1
  43. package/out/lib/adaptive-source.d.ts +39 -0
  44. package/out/lib/adaptive-source.d.ts.map +1 -0
  45. package/out/lib/adaptive-source.js +95 -0
  46. package/out/lib/adaptive-source.js.map +1 -0
  47. package/out/lib/casing.js +3 -3
  48. package/out/lib/casing.js.map +1 -1
  49. package/out/lib/chunks.d.ts +20 -0
  50. package/out/lib/chunks.d.ts.map +1 -0
  51. package/out/lib/chunks.js +36 -0
  52. package/out/lib/chunks.js.map +1 -0
  53. package/out/lib/merge-async-iterators.d.ts.map +1 -1
  54. package/out/lib/merge-async-iterators.js +3 -2
  55. package/out/lib/merge-async-iterators.js.map +1 -1
  56. package/out/lib/parallel-map-runtime.d.ts +3 -1
  57. package/out/lib/parallel-map-runtime.d.ts.map +1 -1
  58. package/out/lib/parallel-map-runtime.js +1 -1
  59. package/out/lib/parallel-map-runtime.js.map +1 -1
  60. package/out/lib/parallel-map-worker-entry.js +3 -2
  61. package/out/lib/parallel-map-worker-entry.js.map +1 -1
  62. package/out/lib/parallel-map-workers.d.ts +57 -0
  63. package/out/lib/parallel-map-workers.d.ts.map +1 -0
  64. package/out/lib/parallel-map-workers.js +82 -0
  65. package/out/lib/parallel-map-workers.js.map +1 -0
  66. package/out/lib/parallel-map.d.ts +17 -34
  67. package/out/lib/parallel-map.d.ts.map +1 -1
  68. package/out/lib/parallel-map.js +18 -62
  69. package/out/lib/parallel-map.js.map +1 -1
  70. package/out/lib/pivot.d.ts +30 -0
  71. package/out/lib/pivot.d.ts.map +1 -0
  72. package/out/lib/pivot.js +47 -0
  73. package/out/lib/pivot.js.map +1 -0
  74. package/out/lib/segment-runtime.d.ts.map +1 -1
  75. package/out/lib/segment-runtime.js +7 -4
  76. package/out/lib/segment-runtime.js.map +1 -1
  77. package/out/lib/segment-worker-entry.js +20 -17
  78. package/out/lib/segment-worker-entry.js.map +1 -1
  79. package/out/lib/segment-workers.d.ts +22 -9
  80. package/out/lib/segment-workers.d.ts.map +1 -1
  81. package/out/lib/segment-workers.js +8 -4
  82. package/out/lib/segment-workers.js.map +1 -1
  83. package/out/lib/segments.d.ts +9 -3
  84. package/out/lib/segments.d.ts.map +1 -1
  85. package/out/lib/segments.js +6 -4
  86. package/out/lib/segments.js.map +1 -1
  87. package/out/lib/shared.d.ts +3 -29
  88. package/out/lib/shared.d.ts.map +1 -1
  89. package/out/lib/shared.js +0 -55
  90. package/out/lib/shared.js.map +1 -1
  91. package/out/lib/take.d.ts +16 -0
  92. package/out/lib/take.d.ts.map +1 -1
  93. package/out/lib/take.js +31 -1
  94. package/out/lib/take.js.map +1 -1
  95. package/out/lib/utils.d.ts +37 -0
  96. package/out/lib/utils.d.ts.map +1 -0
  97. package/out/lib/utils.js +39 -0
  98. package/out/lib/utils.js.map +1 -0
  99. package/out/lib/wasm_module.d.ts +9 -3
  100. package/out/lib/wasm_module.d.ts.map +1 -1
  101. package/out/lib/wasm_module.js.map +1 -1
  102. package/out/lib/writer.d.ts +1 -1
  103. package/out/lib/writer.d.ts.map +1 -1
  104. package/out/lib/writer.js +1 -1
  105. package/out/lib/writer.js.map +1 -1
  106. package/out/lib/zip.d.ts +33 -0
  107. package/out/lib/zip.d.ts.map +1 -0
  108. package/out/lib/zip.js +59 -0
  109. package/out/lib/zip.js.map +1 -0
  110. package/out/node/cli/commands/csv.d.ts +3 -38
  111. package/out/node/cli/commands/csv.d.ts.map +1 -1
  112. package/out/node/cli/commands/csv.js +55 -47
  113. package/out/node/cli/commands/csv.js.map +1 -1
  114. package/out/node/cli/commands/iterate.d.ts +3 -31
  115. package/out/node/cli/commands/iterate.d.ts.map +1 -1
  116. package/out/node/cli/commands/iterate.js +45 -24
  117. package/out/node/cli/commands/iterate.js.map +1 -1
  118. package/out/node/cli/index.js +36 -22
  119. package/out/node/cli/index.js.map +1 -1
  120. package/out/node/cli/parse-args.d.ts +120 -0
  121. package/out/node/cli/parse-args.d.ts.map +1 -0
  122. package/out/node/cli/parse-args.js +118 -0
  123. package/out/node/cli/parse-args.js.map +1 -0
  124. package/out/node/cli/utils.d.ts +79 -28
  125. package/out/node/cli/utils.d.ts.map +1 -1
  126. package/out/node/cli/utils.js +63 -63
  127. package/out/node/cli/utils.js.map +1 -1
  128. package/out/node/fs/index.d.ts.map +1 -1
  129. package/out/node/fs/index.js +7 -3
  130. package/out/node/fs/index.js.map +1 -1
  131. package/package.json +13 -17
@@ -32,7 +32,7 @@ function makeBatcher(source, batchSize) {
32
32
  }
33
33
  batch.push(value);
34
34
  }
35
- return batch.length > 0 ? batch : null;
35
+ return batch.length ? batch : null;
36
36
  });
37
37
  // Keep the chain alive regardless of this pull's outcome so the next call still serializes.
38
38
  chain = pull.then(() => undefined, () => undefined);
@@ -1 +1 @@
1
- {"version":3,"file":"parallel-map-runtime.js","sourceRoot":"","sources":["../../lib/parallel-map-runtime.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AAOhE;;;;GAIG;AACH,SAAS,WAAW,CAAI,MAAsC,EAAE,SAAiB;IAChF,MAAM,QAAQ,GACb,MAAM,CAAC,aAAa,IAAI,MAAM;QAC7B,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE;QAChC,CAAC,CAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAkC,CAAA;IAE9D,IAAI,KAAK,GAAqB,OAAO,CAAC,OAAO,EAAE,CAAA;IAC/C,IAAI,SAAS,GAAG,KAAK,CAAA;IAErB,OAAO,GAAG,EAAE;QACX,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YAClC,IAAI,SAAS;gBAAE,OAAO,IAAI,CAAA;YAE1B,MAAM,KAAK,GAAQ,EAAE,CAAA;YAErB,OAAO,KAAK,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;gBACjC,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;gBAE7C,IAAI,IAAI,EAAE,CAAC;oBACV,SAAS,GAAG,IAAI,CAAA;oBAChB,MAAK;gBACN,CAAC;gBAED,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAClB,CAAC;YAED,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAA;QACvC,CAAC,CAAC,CAAA;QAEF,4FAA4F;QAC5F,KAAK,GAAG,IAAI,CAAC,IAAI,CAChB,GAAG,EAAE,CAAC,SAAS,EACf,GAAG,EAAE,CAAC,SAAS,CACf,CAAA;QAED,OAAO,IAAI,CAAA;IACZ,CAAC,CAAA;AACF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,OAAO,CACtB,OAAgC,EAChC,MAAsC,EACtC,SAAiB;IAEjB,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;IAEhD,KAAK,SAAS,CAAC,CAAC,UAAU,CAAC,MAAwB;QAClD,SAAS,CAAC;YACT,MAAM,KAAK,GAAG,MAAM,SAAS,EAAE,CAAA;YAE/B,IAAI,KAAK,KAAK,IAAI;gBAAE,OAAM;YAE1B,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YAE3C,KAAK,CAAC,CAAC,OAAO,CAAA;QACf,CAAC;IACF,CAAC;IAED,OAAO,mBAAmB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;AACxE,CAAC"}
1
+ {"version":3,"file":"parallel-map-runtime.js","sourceRoot":"","sources":["../../lib/parallel-map-runtime.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AAShE;;;;GAIG;AACH,SAAS,WAAW,CAAI,MAAsC,EAAE,SAAiB;IAChF,MAAM,QAAQ,GACb,MAAM,CAAC,aAAa,IAAI,MAAM;QAC7B,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE;QAChC,CAAC,CAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAkC,CAAA;IAE9D,IAAI,KAAK,GAAqB,OAAO,CAAC,OAAO,EAAE,CAAA;IAC/C,IAAI,SAAS,GAAG,KAAK,CAAA;IAErB,OAAO,GAAG,EAAE;QACX,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YAClC,IAAI,SAAS;gBAAE,OAAO,IAAI,CAAA;YAE1B,MAAM,KAAK,GAAQ,EAAE,CAAA;YAErB,OAAO,KAAK,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;gBACjC,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;gBAE7C,IAAI,IAAI,EAAE,CAAC;oBACV,SAAS,GAAG,IAAI,CAAA;oBAEhB,MAAK;gBACN,CAAC;gBAED,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAClB,CAAC;YAED,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAA;QACnC,CAAC,CAAC,CAAA;QAEF,4FAA4F;QAC5F,KAAK,GAAG,IAAI,CAAC,IAAI,CAChB,GAAG,EAAE,CAAC,SAAS,EACf,GAAG,EAAE,CAAC,SAAS,CACf,CAAA;QAED,OAAO,IAAI,CAAA;IACZ,CAAC,CAAA;AACF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,OAAO,CACtB,OAAgC,EAChC,MAAsC,EACtC,SAAiB;IAEjB,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;IAEhD,KAAK,SAAS,CAAC,CAAC,UAAU,CAAC,MAAwB;QAClD,SAAS,CAAC;YACT,MAAM,KAAK,GAAG,MAAM,SAAS,EAAE,CAAA;YAE/B,IAAI,KAAK,KAAK,IAAI;gBAAE,OAAM;YAE1B,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YAE3C,KAAK,CAAC,CAAC,OAAO,CAAA;QACf,CAAC;IACF,CAAC;IAED,OAAO,mBAAmB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;AACxE,CAAC"}
@@ -19,7 +19,7 @@ port.on("message", async (msg) => {
19
19
  try {
20
20
  const handleItem = await handlerReady;
21
21
  if (typeof handleItem !== "function") {
22
- throw new Error(`Worker module ${data.handlerUrl} has no handleItem export.`);
22
+ throw new TypeError(`Worker module ${data.handlerUrl} has no handleItem export.`);
23
23
  }
24
24
  const results = [];
25
25
  const transfer = [];
@@ -28,8 +28,9 @@ port.on("message", async (msg) => {
28
28
  if (result === undefined)
29
29
  continue;
30
30
  results.push(result);
31
- if (result instanceof Uint8Array)
31
+ if (result instanceof Uint8Array) {
32
32
  transfer.push(result.buffer);
33
+ }
33
34
  }
34
35
  port.postMessage({ type: "result", results }, transfer);
35
36
  }
@@ -1 +1 @@
1
- {"version":3,"file":"parallel-map-worker-entry.js","sourceRoot":"","sources":["../../lib/parallel-map-worker-entry.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AAS5D,MAAM,IAAI,GAAG,UAAwB,CAAA;AACrC,MAAM,IAAI,GAAG,UAAW,CAAA;AAExB,IAAI,SAAS,GAAG,CAAC,CAAA;AACjB,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAChD,CAAC,GAAgE,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,IAAI,GAAG,CAAC,OAAO,CACnG,CAAA;AAED,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,GAAY,EAAE,EAAE;IACzC,MAAM,OAAO,GAAG,GAA0C,CAAA;IAE1D,IAAI,OAAO,EAAE,IAAI,KAAK,OAAO;QAAE,OAAM;IAErC,IAAI,CAAC;QACJ,MAAM,UAAU,GAAG,MAAM,YAAY,CAAA;QAErC,IAAI,OAAO,UAAU,KAAK,UAAU,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CAAC,iBAAiB,IAAI,CAAC,UAAU,4BAA4B,CAAC,CAAA;QAC9E,CAAC;QAED,MAAM,OAAO,GAAc,EAAE,CAAA;QAC7B,MAAM,QAAQ,GAAkB,EAAE,CAAA;QAElC,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAClC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,CAAC,CAAA;YAE7D,IAAI,MAAM,KAAK,SAAS;gBAAE,SAAQ;YAElC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAEpB,IAAI,MAAM,YAAY,UAAU;gBAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,MAAqB,CAAC,CAAA;QAC9E,CAAC;QAED,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,QAAQ,CAAC,CAAA;IACxD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IACrG,CAAC;AACF,CAAC,CAAC,CAAA"}
1
+ {"version":3,"file":"parallel-map-worker-entry.js","sourceRoot":"","sources":["../../lib/parallel-map-worker-entry.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AAS5D,MAAM,IAAI,GAAG,UAAwB,CAAA;AACrC,MAAM,IAAI,GAAG,UAAW,CAAA;AAExB,IAAI,SAAS,GAAG,CAAC,CAAA;AAEjB,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAChD,CAAC,GAAgE,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,IAAI,GAAG,CAAC,OAAO,CACnG,CAAA;AAED,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,GAAY,EAAE,EAAE;IACzC,MAAM,OAAO,GAAG,GAA0C,CAAA;IAE1D,IAAI,OAAO,EAAE,IAAI,KAAK,OAAO;QAAE,OAAM;IAErC,IAAI,CAAC;QACJ,MAAM,UAAU,GAAG,MAAM,YAAY,CAAA;QAErC,IAAI,OAAO,UAAU,KAAK,UAAU,EAAE,CAAC;YACtC,MAAM,IAAI,SAAS,CAAC,iBAAiB,IAAI,CAAC,UAAU,4BAA4B,CAAC,CAAA;QAClF,CAAC;QAED,MAAM,OAAO,GAAc,EAAE,CAAA;QAC7B,MAAM,QAAQ,GAAkB,EAAE,CAAA;QAElC,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAClC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,CAAC,CAAA;YAE7D,IAAI,MAAM,KAAK,SAAS;gBAAE,SAAQ;YAElC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAEpB,IAAI,MAAM,YAAY,UAAU,EAAE,CAAC;gBAClC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,MAAqB,CAAC,CAAA;YAC5C,CAAC;QACF,CAAC;QAED,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,QAAQ,CAAC,CAAA;IACxD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IACrG,CAAC;AACF,CAAC,CAAC,CAAA"}
@@ -0,0 +1,57 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license MIT
4
+ * @author Teffen Ellis, et al.
5
+ */
6
+ /**
7
+ * Per-item handler a parallelMapWorkers worker module exports. `index` is per-worker monotonic.
8
+ */
9
+ export type ParallelHandler<T = unknown, R = unknown> = (item: T, ctx: {
10
+ index: number;
11
+ }) => R | Uint8Array | undefined | Promise<R | Uint8Array | undefined>;
12
+ export interface ParallelMapWorkersOptions {
13
+ /**
14
+ * Worker module path/URL exporting `handleItem(item, ctx)`. Top-level code = per-worker init.
15
+ */
16
+ worker: string | URL;
17
+ /**
18
+ * Number of worker threads in the pool. Keep it SMALL for I/O- or memory-bound handlers (per-row DB lookups, large
19
+ * shared datasets): throughput typically peaks at ~2–3 and _degrades_ past that as workers contend for the shared
20
+ * resource — only CPU-bound handlers scale toward the core count. Also bounded by RAM (each worker re-inits its
21
+ * deps). Sweep it for your workload rather than defaulting to `availableParallelism()`.
22
+ */
23
+ concurrency: number;
24
+ /**
25
+ * Items per dispatched batch. @default 64
26
+ */
27
+ batchSize?: number;
28
+ /**
29
+ * Forwarded to every worker via `workerData.userData` (must be structured-cloneable).
30
+ */
31
+ workerData?: unknown;
32
+ }
33
+ /**
34
+ * Map an async (or sync) iterable through a pool of worker threads. Main pulls items from `source`, batches them, and
35
+ * dispatches each batch to an idle worker running the `worker` handler module; results stream back as a single merged
36
+ * async iterator in **completion order** (not input order). A handler returning `undefined` drops that item; a
37
+ * `Uint8Array` is transferred zero-copy.
38
+ *
39
+ * **When it pays (measure — don't assume).** Threading only wins when per-item work is heavy relative to the ~µs
40
+ * cross-thread dispatch cost (structured-clone in + out). From real workloads: light per-row work — CSV/JSON parse,
41
+ * string normalize, `JSON.parse` — is _dispatch-bound_ and runs **0.3–0.9× of single-threaded (a net loss)**; ms-scale
42
+ * per-row work — model inference, geocoding, crypto/image ops — is where threads win. If the per-row cost isn't clearly
43
+ * milliseconds, benchmark against the single-threaded version before adopting.
44
+ *
45
+ * **Concurrency is not core count.** Heavy work that is I/O- or memory-bound (random reads into one large on-disk DB,
46
+ * anything sharing memory bandwidth) peaks _low_ — often ~2–3 workers — and then _degrades_, because the workers
47
+ * contend for the shared resource, not the CPU. Start small and sweep; don't reach for `availableParallelism()`. Only
48
+ * CPU-bound per-row work (pure compute) scales out toward the core count.
49
+ *
50
+ * All workers are terminated on completion, error, or early `return()`.
51
+ *
52
+ * @see `parallelMap` for the same shape on the caller's thread — it takes a closure, which is why it cannot cross a
53
+ * thread boundary and this takes a module path instead.
54
+ * @see The "Choosing a primitive" table in the README, which keys the decision off per-row work.
55
+ */
56
+ export declare function parallelMapWorkers<T, R = unknown>(source: AsyncIterable<T> | Iterable<T>, options: ParallelMapWorkersOptions): AsyncIterableIterator<R>;
57
+ //# sourceMappingURL=parallel-map-workers.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"parallel-map-workers.d.ts","sourceRoot":"","sources":["../../lib/parallel-map-workers.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH;;GAEG;AACH,MAAM,MAAM,eAAe,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,OAAO,IAAI,CACvD,IAAI,EAAE,CAAC,EACP,GAAG,EAAE;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,KAClB,CAAC,GAAG,UAAU,GAAG,SAAS,GAAG,OAAO,CAAC,CAAC,GAAG,UAAU,GAAG,SAAS,CAAC,CAAA;AAErE,MAAM,WAAW,yBAAyB;IACzC;;OAEG;IACH,MAAM,EAAE,MAAM,GAAG,GAAG,CAAA;IACpB;;;;;OAKG;IACH,WAAW,EAAE,MAAM,CAAA;IACnB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,UAAU,CAAC,EAAE,OAAO,CAAA;CACpB;AAqCD;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,CAAC,GAAG,OAAO,EAChD,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,EACtC,OAAO,EAAE,yBAAyB,GAChC,qBAAqB,CAAC,CAAC,CAAC,CAyB1B"}
@@ -0,0 +1,82 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license MIT
4
+ * @author Teffen Ellis, et al.
5
+ */
6
+ import { Worker } from "node:worker_threads";
7
+ import { runPool } from "./parallel-map-runtime.js";
8
+ /**
9
+ * Wrap a worker as a pool slot: dispatch one batch, await its result. One batch in flight at a time.
10
+ */
11
+ function workerHandle(worker) {
12
+ let pending = null;
13
+ const settle = (fn) => {
14
+ if (!pending)
15
+ return;
16
+ const p = pending;
17
+ pending = null;
18
+ fn(p);
19
+ };
20
+ worker.on("message", (msg) => {
21
+ if (msg?.type === "result") {
22
+ settle((p) => p.resolve(msg.results));
23
+ }
24
+ else if (msg?.type === "error") {
25
+ settle((p) => p.reject(new Error(msg.message)));
26
+ }
27
+ });
28
+ worker.on("error", (error) => settle((p) => p.reject(error instanceof Error ? error : new Error(String(error)))));
29
+ return {
30
+ process(batch) {
31
+ return new Promise((resolve, reject) => {
32
+ pending = { resolve, reject };
33
+ worker.postMessage({ type: "batch", batch });
34
+ });
35
+ },
36
+ };
37
+ }
38
+ /**
39
+ * Map an async (or sync) iterable through a pool of worker threads. Main pulls items from `source`, batches them, and
40
+ * dispatches each batch to an idle worker running the `worker` handler module; results stream back as a single merged
41
+ * async iterator in **completion order** (not input order). A handler returning `undefined` drops that item; a
42
+ * `Uint8Array` is transferred zero-copy.
43
+ *
44
+ * **When it pays (measure — don't assume).** Threading only wins when per-item work is heavy relative to the ~µs
45
+ * cross-thread dispatch cost (structured-clone in + out). From real workloads: light per-row work — CSV/JSON parse,
46
+ * string normalize, `JSON.parse` — is _dispatch-bound_ and runs **0.3–0.9× of single-threaded (a net loss)**; ms-scale
47
+ * per-row work — model inference, geocoding, crypto/image ops — is where threads win. If the per-row cost isn't clearly
48
+ * milliseconds, benchmark against the single-threaded version before adopting.
49
+ *
50
+ * **Concurrency is not core count.** Heavy work that is I/O- or memory-bound (random reads into one large on-disk DB,
51
+ * anything sharing memory bandwidth) peaks _low_ — often ~2–3 workers — and then _degrades_, because the workers
52
+ * contend for the shared resource, not the CPU. Start small and sweep; don't reach for `availableParallelism()`. Only
53
+ * CPU-bound per-row work (pure compute) scales out toward the core count.
54
+ *
55
+ * All workers are terminated on completion, error, or early `return()`.
56
+ *
57
+ * @see `parallelMap` for the same shape on the caller's thread — it takes a closure, which is why it cannot cross a
58
+ * thread boundary and this takes a module path instead.
59
+ * @see The "Choosing a primitive" table in the README, which keys the decision off per-row work.
60
+ */
61
+ export function parallelMapWorkers(source, options) {
62
+ const handlerUrl = options.worker instanceof URL ? options.worker.href : new URL(options.worker, `file://${process.cwd()}/`).href;
63
+ const concurrency = Math.max(1, Math.floor(options.concurrency));
64
+ const batchSize = options.batchSize ?? 64;
65
+ const entryUrl = new URL("./parallel-map-worker-entry.js", import.meta.url);
66
+ const workers = [];
67
+ async function* run() {
68
+ try {
69
+ const pool = Array.from({ length: concurrency }, () => {
70
+ const worker = new Worker(entryUrl, { workerData: { handlerUrl, userData: options.workerData } });
71
+ workers.push(worker);
72
+ return workerHandle(worker);
73
+ });
74
+ yield* runPool(pool, source, batchSize);
75
+ }
76
+ finally {
77
+ await Promise.all(workers.map((worker) => worker.terminate()));
78
+ }
79
+ }
80
+ return run();
81
+ }
82
+ //# sourceMappingURL=parallel-map-workers.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"parallel-map-workers.js","sourceRoot":"","sources":["../../lib/parallel-map-workers.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAE5C,OAAO,EAAmB,OAAO,EAAE,MAAM,2BAA2B,CAAA;AAgCpE;;GAEG;AACH,SAAS,YAAY,CAAO,MAAc;IACzC,IAAI,OAAO,GAA+E,IAAI,CAAA;IAE9F,MAAM,MAAM,GAAG,CAAC,EAA4C,EAAE,EAAE;QAC/D,IAAI,CAAC,OAAO;YAAE,OAAM;QACpB,MAAM,CAAC,GAAG,OAAO,CAAA;QACjB,OAAO,GAAG,IAAI,CAAA;QACd,EAAE,CAAC,CAAC,CAAC,CAAA;IACN,CAAC,CAAA;IAED,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAA0E,EAAE,EAAE;QACnG,IAAI,GAAG,EAAE,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC5B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAA;QACtC,CAAC;aAAM,IAAI,GAAG,EAAE,IAAI,KAAK,OAAO,EAAE,CAAC;YAClC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;QAChD,CAAC;IACF,CAAC,CAAC,CAAA;IAEF,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAc,EAAE,EAAE,CACrC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAClF,CAAA;IAED,OAAO;QACN,OAAO,CAAC,KAAK;YACZ,OAAO,IAAI,OAAO,CAAM,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBAC3C,OAAO,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,CAAA;gBAC7B,MAAM,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAA;YAC7C,CAAC,CAAC,CAAA;QACH,CAAC;KACD,CAAA;AACF,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,UAAU,kBAAkB,CACjC,MAAsC,EACtC,OAAkC;IAElC,MAAM,UAAU,GACf,OAAO,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,CAAA;IAE/G,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAA;IAChE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,EAAE,CAAA;IACzC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,gCAAgC,EAAE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAA;IAC3E,MAAM,OAAO,GAAa,EAAE,CAAA;IAE5B,KAAK,SAAS,CAAC,CAAC,GAAG;QAClB,IAAI,CAAC;YACJ,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE;gBACrD,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAA;gBACjG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;gBAEpB,OAAO,YAAY,CAAO,MAAM,CAAC,CAAA;YAClC,CAAC,CAAC,CAAA;YAEF,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,CAAC,CAAA;QACxC,CAAC;gBAAS,CAAC;YACV,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAA;QAC/D,CAAC;IACF,CAAC;IAED,OAAO,GAAG,EAAE,CAAA;AACb,CAAC"}
@@ -3,43 +3,26 @@
3
3
  * @license MIT
4
4
  * @author Teffen Ellis, et al.
5
5
  */
6
- /** Per-item handler a parallelMap worker module exports. `index` is per-worker monotonic. */
7
- export type ParallelHandler<T = unknown, R = unknown> = (item: T, ctx: {
8
- index: number;
9
- }) => R | Uint8Array | undefined | Promise<R | Uint8Array | undefined>;
10
- export interface ParallelMapOptions {
11
- /** Worker module path/URL exporting `handleItem(item, ctx)`. Top-level code = per-worker init. */
12
- worker: string | URL;
13
- /**
14
- * Number of worker threads in the pool. Keep it SMALL for I/O- or memory-bound handlers (per-row DB lookups, large
15
- * shared datasets): throughput typically peaks at ~2–3 and _degrades_ past that as workers contend for the shared
16
- * resource — only CPU-bound handlers scale toward the core count. Also bounded by RAM (each worker re-inits its
17
- * deps). Sweep it for your workload rather than defaulting to `availableParallelism()`.
18
- */
19
- concurrency: number;
20
- /** Items per dispatched batch. @default 64 */
21
- batchSize?: number;
22
- /** Forwarded to every worker via `workerData.userData` (must be structured-cloneable). */
23
- workerData?: unknown;
24
- }
6
+ import { AsyncSequence, type ParallelMapSequenceOptions } from "./AsyncSequence.js";
25
7
  /**
26
- * Map an async (or sync) iterable through a pool of worker threads. Main pulls items from `source`, batches them, and
27
- * dispatches each batch to an idle worker running the `worker` handler module; results stream back as a single merged
28
- * async iterator in **completion order** (not input order). A handler returning `undefined` drops that item; a
29
- * `Uint8Array` is transferred zero-copy.
8
+ * Map an iterable through a callback with up to `concurrency` calls in flight, yielding in **completion order** — not
9
+ * input order.
30
10
  *
31
- * **When it pays (measure don't assume).** Threading only wins when per-item work is heavy relative to the ~µs
32
- * cross-thread dispatch cost (structured-clone in + out). From real workloads: light per-row work CSV/JSON parse,
33
- * string normalize, `JSON.parse` is _dispatch-bound_ and runs **0.3–0.9× of single-threaded (a net loss)**; ms-scale
34
- * per-row work — model inference, geocoding, crypto/image ops — is where threads win. If the per-row cost isn't clearly
35
- * milliseconds, benchmark against the single-threaded version before adopting.
11
+ * The callback is a closure, so every call runs on the caller's thread. This overlaps _latency_ file reads, network
12
+ * round-trips, anything that spends its time waiting and does nothing for CPU-bound work, which still occupies the
13
+ * one thread it always did.
36
14
  *
37
- * **Concurrency is not core count.** Heavy work that is I/O- or memory-bound (random reads into one large on-disk DB,
38
- * anything sharing memory bandwidth) peaks _low_ — often ~2–3 workers — and then _degrades_, because the workers
39
- * contend for the shared resource, not the CPU. Start small and sweep; don't reach for `availableParallelism()`. Only
40
- * CPU-bound per-row work (pure compute) scales out toward the core count.
15
+ * **Concurrency is not core count.** Work that contends for a shared resource (one disk, one socket pool, one on-disk
16
+ * database) peaks _low_ — often ~2–3 — and then degrades. Sweep it.
41
17
  *
42
- * All workers are terminated on completion, error, or early `return()`.
18
+ * @param source The collection to map.
19
+ * @param fn The callback, receiving `(value, counter)`.
20
+ * @param options Concurrency ceiling and optional abort signal.
21
+ *
22
+ * @yields Each result, in completion order.
23
+ * @see {@linkcode parallelMapWorkers} to run the handler on worker threads instead — it takes a module path rather than
24
+ * a closure, because a closure cannot cross a thread boundary.
25
+ * @see The "Choosing a primitive" table in the README, which keys the decision off per-row work.
43
26
  */
44
- export declare function parallelMap<T, R = unknown>(source: AsyncIterable<T> | Iterable<T>, options: ParallelMapOptions): AsyncIterableIterator<R>;
27
+ export declare function parallelMap<T, U>(source: AsyncIterable<T> | Iterable<T>, fn: (value: T, counter: number) => U | PromiseLike<U>, options: ParallelMapSequenceOptions): AsyncSequence<U>;
45
28
  //# sourceMappingURL=parallel-map.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"parallel-map.d.ts","sourceRoot":"","sources":["../../lib/parallel-map.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH,6FAA6F;AAC7F,MAAM,MAAM,eAAe,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,OAAO,IAAI,CACvD,IAAI,EAAE,CAAC,EACP,GAAG,EAAE;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,KAClB,CAAC,GAAG,UAAU,GAAG,SAAS,GAAG,OAAO,CAAC,CAAC,GAAG,UAAU,GAAG,SAAS,CAAC,CAAA;AAErE,MAAM,WAAW,kBAAkB;IAClC,kGAAkG;IAClG,MAAM,EAAE,MAAM,GAAG,GAAG,CAAA;IACpB;;;;;OAKG;IACH,WAAW,EAAE,MAAM,CAAA;IACnB,8CAA8C;IAC9C,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,0FAA0F;IAC1F,UAAU,CAAC,EAAE,OAAO,CAAA;CACpB;AAgCD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,WAAW,CAAC,CAAC,EAAE,CAAC,GAAG,OAAO,EACzC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,EACtC,OAAO,EAAE,kBAAkB,GACzB,qBAAqB,CAAC,CAAC,CAAC,CAwB1B"}
1
+ {"version":3,"file":"parallel-map.d.ts","sourceRoot":"","sources":["../../lib/parallel-map.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,aAAa,EAAE,KAAK,0BAA0B,EAAE,MAAM,oBAAoB,CAAA;AAEnF;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,WAAW,CAAC,CAAC,EAAE,CAAC,EAC/B,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,EACtC,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,KAAK,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,EACrD,OAAO,EAAE,0BAA0B,GACjC,aAAa,CAAC,CAAC,CAAC,CAElB"}
@@ -3,72 +3,28 @@
3
3
  * @license MIT
4
4
  * @author Teffen Ellis, et al.
5
5
  */
6
- import { Worker } from "node:worker_threads";
7
- import { runPool } from "./parallel-map-runtime.js";
8
- /** Wrap a worker as a pool slot: dispatch one batch, await its result. One batch in flight at a time. */
9
- function workerHandle(worker) {
10
- let pending = null;
11
- const settle = (fn) => {
12
- if (!pending)
13
- return;
14
- const p = pending;
15
- pending = null;
16
- fn(p);
17
- };
18
- worker.on("message", (msg) => {
19
- if (msg?.type === "result")
20
- settle((p) => p.resolve(msg.results));
21
- else if (msg?.type === "error")
22
- settle((p) => p.reject(new Error(msg.message)));
23
- });
24
- worker.on("error", (error) => settle((p) => p.reject(error instanceof Error ? error : new Error(String(error)))));
25
- return {
26
- process(batch) {
27
- return new Promise((resolve, reject) => {
28
- pending = { resolve, reject };
29
- worker.postMessage({ type: "batch", batch });
30
- });
31
- },
32
- };
33
- }
6
+ import { AsyncSequence } from "./AsyncSequence.js";
34
7
  /**
35
- * Map an async (or sync) iterable through a pool of worker threads. Main pulls items from `source`, batches them, and
36
- * dispatches each batch to an idle worker running the `worker` handler module; results stream back as a single merged
37
- * async iterator in **completion order** (not input order). A handler returning `undefined` drops that item; a
38
- * `Uint8Array` is transferred zero-copy.
8
+ * Map an iterable through a callback with up to `concurrency` calls in flight, yielding in **completion order** — not
9
+ * input order.
10
+ *
11
+ * The callback is a closure, so every call runs on the caller's thread. This overlaps _latency_ — file reads, network
12
+ * round-trips, anything that spends its time waiting — and does nothing for CPU-bound work, which still occupies the
13
+ * one thread it always did.
39
14
  *
40
- * **When it pays (measure don't assume).** Threading only wins when per-item work is heavy relative to the ~µs
41
- * cross-thread dispatch cost (structured-clone in + out). From real workloads: light per-row work — CSV/JSON parse,
42
- * string normalize, `JSON.parse` — is _dispatch-bound_ and runs **0.3–0.9× of single-threaded (a net loss)**; ms-scale
43
- * per-row work — model inference, geocoding, crypto/image ops — is where threads win. If the per-row cost isn't clearly
44
- * milliseconds, benchmark against the single-threaded version before adopting.
15
+ * **Concurrency is not core count.** Work that contends for a shared resource (one disk, one socket pool, one on-disk
16
+ * database) peaks _low_ often ~2–3 and then degrades. Sweep it.
45
17
  *
46
- * **Concurrency is not core count.** Heavy work that is I/O- or memory-bound (random reads into one large on-disk DB,
47
- * anything sharing memory bandwidth) peaks _low_ — often ~2–3 workers — and then _degrades_, because the workers
48
- * contend for the shared resource, not the CPU. Start small and sweep; don't reach for `availableParallelism()`. Only
49
- * CPU-bound per-row work (pure compute) scales out toward the core count.
18
+ * @param source The collection to map.
19
+ * @param fn The callback, receiving `(value, counter)`.
20
+ * @param options Concurrency ceiling and optional abort signal.
50
21
  *
51
- * All workers are terminated on completion, error, or early `return()`.
22
+ * @yields Each result, in completion order.
23
+ * @see {@linkcode parallelMapWorkers} to run the handler on worker threads instead — it takes a module path rather than
24
+ * a closure, because a closure cannot cross a thread boundary.
25
+ * @see The "Choosing a primitive" table in the README, which keys the decision off per-row work.
52
26
  */
53
- export function parallelMap(source, options) {
54
- const handlerUrl = options.worker instanceof URL ? options.worker.href : new URL(options.worker, `file://${process.cwd()}/`).href;
55
- const concurrency = Math.max(1, Math.floor(options.concurrency));
56
- const batchSize = options.batchSize ?? 64;
57
- const entryUrl = new URL("./parallel-map-worker-entry.js", import.meta.url);
58
- const workers = [];
59
- async function* run() {
60
- try {
61
- const pool = Array.from({ length: concurrency }, () => {
62
- const worker = new Worker(entryUrl, { workerData: { handlerUrl, userData: options.workerData } });
63
- workers.push(worker);
64
- return workerHandle(worker);
65
- });
66
- yield* runPool(pool, source, batchSize);
67
- }
68
- finally {
69
- await Promise.all(workers.map((worker) => worker.terminate()));
70
- }
71
- }
72
- return run();
27
+ export function parallelMap(source, fn, options) {
28
+ return AsyncSequence.from(source).parallelMap(fn, options);
73
29
  }
74
30
  //# sourceMappingURL=parallel-map.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"parallel-map.js","sourceRoot":"","sources":["../../lib/parallel-map.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAE5C,OAAO,EAAmB,OAAO,EAAE,MAAM,2BAA2B,CAAA;AAwBpE,yGAAyG;AACzG,SAAS,YAAY,CAAO,MAAc;IACzC,IAAI,OAAO,GAA+E,IAAI,CAAA;IAE9F,MAAM,MAAM,GAAG,CAAC,EAA4C,EAAE,EAAE;QAC/D,IAAI,CAAC,OAAO;YAAE,OAAM;QACpB,MAAM,CAAC,GAAG,OAAO,CAAA;QACjB,OAAO,GAAG,IAAI,CAAA;QACd,EAAE,CAAC,CAAC,CAAC,CAAA;IACN,CAAC,CAAA;IAED,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAA0E,EAAE,EAAE;QACnG,IAAI,GAAG,EAAE,IAAI,KAAK,QAAQ;YAAE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAA;aAC5D,IAAI,GAAG,EAAE,IAAI,KAAK,OAAO;YAAE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;IAChF,CAAC,CAAC,CAAA;IAEF,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAc,EAAE,EAAE,CACrC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAClF,CAAA;IAED,OAAO;QACN,OAAO,CAAC,KAAK;YACZ,OAAO,IAAI,OAAO,CAAM,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBAC3C,OAAO,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,CAAA;gBAC7B,MAAM,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAA;YAC7C,CAAC,CAAC,CAAA;QACH,CAAC;KACD,CAAA;AACF,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,WAAW,CAC1B,MAAsC,EACtC,OAA2B;IAE3B,MAAM,UAAU,GACf,OAAO,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,CAAA;IAC/G,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAA;IAChE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,EAAE,CAAA;IACzC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,gCAAgC,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAC3E,MAAM,OAAO,GAAa,EAAE,CAAA;IAE5B,KAAK,SAAS,CAAC,CAAC,GAAG;QAClB,IAAI,CAAC;YACJ,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE;gBACrD,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAA;gBACjG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;gBAEpB,OAAO,YAAY,CAAO,MAAM,CAAC,CAAA;YAClC,CAAC,CAAC,CAAA;YAEF,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,CAAC,CAAA;QACxC,CAAC;gBAAS,CAAC;YACV,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAA;QAC/D,CAAC;IACF,CAAC;IAED,OAAO,GAAG,EAAE,CAAA;AACb,CAAC"}
1
+ {"version":3,"file":"parallel-map.js","sourceRoot":"","sources":["../../lib/parallel-map.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,aAAa,EAAmC,MAAM,oBAAoB,CAAA;AAEnF;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,UAAU,WAAW,CAC1B,MAAsC,EACtC,EAAqD,EACrD,OAAmC;IAEnC,OAAO,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,WAAW,CAAC,EAAE,EAAE,OAAO,CAAC,CAAA;AAC3D,CAAC"}
@@ -0,0 +1,30 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license MIT
4
+ * @author Teffen Ellis, et al.
5
+ */
6
+ /**
7
+ * Given an iterable of values, maps each value to a record of the value and the result of the callback.
8
+ *
9
+ * This is a convenience function when an iterable needs mapping to an object-like structure.
10
+ *
11
+ * ```ts
12
+ * const iterable = ["a", "b", "c"]
13
+ * const result = pivot(iterable, (value) => value.toUpperCase())
14
+ *
15
+ * // { a: "A", b: "B", c: "C" }
16
+ * ```
17
+ *
18
+ * @category Collections
19
+ * @category Object
20
+ */
21
+ export declare function pivot<T extends PropertyKey, C extends (value: T) => Promise<unknown> | unknown>(
22
+ /**
23
+ * The iterable to pivot.
24
+ */
25
+ iterable: Iterable<T>,
26
+ /**
27
+ * The callback to transform each value.
28
+ */
29
+ callback: C): ReturnType<C> extends PromiseLike<infer U> ? Promise<Record<T, U>> : Record<T, ReturnType<C>>;
30
+ //# sourceMappingURL=pivot.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"pivot.d.ts","sourceRoot":"","sources":["../../lib/pivot.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,KAAK,CAAC,CAAC,SAAS,WAAW,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO;AAC9F;;GAEG;AACH,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AACrB;;GAEG;AACH,QAAQ,EAAE,CAAC,GACT,UAAU,CAAC,CAAC,CAAC,SAAS,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CA6B/F"}
@@ -0,0 +1,47 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license MIT
4
+ * @author Teffen Ellis, et al.
5
+ */
6
+ /**
7
+ * Given an iterable of values, maps each value to a record of the value and the result of the callback.
8
+ *
9
+ * This is a convenience function when an iterable needs mapping to an object-like structure.
10
+ *
11
+ * ```ts
12
+ * const iterable = ["a", "b", "c"]
13
+ * const result = pivot(iterable, (value) => value.toUpperCase())
14
+ *
15
+ * // { a: "A", b: "B", c: "C" }
16
+ * ```
17
+ *
18
+ * @category Collections
19
+ * @category Object
20
+ */
21
+ export function pivot(
22
+ /**
23
+ * The iterable to pivot.
24
+ */
25
+ iterable,
26
+ /**
27
+ * The callback to transform each value.
28
+ */
29
+ callback) {
30
+ const entries = [];
31
+ let foundThenable = false;
32
+ for (const value of iterable) {
33
+ const result = callback(value);
34
+ if (result && typeof result === "object" && "then" in result) {
35
+ foundThenable = true;
36
+ }
37
+ entries.push([value, result]);
38
+ }
39
+ if (foundThenable) {
40
+ return Promise.all(entries.map(([key, value]) => {
41
+ return Promise.resolve(value).then((resolvedValue) => [key, resolvedValue]);
42
+ })).then(Object.fromEntries);
43
+ }
44
+ const pivotedRecord = Object.fromEntries(entries);
45
+ return pivotedRecord;
46
+ }
47
+ //# sourceMappingURL=pivot.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"pivot.js","sourceRoot":"","sources":["../../lib/pivot.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,KAAK;AACpB;;GAEG;AACH,QAAqB;AACrB;;GAEG;AACH,QAAW;IAEX,MAAM,OAAO,GAA8B,EAAE,CAAA;IAC7C,IAAI,aAAa,GAAG,KAAK,CAAA;IAEzB,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAA;QAE9B,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,IAAI,MAAM,EAAE,CAAC;YAC9D,aAAa,GAAG,IAAI,CAAA;QACrB,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,MAAuB,CAAC,CAAC,CAAA;IAC/C,CAAC;IAED,IAAI,aAAa,EAAE,CAAC;QACnB,OAAO,OAAO,CAAC,GAAG,CACjB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;YAC5B,OAAO,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,aAAa,CAAU,CAAC,CAAA;QACrF,CAAC,CAAC,CACF,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAEE,CAAA;IAC5B,CAAC;IAED,MAAM,aAAa,GAAG,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAA;IAEjD,OAAO,aAEoB,CAAA;AAC5B,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"segment-runtime.d.ts","sourceRoot":"","sources":["../../lib/segment-runtime.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,MAAM,cAAc,GAAG,CAC5B,MAAM,EAAE,UAAU,EAClB,GAAG,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,KACxC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;AAE/B,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;IAClC,YAAY,EAAE,cAAc,CAAA;IAC5B,YAAY,EAAE,MAAM,CAAA;IACpB,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;IACnB,IAAI,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,IAAI,CAAA;IACzD,UAAU,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;IAC/B,QAAQ,EAAE,MAAM,MAAM,CAAA;CACtB;AAED;;;;;GAKG;AACH,wBAAsB,UAAU,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CA6BhE"}
1
+ {"version":3,"file":"segment-runtime.d.ts","sourceRoot":"","sources":["../../lib/segment-runtime.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,MAAM,cAAc,GAAG,CAC5B,MAAM,EAAE,UAAU,EAClB,GAAG,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,KACxC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;AAE/B,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;IAClC,YAAY,EAAE,cAAc,CAAA;IAC5B,YAAY,EAAE,MAAM,CAAA;IACpB,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;IACnB,IAAI,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,IAAI,CAAA;IACzD,UAAU,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;IAC/B,QAAQ,EAAE,MAAM,MAAM,CAAA;CACtB;AAED;;;;;GAKG;AACH,wBAAsB,UAAU,CAAC,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAoChE"}
@@ -14,10 +14,11 @@ export async function runSegment(io) {
14
14
  let transfer = [];
15
15
  let index = 0;
16
16
  const flush = async () => {
17
- if (batch.length === 0)
17
+ if (!batch.length)
18
18
  return;
19
- while (io.inFlight() >= io.maxInFlight)
19
+ while (io.inFlight() >= io.maxInFlight) {
20
20
  await io.waitForAck();
21
+ }
21
22
  io.post(batch, transfer);
22
23
  batch = [];
23
24
  transfer = [];
@@ -28,10 +29,12 @@ export async function runSegment(io) {
28
29
  if (result === undefined)
29
30
  continue;
30
31
  batch.push(result);
31
- if (result instanceof Uint8Array)
32
+ if (result instanceof Uint8Array) {
32
33
  transfer.push(result.buffer);
33
- if (batch.length >= io.batchSize)
34
+ }
35
+ if (batch.length >= io.batchSize) {
34
36
  await flush();
37
+ }
35
38
  }
36
39
  await flush();
37
40
  }
@@ -1 +1 @@
1
- {"version":3,"file":"segment-runtime.js","sourceRoot":"","sources":["../../lib/segment-runtime.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAkBH;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,EAAgB;IAChD,IAAI,KAAK,GAAc,EAAE,CAAA;IACzB,IAAI,QAAQ,GAAkB,EAAE,CAAA;IAChC,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,MAAM,KAAK,GAAG,KAAK,IAAmB,EAAE;QACvC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAM;QAE9B,OAAO,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,WAAW;YAAE,MAAM,EAAE,CAAC,UAAU,EAAE,CAAA;QAE7D,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;QACxB,KAAK,GAAG,EAAE,CAAA;QACV,QAAQ,GAAG,EAAE,CAAA;IACd,CAAC,CAAA;IAED,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,YAAY,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,CAAC,YAAY,EAAE,CAAC,CAAA;QACtF,KAAK,EAAE,CAAA;QAEP,IAAI,MAAM,KAAK,SAAS;YAAE,SAAQ;QAElC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAElB,IAAI,MAAM,YAAY,UAAU;YAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,MAAqB,CAAC,CAAA;QAE7E,IAAI,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC,SAAS;YAAE,MAAM,KAAK,EAAE,CAAA;IAChD,CAAC;IAED,MAAM,KAAK,EAAE,CAAA;AACd,CAAC"}
1
+ {"version":3,"file":"segment-runtime.js","sourceRoot":"","sources":["../../lib/segment-runtime.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAkBH;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,EAAgB;IAChD,IAAI,KAAK,GAAc,EAAE,CAAA;IACzB,IAAI,QAAQ,GAAkB,EAAE,CAAA;IAChC,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,MAAM,KAAK,GAAG,KAAK,IAAmB,EAAE;QACvC,IAAI,CAAC,KAAK,CAAC,MAAM;YAAE,OAAM;QAEzB,OAAO,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YACxC,MAAM,EAAE,CAAC,UAAU,EAAE,CAAA;QACtB,CAAC;QAED,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;QACxB,KAAK,GAAG,EAAE,CAAA;QACV,QAAQ,GAAG,EAAE,CAAA;IACd,CAAC,CAAA;IAED,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,YAAY,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,CAAC,YAAY,EAAE,CAAC,CAAA;QAEtF,KAAK,EAAE,CAAA;QAEP,IAAI,MAAM,KAAK,SAAS;YAAE,SAAQ;QAElC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAElB,IAAI,MAAM,YAAY,UAAU,EAAE,CAAC;YAClC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,MAAqB,CAAC,CAAA;QAC5C,CAAC;QAED,IAAI,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC,SAAS,EAAE,CAAC;YAClC,MAAM,KAAK,EAAE,CAAA;QACd,CAAC;IACF,CAAC;IAED,MAAM,KAAK,EAAE,CAAA;AACd,CAAC"}
@@ -32,25 +32,28 @@ async function main() {
32
32
  const { createChunkIterator } = await import("spliterator/node/fs");
33
33
  const chunkIterator = await createChunkIterator(data.source, { start: data.start, end: data.end - 1 });
34
34
  const records = new AsyncSpliterator(chunkIterator, { delimiter: data.delimiter, autoDispose: true });
35
- try {
36
- await runSegment({
37
- records,
38
- handleRecord,
39
- segmentIndex: data.segmentIndex,
40
- batchSize: data.batchSize,
41
- maxInFlight: data.maxInFlight,
42
- post: (batch, transfer) => {
43
- posted++;
44
- port.postMessage({ type: "batch", records: batch }, transfer);
45
- },
46
- waitForAck: () => new Promise((resolve) => (wakeAck = resolve)),
47
- inFlight: () => posted - acked,
48
- });
35
+ return runSegment({
36
+ records,
37
+ handleRecord,
38
+ segmentIndex: data.segmentIndex,
39
+ batchSize: data.batchSize,
40
+ maxInFlight: data.maxInFlight,
41
+ post: (batch, transfer) => {
42
+ posted++;
43
+ port.postMessage({ type: "batch", records: batch }, transfer);
44
+ },
45
+ waitForAck: () => new Promise((resolve) => {
46
+ wakeAck = resolve;
47
+ }),
48
+ inFlight: () => posted - acked,
49
+ })
50
+ .then(() => {
49
51
  port.postMessage({ type: "done" });
50
- }
51
- catch (error) {
52
+ return void 0;
53
+ })
54
+ .catch((error) => {
52
55
  port.postMessage({ type: "error", message: error instanceof Error ? error.message : String(error) });
53
- }
56
+ });
54
57
  }
55
58
  void main();
56
59
  //# sourceMappingURL=segment-worker-entry.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"segment-worker-entry.js","sourceRoot":"","sources":["../../lib/segment-worker-entry.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AAE5D,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AACxD,OAAO,EAAE,UAAU,EAAuB,MAAM,sBAAsB,CAAA;AActE,KAAK,UAAU,IAAI;IAClB,MAAM,IAAI,GAAG,UAAwB,CAAA;IACrC,MAAM,IAAI,GAAG,UAAW,CAAA;IAExB,+FAA+F;IAC/F,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,OAAiC,CAAA;IAErC,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAY,EAAE,EAAE;QACnC,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;YACnB,KAAK,EAAE,CAAA;YACP,OAAO,EAAE,EAAE,CAAA;YACX,OAAO,GAAG,SAAS,CAAA;QACpB,CAAC;IACF,CAAC,CAAC,CAAA;IAEF,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAgE,CAAA;IAC1G,MAAM,YAAY,GAAG,GAAG,CAAC,YAAY,IAAI,GAAG,CAAC,OAAO,CAAA;IAEpD,IAAI,OAAO,YAAY,KAAK,UAAU,EAAE,CAAC;QACxC,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,iBAAiB,IAAI,CAAC,UAAU,8BAA8B,EAAE,CAAC,CAAA;QAE5G,OAAM;IACP,CAAC;IAED,MAAM,EAAE,mBAAmB,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAA;IACnE,MAAM,aAAa,GAAG,MAAM,mBAAmB,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAA;IACtG,MAAM,OAAO,GAAG,IAAI,gBAAgB,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,SAAkB,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAA;IAE9G,IAAI,CAAC;QACJ,MAAM,UAAU,CAAC;YAChB,OAAO;YACP,YAAY;YACZ,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,IAAI,EAAE,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;gBACzB,MAAM,EAAE,CAAA;gBACR,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,CAAC,CAAA;YAC9D,CAAC;YACD,UAAU,EAAE,GAAG,EAAE,CAAC,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,GAAG,OAAO,CAAC,CAAC;YACrE,QAAQ,EAAE,GAAG,EAAE,CAAC,MAAM,GAAG,KAAK;SAC9B,CAAC,CAAA;QAEF,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAA;IACnC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IACrG,CAAC;AACF,CAAC;AAED,KAAK,IAAI,EAAE,CAAA"}
1
+ {"version":3,"file":"segment-worker-entry.js","sourceRoot":"","sources":["../../lib/segment-worker-entry.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AAE5D,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AACxD,OAAO,EAAE,UAAU,EAAuB,MAAM,sBAAsB,CAAA;AActE,KAAK,UAAU,IAAI;IAClB,MAAM,IAAI,GAAG,UAAwB,CAAA;IACrC,MAAM,IAAI,GAAG,UAAW,CAAA;IAExB,+FAA+F;IAC/F,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,OAAiC,CAAA;IAErC,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAY,EAAE,EAAE;QACnC,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;YACnB,KAAK,EAAE,CAAA;YACP,OAAO,EAAE,EAAE,CAAA;YACX,OAAO,GAAG,SAAS,CAAA;QACpB,CAAC;IACF,CAAC,CAAC,CAAA;IAEF,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAgE,CAAA;IAC1G,MAAM,YAAY,GAAG,GAAG,CAAC,YAAY,IAAI,GAAG,CAAC,OAAO,CAAA;IAEpD,IAAI,OAAO,YAAY,KAAK,UAAU,EAAE,CAAC;QACxC,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,iBAAiB,IAAI,CAAC,UAAU,8BAA8B,EAAE,CAAC,CAAA;QAE5G,OAAM;IACP,CAAC;IAED,MAAM,EAAE,mBAAmB,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAA;IACnE,MAAM,aAAa,GAAG,MAAM,mBAAmB,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAA;IACtG,MAAM,OAAO,GAAG,IAAI,gBAAgB,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,SAAkB,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAA;IAE9G,OAAO,UAAU,CAAC;QACjB,OAAO;QACP,YAAY;QACZ,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,IAAI,EAAE,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;YACzB,MAAM,EAAE,CAAA;YACR,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,CAAC,CAAA;QAC9D,CAAC;QACD,UAAU,EAAE,GAAG,EAAE,CAChB,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAC7B,OAAO,GAAG,OAAO,CAAA;QAClB,CAAC,CAAC;QACH,QAAQ,EAAE,GAAG,EAAE,CAAC,MAAM,GAAG,KAAK;KAC9B,CAAC;SACA,IAAI,CAAC,GAAG,EAAE;QACV,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAA;QAElC,OAAO,KAAK,CAAC,CAAA;IACd,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QAChB,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IACrG,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,KAAK,IAAI,EAAE,CAAA"}
@@ -4,9 +4,8 @@
4
4
  * @author Teffen Ellis, et al.
5
5
  */
6
6
  import type { CharacterSequenceInput } from "./CharacterSequence.js";
7
- import { mergeAsyncIterators } from "./merge-async-iterators.js";
8
7
  import type { AsyncDataResource } from "./shared.js";
9
- export { mergeAsyncIterators };
8
+ export { mergeAsyncIterators } from "./merge-async-iterators.js";
10
9
  export interface MinimalWorker {
11
10
  on(event: "message", cb: (msg: unknown) => void): void;
12
11
  on(event: "error", cb: (err: Error) => void): void;
@@ -19,19 +18,33 @@ export interface MinimalWorker {
19
18
  */
20
19
  export declare function workerToIterable<R>(worker: MinimalWorker, onBatchConsumed: () => void): AsyncIterableIterator<R>;
21
20
  export interface AsManyWorkersOptions {
22
- /** Module path or URL exporting `handleRecord(bytes, ctx)`. Runs once per worker at import. */
21
+ /**
22
+ * Module path or URL exporting `handleRecord(bytes, ctx)`. Runs once per worker at import.
23
+ */
23
24
  worker: string | URL;
24
- /** The record delimiter. @default LineFeed */
25
+ /**
26
+ * The record delimiter. @default LineFeed
27
+ */
25
28
  delimiter?: CharacterSequenceInput;
26
- /** Desired number of segments/workers. Clamped to ≥ 1; fewer may run. */
29
+ /**
30
+ * Desired number of segments/workers. Clamped to ≥ 1; fewer may run.
31
+ */
27
32
  concurrency: number;
28
- /** Bytes read at each ideal boundary to find the next delimiter. @default 65536 */
33
+ /**
34
+ * Bytes read at each ideal boundary to find the next delimiter. @default 65536
35
+ */
29
36
  probeSize?: number;
30
- /** Results per message. @default 256 */
37
+ /**
38
+ * Results per message. @default 256
39
+ */
31
40
  batchSize?: number;
32
- /** Unacked batches per worker before it pauses (bounds memory). @default 8 */
41
+ /**
42
+ * Unacked batches per worker before it pauses (bounds memory). @default 8
43
+ */
33
44
  maxInFlight?: number;
34
- /** Forwarded to every worker via `workerData.userData`. */
45
+ /**
46
+ * Forwarded to every worker via `workerData.userData`.
47
+ */
35
48
  workerData?: unknown;
36
49
  }
37
50
  /**