readline-pager 0.6.5 → 0.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -12,7 +12,7 @@
12
12
  <img src="https://img.shields.io/github/stars/devmor-j/readline-pager" alt="stars">
13
13
  </p>
14
14
 
15
- ⚡ High-performance paginated file reader for Node.js. Efficiently process large text files without loading them into memory.
15
+ ⚡ High-performance paginated file reader for Node.js. Process large text files efficiently without loading them into memory.
16
16
 
17
17
  - 📦 Zero dependencies
18
18
  - ⚡ Up to ~3× faster than Node.js `readline`
@@ -20,7 +20,7 @@
20
20
  - 🔁 Async (`for await...of`) and sync (`for...of`) iteration
21
21
  - 📄 Page-based reading with manual control (`next`, `nextSync`)
22
22
  - 🔀 Forward and backward reading support
23
- - 🧪 Fully typed with high test coverage (>90%)
23
+ - 🧪 Fully typed with over 90% test coverage
24
24
 
25
25
  > **Important:**
26
26
  > Performance depends heavily on the `chunkSize` option. Tune it for your storage device. A value of **64 KiB** is usually a good starting point. Increasing it may improve throughput until you reach the best value for your hardware.
@@ -69,7 +69,7 @@ const pager = createPager("./bigfile.txt");
69
69
  while ((page = pager.nextSync()) !== null) {}
70
70
 
71
71
  // Native C++
72
- for await (const page of createNativePager("./bigfile.txt")) {
72
+ for (const page of createNativePager("./bigfile.txt")) {
73
73
  }
74
74
  ```
75
75
 
@@ -79,38 +79,37 @@ for await (const page of createNativePager("./bigfile.txt")) {
79
79
 
80
80
  ```ts
81
81
  createPager(filepath, {
82
- chunkSize?: number, // default: 64 * 1024 (64 KiB)
83
- pageSize?: number, // default: 1_000
84
- delimiter?: string, // default: "\n"
85
- prefetch?: number, // default: 8
86
- backward?: boolean, // default: false
87
- useWorker?: boolean, // default: false
88
- tryNative?: boolean, // default: true
82
+ chunkSize?: number, // default: 64 * 1024 (64 KiB)
83
+ pageSize?: number, // default: 1_000
84
+ delimiter?: string, // default: "\n"
85
+ prefetch?: number, // default: 8
86
+ backward?: boolean, // default: false
87
+ output?: "string" | "buffer", // default: "string"
89
88
  });
90
89
 
91
90
  createNativePager(filepath, {
92
- pageSize?: number, // default: 1_000
93
- delimiter?: string, // default: "\n"
94
- backward?: boolean, // default: false
91
+ pageSize?: number, // default: 1_000
92
+ delimiter?: string, // default: "\n"
93
+ backward?: boolean, // default: false
94
+ output?: "string" | "buffer", // default: "string"
95
95
  });
96
96
  ```
97
97
 
98
- - `chunkSize` — number of bytes read per I/O operation.
99
- - `pageSize` — number of lines per page.
100
- - `delimiter` — line separator.
101
- - `prefetch` — maximum number of pages buffered internally.
102
- - `backward` — read the file from end to start.
103
- - `useWorker` — offload reading to a worker thread (forward reading only).
104
- - `tryNative` — attempts to use the native reader, falls back to the non-native version if it fails.
98
+ - `chunkSize` — Number of bytes read per I/O operation.
99
+ - `pageSize` — Number of lines per page.
100
+ - `delimiter` — Line separator.
101
+ - `prefetch` — Maximum number of pages buffered internally.
102
+ - `backward` — Read the file from end to start.
103
+ - `output` — Controls the page data type.
105
104
 
106
105
  > **Note:**
107
- > `createNativePager` requires x86 AVX2 or ARM NEON CPU instruction set extensions and will throw if they are not available. It also does **not** support multi-character delimiters due to fast SIMD-based scanning.
106
+ > `createNativePager` requires x86 AVX2 or ARM NEON CPU instruction set extensions and will throw if they are not available. It also does **not** support multi-character delimiters because it uses fast SIMD-based scanning.
108
107
 
109
108
  ---
110
109
 
111
110
  ## 📚 API
112
111
 
113
- ### `pager.next(): Promise<string[] | null>`
112
+ ### `pager.next(): Promise<string[] | Buffer | null>`
114
113
 
115
114
  Returns the next page asynchronously.
116
115
 
@@ -124,7 +123,7 @@ Empty lines are preserved.
124
123
  > - A completely empty file (`0` bytes) produces `[""]` on the first read.
125
124
  > - A file containing multiple empty lines returns each line as an empty string.
126
125
 
127
- ### `pager.nextSync(): string[] | null`
126
+ ### `pager.nextSync(): string[] | Buffer | null`
128
127
 
129
128
  Synchronous version of `pager.next()`.
130
129
 
package/dist/main.cjs CHANGED
@@ -5,7 +5,6 @@ Object.defineProperties(exports, {
5
5
  const require_native = require("./native.cjs");
6
6
  let node_fs = require("node:fs");
7
7
  let node_fs_promises = require("node:fs/promises");
8
- let node_worker_threads = require("node:worker_threads");
9
8
  //#region src/helper.ts
10
9
  function createRingBuffer(capacity) {
11
10
  if (!Number.isFinite(capacity) || capacity <= 0) throw new RangeError("capacity must be a positive number");
@@ -93,8 +92,10 @@ function createRingBuffer(capacity) {
93
92
  //#endregion
94
93
  //#region src/reader/backward.reader.ts
95
94
  function createBackwardReader(filepath, options) {
96
- const { chunkSize, pageSize, delimiter, prefetch } = options;
95
+ const { chunkSize, pageSize, delimiter, prefetch, output } = options;
97
96
  const pageQueue = createRingBuffer(Math.max(2, prefetch + 1));
97
+ const isBufferOutput = output === "buffer";
98
+ const emptyPage = isBufferOutput ? Buffer.allocUnsafe(0) : [""];
98
99
  const local = [];
99
100
  let fd = null;
100
101
  let fdSync = null;
@@ -116,13 +117,15 @@ function createBackwardReader(filepath, options) {
116
117
  function flushTail() {
117
118
  if (flushed) return;
118
119
  flushed = true;
119
- if (buffer.length > 0) local.push(buffer);
120
- else if (startsWithDelimiter) local.push("");
121
- buffer = "";
122
- while (local.length > 0) {
123
- const page = local.slice(local.length - Math.min(pageSize, local.length));
124
- local.length -= page.length;
125
- pageQueue.push(page);
120
+ if (!isBufferOutput) {
121
+ if (buffer.length > 0) local.push(buffer);
122
+ else if (startsWithDelimiter) local.push("");
123
+ buffer = "";
124
+ while (local.length > 0) {
125
+ const page = local.slice(local.length - Math.min(pageSize, local.length));
126
+ local.length -= page.length;
127
+ pageQueue.push(page);
128
+ }
126
129
  }
127
130
  done = true;
128
131
  pageQueue.wake();
@@ -130,7 +133,7 @@ function createBackwardReader(filepath, options) {
130
133
  fdSync = (0, node_fs.openSync)(filepath, "r");
131
134
  pos = (0, node_fs.statSync)(filepath).size;
132
135
  if (pos === 0) {
133
- pageQueue.push([""]);
136
+ pageQueue.push(emptyPage);
134
137
  done = true;
135
138
  flushed = true;
136
139
  pageQueue.wake();
@@ -140,7 +143,7 @@ function createBackwardReader(filepath, options) {
140
143
  pos = (await fd.stat()).size;
141
144
  if (pos === 0) {
142
145
  if (!done) {
143
- pageQueue.push([""]);
146
+ pageQueue.push(emptyPage);
144
147
  done = true;
145
148
  flushed = true;
146
149
  }
@@ -159,9 +162,12 @@ function createBackwardReader(filepath, options) {
159
162
  pos -= readSize;
160
163
  const buf = Buffer.allocUnsafe(readSize);
161
164
  const { bytesRead } = await fd.read(buf, 0, readSize, pos);
162
- buffer = buf.toString("utf8", 0, bytesRead) + buffer;
163
- if (pos === 0 && buffer.startsWith(delimiter)) startsWithDelimiter = true;
164
- consumeBuffer();
165
+ if (isBufferOutput) pageQueue.push(buf.subarray(0, bytesRead));
166
+ else {
167
+ buffer = buf.toString("utf8", 0, bytesRead) + buffer;
168
+ if (pos === 0 && buffer.startsWith(delimiter)) startsWithDelimiter = true;
169
+ consumeBuffer();
170
+ }
165
171
  }
166
172
  if (pos === 0 && !flushed) {
167
173
  flushTail();
@@ -183,9 +189,12 @@ function createBackwardReader(filepath, options) {
183
189
  pos -= readSize;
184
190
  const buf = Buffer.allocUnsafe(readSize);
185
191
  const bytesRead = (0, node_fs.readSync)(fdSync, buf, 0, readSize, pos);
186
- buffer = buf.toString("utf8", 0, bytesRead) + buffer;
187
- if (pos === 0 && buffer.startsWith(delimiter)) startsWithDelimiter = true;
188
- consumeBuffer();
192
+ if (isBufferOutput) pageQueue.push(buf.subarray(0, bytesRead));
193
+ else {
194
+ buffer = buf.toString("utf8", 0, bytesRead) + buffer;
195
+ if (pos === 0 && buffer.startsWith(delimiter)) startsWithDelimiter = true;
196
+ consumeBuffer();
197
+ }
189
198
  }
190
199
  if (pos === 0 && !flushed) {
191
200
  flushTail();
@@ -269,8 +278,10 @@ function createBackwardReader(filepath, options) {
269
278
  //#endregion
270
279
  //#region src/reader/forward.reader.ts
271
280
  function createForwardReader(filepath, options) {
272
- const { chunkSize, pageSize, delimiter, prefetch } = options;
281
+ const { chunkSize, pageSize, delimiter, prefetch, output } = options;
273
282
  const pageQueue = createRingBuffer(Math.max(2, prefetch + 1));
283
+ const isBufferOutput = output === "buffer";
284
+ const emptyPage = isBufferOutput ? Buffer.allocUnsafe(0) : [""];
274
285
  const local = [];
275
286
  let fd = null;
276
287
  let fdSync = null;
@@ -292,16 +303,18 @@ function createForwardReader(filepath, options) {
292
303
  function flushTail() {
293
304
  if (flushed) return;
294
305
  flushed = true;
295
- local.push(buffer.length > 0 ? buffer : "");
296
- buffer = "";
297
- while (local.length > 0) pageQueue.push(local.splice(0, pageSize));
306
+ if (!isBufferOutput) {
307
+ local.push(buffer.length > 0 ? buffer : "");
308
+ buffer = "";
309
+ while (local.length > 0) pageQueue.push(local.splice(0, pageSize));
310
+ }
298
311
  done = true;
299
312
  pageQueue.wake();
300
313
  }
301
314
  fdSync = (0, node_fs.openSync)(filepath, "r");
302
315
  size = (0, node_fs.statSync)(filepath).size;
303
316
  if (size === 0) {
304
- pageQueue.push([""]);
317
+ pageQueue.push(emptyPage);
305
318
  done = true;
306
319
  flushed = true;
307
320
  pageQueue.wake();
@@ -311,7 +324,7 @@ function createForwardReader(filepath, options) {
311
324
  size = (await fd.stat()).size;
312
325
  if (size === 0) {
313
326
  if (!done) {
314
- pageQueue.push([""]);
327
+ pageQueue.push(emptyPage);
315
328
  done = true;
316
329
  flushed = true;
317
330
  }
@@ -330,8 +343,11 @@ function createForwardReader(filepath, options) {
330
343
  const buf = Buffer.allocUnsafe(readSize);
331
344
  const { bytesRead } = await fd.read(buf, 0, readSize, pos);
332
345
  pos += bytesRead;
333
- buffer = buffer + buf.toString("utf8", 0, bytesRead);
334
- consumeBuffer();
346
+ if (isBufferOutput) pageQueue.push(buf.subarray(0, bytesRead));
347
+ else {
348
+ buffer = buffer + buf.toString("utf8", 0, bytesRead);
349
+ consumeBuffer();
350
+ }
335
351
  }
336
352
  if (pos >= size && !flushed) {
337
353
  flushTail();
@@ -353,8 +369,11 @@ function createForwardReader(filepath, options) {
353
369
  const buf = Buffer.allocUnsafe(readSize);
354
370
  const bytesRead = (0, node_fs.readSync)(fdSync, buf, 0, readSize, pos);
355
371
  pos += bytesRead;
356
- buffer = buffer + buf.toString("utf8", 0, bytesRead);
357
- consumeBuffer();
372
+ if (isBufferOutput) pageQueue.push(buf.subarray(0, bytesRead));
373
+ else {
374
+ buffer = buffer + buf.toString("utf8", 0, bytesRead);
375
+ consumeBuffer();
376
+ }
358
377
  }
359
378
  if (pos >= size && !flushed) {
360
379
  flushTail();
@@ -436,117 +455,20 @@ function createForwardReader(filepath, options) {
436
455
  };
437
456
  }
438
457
  //#endregion
439
- //#region src/reader/worker.reader.ts
440
- const workerFile = new URL("./worker.mjs", require("url").pathToFileURL(__filename).href);
441
- function createWorkerReader(filepath, options) {
442
- const { prefetch } = options;
443
- let done = false;
444
- let closed = false;
445
- const pageQueue = createRingBuffer(Math.max(2, prefetch + 1));
446
- const worker = new node_worker_threads.Worker(new URL(workerFile, require("url").pathToFileURL(__filename).href), { workerData: {
447
- filepath,
448
- options
449
- } });
450
- worker.on("message", (msg) => {
451
- switch (msg.type) {
452
- case "page":
453
- pageQueue.push(msg.data);
454
- break;
455
- case "done":
456
- case "error":
457
- done = true;
458
- pageQueue.wake();
459
- break;
460
- }
461
- });
462
- worker.on("error", () => {
463
- done = true;
464
- pageQueue.wake();
465
- });
466
- worker.on("exit", () => {
467
- done = true;
468
- pageQueue.wake();
469
- });
470
- async function next() {
471
- if (closed) return null;
472
- return pageQueue.shift(done);
473
- }
474
- function nextSync() {
475
- if (closed) return null;
476
- return pageQueue.shiftSync();
477
- }
478
- async function close() {
479
- if (closed) return;
480
- closed = true;
481
- done = true;
482
- pageQueue.clear();
483
- try {
484
- await worker.terminate();
485
- } catch {}
486
- }
487
- function tryClose() {
488
- close().catch(() => {});
489
- }
490
- return {
491
- next,
492
- nextSync,
493
- close,
494
- async *[Symbol.asyncIterator]() {
495
- try {
496
- while (true) {
497
- const p = await next();
498
- if (!p) break;
499
- yield p;
500
- }
501
- } finally {
502
- tryClose();
503
- }
504
- },
505
- *[Symbol.iterator]() {
506
- try {
507
- while (true) {
508
- const p = nextSync();
509
- if (!p) break;
510
- yield p;
511
- }
512
- } finally {
513
- tryClose();
514
- }
515
- }
516
- };
517
- }
518
- //#endregion
519
458
  //#region src/main.ts
520
459
  function createPager(filepath, options = {}) {
521
- const { chunkSize = 64 * 1024, pageSize = 1e3, delimiter = "\n", prefetch = 8, backward = false, useWorker = false, tryNative = true } = options;
460
+ const { chunkSize = 64 * 1024, pageSize = 1e3, delimiter = "\n", prefetch = 8, backward = false, output = "string" } = options;
522
461
  if (!filepath) throw new Error("filepath required");
523
462
  if (pageSize < 1) throw new RangeError("pageSize must be >= 1");
524
463
  if (prefetch < 1) throw new RangeError("prefetch must be >= 1");
525
- if (useWorker) {
526
- if (backward) throw new Error("backward not supported with useWorker");
527
- if (tryNative) throw new Error("tryNative not supported with useWorker");
528
- }
529
- if (tryNative) {
530
- if (delimiter.length !== 1) throw new RangeError("native reader only supports single-character delimiters");
531
- }
532
464
  const readerOptions = {
533
465
  chunkSize,
534
466
  pageSize,
535
467
  prefetch,
536
- delimiter
468
+ delimiter,
469
+ output
537
470
  };
538
- let nativeReader;
539
- if (tryNative) {
540
- const nativeOptions = {
541
- pageSize,
542
- delimiter,
543
- backward
544
- };
545
- try {
546
- nativeReader = require_native.createNativePager(filepath, nativeOptions);
547
- } catch {}
548
- }
549
- const reader = tryNative && nativeReader ? nativeReader : useWorker ? createWorkerReader(filepath, readerOptions) : backward ? createBackwardReader(filepath, readerOptions) : createForwardReader(filepath, readerOptions);
471
+ const reader = backward ? createBackwardReader(filepath, readerOptions) : createForwardReader(filepath, readerOptions);
550
472
  if (process.env.PAGER_TEST_CLEANUPS) {
551
473
  globalThis.__pager_test_cleanups__ ??= [];
552
474
  globalThis.__pager_test_cleanups__.push(reader.close);
package/dist/main.d.cts CHANGED
@@ -1,6 +1,9 @@
1
- import { a as PagerOptions, i as Pager, n as NativeAddon, o as ReaderOptions, r as NativeReaderOptions, s as WorkerMessage, t as createNativePager } from "./native-_NmVYcF6.cjs";
1
+ import { a as Output, c as PagerOptions, i as NativeReaderOptions, l as ReaderOptions, n as AsyncFunction, o as PageOutput, r as NativeAddon, s as Pager, t as createNativePager, u as ResolvePageOutput } from "./native-7x1Vhdaw.cjs";
2
2
 
3
3
  //#region src/main.d.ts
4
- declare function createPager(filepath: string, options?: PagerOptions): Pager;
4
+ declare function createPager<T extends Output>(filepath: string, options: PagerOptions & {
5
+ output: T;
6
+ }): Pager<T>;
7
+ declare function createPager(filepath: string, options?: PagerOptions): Pager<"string">;
5
8
  //#endregion
6
- export { NativeAddon, NativeReaderOptions, Pager, PagerOptions, ReaderOptions, WorkerMessage, createNativePager, createPager, createPager as default };
9
+ export { AsyncFunction, NativeAddon, NativeReaderOptions, Output, PageOutput, Pager, PagerOptions, ReaderOptions, ResolvePageOutput, createNativePager, createPager, createPager as default };
package/dist/main.d.mts CHANGED
@@ -1,6 +1,9 @@
1
- import { a as PagerOptions, i as Pager, n as NativeAddon, o as ReaderOptions, r as NativeReaderOptions, s as WorkerMessage, t as createNativePager } from "./native-DGzYrMHK.mjs";
1
+ import { a as Output, c as PagerOptions, i as NativeReaderOptions, l as ReaderOptions, n as AsyncFunction, o as PageOutput, r as NativeAddon, s as Pager, t as createNativePager, u as ResolvePageOutput } from "./native-D6SBIgoE.mjs";
2
2
 
3
3
  //#region src/main.d.ts
4
- declare function createPager(filepath: string, options?: PagerOptions): Pager;
4
+ declare function createPager<T extends Output>(filepath: string, options: PagerOptions & {
5
+ output: T;
6
+ }): Pager<T>;
7
+ declare function createPager(filepath: string, options?: PagerOptions): Pager<"string">;
5
8
  //#endregion
6
- export { NativeAddon, NativeReaderOptions, Pager, PagerOptions, ReaderOptions, WorkerMessage, createNativePager, createPager, createPager as default };
9
+ export { AsyncFunction, NativeAddon, NativeReaderOptions, Output, PageOutput, Pager, PagerOptions, ReaderOptions, ResolvePageOutput, createNativePager, createPager, createPager as default };
package/dist/main.mjs CHANGED
@@ -1,11 +1,6 @@
1
1
  import { createNativePager } from "./native.mjs";
2
- import { createRequire } from "node:module";
3
2
  import { closeSync, openSync, readSync, statSync } from "node:fs";
4
3
  import { open } from "node:fs/promises";
5
- import { Worker } from "node:worker_threads";
6
- //#region \0rolldown/runtime.js
7
- var __require = /* @__PURE__ */ createRequire(import.meta.url);
8
- //#endregion
9
4
  //#region src/helper.ts
10
5
  function createRingBuffer(capacity) {
11
6
  if (!Number.isFinite(capacity) || capacity <= 0) throw new RangeError("capacity must be a positive number");
@@ -93,8 +88,10 @@ function createRingBuffer(capacity) {
93
88
  //#endregion
94
89
  //#region src/reader/backward.reader.ts
95
90
  function createBackwardReader(filepath, options) {
96
- const { chunkSize, pageSize, delimiter, prefetch } = options;
91
+ const { chunkSize, pageSize, delimiter, prefetch, output } = options;
97
92
  const pageQueue = createRingBuffer(Math.max(2, prefetch + 1));
93
+ const isBufferOutput = output === "buffer";
94
+ const emptyPage = isBufferOutput ? Buffer.allocUnsafe(0) : [""];
98
95
  const local = [];
99
96
  let fd = null;
100
97
  let fdSync = null;
@@ -116,13 +113,15 @@ function createBackwardReader(filepath, options) {
116
113
  function flushTail() {
117
114
  if (flushed) return;
118
115
  flushed = true;
119
- if (buffer.length > 0) local.push(buffer);
120
- else if (startsWithDelimiter) local.push("");
121
- buffer = "";
122
- while (local.length > 0) {
123
- const page = local.slice(local.length - Math.min(pageSize, local.length));
124
- local.length -= page.length;
125
- pageQueue.push(page);
116
+ if (!isBufferOutput) {
117
+ if (buffer.length > 0) local.push(buffer);
118
+ else if (startsWithDelimiter) local.push("");
119
+ buffer = "";
120
+ while (local.length > 0) {
121
+ const page = local.slice(local.length - Math.min(pageSize, local.length));
122
+ local.length -= page.length;
123
+ pageQueue.push(page);
124
+ }
126
125
  }
127
126
  done = true;
128
127
  pageQueue.wake();
@@ -130,7 +129,7 @@ function createBackwardReader(filepath, options) {
130
129
  fdSync = openSync(filepath, "r");
131
130
  pos = statSync(filepath).size;
132
131
  if (pos === 0) {
133
- pageQueue.push([""]);
132
+ pageQueue.push(emptyPage);
134
133
  done = true;
135
134
  flushed = true;
136
135
  pageQueue.wake();
@@ -140,7 +139,7 @@ function createBackwardReader(filepath, options) {
140
139
  pos = (await fd.stat()).size;
141
140
  if (pos === 0) {
142
141
  if (!done) {
143
- pageQueue.push([""]);
142
+ pageQueue.push(emptyPage);
144
143
  done = true;
145
144
  flushed = true;
146
145
  }
@@ -159,9 +158,12 @@ function createBackwardReader(filepath, options) {
159
158
  pos -= readSize;
160
159
  const buf = Buffer.allocUnsafe(readSize);
161
160
  const { bytesRead } = await fd.read(buf, 0, readSize, pos);
162
- buffer = buf.toString("utf8", 0, bytesRead) + buffer;
163
- if (pos === 0 && buffer.startsWith(delimiter)) startsWithDelimiter = true;
164
- consumeBuffer();
161
+ if (isBufferOutput) pageQueue.push(buf.subarray(0, bytesRead));
162
+ else {
163
+ buffer = buf.toString("utf8", 0, bytesRead) + buffer;
164
+ if (pos === 0 && buffer.startsWith(delimiter)) startsWithDelimiter = true;
165
+ consumeBuffer();
166
+ }
165
167
  }
166
168
  if (pos === 0 && !flushed) {
167
169
  flushTail();
@@ -183,9 +185,12 @@ function createBackwardReader(filepath, options) {
183
185
  pos -= readSize;
184
186
  const buf = Buffer.allocUnsafe(readSize);
185
187
  const bytesRead = readSync(fdSync, buf, 0, readSize, pos);
186
- buffer = buf.toString("utf8", 0, bytesRead) + buffer;
187
- if (pos === 0 && buffer.startsWith(delimiter)) startsWithDelimiter = true;
188
- consumeBuffer();
188
+ if (isBufferOutput) pageQueue.push(buf.subarray(0, bytesRead));
189
+ else {
190
+ buffer = buf.toString("utf8", 0, bytesRead) + buffer;
191
+ if (pos === 0 && buffer.startsWith(delimiter)) startsWithDelimiter = true;
192
+ consumeBuffer();
193
+ }
189
194
  }
190
195
  if (pos === 0 && !flushed) {
191
196
  flushTail();
@@ -269,8 +274,10 @@ function createBackwardReader(filepath, options) {
269
274
  //#endregion
270
275
  //#region src/reader/forward.reader.ts
271
276
  function createForwardReader(filepath, options) {
272
- const { chunkSize, pageSize, delimiter, prefetch } = options;
277
+ const { chunkSize, pageSize, delimiter, prefetch, output } = options;
273
278
  const pageQueue = createRingBuffer(Math.max(2, prefetch + 1));
279
+ const isBufferOutput = output === "buffer";
280
+ const emptyPage = isBufferOutput ? Buffer.allocUnsafe(0) : [""];
274
281
  const local = [];
275
282
  let fd = null;
276
283
  let fdSync = null;
@@ -292,16 +299,18 @@ function createForwardReader(filepath, options) {
292
299
  function flushTail() {
293
300
  if (flushed) return;
294
301
  flushed = true;
295
- local.push(buffer.length > 0 ? buffer : "");
296
- buffer = "";
297
- while (local.length > 0) pageQueue.push(local.splice(0, pageSize));
302
+ if (!isBufferOutput) {
303
+ local.push(buffer.length > 0 ? buffer : "");
304
+ buffer = "";
305
+ while (local.length > 0) pageQueue.push(local.splice(0, pageSize));
306
+ }
298
307
  done = true;
299
308
  pageQueue.wake();
300
309
  }
301
310
  fdSync = openSync(filepath, "r");
302
311
  size = statSync(filepath).size;
303
312
  if (size === 0) {
304
- pageQueue.push([""]);
313
+ pageQueue.push(emptyPage);
305
314
  done = true;
306
315
  flushed = true;
307
316
  pageQueue.wake();
@@ -311,7 +320,7 @@ function createForwardReader(filepath, options) {
311
320
  size = (await fd.stat()).size;
312
321
  if (size === 0) {
313
322
  if (!done) {
314
- pageQueue.push([""]);
323
+ pageQueue.push(emptyPage);
315
324
  done = true;
316
325
  flushed = true;
317
326
  }
@@ -330,8 +339,11 @@ function createForwardReader(filepath, options) {
330
339
  const buf = Buffer.allocUnsafe(readSize);
331
340
  const { bytesRead } = await fd.read(buf, 0, readSize, pos);
332
341
  pos += bytesRead;
333
- buffer = buffer + buf.toString("utf8", 0, bytesRead);
334
- consumeBuffer();
342
+ if (isBufferOutput) pageQueue.push(buf.subarray(0, bytesRead));
343
+ else {
344
+ buffer = buffer + buf.toString("utf8", 0, bytesRead);
345
+ consumeBuffer();
346
+ }
335
347
  }
336
348
  if (pos >= size && !flushed) {
337
349
  flushTail();
@@ -353,8 +365,11 @@ function createForwardReader(filepath, options) {
353
365
  const buf = Buffer.allocUnsafe(readSize);
354
366
  const bytesRead = readSync(fdSync, buf, 0, readSize, pos);
355
367
  pos += bytesRead;
356
- buffer = buffer + buf.toString("utf8", 0, bytesRead);
357
- consumeBuffer();
368
+ if (isBufferOutput) pageQueue.push(buf.subarray(0, bytesRead));
369
+ else {
370
+ buffer = buffer + buf.toString("utf8", 0, bytesRead);
371
+ consumeBuffer();
372
+ }
358
373
  }
359
374
  if (pos >= size && !flushed) {
360
375
  flushTail();
@@ -436,117 +451,20 @@ function createForwardReader(filepath, options) {
436
451
  };
437
452
  }
438
453
  //#endregion
439
- //#region src/reader/worker.reader.ts
440
- const workerFile = typeof import.meta !== "undefined" ? new URL("./worker.mjs", import.meta.url) : __require.resolve("./worker.cjs");
441
- function createWorkerReader(filepath, options) {
442
- const { prefetch } = options;
443
- let done = false;
444
- let closed = false;
445
- const pageQueue = createRingBuffer(Math.max(2, prefetch + 1));
446
- const worker = new Worker(new URL(workerFile, import.meta.url), { workerData: {
447
- filepath,
448
- options
449
- } });
450
- worker.on("message", (msg) => {
451
- switch (msg.type) {
452
- case "page":
453
- pageQueue.push(msg.data);
454
- break;
455
- case "done":
456
- case "error":
457
- done = true;
458
- pageQueue.wake();
459
- break;
460
- }
461
- });
462
- worker.on("error", () => {
463
- done = true;
464
- pageQueue.wake();
465
- });
466
- worker.on("exit", () => {
467
- done = true;
468
- pageQueue.wake();
469
- });
470
- async function next() {
471
- if (closed) return null;
472
- return pageQueue.shift(done);
473
- }
474
- function nextSync() {
475
- if (closed) return null;
476
- return pageQueue.shiftSync();
477
- }
478
- async function close() {
479
- if (closed) return;
480
- closed = true;
481
- done = true;
482
- pageQueue.clear();
483
- try {
484
- await worker.terminate();
485
- } catch {}
486
- }
487
- function tryClose() {
488
- close().catch(() => {});
489
- }
490
- return {
491
- next,
492
- nextSync,
493
- close,
494
- async *[Symbol.asyncIterator]() {
495
- try {
496
- while (true) {
497
- const p = await next();
498
- if (!p) break;
499
- yield p;
500
- }
501
- } finally {
502
- tryClose();
503
- }
504
- },
505
- *[Symbol.iterator]() {
506
- try {
507
- while (true) {
508
- const p = nextSync();
509
- if (!p) break;
510
- yield p;
511
- }
512
- } finally {
513
- tryClose();
514
- }
515
- }
516
- };
517
- }
518
- //#endregion
519
454
  //#region src/main.ts
520
455
  function createPager(filepath, options = {}) {
521
- const { chunkSize = 64 * 1024, pageSize = 1e3, delimiter = "\n", prefetch = 8, backward = false, useWorker = false, tryNative = true } = options;
456
+ const { chunkSize = 64 * 1024, pageSize = 1e3, delimiter = "\n", prefetch = 8, backward = false, output = "string" } = options;
522
457
  if (!filepath) throw new Error("filepath required");
523
458
  if (pageSize < 1) throw new RangeError("pageSize must be >= 1");
524
459
  if (prefetch < 1) throw new RangeError("prefetch must be >= 1");
525
- if (useWorker) {
526
- if (backward) throw new Error("backward not supported with useWorker");
527
- if (tryNative) throw new Error("tryNative not supported with useWorker");
528
- }
529
- if (tryNative) {
530
- if (delimiter.length !== 1) throw new RangeError("native reader only supports single-character delimiters");
531
- }
532
460
  const readerOptions = {
533
461
  chunkSize,
534
462
  pageSize,
535
463
  prefetch,
536
- delimiter
464
+ delimiter,
465
+ output
537
466
  };
538
- let nativeReader;
539
- if (tryNative) {
540
- const nativeOptions = {
541
- pageSize,
542
- delimiter,
543
- backward
544
- };
545
- try {
546
- nativeReader = createNativePager(filepath, nativeOptions);
547
- } catch {}
548
- }
549
- const reader = tryNative && nativeReader ? nativeReader : useWorker ? createWorkerReader(filepath, readerOptions) : backward ? createBackwardReader(filepath, readerOptions) : createForwardReader(filepath, readerOptions);
467
+ const reader = backward ? createBackwardReader(filepath, readerOptions) : createForwardReader(filepath, readerOptions);
550
468
  if (process.env.PAGER_TEST_CLEANUPS) {
551
469
  globalThis.__pager_test_cleanups__ ??= [];
552
470
  globalThis.__pager_test_cleanups__.push(reader.close);
@@ -0,0 +1,44 @@
1
+ //#region src/types.d.ts
2
+ type AsyncFunction = () => Promise<void>;
3
+ type Output = "string" | "buffer";
4
+ type PageOutput = string[] | Buffer;
5
+ type ResolvePageOutput<T extends Output> = T extends "buffer" ? Buffer : string[];
6
+ interface ReaderOptions {
7
+ chunkSize: number;
8
+ pageSize: number;
9
+ delimiter: string;
10
+ prefetch: number;
11
+ output: Output;
12
+ }
13
+ interface NativeReaderOptions {
14
+ pageSize: number;
15
+ delimiter: string;
16
+ backward: boolean;
17
+ output: Output;
18
+ }
19
+ type PagerOptions = Partial<ReaderOptions> & {
20
+ backward?: boolean;
21
+ };
22
+ type AddonFD = object | null;
23
+ type AddonData = Buffer | null;
24
+ interface NativeAddon {
25
+ open: (filepath: string, pageSize: number, delimiter: string, backward: boolean) => AddonFD;
26
+ next: (fd: AddonFD) => Promise<AddonData>;
27
+ nextSync: (fd: AddonFD) => AddonData;
28
+ close: (fd: AddonFD) => Promise<void>;
29
+ }
30
+ interface Pager<T extends Output = "string"> {
31
+ next(): Promise<ResolvePageOutput<T> | null>;
32
+ nextSync(): ResolvePageOutput<T> | null;
33
+ close(): Promise<void>;
34
+ [Symbol.asyncIterator](): AsyncIterator<ResolvePageOutput<T>>;
35
+ [Symbol.iterator](): Iterator<ResolvePageOutput<T>>;
36
+ }
37
+ //#endregion
38
+ //#region src/native.d.ts
39
+ declare function createNativePager<T extends Output>(filepath: string, options: Partial<NativeReaderOptions> & {
40
+ output: T;
41
+ }): Pager<T>;
42
+ declare function createNativePager(filepath: string, options?: Partial<NativeReaderOptions>): Pager<"string">;
43
+ //#endregion
44
+ export { Output as a, PagerOptions as c, NativeReaderOptions as i, ReaderOptions as l, AsyncFunction as n, PageOutput as o, NativeAddon as r, Pager as s, createNativePager as t, ResolvePageOutput as u };
@@ -0,0 +1,44 @@
1
+ //#region src/types.d.ts
2
+ type AsyncFunction = () => Promise<void>;
3
+ type Output = "string" | "buffer";
4
+ type PageOutput = string[] | Buffer;
5
+ type ResolvePageOutput<T extends Output> = T extends "buffer" ? Buffer : string[];
6
+ interface ReaderOptions {
7
+ chunkSize: number;
8
+ pageSize: number;
9
+ delimiter: string;
10
+ prefetch: number;
11
+ output: Output;
12
+ }
13
+ interface NativeReaderOptions {
14
+ pageSize: number;
15
+ delimiter: string;
16
+ backward: boolean;
17
+ output: Output;
18
+ }
19
+ type PagerOptions = Partial<ReaderOptions> & {
20
+ backward?: boolean;
21
+ };
22
+ type AddonFD = object | null;
23
+ type AddonData = Buffer | null;
24
+ interface NativeAddon {
25
+ open: (filepath: string, pageSize: number, delimiter: string, backward: boolean) => AddonFD;
26
+ next: (fd: AddonFD) => Promise<AddonData>;
27
+ nextSync: (fd: AddonFD) => AddonData;
28
+ close: (fd: AddonFD) => Promise<void>;
29
+ }
30
+ interface Pager<T extends Output = "string"> {
31
+ next(): Promise<ResolvePageOutput<T> | null>;
32
+ nextSync(): ResolvePageOutput<T> | null;
33
+ close(): Promise<void>;
34
+ [Symbol.asyncIterator](): AsyncIterator<ResolvePageOutput<T>>;
35
+ [Symbol.iterator](): Iterator<ResolvePageOutput<T>>;
36
+ }
37
+ //#endregion
38
+ //#region src/native.d.ts
39
+ declare function createNativePager<T extends Output>(filepath: string, options: Partial<NativeReaderOptions> & {
40
+ output: T;
41
+ }): Pager<T>;
42
+ declare function createNativePager(filepath: string, options?: Partial<NativeReaderOptions>): Pager<"string">;
43
+ //#endregion
44
+ export { Output as a, PagerOptions as c, NativeReaderOptions as i, ReaderOptions as l, AsyncFunction as n, PageOutput as o, NativeAddon as r, Pager as s, createNativePager as t, ResolvePageOutput as u };
package/dist/native.cjs CHANGED
@@ -28,11 +28,12 @@ function loadNativeAddon() {
28
28
  }
29
29
  }
30
30
  function createNativePager(filepath, options = {}) {
31
- const { pageSize = 1e3, delimiter = "\n", backward = false } = options;
31
+ const { pageSize = 1e3, delimiter = "\n", backward = false, output = "string" } = options;
32
32
  if (!filepath) throw new Error("filepath required");
33
33
  if (pageSize < 1) throw new RangeError("pageSize must be >= 1");
34
34
  if (delimiter?.length > 1) throw new RangeError("native reader only supports single-character delimiters");
35
35
  const nativeReader = loadNativeAddon();
36
+ const isBufferOutput = output === "buffer";
36
37
  if (process.env.PAGER_TEST_CLEANUPS) {
37
38
  globalThis.__pager_test_cleanups__ ??= [];
38
39
  globalThis.__pager_test_cleanups__.push(nativeReader.close);
@@ -41,15 +42,15 @@ function createNativePager(filepath, options = {}) {
41
42
  let closed = false;
42
43
  async function next() {
43
44
  if (closed || !fd) return null;
44
- const data = await nativeReader.next(fd);
45
- if (!data) return null;
46
- return data.toString("utf8").split(delimiter);
45
+ const buf = await nativeReader.next(fd);
46
+ if (!buf) return null;
47
+ return isBufferOutput ? buf : buf.toString("utf8").split(delimiter);
47
48
  }
48
49
  function nextSync() {
49
50
  if (closed || !fd) return null;
50
- const data = nativeReader.nextSync(fd);
51
- if (!data) return null;
52
- return data.toString("utf8").split(delimiter);
51
+ const buf = nativeReader.nextSync(fd);
52
+ if (!buf) return null;
53
+ return isBufferOutput ? buf : buf.toString("utf8").split(delimiter);
53
54
  }
54
55
  async function close() {
55
56
  if (closed || !fd) return;
package/dist/native.d.cts CHANGED
@@ -1,2 +1,2 @@
1
- import { t as createNativePager } from "./native-_NmVYcF6.cjs";
1
+ import { t as createNativePager } from "./native-7x1Vhdaw.cjs";
2
2
  export { createNativePager };
package/dist/native.d.mts CHANGED
@@ -1,2 +1,2 @@
1
- import { t as createNativePager } from "./native-DGzYrMHK.mjs";
1
+ import { t as createNativePager } from "./native-D6SBIgoE.mjs";
2
2
  export { createNativePager };
package/dist/native.mjs CHANGED
@@ -27,11 +27,12 @@ function loadNativeAddon() {
27
27
  }
28
28
  }
29
29
  function createNativePager(filepath, options = {}) {
30
- const { pageSize = 1e3, delimiter = "\n", backward = false } = options;
30
+ const { pageSize = 1e3, delimiter = "\n", backward = false, output = "string" } = options;
31
31
  if (!filepath) throw new Error("filepath required");
32
32
  if (pageSize < 1) throw new RangeError("pageSize must be >= 1");
33
33
  if (delimiter?.length > 1) throw new RangeError("native reader only supports single-character delimiters");
34
34
  const nativeReader = loadNativeAddon();
35
+ const isBufferOutput = output === "buffer";
35
36
  if (process.env.PAGER_TEST_CLEANUPS) {
36
37
  globalThis.__pager_test_cleanups__ ??= [];
37
38
  globalThis.__pager_test_cleanups__.push(nativeReader.close);
@@ -40,15 +41,15 @@ function createNativePager(filepath, options = {}) {
40
41
  let closed = false;
41
42
  async function next() {
42
43
  if (closed || !fd) return null;
43
- const data = await nativeReader.next(fd);
44
- if (!data) return null;
45
- return data.toString("utf8").split(delimiter);
44
+ const buf = await nativeReader.next(fd);
45
+ if (!buf) return null;
46
+ return isBufferOutput ? buf : buf.toString("utf8").split(delimiter);
46
47
  }
47
48
  function nextSync() {
48
49
  if (closed || !fd) return null;
49
- const data = nativeReader.nextSync(fd);
50
- if (!data) return null;
51
- return data.toString("utf8").split(delimiter);
50
+ const buf = nativeReader.nextSync(fd);
51
+ if (!buf) return null;
52
+ return isBufferOutput ? buf : buf.toString("utf8").split(delimiter);
52
53
  }
53
54
  async function close() {
54
55
  if (closed || !fd) return;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "readline-pager",
3
- "version": "0.6.5",
3
+ "version": "0.7.2",
4
4
  "scripts": {
5
5
  "build:js": "tsdown",
6
6
  "build:native": "node-gyp rebuild",
@@ -13,20 +13,20 @@
13
13
  "benchmark:bun": "bun test/benchmark.ts"
14
14
  },
15
15
  "devDependencies": {
16
- "@types/bun": "~1.3.11",
16
+ "@types/bun": "~1.3.12",
17
17
  "@types/deno": "~2.5.0",
18
- "@types/node": "~25.5.0",
18
+ "@types/node": "~25.6.0",
19
19
  "node-gyp": "~12.2.0",
20
- "prettier": "~3.8.1",
20
+ "prettier": "~3.8.2",
21
21
  "prettier-plugin-organize-imports": "~4.3.0",
22
- "tsdown": "~0.21.6",
22
+ "tsdown": "~0.21.7",
23
23
  "typescript": "~6.0.2"
24
24
  },
25
25
  "optionalDependencies": {
26
- "@devmor-j/readline-pager-linux-arm64": "0.6.5",
27
- "@devmor-j/readline-pager-linux-musl-arm64": "0.6.5",
28
- "@devmor-j/readline-pager-linux-musl-x64": "0.6.5",
29
- "@devmor-j/readline-pager-linux-x64": "0.6.5"
26
+ "@devmor-j/readline-pager-linux-arm64": "0.7.2",
27
+ "@devmor-j/readline-pager-linux-musl-arm64": "0.7.2",
28
+ "@devmor-j/readline-pager-linux-musl-x64": "0.7.2",
29
+ "@devmor-j/readline-pager-linux-x64": "0.7.2"
30
30
  },
31
31
  "gypfile": false,
32
32
  "type": "module",
@@ -60,7 +60,7 @@
60
60
  "avx2",
61
61
  "neon"
62
62
  ],
63
- "description": "High-performance paginated file reader for Node.js. Efficiently process large text files without loading them into memory.",
63
+ "description": "High-performance paginated file reader for Node.js. Process large text files efficiently without loading them into memory.",
64
64
  "repository": {
65
65
  "type": "git",
66
66
  "url": "git+https://github.com/devmor-j/readline-pager.git"
@@ -1,46 +0,0 @@
1
- //#region src/types.d.ts
2
- interface ReaderOptions {
3
- chunkSize: number;
4
- pageSize: number;
5
- delimiter: string;
6
- prefetch: number;
7
- }
8
- interface PagerOptions extends Partial<ReaderOptions> {
9
- backward?: boolean;
10
- useWorker?: boolean;
11
- tryNative?: boolean;
12
- }
13
- interface NativeReaderOptions {
14
- pageSize: number;
15
- delimiter: string;
16
- backward: boolean;
17
- }
18
- type WorkerMessage = {
19
- type: "page";
20
- data: string[];
21
- } | {
22
- type: "error";
23
- error: unknown;
24
- } | {
25
- type: "done";
26
- };
27
- type AddonFD = object | null;
28
- type AddonData = Buffer | null;
29
- interface NativeAddon {
30
- open: (filepath: string, pageSize: number, delimiter: string, backward: boolean) => AddonFD;
31
- next: (fd: AddonFD) => Promise<AddonData>;
32
- nextSync: (fd: AddonFD) => AddonData;
33
- close: (fd: AddonFD) => Promise<void>;
34
- }
35
- interface Pager {
36
- next(): Promise<string[] | null>;
37
- nextSync(): string[] | null;
38
- close(): Promise<void>;
39
- [Symbol.asyncIterator](): AsyncIterator<string[]>;
40
- [Symbol.iterator](): Iterator<string[]>;
41
- }
42
- //#endregion
43
- //#region src/native.d.ts
44
- declare function createNativePager(filepath: string, options?: Partial<NativeReaderOptions>): Pager;
45
- //#endregion
46
- export { PagerOptions as a, Pager as i, NativeAddon as n, ReaderOptions as o, NativeReaderOptions as r, WorkerMessage as s, createNativePager as t };
@@ -1,46 +0,0 @@
1
- //#region src/types.d.ts
2
- interface ReaderOptions {
3
- chunkSize: number;
4
- pageSize: number;
5
- delimiter: string;
6
- prefetch: number;
7
- }
8
- interface PagerOptions extends Partial<ReaderOptions> {
9
- backward?: boolean;
10
- useWorker?: boolean;
11
- tryNative?: boolean;
12
- }
13
- interface NativeReaderOptions {
14
- pageSize: number;
15
- delimiter: string;
16
- backward: boolean;
17
- }
18
- type WorkerMessage = {
19
- type: "page";
20
- data: string[];
21
- } | {
22
- type: "error";
23
- error: unknown;
24
- } | {
25
- type: "done";
26
- };
27
- type AddonFD = object | null;
28
- type AddonData = Buffer | null;
29
- interface NativeAddon {
30
- open: (filepath: string, pageSize: number, delimiter: string, backward: boolean) => AddonFD;
31
- next: (fd: AddonFD) => Promise<AddonData>;
32
- nextSync: (fd: AddonFD) => AddonData;
33
- close: (fd: AddonFD) => Promise<void>;
34
- }
35
- interface Pager {
36
- next(): Promise<string[] | null>;
37
- nextSync(): string[] | null;
38
- close(): Promise<void>;
39
- [Symbol.asyncIterator](): AsyncIterator<string[]>;
40
- [Symbol.iterator](): Iterator<string[]>;
41
- }
42
- //#endregion
43
- //#region src/native.d.ts
44
- declare function createNativePager(filepath: string, options?: Partial<NativeReaderOptions>): Pager;
45
- //#endregion
46
- export { PagerOptions as a, Pager as i, NativeAddon as n, ReaderOptions as o, NativeReaderOptions as r, WorkerMessage as s, createNativePager as t };
package/dist/worker.cjs DELETED
@@ -1,52 +0,0 @@
1
- let node_fs_promises = require("node:fs/promises");
2
- let node_worker_threads = require("node:worker_threads");
3
- //#region src/worker.ts
4
- const { filepath, options } = node_worker_threads.workerData;
5
- const { chunkSize, pageSize, delimiter } = options;
6
- const backpressure = pageSize * 8;
7
- function post(msg) {
8
- if (!node_worker_threads.parentPort) process.exit(1);
9
- node_worker_threads.parentPort.postMessage(msg);
10
- }
11
- (async () => {
12
- try {
13
- const fd = await (0, node_fs_promises.open)(filepath, "r");
14
- const { size } = await fd.stat();
15
- let pos = 0;
16
- let buffer = "";
17
- const local = [];
18
- while (pos < size) {
19
- const readSize = Math.min(chunkSize, size - pos);
20
- const buf = Buffer.allocUnsafe(readSize);
21
- const { bytesRead } = await fd.read(buf, 0, readSize, pos);
22
- pos += bytesRead;
23
- buffer = buffer + buf.toString("utf8", 0, bytesRead);
24
- let idx;
25
- while ((idx = buffer.indexOf(delimiter)) !== -1) {
26
- const line = buffer.slice(0, idx);
27
- buffer = buffer.slice(idx + delimiter.length);
28
- local.push(line);
29
- if (local.length >= pageSize) {
30
- post({
31
- type: "page",
32
- data: local.splice(0, pageSize)
33
- });
34
- if (local.length > backpressure) await new Promise((r) => setImmediate(r));
35
- }
36
- }
37
- }
38
- local.push(buffer);
39
- while (local.length > 0) post({
40
- type: "page",
41
- data: local.splice(0, pageSize)
42
- });
43
- post({ type: "done" });
44
- await fd.close();
45
- } catch (err) {
46
- post({
47
- type: "error",
48
- error: err
49
- });
50
- }
51
- })();
52
- //#endregion
package/dist/worker.d.cts DELETED
@@ -1 +0,0 @@
1
- export { };
package/dist/worker.d.mts DELETED
@@ -1 +0,0 @@
1
- export { };
package/dist/worker.mjs DELETED
@@ -1,53 +0,0 @@
1
- import { open } from "node:fs/promises";
2
- import { parentPort, workerData } from "node:worker_threads";
3
- //#region src/worker.ts
4
- const { filepath, options } = workerData;
5
- const { chunkSize, pageSize, delimiter } = options;
6
- const backpressure = pageSize * 8;
7
- function post(msg) {
8
- if (!parentPort) process.exit(1);
9
- parentPort.postMessage(msg);
10
- }
11
- (async () => {
12
- try {
13
- const fd = await open(filepath, "r");
14
- const { size } = await fd.stat();
15
- let pos = 0;
16
- let buffer = "";
17
- const local = [];
18
- while (pos < size) {
19
- const readSize = Math.min(chunkSize, size - pos);
20
- const buf = Buffer.allocUnsafe(readSize);
21
- const { bytesRead } = await fd.read(buf, 0, readSize, pos);
22
- pos += bytesRead;
23
- buffer = buffer + buf.toString("utf8", 0, bytesRead);
24
- let idx;
25
- while ((idx = buffer.indexOf(delimiter)) !== -1) {
26
- const line = buffer.slice(0, idx);
27
- buffer = buffer.slice(idx + delimiter.length);
28
- local.push(line);
29
- if (local.length >= pageSize) {
30
- post({
31
- type: "page",
32
- data: local.splice(0, pageSize)
33
- });
34
- if (local.length > backpressure) await new Promise((r) => setImmediate(r));
35
- }
36
- }
37
- }
38
- local.push(buffer);
39
- while (local.length > 0) post({
40
- type: "page",
41
- data: local.splice(0, pageSize)
42
- });
43
- post({ type: "done" });
44
- await fd.close();
45
- } catch (err) {
46
- post({
47
- type: "error",
48
- error: err
49
- });
50
- }
51
- })();
52
- //#endregion
53
- export {};