readline-pager 0.6.5 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -23
- package/dist/main.cjs +50 -128
- package/dist/main.d.cts +6 -3
- package/dist/main.d.mts +6 -3
- package/dist/main.mjs +50 -132
- package/dist/native-7x1Vhdaw.d.cts +44 -0
- package/dist/native-D6SBIgoE.d.mts +44 -0
- package/dist/native.cjs +8 -7
- package/dist/native.d.cts +1 -1
- package/dist/native.d.mts +1 -1
- package/dist/native.mjs +8 -7
- package/package.json +10 -10
- package/dist/native-DGzYrMHK.d.mts +0 -46
- package/dist/native-_NmVYcF6.d.cts +0 -46
- package/dist/worker.cjs +0 -52
- package/dist/worker.d.cts +0 -1
- package/dist/worker.d.mts +0 -1
- package/dist/worker.mjs +0 -53
package/README.md
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
<img src="https://img.shields.io/github/stars/devmor-j/readline-pager" alt="stars">
|
|
13
13
|
</p>
|
|
14
14
|
|
|
15
|
-
⚡ High-performance paginated file reader for Node.js.
|
|
15
|
+
⚡ High-performance paginated file reader for Node.js. Process large text files efficiently without loading them into memory.
|
|
16
16
|
|
|
17
17
|
- 📦 Zero dependencies
|
|
18
18
|
- ⚡ Up to ~3× faster than Node.js `readline`
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
- 🔁 Async (`for await...of`) and sync (`for...of`) iteration
|
|
21
21
|
- 📄 Page-based reading with manual control (`next`, `nextSync`)
|
|
22
22
|
- 🔀 Forward and backward reading support
|
|
23
|
-
- 🧪 Fully typed with
|
|
23
|
+
- 🧪 Fully typed with over 90% test coverage
|
|
24
24
|
|
|
25
25
|
> **Important:**
|
|
26
26
|
> Performance depends heavily on the `chunkSize` option. Tune it for your storage device. A value of **64 KiB** is usually a good starting point. Increasing it may improve throughput until you reach the best value for your hardware.
|
|
@@ -69,7 +69,7 @@ const pager = createPager("./bigfile.txt");
|
|
|
69
69
|
while ((page = pager.nextSync()) !== null) {}
|
|
70
70
|
|
|
71
71
|
// Native C++
|
|
72
|
-
for
|
|
72
|
+
for (const page of createNativePager("./bigfile.txt")) {
|
|
73
73
|
}
|
|
74
74
|
```
|
|
75
75
|
|
|
@@ -79,38 +79,37 @@ for await (const page of createNativePager("./bigfile.txt")) {
|
|
|
79
79
|
|
|
80
80
|
```ts
|
|
81
81
|
createPager(filepath, {
|
|
82
|
-
chunkSize?: number,
|
|
83
|
-
pageSize?: number,
|
|
84
|
-
delimiter?: string,
|
|
85
|
-
prefetch?: number,
|
|
86
|
-
backward?: boolean,
|
|
87
|
-
|
|
88
|
-
tryNative?: boolean, // default: true
|
|
82
|
+
chunkSize?: number, // default: 64 * 1024 (64 KiB)
|
|
83
|
+
pageSize?: number, // default: 1_000
|
|
84
|
+
delimiter?: string, // default: "\n"
|
|
85
|
+
prefetch?: number, // default: 8
|
|
86
|
+
backward?: boolean, // default: false
|
|
87
|
+
output?: "string" | "buffer", // default: "string"
|
|
89
88
|
});
|
|
90
89
|
|
|
91
90
|
createNativePager(filepath, {
|
|
92
|
-
pageSize?: number,
|
|
93
|
-
delimiter?: string,
|
|
94
|
-
backward?: boolean,
|
|
91
|
+
pageSize?: number, // default: 1_000
|
|
92
|
+
delimiter?: string, // default: "\n"
|
|
93
|
+
backward?: boolean, // default: false
|
|
94
|
+
output?: "string" | "buffer", // default: "string"
|
|
95
95
|
});
|
|
96
96
|
```
|
|
97
97
|
|
|
98
|
-
- `chunkSize` —
|
|
99
|
-
- `pageSize` —
|
|
100
|
-
- `delimiter` —
|
|
101
|
-
- `prefetch` —
|
|
102
|
-
- `backward` —
|
|
103
|
-
- `
|
|
104
|
-
- `tryNative` — attempts to use the native reader, falls back to the non-native version if it fails.
|
|
98
|
+
- `chunkSize` — Number of bytes read per I/O operation.
|
|
99
|
+
- `pageSize` — Number of lines per page.
|
|
100
|
+
- `delimiter` — Line separator.
|
|
101
|
+
- `prefetch` — Maximum number of pages buffered internally.
|
|
102
|
+
- `backward` — Read the file from end to start.
|
|
103
|
+
- `output` — Controls the page data type.
|
|
105
104
|
|
|
106
105
|
> **Note:**
|
|
107
|
-
> `createNativePager` requires x86 AVX2 or ARM NEON CPU instruction set extensions and will throw if they are not available. It also does **not** support multi-character delimiters
|
|
106
|
+
> `createNativePager` requires x86 AVX2 or ARM NEON CPU instruction set extensions and will throw if they are not available. It also does **not** support multi-character delimiters because it uses fast SIMD-based scanning.
|
|
108
107
|
|
|
109
108
|
---
|
|
110
109
|
|
|
111
110
|
## 📚 API
|
|
112
111
|
|
|
113
|
-
### `pager.next(): Promise<string[] | null>`
|
|
112
|
+
### `pager.next(): Promise<string[] | Buffer | null>`
|
|
114
113
|
|
|
115
114
|
Returns the next page asynchronously.
|
|
116
115
|
|
|
@@ -124,7 +123,7 @@ Empty lines are preserved.
|
|
|
124
123
|
> - A completely empty file (`0` bytes) produces `[""]` on the first read.
|
|
125
124
|
> - A file containing multiple empty lines returns each line as an empty string.
|
|
126
125
|
|
|
127
|
-
### `pager.nextSync(): string[] | null`
|
|
126
|
+
### `pager.nextSync(): string[] | Buffer | null`
|
|
128
127
|
|
|
129
128
|
Synchronous version of `pager.next()`.
|
|
130
129
|
|
package/dist/main.cjs
CHANGED
|
@@ -5,7 +5,6 @@ Object.defineProperties(exports, {
|
|
|
5
5
|
const require_native = require("./native.cjs");
|
|
6
6
|
let node_fs = require("node:fs");
|
|
7
7
|
let node_fs_promises = require("node:fs/promises");
|
|
8
|
-
let node_worker_threads = require("node:worker_threads");
|
|
9
8
|
//#region src/helper.ts
|
|
10
9
|
function createRingBuffer(capacity) {
|
|
11
10
|
if (!Number.isFinite(capacity) || capacity <= 0) throw new RangeError("capacity must be a positive number");
|
|
@@ -93,8 +92,10 @@ function createRingBuffer(capacity) {
|
|
|
93
92
|
//#endregion
|
|
94
93
|
//#region src/reader/backward.reader.ts
|
|
95
94
|
function createBackwardReader(filepath, options) {
|
|
96
|
-
const { chunkSize, pageSize, delimiter, prefetch } = options;
|
|
95
|
+
const { chunkSize, pageSize, delimiter, prefetch, output } = options;
|
|
97
96
|
const pageQueue = createRingBuffer(Math.max(2, prefetch + 1));
|
|
97
|
+
const isBufferOutput = output === "buffer";
|
|
98
|
+
const emptyPage = isBufferOutput ? Buffer.allocUnsafe(0) : [""];
|
|
98
99
|
const local = [];
|
|
99
100
|
let fd = null;
|
|
100
101
|
let fdSync = null;
|
|
@@ -116,13 +117,15 @@ function createBackwardReader(filepath, options) {
|
|
|
116
117
|
function flushTail() {
|
|
117
118
|
if (flushed) return;
|
|
118
119
|
flushed = true;
|
|
119
|
-
if (
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
120
|
+
if (!isBufferOutput) {
|
|
121
|
+
if (buffer.length > 0) local.push(buffer);
|
|
122
|
+
else if (startsWithDelimiter) local.push("");
|
|
123
|
+
buffer = "";
|
|
124
|
+
while (local.length > 0) {
|
|
125
|
+
const page = local.slice(local.length - Math.min(pageSize, local.length));
|
|
126
|
+
local.length -= page.length;
|
|
127
|
+
pageQueue.push(page);
|
|
128
|
+
}
|
|
126
129
|
}
|
|
127
130
|
done = true;
|
|
128
131
|
pageQueue.wake();
|
|
@@ -130,7 +133,7 @@ function createBackwardReader(filepath, options) {
|
|
|
130
133
|
fdSync = (0, node_fs.openSync)(filepath, "r");
|
|
131
134
|
pos = (0, node_fs.statSync)(filepath).size;
|
|
132
135
|
if (pos === 0) {
|
|
133
|
-
pageQueue.push(
|
|
136
|
+
pageQueue.push(emptyPage);
|
|
134
137
|
done = true;
|
|
135
138
|
flushed = true;
|
|
136
139
|
pageQueue.wake();
|
|
@@ -140,7 +143,7 @@ function createBackwardReader(filepath, options) {
|
|
|
140
143
|
pos = (await fd.stat()).size;
|
|
141
144
|
if (pos === 0) {
|
|
142
145
|
if (!done) {
|
|
143
|
-
pageQueue.push(
|
|
146
|
+
pageQueue.push(emptyPage);
|
|
144
147
|
done = true;
|
|
145
148
|
flushed = true;
|
|
146
149
|
}
|
|
@@ -159,9 +162,12 @@ function createBackwardReader(filepath, options) {
|
|
|
159
162
|
pos -= readSize;
|
|
160
163
|
const buf = Buffer.allocUnsafe(readSize);
|
|
161
164
|
const { bytesRead } = await fd.read(buf, 0, readSize, pos);
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
+
if (isBufferOutput) pageQueue.push(buf.subarray(0, bytesRead));
|
|
166
|
+
else {
|
|
167
|
+
buffer = buf.toString("utf8", 0, bytesRead) + buffer;
|
|
168
|
+
if (pos === 0 && buffer.startsWith(delimiter)) startsWithDelimiter = true;
|
|
169
|
+
consumeBuffer();
|
|
170
|
+
}
|
|
165
171
|
}
|
|
166
172
|
if (pos === 0 && !flushed) {
|
|
167
173
|
flushTail();
|
|
@@ -183,9 +189,12 @@ function createBackwardReader(filepath, options) {
|
|
|
183
189
|
pos -= readSize;
|
|
184
190
|
const buf = Buffer.allocUnsafe(readSize);
|
|
185
191
|
const bytesRead = (0, node_fs.readSync)(fdSync, buf, 0, readSize, pos);
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
192
|
+
if (isBufferOutput) pageQueue.push(buf.subarray(0, bytesRead));
|
|
193
|
+
else {
|
|
194
|
+
buffer = buf.toString("utf8", 0, bytesRead) + buffer;
|
|
195
|
+
if (pos === 0 && buffer.startsWith(delimiter)) startsWithDelimiter = true;
|
|
196
|
+
consumeBuffer();
|
|
197
|
+
}
|
|
189
198
|
}
|
|
190
199
|
if (pos === 0 && !flushed) {
|
|
191
200
|
flushTail();
|
|
@@ -269,8 +278,10 @@ function createBackwardReader(filepath, options) {
|
|
|
269
278
|
//#endregion
|
|
270
279
|
//#region src/reader/forward.reader.ts
|
|
271
280
|
function createForwardReader(filepath, options) {
|
|
272
|
-
const { chunkSize, pageSize, delimiter, prefetch } = options;
|
|
281
|
+
const { chunkSize, pageSize, delimiter, prefetch, output } = options;
|
|
273
282
|
const pageQueue = createRingBuffer(Math.max(2, prefetch + 1));
|
|
283
|
+
const isBufferOutput = output === "buffer";
|
|
284
|
+
const emptyPage = isBufferOutput ? Buffer.allocUnsafe(0) : [""];
|
|
274
285
|
const local = [];
|
|
275
286
|
let fd = null;
|
|
276
287
|
let fdSync = null;
|
|
@@ -292,16 +303,18 @@ function createForwardReader(filepath, options) {
|
|
|
292
303
|
function flushTail() {
|
|
293
304
|
if (flushed) return;
|
|
294
305
|
flushed = true;
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
306
|
+
if (!isBufferOutput) {
|
|
307
|
+
local.push(buffer.length > 0 ? buffer : "");
|
|
308
|
+
buffer = "";
|
|
309
|
+
while (local.length > 0) pageQueue.push(local.splice(0, pageSize));
|
|
310
|
+
}
|
|
298
311
|
done = true;
|
|
299
312
|
pageQueue.wake();
|
|
300
313
|
}
|
|
301
314
|
fdSync = (0, node_fs.openSync)(filepath, "r");
|
|
302
315
|
size = (0, node_fs.statSync)(filepath).size;
|
|
303
316
|
if (size === 0) {
|
|
304
|
-
pageQueue.push(
|
|
317
|
+
pageQueue.push(emptyPage);
|
|
305
318
|
done = true;
|
|
306
319
|
flushed = true;
|
|
307
320
|
pageQueue.wake();
|
|
@@ -311,7 +324,7 @@ function createForwardReader(filepath, options) {
|
|
|
311
324
|
size = (await fd.stat()).size;
|
|
312
325
|
if (size === 0) {
|
|
313
326
|
if (!done) {
|
|
314
|
-
pageQueue.push(
|
|
327
|
+
pageQueue.push(emptyPage);
|
|
315
328
|
done = true;
|
|
316
329
|
flushed = true;
|
|
317
330
|
}
|
|
@@ -330,8 +343,11 @@ function createForwardReader(filepath, options) {
|
|
|
330
343
|
const buf = Buffer.allocUnsafe(readSize);
|
|
331
344
|
const { bytesRead } = await fd.read(buf, 0, readSize, pos);
|
|
332
345
|
pos += bytesRead;
|
|
333
|
-
|
|
334
|
-
|
|
346
|
+
if (isBufferOutput) pageQueue.push(buf.subarray(0, bytesRead));
|
|
347
|
+
else {
|
|
348
|
+
buffer = buffer + buf.toString("utf8", 0, bytesRead);
|
|
349
|
+
consumeBuffer();
|
|
350
|
+
}
|
|
335
351
|
}
|
|
336
352
|
if (pos >= size && !flushed) {
|
|
337
353
|
flushTail();
|
|
@@ -353,8 +369,11 @@ function createForwardReader(filepath, options) {
|
|
|
353
369
|
const buf = Buffer.allocUnsafe(readSize);
|
|
354
370
|
const bytesRead = (0, node_fs.readSync)(fdSync, buf, 0, readSize, pos);
|
|
355
371
|
pos += bytesRead;
|
|
356
|
-
|
|
357
|
-
|
|
372
|
+
if (isBufferOutput) pageQueue.push(buf.subarray(0, bytesRead));
|
|
373
|
+
else {
|
|
374
|
+
buffer = buffer + buf.toString("utf8", 0, bytesRead);
|
|
375
|
+
consumeBuffer();
|
|
376
|
+
}
|
|
358
377
|
}
|
|
359
378
|
if (pos >= size && !flushed) {
|
|
360
379
|
flushTail();
|
|
@@ -436,117 +455,20 @@ function createForwardReader(filepath, options) {
|
|
|
436
455
|
};
|
|
437
456
|
}
|
|
438
457
|
//#endregion
|
|
439
|
-
//#region src/reader/worker.reader.ts
|
|
440
|
-
const workerFile = new URL("./worker.mjs", require("url").pathToFileURL(__filename).href);
|
|
441
|
-
function createWorkerReader(filepath, options) {
|
|
442
|
-
const { prefetch } = options;
|
|
443
|
-
let done = false;
|
|
444
|
-
let closed = false;
|
|
445
|
-
const pageQueue = createRingBuffer(Math.max(2, prefetch + 1));
|
|
446
|
-
const worker = new node_worker_threads.Worker(new URL(workerFile, require("url").pathToFileURL(__filename).href), { workerData: {
|
|
447
|
-
filepath,
|
|
448
|
-
options
|
|
449
|
-
} });
|
|
450
|
-
worker.on("message", (msg) => {
|
|
451
|
-
switch (msg.type) {
|
|
452
|
-
case "page":
|
|
453
|
-
pageQueue.push(msg.data);
|
|
454
|
-
break;
|
|
455
|
-
case "done":
|
|
456
|
-
case "error":
|
|
457
|
-
done = true;
|
|
458
|
-
pageQueue.wake();
|
|
459
|
-
break;
|
|
460
|
-
}
|
|
461
|
-
});
|
|
462
|
-
worker.on("error", () => {
|
|
463
|
-
done = true;
|
|
464
|
-
pageQueue.wake();
|
|
465
|
-
});
|
|
466
|
-
worker.on("exit", () => {
|
|
467
|
-
done = true;
|
|
468
|
-
pageQueue.wake();
|
|
469
|
-
});
|
|
470
|
-
async function next() {
|
|
471
|
-
if (closed) return null;
|
|
472
|
-
return pageQueue.shift(done);
|
|
473
|
-
}
|
|
474
|
-
function nextSync() {
|
|
475
|
-
if (closed) return null;
|
|
476
|
-
return pageQueue.shiftSync();
|
|
477
|
-
}
|
|
478
|
-
async function close() {
|
|
479
|
-
if (closed) return;
|
|
480
|
-
closed = true;
|
|
481
|
-
done = true;
|
|
482
|
-
pageQueue.clear();
|
|
483
|
-
try {
|
|
484
|
-
await worker.terminate();
|
|
485
|
-
} catch {}
|
|
486
|
-
}
|
|
487
|
-
function tryClose() {
|
|
488
|
-
close().catch(() => {});
|
|
489
|
-
}
|
|
490
|
-
return {
|
|
491
|
-
next,
|
|
492
|
-
nextSync,
|
|
493
|
-
close,
|
|
494
|
-
async *[Symbol.asyncIterator]() {
|
|
495
|
-
try {
|
|
496
|
-
while (true) {
|
|
497
|
-
const p = await next();
|
|
498
|
-
if (!p) break;
|
|
499
|
-
yield p;
|
|
500
|
-
}
|
|
501
|
-
} finally {
|
|
502
|
-
tryClose();
|
|
503
|
-
}
|
|
504
|
-
},
|
|
505
|
-
*[Symbol.iterator]() {
|
|
506
|
-
try {
|
|
507
|
-
while (true) {
|
|
508
|
-
const p = nextSync();
|
|
509
|
-
if (!p) break;
|
|
510
|
-
yield p;
|
|
511
|
-
}
|
|
512
|
-
} finally {
|
|
513
|
-
tryClose();
|
|
514
|
-
}
|
|
515
|
-
}
|
|
516
|
-
};
|
|
517
|
-
}
|
|
518
|
-
//#endregion
|
|
519
458
|
//#region src/main.ts
|
|
520
459
|
function createPager(filepath, options = {}) {
|
|
521
|
-
const { chunkSize = 64 * 1024, pageSize = 1e3, delimiter = "\n", prefetch = 8, backward = false,
|
|
460
|
+
const { chunkSize = 64 * 1024, pageSize = 1e3, delimiter = "\n", prefetch = 8, backward = false, output = "string" } = options;
|
|
522
461
|
if (!filepath) throw new Error("filepath required");
|
|
523
462
|
if (pageSize < 1) throw new RangeError("pageSize must be >= 1");
|
|
524
463
|
if (prefetch < 1) throw new RangeError("prefetch must be >= 1");
|
|
525
|
-
if (useWorker) {
|
|
526
|
-
if (backward) throw new Error("backward not supported with useWorker");
|
|
527
|
-
if (tryNative) throw new Error("tryNative not supported with useWorker");
|
|
528
|
-
}
|
|
529
|
-
if (tryNative) {
|
|
530
|
-
if (delimiter.length !== 1) throw new RangeError("native reader only supports single-character delimiters");
|
|
531
|
-
}
|
|
532
464
|
const readerOptions = {
|
|
533
465
|
chunkSize,
|
|
534
466
|
pageSize,
|
|
535
467
|
prefetch,
|
|
536
|
-
delimiter
|
|
468
|
+
delimiter,
|
|
469
|
+
output
|
|
537
470
|
};
|
|
538
|
-
|
|
539
|
-
if (tryNative) {
|
|
540
|
-
const nativeOptions = {
|
|
541
|
-
pageSize,
|
|
542
|
-
delimiter,
|
|
543
|
-
backward
|
|
544
|
-
};
|
|
545
|
-
try {
|
|
546
|
-
nativeReader = require_native.createNativePager(filepath, nativeOptions);
|
|
547
|
-
} catch {}
|
|
548
|
-
}
|
|
549
|
-
const reader = tryNative && nativeReader ? nativeReader : useWorker ? createWorkerReader(filepath, readerOptions) : backward ? createBackwardReader(filepath, readerOptions) : createForwardReader(filepath, readerOptions);
|
|
471
|
+
const reader = backward ? createBackwardReader(filepath, readerOptions) : createForwardReader(filepath, readerOptions);
|
|
550
472
|
if (process.env.PAGER_TEST_CLEANUPS) {
|
|
551
473
|
globalThis.__pager_test_cleanups__ ??= [];
|
|
552
474
|
globalThis.__pager_test_cleanups__.push(reader.close);
|
package/dist/main.d.cts
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
|
-
import { a as PagerOptions, i as
|
|
1
|
+
import { a as Output, c as PagerOptions, i as NativeReaderOptions, l as ReaderOptions, n as AsyncFunction, o as PageOutput, r as NativeAddon, s as Pager, t as createNativePager, u as ResolvePageOutput } from "./native-7x1Vhdaw.cjs";
|
|
2
2
|
|
|
3
3
|
//#region src/main.d.ts
|
|
4
|
-
declare function createPager(filepath: string, options
|
|
4
|
+
declare function createPager<T extends Output>(filepath: string, options: PagerOptions & {
|
|
5
|
+
output: T;
|
|
6
|
+
}): Pager<T>;
|
|
7
|
+
declare function createPager(filepath: string, options?: PagerOptions): Pager<"string">;
|
|
5
8
|
//#endregion
|
|
6
|
-
export { NativeAddon, NativeReaderOptions, Pager, PagerOptions, ReaderOptions,
|
|
9
|
+
export { AsyncFunction, NativeAddon, NativeReaderOptions, Output, PageOutput, Pager, PagerOptions, ReaderOptions, ResolvePageOutput, createNativePager, createPager, createPager as default };
|
package/dist/main.d.mts
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
|
-
import { a as PagerOptions, i as
|
|
1
|
+
import { a as Output, c as PagerOptions, i as NativeReaderOptions, l as ReaderOptions, n as AsyncFunction, o as PageOutput, r as NativeAddon, s as Pager, t as createNativePager, u as ResolvePageOutput } from "./native-D6SBIgoE.mjs";
|
|
2
2
|
|
|
3
3
|
//#region src/main.d.ts
|
|
4
|
-
declare function createPager(filepath: string, options
|
|
4
|
+
declare function createPager<T extends Output>(filepath: string, options: PagerOptions & {
|
|
5
|
+
output: T;
|
|
6
|
+
}): Pager<T>;
|
|
7
|
+
declare function createPager(filepath: string, options?: PagerOptions): Pager<"string">;
|
|
5
8
|
//#endregion
|
|
6
|
-
export { NativeAddon, NativeReaderOptions, Pager, PagerOptions, ReaderOptions,
|
|
9
|
+
export { AsyncFunction, NativeAddon, NativeReaderOptions, Output, PageOutput, Pager, PagerOptions, ReaderOptions, ResolvePageOutput, createNativePager, createPager, createPager as default };
|
package/dist/main.mjs
CHANGED
|
@@ -1,11 +1,6 @@
|
|
|
1
1
|
import { createNativePager } from "./native.mjs";
|
|
2
|
-
import { createRequire } from "node:module";
|
|
3
2
|
import { closeSync, openSync, readSync, statSync } from "node:fs";
|
|
4
3
|
import { open } from "node:fs/promises";
|
|
5
|
-
import { Worker } from "node:worker_threads";
|
|
6
|
-
//#region \0rolldown/runtime.js
|
|
7
|
-
var __require = /* @__PURE__ */ createRequire(import.meta.url);
|
|
8
|
-
//#endregion
|
|
9
4
|
//#region src/helper.ts
|
|
10
5
|
function createRingBuffer(capacity) {
|
|
11
6
|
if (!Number.isFinite(capacity) || capacity <= 0) throw new RangeError("capacity must be a positive number");
|
|
@@ -93,8 +88,10 @@ function createRingBuffer(capacity) {
|
|
|
93
88
|
//#endregion
|
|
94
89
|
//#region src/reader/backward.reader.ts
|
|
95
90
|
function createBackwardReader(filepath, options) {
|
|
96
|
-
const { chunkSize, pageSize, delimiter, prefetch } = options;
|
|
91
|
+
const { chunkSize, pageSize, delimiter, prefetch, output } = options;
|
|
97
92
|
const pageQueue = createRingBuffer(Math.max(2, prefetch + 1));
|
|
93
|
+
const isBufferOutput = output === "buffer";
|
|
94
|
+
const emptyPage = isBufferOutput ? Buffer.allocUnsafe(0) : [""];
|
|
98
95
|
const local = [];
|
|
99
96
|
let fd = null;
|
|
100
97
|
let fdSync = null;
|
|
@@ -116,13 +113,15 @@ function createBackwardReader(filepath, options) {
|
|
|
116
113
|
function flushTail() {
|
|
117
114
|
if (flushed) return;
|
|
118
115
|
flushed = true;
|
|
119
|
-
if (
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
116
|
+
if (!isBufferOutput) {
|
|
117
|
+
if (buffer.length > 0) local.push(buffer);
|
|
118
|
+
else if (startsWithDelimiter) local.push("");
|
|
119
|
+
buffer = "";
|
|
120
|
+
while (local.length > 0) {
|
|
121
|
+
const page = local.slice(local.length - Math.min(pageSize, local.length));
|
|
122
|
+
local.length -= page.length;
|
|
123
|
+
pageQueue.push(page);
|
|
124
|
+
}
|
|
126
125
|
}
|
|
127
126
|
done = true;
|
|
128
127
|
pageQueue.wake();
|
|
@@ -130,7 +129,7 @@ function createBackwardReader(filepath, options) {
|
|
|
130
129
|
fdSync = openSync(filepath, "r");
|
|
131
130
|
pos = statSync(filepath).size;
|
|
132
131
|
if (pos === 0) {
|
|
133
|
-
pageQueue.push(
|
|
132
|
+
pageQueue.push(emptyPage);
|
|
134
133
|
done = true;
|
|
135
134
|
flushed = true;
|
|
136
135
|
pageQueue.wake();
|
|
@@ -140,7 +139,7 @@ function createBackwardReader(filepath, options) {
|
|
|
140
139
|
pos = (await fd.stat()).size;
|
|
141
140
|
if (pos === 0) {
|
|
142
141
|
if (!done) {
|
|
143
|
-
pageQueue.push(
|
|
142
|
+
pageQueue.push(emptyPage);
|
|
144
143
|
done = true;
|
|
145
144
|
flushed = true;
|
|
146
145
|
}
|
|
@@ -159,9 +158,12 @@ function createBackwardReader(filepath, options) {
|
|
|
159
158
|
pos -= readSize;
|
|
160
159
|
const buf = Buffer.allocUnsafe(readSize);
|
|
161
160
|
const { bytesRead } = await fd.read(buf, 0, readSize, pos);
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
161
|
+
if (isBufferOutput) pageQueue.push(buf.subarray(0, bytesRead));
|
|
162
|
+
else {
|
|
163
|
+
buffer = buf.toString("utf8", 0, bytesRead) + buffer;
|
|
164
|
+
if (pos === 0 && buffer.startsWith(delimiter)) startsWithDelimiter = true;
|
|
165
|
+
consumeBuffer();
|
|
166
|
+
}
|
|
165
167
|
}
|
|
166
168
|
if (pos === 0 && !flushed) {
|
|
167
169
|
flushTail();
|
|
@@ -183,9 +185,12 @@ function createBackwardReader(filepath, options) {
|
|
|
183
185
|
pos -= readSize;
|
|
184
186
|
const buf = Buffer.allocUnsafe(readSize);
|
|
185
187
|
const bytesRead = readSync(fdSync, buf, 0, readSize, pos);
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
188
|
+
if (isBufferOutput) pageQueue.push(buf.subarray(0, bytesRead));
|
|
189
|
+
else {
|
|
190
|
+
buffer = buf.toString("utf8", 0, bytesRead) + buffer;
|
|
191
|
+
if (pos === 0 && buffer.startsWith(delimiter)) startsWithDelimiter = true;
|
|
192
|
+
consumeBuffer();
|
|
193
|
+
}
|
|
189
194
|
}
|
|
190
195
|
if (pos === 0 && !flushed) {
|
|
191
196
|
flushTail();
|
|
@@ -269,8 +274,10 @@ function createBackwardReader(filepath, options) {
|
|
|
269
274
|
//#endregion
|
|
270
275
|
//#region src/reader/forward.reader.ts
|
|
271
276
|
function createForwardReader(filepath, options) {
|
|
272
|
-
const { chunkSize, pageSize, delimiter, prefetch } = options;
|
|
277
|
+
const { chunkSize, pageSize, delimiter, prefetch, output } = options;
|
|
273
278
|
const pageQueue = createRingBuffer(Math.max(2, prefetch + 1));
|
|
279
|
+
const isBufferOutput = output === "buffer";
|
|
280
|
+
const emptyPage = isBufferOutput ? Buffer.allocUnsafe(0) : [""];
|
|
274
281
|
const local = [];
|
|
275
282
|
let fd = null;
|
|
276
283
|
let fdSync = null;
|
|
@@ -292,16 +299,18 @@ function createForwardReader(filepath, options) {
|
|
|
292
299
|
function flushTail() {
|
|
293
300
|
if (flushed) return;
|
|
294
301
|
flushed = true;
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
302
|
+
if (!isBufferOutput) {
|
|
303
|
+
local.push(buffer.length > 0 ? buffer : "");
|
|
304
|
+
buffer = "";
|
|
305
|
+
while (local.length > 0) pageQueue.push(local.splice(0, pageSize));
|
|
306
|
+
}
|
|
298
307
|
done = true;
|
|
299
308
|
pageQueue.wake();
|
|
300
309
|
}
|
|
301
310
|
fdSync = openSync(filepath, "r");
|
|
302
311
|
size = statSync(filepath).size;
|
|
303
312
|
if (size === 0) {
|
|
304
|
-
pageQueue.push(
|
|
313
|
+
pageQueue.push(emptyPage);
|
|
305
314
|
done = true;
|
|
306
315
|
flushed = true;
|
|
307
316
|
pageQueue.wake();
|
|
@@ -311,7 +320,7 @@ function createForwardReader(filepath, options) {
|
|
|
311
320
|
size = (await fd.stat()).size;
|
|
312
321
|
if (size === 0) {
|
|
313
322
|
if (!done) {
|
|
314
|
-
pageQueue.push(
|
|
323
|
+
pageQueue.push(emptyPage);
|
|
315
324
|
done = true;
|
|
316
325
|
flushed = true;
|
|
317
326
|
}
|
|
@@ -330,8 +339,11 @@ function createForwardReader(filepath, options) {
|
|
|
330
339
|
const buf = Buffer.allocUnsafe(readSize);
|
|
331
340
|
const { bytesRead } = await fd.read(buf, 0, readSize, pos);
|
|
332
341
|
pos += bytesRead;
|
|
333
|
-
|
|
334
|
-
|
|
342
|
+
if (isBufferOutput) pageQueue.push(buf.subarray(0, bytesRead));
|
|
343
|
+
else {
|
|
344
|
+
buffer = buffer + buf.toString("utf8", 0, bytesRead);
|
|
345
|
+
consumeBuffer();
|
|
346
|
+
}
|
|
335
347
|
}
|
|
336
348
|
if (pos >= size && !flushed) {
|
|
337
349
|
flushTail();
|
|
@@ -353,8 +365,11 @@ function createForwardReader(filepath, options) {
|
|
|
353
365
|
const buf = Buffer.allocUnsafe(readSize);
|
|
354
366
|
const bytesRead = readSync(fdSync, buf, 0, readSize, pos);
|
|
355
367
|
pos += bytesRead;
|
|
356
|
-
|
|
357
|
-
|
|
368
|
+
if (isBufferOutput) pageQueue.push(buf.subarray(0, bytesRead));
|
|
369
|
+
else {
|
|
370
|
+
buffer = buffer + buf.toString("utf8", 0, bytesRead);
|
|
371
|
+
consumeBuffer();
|
|
372
|
+
}
|
|
358
373
|
}
|
|
359
374
|
if (pos >= size && !flushed) {
|
|
360
375
|
flushTail();
|
|
@@ -436,117 +451,20 @@ function createForwardReader(filepath, options) {
|
|
|
436
451
|
};
|
|
437
452
|
}
|
|
438
453
|
//#endregion
|
|
439
|
-
//#region src/reader/worker.reader.ts
|
|
440
|
-
const workerFile = typeof import.meta !== "undefined" ? new URL("./worker.mjs", import.meta.url) : __require.resolve("./worker.cjs");
|
|
441
|
-
function createWorkerReader(filepath, options) {
|
|
442
|
-
const { prefetch } = options;
|
|
443
|
-
let done = false;
|
|
444
|
-
let closed = false;
|
|
445
|
-
const pageQueue = createRingBuffer(Math.max(2, prefetch + 1));
|
|
446
|
-
const worker = new Worker(new URL(workerFile, import.meta.url), { workerData: {
|
|
447
|
-
filepath,
|
|
448
|
-
options
|
|
449
|
-
} });
|
|
450
|
-
worker.on("message", (msg) => {
|
|
451
|
-
switch (msg.type) {
|
|
452
|
-
case "page":
|
|
453
|
-
pageQueue.push(msg.data);
|
|
454
|
-
break;
|
|
455
|
-
case "done":
|
|
456
|
-
case "error":
|
|
457
|
-
done = true;
|
|
458
|
-
pageQueue.wake();
|
|
459
|
-
break;
|
|
460
|
-
}
|
|
461
|
-
});
|
|
462
|
-
worker.on("error", () => {
|
|
463
|
-
done = true;
|
|
464
|
-
pageQueue.wake();
|
|
465
|
-
});
|
|
466
|
-
worker.on("exit", () => {
|
|
467
|
-
done = true;
|
|
468
|
-
pageQueue.wake();
|
|
469
|
-
});
|
|
470
|
-
async function next() {
|
|
471
|
-
if (closed) return null;
|
|
472
|
-
return pageQueue.shift(done);
|
|
473
|
-
}
|
|
474
|
-
function nextSync() {
|
|
475
|
-
if (closed) return null;
|
|
476
|
-
return pageQueue.shiftSync();
|
|
477
|
-
}
|
|
478
|
-
async function close() {
|
|
479
|
-
if (closed) return;
|
|
480
|
-
closed = true;
|
|
481
|
-
done = true;
|
|
482
|
-
pageQueue.clear();
|
|
483
|
-
try {
|
|
484
|
-
await worker.terminate();
|
|
485
|
-
} catch {}
|
|
486
|
-
}
|
|
487
|
-
function tryClose() {
|
|
488
|
-
close().catch(() => {});
|
|
489
|
-
}
|
|
490
|
-
return {
|
|
491
|
-
next,
|
|
492
|
-
nextSync,
|
|
493
|
-
close,
|
|
494
|
-
async *[Symbol.asyncIterator]() {
|
|
495
|
-
try {
|
|
496
|
-
while (true) {
|
|
497
|
-
const p = await next();
|
|
498
|
-
if (!p) break;
|
|
499
|
-
yield p;
|
|
500
|
-
}
|
|
501
|
-
} finally {
|
|
502
|
-
tryClose();
|
|
503
|
-
}
|
|
504
|
-
},
|
|
505
|
-
*[Symbol.iterator]() {
|
|
506
|
-
try {
|
|
507
|
-
while (true) {
|
|
508
|
-
const p = nextSync();
|
|
509
|
-
if (!p) break;
|
|
510
|
-
yield p;
|
|
511
|
-
}
|
|
512
|
-
} finally {
|
|
513
|
-
tryClose();
|
|
514
|
-
}
|
|
515
|
-
}
|
|
516
|
-
};
|
|
517
|
-
}
|
|
518
|
-
//#endregion
|
|
519
454
|
//#region src/main.ts
|
|
520
455
|
function createPager(filepath, options = {}) {
|
|
521
|
-
const { chunkSize = 64 * 1024, pageSize = 1e3, delimiter = "\n", prefetch = 8, backward = false,
|
|
456
|
+
const { chunkSize = 64 * 1024, pageSize = 1e3, delimiter = "\n", prefetch = 8, backward = false, output = "string" } = options;
|
|
522
457
|
if (!filepath) throw new Error("filepath required");
|
|
523
458
|
if (pageSize < 1) throw new RangeError("pageSize must be >= 1");
|
|
524
459
|
if (prefetch < 1) throw new RangeError("prefetch must be >= 1");
|
|
525
|
-
if (useWorker) {
|
|
526
|
-
if (backward) throw new Error("backward not supported with useWorker");
|
|
527
|
-
if (tryNative) throw new Error("tryNative not supported with useWorker");
|
|
528
|
-
}
|
|
529
|
-
if (tryNative) {
|
|
530
|
-
if (delimiter.length !== 1) throw new RangeError("native reader only supports single-character delimiters");
|
|
531
|
-
}
|
|
532
460
|
const readerOptions = {
|
|
533
461
|
chunkSize,
|
|
534
462
|
pageSize,
|
|
535
463
|
prefetch,
|
|
536
|
-
delimiter
|
|
464
|
+
delimiter,
|
|
465
|
+
output
|
|
537
466
|
};
|
|
538
|
-
|
|
539
|
-
if (tryNative) {
|
|
540
|
-
const nativeOptions = {
|
|
541
|
-
pageSize,
|
|
542
|
-
delimiter,
|
|
543
|
-
backward
|
|
544
|
-
};
|
|
545
|
-
try {
|
|
546
|
-
nativeReader = createNativePager(filepath, nativeOptions);
|
|
547
|
-
} catch {}
|
|
548
|
-
}
|
|
549
|
-
const reader = tryNative && nativeReader ? nativeReader : useWorker ? createWorkerReader(filepath, readerOptions) : backward ? createBackwardReader(filepath, readerOptions) : createForwardReader(filepath, readerOptions);
|
|
467
|
+
const reader = backward ? createBackwardReader(filepath, readerOptions) : createForwardReader(filepath, readerOptions);
|
|
550
468
|
if (process.env.PAGER_TEST_CLEANUPS) {
|
|
551
469
|
globalThis.__pager_test_cleanups__ ??= [];
|
|
552
470
|
globalThis.__pager_test_cleanups__.push(reader.close);
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
//#region src/types.d.ts
|
|
2
|
+
type AsyncFunction = () => Promise<void>;
|
|
3
|
+
type Output = "string" | "buffer";
|
|
4
|
+
type PageOutput = string[] | Buffer;
|
|
5
|
+
type ResolvePageOutput<T extends Output> = T extends "buffer" ? Buffer : string[];
|
|
6
|
+
interface ReaderOptions {
|
|
7
|
+
chunkSize: number;
|
|
8
|
+
pageSize: number;
|
|
9
|
+
delimiter: string;
|
|
10
|
+
prefetch: number;
|
|
11
|
+
output: Output;
|
|
12
|
+
}
|
|
13
|
+
interface NativeReaderOptions {
|
|
14
|
+
pageSize: number;
|
|
15
|
+
delimiter: string;
|
|
16
|
+
backward: boolean;
|
|
17
|
+
output: Output;
|
|
18
|
+
}
|
|
19
|
+
type PagerOptions = Partial<ReaderOptions> & {
|
|
20
|
+
backward?: boolean;
|
|
21
|
+
};
|
|
22
|
+
type AddonFD = object | null;
|
|
23
|
+
type AddonData = Buffer | null;
|
|
24
|
+
interface NativeAddon {
|
|
25
|
+
open: (filepath: string, pageSize: number, delimiter: string, backward: boolean) => AddonFD;
|
|
26
|
+
next: (fd: AddonFD) => Promise<AddonData>;
|
|
27
|
+
nextSync: (fd: AddonFD) => AddonData;
|
|
28
|
+
close: (fd: AddonFD) => Promise<void>;
|
|
29
|
+
}
|
|
30
|
+
interface Pager<T extends Output = "string"> {
|
|
31
|
+
next(): Promise<ResolvePageOutput<T> | null>;
|
|
32
|
+
nextSync(): ResolvePageOutput<T> | null;
|
|
33
|
+
close(): Promise<void>;
|
|
34
|
+
[Symbol.asyncIterator](): AsyncIterator<ResolvePageOutput<T>>;
|
|
35
|
+
[Symbol.iterator](): Iterator<ResolvePageOutput<T>>;
|
|
36
|
+
}
|
|
37
|
+
//#endregion
|
|
38
|
+
//#region src/native.d.ts
|
|
39
|
+
declare function createNativePager<T extends Output>(filepath: string, options: Partial<NativeReaderOptions> & {
|
|
40
|
+
output: T;
|
|
41
|
+
}): Pager<T>;
|
|
42
|
+
declare function createNativePager(filepath: string, options?: Partial<NativeReaderOptions>): Pager<"string">;
|
|
43
|
+
//#endregion
|
|
44
|
+
export { Output as a, PagerOptions as c, NativeReaderOptions as i, ReaderOptions as l, AsyncFunction as n, PageOutput as o, NativeAddon as r, Pager as s, createNativePager as t, ResolvePageOutput as u };
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
//#region src/types.d.ts
|
|
2
|
+
type AsyncFunction = () => Promise<void>;
|
|
3
|
+
type Output = "string" | "buffer";
|
|
4
|
+
type PageOutput = string[] | Buffer;
|
|
5
|
+
type ResolvePageOutput<T extends Output> = T extends "buffer" ? Buffer : string[];
|
|
6
|
+
interface ReaderOptions {
|
|
7
|
+
chunkSize: number;
|
|
8
|
+
pageSize: number;
|
|
9
|
+
delimiter: string;
|
|
10
|
+
prefetch: number;
|
|
11
|
+
output: Output;
|
|
12
|
+
}
|
|
13
|
+
interface NativeReaderOptions {
|
|
14
|
+
pageSize: number;
|
|
15
|
+
delimiter: string;
|
|
16
|
+
backward: boolean;
|
|
17
|
+
output: Output;
|
|
18
|
+
}
|
|
19
|
+
type PagerOptions = Partial<ReaderOptions> & {
|
|
20
|
+
backward?: boolean;
|
|
21
|
+
};
|
|
22
|
+
type AddonFD = object | null;
|
|
23
|
+
type AddonData = Buffer | null;
|
|
24
|
+
interface NativeAddon {
|
|
25
|
+
open: (filepath: string, pageSize: number, delimiter: string, backward: boolean) => AddonFD;
|
|
26
|
+
next: (fd: AddonFD) => Promise<AddonData>;
|
|
27
|
+
nextSync: (fd: AddonFD) => AddonData;
|
|
28
|
+
close: (fd: AddonFD) => Promise<void>;
|
|
29
|
+
}
|
|
30
|
+
interface Pager<T extends Output = "string"> {
|
|
31
|
+
next(): Promise<ResolvePageOutput<T> | null>;
|
|
32
|
+
nextSync(): ResolvePageOutput<T> | null;
|
|
33
|
+
close(): Promise<void>;
|
|
34
|
+
[Symbol.asyncIterator](): AsyncIterator<ResolvePageOutput<T>>;
|
|
35
|
+
[Symbol.iterator](): Iterator<ResolvePageOutput<T>>;
|
|
36
|
+
}
|
|
37
|
+
//#endregion
|
|
38
|
+
//#region src/native.d.ts
|
|
39
|
+
declare function createNativePager<T extends Output>(filepath: string, options: Partial<NativeReaderOptions> & {
|
|
40
|
+
output: T;
|
|
41
|
+
}): Pager<T>;
|
|
42
|
+
declare function createNativePager(filepath: string, options?: Partial<NativeReaderOptions>): Pager<"string">;
|
|
43
|
+
//#endregion
|
|
44
|
+
export { Output as a, PagerOptions as c, NativeReaderOptions as i, ReaderOptions as l, AsyncFunction as n, PageOutput as o, NativeAddon as r, Pager as s, createNativePager as t, ResolvePageOutput as u };
|
package/dist/native.cjs
CHANGED
|
@@ -28,11 +28,12 @@ function loadNativeAddon() {
|
|
|
28
28
|
}
|
|
29
29
|
}
|
|
30
30
|
function createNativePager(filepath, options = {}) {
|
|
31
|
-
const { pageSize = 1e3, delimiter = "\n", backward = false } = options;
|
|
31
|
+
const { pageSize = 1e3, delimiter = "\n", backward = false, output = "string" } = options;
|
|
32
32
|
if (!filepath) throw new Error("filepath required");
|
|
33
33
|
if (pageSize < 1) throw new RangeError("pageSize must be >= 1");
|
|
34
34
|
if (delimiter?.length > 1) throw new RangeError("native reader only supports single-character delimiters");
|
|
35
35
|
const nativeReader = loadNativeAddon();
|
|
36
|
+
const isBufferOutput = output === "buffer";
|
|
36
37
|
if (process.env.PAGER_TEST_CLEANUPS) {
|
|
37
38
|
globalThis.__pager_test_cleanups__ ??= [];
|
|
38
39
|
globalThis.__pager_test_cleanups__.push(nativeReader.close);
|
|
@@ -41,15 +42,15 @@ function createNativePager(filepath, options = {}) {
|
|
|
41
42
|
let closed = false;
|
|
42
43
|
async function next() {
|
|
43
44
|
if (closed || !fd) return null;
|
|
44
|
-
const
|
|
45
|
-
if (!
|
|
46
|
-
return
|
|
45
|
+
const buf = await nativeReader.next(fd);
|
|
46
|
+
if (!buf) return null;
|
|
47
|
+
return isBufferOutput ? buf : buf.toString("utf8").split(delimiter);
|
|
47
48
|
}
|
|
48
49
|
function nextSync() {
|
|
49
50
|
if (closed || !fd) return null;
|
|
50
|
-
const
|
|
51
|
-
if (!
|
|
52
|
-
return
|
|
51
|
+
const buf = nativeReader.nextSync(fd);
|
|
52
|
+
if (!buf) return null;
|
|
53
|
+
return isBufferOutput ? buf : buf.toString("utf8").split(delimiter);
|
|
53
54
|
}
|
|
54
55
|
async function close() {
|
|
55
56
|
if (closed || !fd) return;
|
package/dist/native.d.cts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { t as createNativePager } from "./native-
|
|
1
|
+
import { t as createNativePager } from "./native-7x1Vhdaw.cjs";
|
|
2
2
|
export { createNativePager };
|
package/dist/native.d.mts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { t as createNativePager } from "./native-
|
|
1
|
+
import { t as createNativePager } from "./native-D6SBIgoE.mjs";
|
|
2
2
|
export { createNativePager };
|
package/dist/native.mjs
CHANGED
|
@@ -27,11 +27,12 @@ function loadNativeAddon() {
|
|
|
27
27
|
}
|
|
28
28
|
}
|
|
29
29
|
function createNativePager(filepath, options = {}) {
|
|
30
|
-
const { pageSize = 1e3, delimiter = "\n", backward = false } = options;
|
|
30
|
+
const { pageSize = 1e3, delimiter = "\n", backward = false, output = "string" } = options;
|
|
31
31
|
if (!filepath) throw new Error("filepath required");
|
|
32
32
|
if (pageSize < 1) throw new RangeError("pageSize must be >= 1");
|
|
33
33
|
if (delimiter?.length > 1) throw new RangeError("native reader only supports single-character delimiters");
|
|
34
34
|
const nativeReader = loadNativeAddon();
|
|
35
|
+
const isBufferOutput = output === "buffer";
|
|
35
36
|
if (process.env.PAGER_TEST_CLEANUPS) {
|
|
36
37
|
globalThis.__pager_test_cleanups__ ??= [];
|
|
37
38
|
globalThis.__pager_test_cleanups__.push(nativeReader.close);
|
|
@@ -40,15 +41,15 @@ function createNativePager(filepath, options = {}) {
|
|
|
40
41
|
let closed = false;
|
|
41
42
|
async function next() {
|
|
42
43
|
if (closed || !fd) return null;
|
|
43
|
-
const
|
|
44
|
-
if (!
|
|
45
|
-
return
|
|
44
|
+
const buf = await nativeReader.next(fd);
|
|
45
|
+
if (!buf) return null;
|
|
46
|
+
return isBufferOutput ? buf : buf.toString("utf8").split(delimiter);
|
|
46
47
|
}
|
|
47
48
|
function nextSync() {
|
|
48
49
|
if (closed || !fd) return null;
|
|
49
|
-
const
|
|
50
|
-
if (!
|
|
51
|
-
return
|
|
50
|
+
const buf = nativeReader.nextSync(fd);
|
|
51
|
+
if (!buf) return null;
|
|
52
|
+
return isBufferOutput ? buf : buf.toString("utf8").split(delimiter);
|
|
52
53
|
}
|
|
53
54
|
async function close() {
|
|
54
55
|
if (closed || !fd) return;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "readline-pager",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.2",
|
|
4
4
|
"scripts": {
|
|
5
5
|
"build:js": "tsdown",
|
|
6
6
|
"build:native": "node-gyp rebuild",
|
|
@@ -13,20 +13,20 @@
|
|
|
13
13
|
"benchmark:bun": "bun test/benchmark.ts"
|
|
14
14
|
},
|
|
15
15
|
"devDependencies": {
|
|
16
|
-
"@types/bun": "~1.3.
|
|
16
|
+
"@types/bun": "~1.3.12",
|
|
17
17
|
"@types/deno": "~2.5.0",
|
|
18
|
-
"@types/node": "~25.
|
|
18
|
+
"@types/node": "~25.6.0",
|
|
19
19
|
"node-gyp": "~12.2.0",
|
|
20
|
-
"prettier": "~3.8.
|
|
20
|
+
"prettier": "~3.8.2",
|
|
21
21
|
"prettier-plugin-organize-imports": "~4.3.0",
|
|
22
|
-
"tsdown": "~0.21.
|
|
22
|
+
"tsdown": "~0.21.7",
|
|
23
23
|
"typescript": "~6.0.2"
|
|
24
24
|
},
|
|
25
25
|
"optionalDependencies": {
|
|
26
|
-
"@devmor-j/readline-pager-linux-arm64": "0.
|
|
27
|
-
"@devmor-j/readline-pager-linux-musl-arm64": "0.
|
|
28
|
-
"@devmor-j/readline-pager-linux-musl-x64": "0.
|
|
29
|
-
"@devmor-j/readline-pager-linux-x64": "0.
|
|
26
|
+
"@devmor-j/readline-pager-linux-arm64": "0.7.2",
|
|
27
|
+
"@devmor-j/readline-pager-linux-musl-arm64": "0.7.2",
|
|
28
|
+
"@devmor-j/readline-pager-linux-musl-x64": "0.7.2",
|
|
29
|
+
"@devmor-j/readline-pager-linux-x64": "0.7.2"
|
|
30
30
|
},
|
|
31
31
|
"gypfile": false,
|
|
32
32
|
"type": "module",
|
|
@@ -60,7 +60,7 @@
|
|
|
60
60
|
"avx2",
|
|
61
61
|
"neon"
|
|
62
62
|
],
|
|
63
|
-
"description": "High-performance paginated file reader for Node.js.
|
|
63
|
+
"description": "High-performance paginated file reader for Node.js. Process large text files efficiently without loading them into memory.",
|
|
64
64
|
"repository": {
|
|
65
65
|
"type": "git",
|
|
66
66
|
"url": "git+https://github.com/devmor-j/readline-pager.git"
|
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
//#region src/types.d.ts
|
|
2
|
-
interface ReaderOptions {
|
|
3
|
-
chunkSize: number;
|
|
4
|
-
pageSize: number;
|
|
5
|
-
delimiter: string;
|
|
6
|
-
prefetch: number;
|
|
7
|
-
}
|
|
8
|
-
interface PagerOptions extends Partial<ReaderOptions> {
|
|
9
|
-
backward?: boolean;
|
|
10
|
-
useWorker?: boolean;
|
|
11
|
-
tryNative?: boolean;
|
|
12
|
-
}
|
|
13
|
-
interface NativeReaderOptions {
|
|
14
|
-
pageSize: number;
|
|
15
|
-
delimiter: string;
|
|
16
|
-
backward: boolean;
|
|
17
|
-
}
|
|
18
|
-
type WorkerMessage = {
|
|
19
|
-
type: "page";
|
|
20
|
-
data: string[];
|
|
21
|
-
} | {
|
|
22
|
-
type: "error";
|
|
23
|
-
error: unknown;
|
|
24
|
-
} | {
|
|
25
|
-
type: "done";
|
|
26
|
-
};
|
|
27
|
-
type AddonFD = object | null;
|
|
28
|
-
type AddonData = Buffer | null;
|
|
29
|
-
interface NativeAddon {
|
|
30
|
-
open: (filepath: string, pageSize: number, delimiter: string, backward: boolean) => AddonFD;
|
|
31
|
-
next: (fd: AddonFD) => Promise<AddonData>;
|
|
32
|
-
nextSync: (fd: AddonFD) => AddonData;
|
|
33
|
-
close: (fd: AddonFD) => Promise<void>;
|
|
34
|
-
}
|
|
35
|
-
interface Pager {
|
|
36
|
-
next(): Promise<string[] | null>;
|
|
37
|
-
nextSync(): string[] | null;
|
|
38
|
-
close(): Promise<void>;
|
|
39
|
-
[Symbol.asyncIterator](): AsyncIterator<string[]>;
|
|
40
|
-
[Symbol.iterator](): Iterator<string[]>;
|
|
41
|
-
}
|
|
42
|
-
//#endregion
|
|
43
|
-
//#region src/native.d.ts
|
|
44
|
-
declare function createNativePager(filepath: string, options?: Partial<NativeReaderOptions>): Pager;
|
|
45
|
-
//#endregion
|
|
46
|
-
export { PagerOptions as a, Pager as i, NativeAddon as n, ReaderOptions as o, NativeReaderOptions as r, WorkerMessage as s, createNativePager as t };
|
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
//#region src/types.d.ts
|
|
2
|
-
interface ReaderOptions {
|
|
3
|
-
chunkSize: number;
|
|
4
|
-
pageSize: number;
|
|
5
|
-
delimiter: string;
|
|
6
|
-
prefetch: number;
|
|
7
|
-
}
|
|
8
|
-
interface PagerOptions extends Partial<ReaderOptions> {
|
|
9
|
-
backward?: boolean;
|
|
10
|
-
useWorker?: boolean;
|
|
11
|
-
tryNative?: boolean;
|
|
12
|
-
}
|
|
13
|
-
interface NativeReaderOptions {
|
|
14
|
-
pageSize: number;
|
|
15
|
-
delimiter: string;
|
|
16
|
-
backward: boolean;
|
|
17
|
-
}
|
|
18
|
-
type WorkerMessage = {
|
|
19
|
-
type: "page";
|
|
20
|
-
data: string[];
|
|
21
|
-
} | {
|
|
22
|
-
type: "error";
|
|
23
|
-
error: unknown;
|
|
24
|
-
} | {
|
|
25
|
-
type: "done";
|
|
26
|
-
};
|
|
27
|
-
type AddonFD = object | null;
|
|
28
|
-
type AddonData = Buffer | null;
|
|
29
|
-
interface NativeAddon {
|
|
30
|
-
open: (filepath: string, pageSize: number, delimiter: string, backward: boolean) => AddonFD;
|
|
31
|
-
next: (fd: AddonFD) => Promise<AddonData>;
|
|
32
|
-
nextSync: (fd: AddonFD) => AddonData;
|
|
33
|
-
close: (fd: AddonFD) => Promise<void>;
|
|
34
|
-
}
|
|
35
|
-
interface Pager {
|
|
36
|
-
next(): Promise<string[] | null>;
|
|
37
|
-
nextSync(): string[] | null;
|
|
38
|
-
close(): Promise<void>;
|
|
39
|
-
[Symbol.asyncIterator](): AsyncIterator<string[]>;
|
|
40
|
-
[Symbol.iterator](): Iterator<string[]>;
|
|
41
|
-
}
|
|
42
|
-
//#endregion
|
|
43
|
-
//#region src/native.d.ts
|
|
44
|
-
declare function createNativePager(filepath: string, options?: Partial<NativeReaderOptions>): Pager;
|
|
45
|
-
//#endregion
|
|
46
|
-
export { PagerOptions as a, Pager as i, NativeAddon as n, ReaderOptions as o, NativeReaderOptions as r, WorkerMessage as s, createNativePager as t };
|
package/dist/worker.cjs
DELETED
|
@@ -1,52 +0,0 @@
|
|
|
1
|
-
let node_fs_promises = require("node:fs/promises");
|
|
2
|
-
let node_worker_threads = require("node:worker_threads");
|
|
3
|
-
//#region src/worker.ts
|
|
4
|
-
const { filepath, options } = node_worker_threads.workerData;
|
|
5
|
-
const { chunkSize, pageSize, delimiter } = options;
|
|
6
|
-
const backpressure = pageSize * 8;
|
|
7
|
-
function post(msg) {
|
|
8
|
-
if (!node_worker_threads.parentPort) process.exit(1);
|
|
9
|
-
node_worker_threads.parentPort.postMessage(msg);
|
|
10
|
-
}
|
|
11
|
-
(async () => {
|
|
12
|
-
try {
|
|
13
|
-
const fd = await (0, node_fs_promises.open)(filepath, "r");
|
|
14
|
-
const { size } = await fd.stat();
|
|
15
|
-
let pos = 0;
|
|
16
|
-
let buffer = "";
|
|
17
|
-
const local = [];
|
|
18
|
-
while (pos < size) {
|
|
19
|
-
const readSize = Math.min(chunkSize, size - pos);
|
|
20
|
-
const buf = Buffer.allocUnsafe(readSize);
|
|
21
|
-
const { bytesRead } = await fd.read(buf, 0, readSize, pos);
|
|
22
|
-
pos += bytesRead;
|
|
23
|
-
buffer = buffer + buf.toString("utf8", 0, bytesRead);
|
|
24
|
-
let idx;
|
|
25
|
-
while ((idx = buffer.indexOf(delimiter)) !== -1) {
|
|
26
|
-
const line = buffer.slice(0, idx);
|
|
27
|
-
buffer = buffer.slice(idx + delimiter.length);
|
|
28
|
-
local.push(line);
|
|
29
|
-
if (local.length >= pageSize) {
|
|
30
|
-
post({
|
|
31
|
-
type: "page",
|
|
32
|
-
data: local.splice(0, pageSize)
|
|
33
|
-
});
|
|
34
|
-
if (local.length > backpressure) await new Promise((r) => setImmediate(r));
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
local.push(buffer);
|
|
39
|
-
while (local.length > 0) post({
|
|
40
|
-
type: "page",
|
|
41
|
-
data: local.splice(0, pageSize)
|
|
42
|
-
});
|
|
43
|
-
post({ type: "done" });
|
|
44
|
-
await fd.close();
|
|
45
|
-
} catch (err) {
|
|
46
|
-
post({
|
|
47
|
-
type: "error",
|
|
48
|
-
error: err
|
|
49
|
-
});
|
|
50
|
-
}
|
|
51
|
-
})();
|
|
52
|
-
//#endregion
|
package/dist/worker.d.cts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export { };
|
package/dist/worker.d.mts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export { };
|
package/dist/worker.mjs
DELETED
|
@@ -1,53 +0,0 @@
|
|
|
1
|
-
import { open } from "node:fs/promises";
|
|
2
|
-
import { parentPort, workerData } from "node:worker_threads";
|
|
3
|
-
//#region src/worker.ts
|
|
4
|
-
const { filepath, options } = workerData;
|
|
5
|
-
const { chunkSize, pageSize, delimiter } = options;
|
|
6
|
-
const backpressure = pageSize * 8;
|
|
7
|
-
function post(msg) {
|
|
8
|
-
if (!parentPort) process.exit(1);
|
|
9
|
-
parentPort.postMessage(msg);
|
|
10
|
-
}
|
|
11
|
-
(async () => {
|
|
12
|
-
try {
|
|
13
|
-
const fd = await open(filepath, "r");
|
|
14
|
-
const { size } = await fd.stat();
|
|
15
|
-
let pos = 0;
|
|
16
|
-
let buffer = "";
|
|
17
|
-
const local = [];
|
|
18
|
-
while (pos < size) {
|
|
19
|
-
const readSize = Math.min(chunkSize, size - pos);
|
|
20
|
-
const buf = Buffer.allocUnsafe(readSize);
|
|
21
|
-
const { bytesRead } = await fd.read(buf, 0, readSize, pos);
|
|
22
|
-
pos += bytesRead;
|
|
23
|
-
buffer = buffer + buf.toString("utf8", 0, bytesRead);
|
|
24
|
-
let idx;
|
|
25
|
-
while ((idx = buffer.indexOf(delimiter)) !== -1) {
|
|
26
|
-
const line = buffer.slice(0, idx);
|
|
27
|
-
buffer = buffer.slice(idx + delimiter.length);
|
|
28
|
-
local.push(line);
|
|
29
|
-
if (local.length >= pageSize) {
|
|
30
|
-
post({
|
|
31
|
-
type: "page",
|
|
32
|
-
data: local.splice(0, pageSize)
|
|
33
|
-
});
|
|
34
|
-
if (local.length > backpressure) await new Promise((r) => setImmediate(r));
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
local.push(buffer);
|
|
39
|
-
while (local.length > 0) post({
|
|
40
|
-
type: "page",
|
|
41
|
-
data: local.splice(0, pageSize)
|
|
42
|
-
});
|
|
43
|
-
post({ type: "done" });
|
|
44
|
-
await fd.close();
|
|
45
|
-
} catch (err) {
|
|
46
|
-
post({
|
|
47
|
-
type: "error",
|
|
48
|
-
error: err
|
|
49
|
-
});
|
|
50
|
-
}
|
|
51
|
-
})();
|
|
52
|
-
//#endregion
|
|
53
|
-
export {};
|