readline-pager 0.3.1 → 0.4.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +65 -61
- package/dist/main.cjs +16 -21
- package/dist/main.d.cts +3 -18
- package/dist/main.d.mts +3 -18
- package/dist/main.mjs +16 -22
- package/dist/native-BNwCco1j.d.cts +27 -0
- package/dist/native-BWytCdQz.d.mts +27 -0
- package/dist/native.cjs +91 -0
- package/dist/native.d.cts +2 -0
- package/dist/native.d.mts +2 -0
- package/dist/native.mjs +90 -0
- package/dist/worker.cjs +2 -1
- package/dist/worker.mjs +2 -1
- package/package.json +30 -12
package/README.md
CHANGED
|
@@ -12,18 +12,18 @@
|
|
|
12
12
|
<img src="https://img.shields.io/github/stars/devmor-j/readline-pager" alt="stars">
|
|
13
13
|
</p>
|
|
14
14
|
|
|
15
|
-
⚡
|
|
15
|
+
⚡ High-performance paginated file reader for Node.js. Efficiently process large text files without loading them into memory.
|
|
16
16
|
|
|
17
|
-
-
|
|
18
|
-
-
|
|
19
|
-
-
|
|
20
|
-
-
|
|
21
|
-
-
|
|
22
|
-
-
|
|
23
|
-
-
|
|
17
|
+
- 📦 Zero dependencies
|
|
18
|
+
- ⚡ Up to ~3× faster than Node.js `readline`
|
|
19
|
+
- 🚀 Up to ~6× faster with optional native C++ acceleration
|
|
20
|
+
- 🔁 Async (`for await...of`) and sync (`for...of`) iteration
|
|
21
|
+
- 📄 Page-based reading with manual control (`next`, `nextSync`)
|
|
22
|
+
- 🔀 Forward and backward reading support
|
|
23
|
+
- 🧪 Fully typed with high test coverage (~95%)
|
|
24
24
|
|
|
25
25
|
> **Important:**
|
|
26
|
-
> Performance depends heavily on the `chunkSize` option. Tune it for your
|
|
26
|
+
> Performance depends heavily on the `chunkSize` option. Tune it for your storage device. A value of **64 KiB** is usually a good starting point. Increasing it may improve throughput until you reach the best value for your hardware.
|
|
27
27
|
|
|
28
28
|
---
|
|
29
29
|
|
|
@@ -39,42 +39,52 @@ npm install readline-pager
|
|
|
39
39
|
|
|
40
40
|
```ts
|
|
41
41
|
import { createPager } from "readline-pager";
|
|
42
|
-
// const { createPager } = require("readline-pager");
|
|
43
42
|
|
|
44
|
-
const
|
|
45
|
-
|
|
46
|
-
// Async iteration
|
|
47
|
-
for await (const page of pager) {
|
|
48
|
-
console.log(page[0]); // first line of the current page
|
|
43
|
+
for await (const page of createPager("./bigfile.txt")) {
|
|
44
|
+
console.log(page[0]);
|
|
49
45
|
}
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
---
|
|
49
|
+
|
|
50
|
+
### Other usage patterns
|
|
51
|
+
|
|
52
|
+
```ts
|
|
53
|
+
import { createPager, createNativePager } from "readline-pager";
|
|
50
54
|
|
|
51
55
|
// Sync iteration
|
|
52
|
-
for (const page of
|
|
56
|
+
for (const page of createPager("./bigfile.txt")) {
|
|
53
57
|
}
|
|
54
58
|
|
|
55
|
-
// Manual
|
|
59
|
+
// Manual async
|
|
60
|
+
const pager = createPager("./bigfile.txt");
|
|
56
61
|
while (true) {
|
|
57
62
|
const page = await pager.next();
|
|
58
63
|
if (!page) break;
|
|
59
64
|
}
|
|
60
65
|
|
|
61
|
-
// Manual
|
|
66
|
+
// Manual sync
|
|
62
67
|
let page;
|
|
63
|
-
|
|
64
|
-
|
|
68
|
+
const pager = createPager("./bigfile.txt");
|
|
69
|
+
while ((page = pager.nextSync()) !== null) {}
|
|
70
|
+
|
|
71
|
+
// Native C++ (fastest)
|
|
72
|
+
for await (const page of createNativePager("./bigfile.txt")) {
|
|
65
73
|
}
|
|
66
74
|
```
|
|
67
75
|
|
|
76
|
+
---
|
|
77
|
+
|
|
68
78
|
## ⚙️ Options
|
|
69
79
|
|
|
70
80
|
```ts
|
|
71
81
|
createPager(filepath, {
|
|
72
|
-
chunkSize?: number,
|
|
73
|
-
pageSize?: number,
|
|
74
|
-
delimiter?: string,
|
|
75
|
-
prefetch?: number,
|
|
76
|
-
backward?: boolean,
|
|
77
|
-
useWorker?: boolean,
|
|
82
|
+
chunkSize?: number, // default: 64 * 1024 (64 KiB)
|
|
83
|
+
pageSize?: number, // default: 1_000
|
|
84
|
+
delimiter?: string, // default: "\n"
|
|
85
|
+
prefetch?: number, // default: 8
|
|
86
|
+
backward?: boolean, // default: false
|
|
87
|
+
useWorker?: boolean, // default: false
|
|
78
88
|
});
|
|
79
89
|
```
|
|
80
90
|
|
|
@@ -82,7 +92,7 @@ createPager(filepath, {
|
|
|
82
92
|
- `pageSize` — number of lines per page.
|
|
83
93
|
- `delimiter` — line separator.
|
|
84
94
|
- `prefetch` — maximum number of pages buffered internally.
|
|
85
|
-
- `backward` — read the file from end
|
|
95
|
+
- `backward` — read the file from end to start.
|
|
86
96
|
- `useWorker` — offload reading to a worker thread (forward reading only).
|
|
87
97
|
|
|
88
98
|
---
|
|
@@ -113,58 +123,52 @@ Stops reading and releases resources asynchronously. Safe to call at any time.
|
|
|
113
123
|
|
|
114
124
|
---
|
|
115
125
|
|
|
116
|
-
|
|
117
|
-
|
|
126
|
+
### `createNativePager(filepath, options?): Pager`
|
|
127
|
+
|
|
128
|
+
Creates a pager backed by the optional native C++ addon.
|
|
118
129
|
|
|
119
|
-
|
|
120
|
-
|
|
130
|
+
If the native addon is not available for the current platform, this function throws.
|
|
131
|
+
|
|
132
|
+
---
|
|
133
|
+
|
|
134
|
+
> **Note:**
|
|
135
|
+
> Unlike Node.js `readline`, which may skip empty files or leading empty lines, `readline-pager` always returns all lines.
|
|
136
|
+
>
|
|
137
|
+
> - A completely empty file (`0` bytes) produces `[""]` on the first read.
|
|
138
|
+
> - A file containing multiple empty lines returns each line as an empty string.
|
|
139
|
+
|
|
140
|
+
---
|
|
121
141
|
|
|
122
142
|
## 📊 Benchmark
|
|
123
143
|
|
|
124
|
-
Run the
|
|
144
|
+
Run the benchmark locally:
|
|
125
145
|
|
|
126
146
|
```bash
|
|
127
|
-
|
|
128
|
-
npm run benchmark
|
|
147
|
+
npm run benchmark:node
|
|
129
148
|
|
|
130
149
|
# or customize with args
|
|
131
150
|
node test/_benchmark.ts --lines=20000 --page-size=500 --backward
|
|
132
151
|
```
|
|
133
152
|
|
|
134
|
-
> Test setup: generated text files
|
|
135
|
-
>
|
|
136
|
-
> The **Average Throughput (MB/s)** is computed for two strategies: reading files line by line and page by page.
|
|
137
|
-
>
|
|
138
|
-
> In addition to _Node_, the two other popular JavaScript runtimes were also tested with `readline-pager`.
|
|
139
|
-
|
|
140
|
-
### Line by line
|
|
141
|
-
|
|
142
|
-
| Runtime / Method | 1M lines (35 MB) | 10M lines (353 MB) | 100M lines (3,529 MB) | 1,000M lines (35,286 MB) |
|
|
143
|
-
| ---------------- | ---------------: | -----------------: | --------------------: | -----------------------: |
|
|
144
|
-
| Node — node:line | 369 | 435 | 455 | 455 |
|
|
145
|
-
| Deno — node:line | 203 | 230 | 230 | 229 |
|
|
146
|
-
| Deno — deno:line | 738 | 901 | 915 | 809 |
|
|
147
|
-
| Bun — node:line | 246 | 279 | 283 | 280 |
|
|
148
|
-
| Bun — bun:line | 938 | 1,540 | 1,668 | 1,315 |
|
|
153
|
+
> Test setup: generated text files (UUID lines), NVMe SSD, Node.js runtime.
|
|
154
|
+
> Results are averaged across multiple runs. Actual performance depends on hardware.
|
|
149
155
|
|
|
150
|
-
|
|
156
|
+
---
|
|
151
157
|
|
|
152
|
-
|
|
153
|
-
| --------------------- | ---------------: | -----------------: | --------------------: | -----------------------: |
|
|
154
|
-
| Node — readline-pager | 1,053 | 1,311 | 1,278 | 936 |
|
|
155
|
-
| Deno — deno:page | 852 | 909 | 908 | 783 |
|
|
156
|
-
| Deno — readline-pager | 1,131 | 1,268 | 1,271 | 911 |
|
|
157
|
-
| Bun — bun:page | 411 | 440 | 449 | 428 |
|
|
158
|
-
| Bun — readline-pager | 827 | 1,021 | 1,040 | 804 |
|
|
158
|
+
### ⚡ Throughput (MB/s)
|
|
159
159
|
|
|
160
|
-
|
|
160
|
+
| Method | 1M lines (35 MB) | 10M lines (353 MB) | 100M lines (3.5 GB) | 1B lines (35.3 GB) |
|
|
161
|
+
| ---------------------- | ---------------: | -----------------: | ------------------: | -----------------: |
|
|
162
|
+
| `readline` | ~370 MB/s | ~460 MB/s | ~460 MB/s | ~460 MB/s |
|
|
163
|
+
| `readline-pager` (JS) | ~1100 MB/s | ~1300 MB/s | ~1300 MB/s | ~1150 MB/s |
|
|
164
|
+
| `readline-pager` (C++) | ~2200 MB/s | ~2500 MB/s | ~2500 MB/s | ~2450 MB/s |
|
|
161
165
|
|
|
162
166
|
---
|
|
163
167
|
|
|
164
168
|
## 🛠 Development & Contributing
|
|
165
169
|
|
|
166
|
-
- Minimum supported Node.js: **v18.12
|
|
167
|
-
- Development/test environment: **Node v25.
|
|
170
|
+
- Minimum supported Node.js: **v18.12**
|
|
171
|
+
- Development/test environment: **Node v25.8** and **TypeScript v6.0**
|
|
168
172
|
|
|
169
173
|
Run tests:
|
|
170
174
|
|
|
@@ -173,7 +177,7 @@ npm ci
|
|
|
173
177
|
npm test
|
|
174
178
|
```
|
|
175
179
|
|
|
176
|
-
Contributions are welcome
|
|
180
|
+
Contributions are welcome. Open an issue or submit a PR.
|
|
177
181
|
|
|
178
182
|
---
|
|
179
183
|
|
package/dist/main.cjs
CHANGED
|
@@ -2,6 +2,7 @@ Object.defineProperties(exports, {
|
|
|
2
2
|
__esModule: { value: true },
|
|
3
3
|
[Symbol.toStringTag]: { value: "Module" }
|
|
4
4
|
});
|
|
5
|
+
const require_native = require("./native.cjs");
|
|
5
6
|
let node_fs = require("node:fs");
|
|
6
7
|
let node_fs_promises = require("node:fs/promises");
|
|
7
8
|
let node_worker_threads = require("node:worker_threads");
|
|
@@ -47,7 +48,7 @@ function createRingBuffer(capacity) {
|
|
|
47
48
|
}
|
|
48
49
|
return v;
|
|
49
50
|
}
|
|
50
|
-
async function shift(done
|
|
51
|
+
async function shift(done) {
|
|
51
52
|
if (count) return shiftSync();
|
|
52
53
|
if (done) return null;
|
|
53
54
|
await new Promise((r) => {
|
|
@@ -214,7 +215,7 @@ function createBackwardReader(filepath, options) {
|
|
|
214
215
|
}
|
|
215
216
|
async function next() {
|
|
216
217
|
if (closed) return null;
|
|
217
|
-
return
|
|
218
|
+
return pageQueue.shift(done);
|
|
218
219
|
}
|
|
219
220
|
function nextSync() {
|
|
220
221
|
if (closed) return null;
|
|
@@ -394,7 +395,7 @@ function createForwardReader(filepath, options) {
|
|
|
394
395
|
}
|
|
395
396
|
async function next() {
|
|
396
397
|
if (closed) return null;
|
|
397
|
-
return
|
|
398
|
+
return pageQueue.shift(done);
|
|
398
399
|
}
|
|
399
400
|
function nextSync() {
|
|
400
401
|
if (closed) return null;
|
|
@@ -460,16 +461,13 @@ function createForwardReader(filepath, options) {
|
|
|
460
461
|
//#region src/reader/worker.reader.ts
|
|
461
462
|
const workerFile = new URL("./worker.mjs", require("url").pathToFileURL(__filename).href);
|
|
462
463
|
function createWorkerReader(filepath, options) {
|
|
463
|
-
const {
|
|
464
|
+
const { prefetch } = options;
|
|
464
465
|
const pageQueue = createRingBuffer(Math.max(2, prefetch + 1));
|
|
465
466
|
let done = false;
|
|
466
467
|
let closed = false;
|
|
467
468
|
const worker = new node_worker_threads.Worker(new URL(workerFile, require("url").pathToFileURL(__filename).href), { workerData: {
|
|
468
469
|
filepath,
|
|
469
|
-
|
|
470
|
-
pageSize,
|
|
471
|
-
delimiter,
|
|
472
|
-
prefetch
|
|
470
|
+
options
|
|
473
471
|
} });
|
|
474
472
|
worker.on("message", (msg) => {
|
|
475
473
|
if (msg.type === "page") pageQueue.push(msg.data);
|
|
@@ -488,7 +486,7 @@ function createWorkerReader(filepath, options) {
|
|
|
488
486
|
});
|
|
489
487
|
async function next() {
|
|
490
488
|
if (closed) return null;
|
|
491
|
-
return
|
|
489
|
+
return pageQueue.shift(done);
|
|
492
490
|
}
|
|
493
491
|
function nextSync() {
|
|
494
492
|
if (closed) return null;
|
|
@@ -539,23 +537,20 @@ function createPager(filepath, options = {}) {
|
|
|
539
537
|
if (pageSize < 1) throw new RangeError("pageSize must be >= 1");
|
|
540
538
|
if (prefetch < 1) throw new RangeError("prefetch must be >= 1");
|
|
541
539
|
if (backward && useWorker) throw new Error("backward not supported with useWorker");
|
|
542
|
-
|
|
543
|
-
chunkSize,
|
|
544
|
-
pageSize,
|
|
545
|
-
prefetch,
|
|
546
|
-
delimiter
|
|
547
|
-
}) : backward ? createBackwardReader(filepath, {
|
|
540
|
+
const _options = {
|
|
548
541
|
chunkSize,
|
|
549
542
|
pageSize,
|
|
550
543
|
prefetch,
|
|
551
544
|
delimiter
|
|
552
|
-
}
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
}
|
|
545
|
+
};
|
|
546
|
+
const reader = useWorker ? createWorkerReader(filepath, _options) : backward ? createBackwardReader(filepath, _options) : createForwardReader(filepath, _options);
|
|
547
|
+
if (process.env.TEST_CLEANUPS) {
|
|
548
|
+
globalThis.__test_cleanups__ ??= [];
|
|
549
|
+
globalThis.__test_cleanups__.push(reader.close);
|
|
550
|
+
}
|
|
551
|
+
return reader;
|
|
558
552
|
}
|
|
559
553
|
//#endregion
|
|
554
|
+
exports.createNativePager = require_native.createNativePager;
|
|
560
555
|
exports.createPager = createPager;
|
|
561
556
|
exports.default = createPager;
|
package/dist/main.d.cts
CHANGED
|
@@ -1,21 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
chunkSize: number;
|
|
4
|
-
pageSize: number;
|
|
5
|
-
delimiter: string;
|
|
6
|
-
prefetch: number;
|
|
7
|
-
}
|
|
8
|
-
interface PagerOptions extends Partial<ReaderOptions> {
|
|
9
|
-
backward?: boolean;
|
|
10
|
-
useWorker?: boolean;
|
|
11
|
-
}
|
|
12
|
-
interface Pager extends AsyncIterable<string[]>, Iterable<string[]> {
|
|
13
|
-
next(): Promise<string[] | null>;
|
|
14
|
-
nextSync(): string[] | null;
|
|
15
|
-
close(): Promise<void>;
|
|
16
|
-
}
|
|
17
|
-
//#endregion
|
|
1
|
+
import { i as ReaderOptions, n as Pager, r as PagerOptions, t as createNativePager } from "./native-BNwCco1j.cjs";
|
|
2
|
+
|
|
18
3
|
//#region src/main.d.ts
|
|
19
4
|
declare function createPager(filepath: string, options?: PagerOptions): Pager;
|
|
20
5
|
//#endregion
|
|
21
|
-
export { Pager, PagerOptions, ReaderOptions, createPager, createPager as default };
|
|
6
|
+
export { Pager, PagerOptions, ReaderOptions, createNativePager, createPager, createPager as default };
|
package/dist/main.d.mts
CHANGED
|
@@ -1,21 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
chunkSize: number;
|
|
4
|
-
pageSize: number;
|
|
5
|
-
delimiter: string;
|
|
6
|
-
prefetch: number;
|
|
7
|
-
}
|
|
8
|
-
interface PagerOptions extends Partial<ReaderOptions> {
|
|
9
|
-
backward?: boolean;
|
|
10
|
-
useWorker?: boolean;
|
|
11
|
-
}
|
|
12
|
-
interface Pager extends AsyncIterable<string[]>, Iterable<string[]> {
|
|
13
|
-
next(): Promise<string[] | null>;
|
|
14
|
-
nextSync(): string[] | null;
|
|
15
|
-
close(): Promise<void>;
|
|
16
|
-
}
|
|
17
|
-
//#endregion
|
|
1
|
+
import { i as ReaderOptions, n as Pager, r as PagerOptions, t as createNativePager } from "./native-BWytCdQz.mjs";
|
|
2
|
+
|
|
18
3
|
//#region src/main.d.ts
|
|
19
4
|
declare function createPager(filepath: string, options?: PagerOptions): Pager;
|
|
20
5
|
//#endregion
|
|
21
|
-
export { Pager, PagerOptions, ReaderOptions, createPager, createPager as default };
|
|
6
|
+
export { Pager, PagerOptions, ReaderOptions, createNativePager, createPager, createPager as default };
|
package/dist/main.mjs
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { createNativePager } from "./native.mjs";
|
|
1
2
|
import { createRequire } from "node:module";
|
|
2
3
|
import { closeSync, openSync, readSync, statSync } from "node:fs";
|
|
3
4
|
import { open } from "node:fs/promises";
|
|
@@ -47,7 +48,7 @@ function createRingBuffer(capacity) {
|
|
|
47
48
|
}
|
|
48
49
|
return v;
|
|
49
50
|
}
|
|
50
|
-
async function shift(done
|
|
51
|
+
async function shift(done) {
|
|
51
52
|
if (count) return shiftSync();
|
|
52
53
|
if (done) return null;
|
|
53
54
|
await new Promise((r) => {
|
|
@@ -214,7 +215,7 @@ function createBackwardReader(filepath, options) {
|
|
|
214
215
|
}
|
|
215
216
|
async function next() {
|
|
216
217
|
if (closed) return null;
|
|
217
|
-
return
|
|
218
|
+
return pageQueue.shift(done);
|
|
218
219
|
}
|
|
219
220
|
function nextSync() {
|
|
220
221
|
if (closed) return null;
|
|
@@ -394,7 +395,7 @@ function createForwardReader(filepath, options) {
|
|
|
394
395
|
}
|
|
395
396
|
async function next() {
|
|
396
397
|
if (closed) return null;
|
|
397
|
-
return
|
|
398
|
+
return pageQueue.shift(done);
|
|
398
399
|
}
|
|
399
400
|
function nextSync() {
|
|
400
401
|
if (closed) return null;
|
|
@@ -460,16 +461,13 @@ function createForwardReader(filepath, options) {
|
|
|
460
461
|
//#region src/reader/worker.reader.ts
|
|
461
462
|
const workerFile = typeof import.meta !== "undefined" ? new URL("./worker.mjs", import.meta.url) : __require.resolve("./worker.cjs");
|
|
462
463
|
function createWorkerReader(filepath, options) {
|
|
463
|
-
const {
|
|
464
|
+
const { prefetch } = options;
|
|
464
465
|
const pageQueue = createRingBuffer(Math.max(2, prefetch + 1));
|
|
465
466
|
let done = false;
|
|
466
467
|
let closed = false;
|
|
467
468
|
const worker = new Worker(new URL(workerFile, import.meta.url), { workerData: {
|
|
468
469
|
filepath,
|
|
469
|
-
|
|
470
|
-
pageSize,
|
|
471
|
-
delimiter,
|
|
472
|
-
prefetch
|
|
470
|
+
options
|
|
473
471
|
} });
|
|
474
472
|
worker.on("message", (msg) => {
|
|
475
473
|
if (msg.type === "page") pageQueue.push(msg.data);
|
|
@@ -488,7 +486,7 @@ function createWorkerReader(filepath, options) {
|
|
|
488
486
|
});
|
|
489
487
|
async function next() {
|
|
490
488
|
if (closed) return null;
|
|
491
|
-
return
|
|
489
|
+
return pageQueue.shift(done);
|
|
492
490
|
}
|
|
493
491
|
function nextSync() {
|
|
494
492
|
if (closed) return null;
|
|
@@ -539,22 +537,18 @@ function createPager(filepath, options = {}) {
|
|
|
539
537
|
if (pageSize < 1) throw new RangeError("pageSize must be >= 1");
|
|
540
538
|
if (prefetch < 1) throw new RangeError("prefetch must be >= 1");
|
|
541
539
|
if (backward && useWorker) throw new Error("backward not supported with useWorker");
|
|
542
|
-
|
|
543
|
-
chunkSize,
|
|
544
|
-
pageSize,
|
|
545
|
-
prefetch,
|
|
546
|
-
delimiter
|
|
547
|
-
}) : backward ? createBackwardReader(filepath, {
|
|
540
|
+
const _options = {
|
|
548
541
|
chunkSize,
|
|
549
542
|
pageSize,
|
|
550
543
|
prefetch,
|
|
551
544
|
delimiter
|
|
552
|
-
}
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
}
|
|
545
|
+
};
|
|
546
|
+
const reader = useWorker ? createWorkerReader(filepath, _options) : backward ? createBackwardReader(filepath, _options) : createForwardReader(filepath, _options);
|
|
547
|
+
if (process.env.TEST_CLEANUPS) {
|
|
548
|
+
globalThis.__test_cleanups__ ??= [];
|
|
549
|
+
globalThis.__test_cleanups__.push(reader.close);
|
|
550
|
+
}
|
|
551
|
+
return reader;
|
|
558
552
|
}
|
|
559
553
|
//#endregion
|
|
560
|
-
export { createPager, createPager as default };
|
|
554
|
+
export { createNativePager, createPager, createPager as default };
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
//#region src/types.d.ts
|
|
2
|
+
interface ReaderOptions {
|
|
3
|
+
chunkSize: number;
|
|
4
|
+
pageSize: number;
|
|
5
|
+
delimiter: string;
|
|
6
|
+
prefetch: number;
|
|
7
|
+
}
|
|
8
|
+
interface PagerOptions extends Partial<ReaderOptions> {
|
|
9
|
+
backward?: boolean;
|
|
10
|
+
useWorker?: boolean;
|
|
11
|
+
}
|
|
12
|
+
interface Pager extends AsyncIterable<string[]>, Iterable<string[]> {
|
|
13
|
+
next(): Promise<string[] | null>;
|
|
14
|
+
nextSync(): string[] | null;
|
|
15
|
+
close(): Promise<void>;
|
|
16
|
+
}
|
|
17
|
+
//#endregion
|
|
18
|
+
//#region src/native.d.ts
|
|
19
|
+
declare function createNativePager(filepath: string, {
|
|
20
|
+
pageSize,
|
|
21
|
+
delimiter
|
|
22
|
+
}?: {
|
|
23
|
+
pageSize?: number | undefined;
|
|
24
|
+
delimiter?: string | undefined;
|
|
25
|
+
}): Pager;
|
|
26
|
+
//#endregion
|
|
27
|
+
export { ReaderOptions as i, Pager as n, PagerOptions as r, createNativePager as t };
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
//#region src/types.d.ts
|
|
2
|
+
interface ReaderOptions {
|
|
3
|
+
chunkSize: number;
|
|
4
|
+
pageSize: number;
|
|
5
|
+
delimiter: string;
|
|
6
|
+
prefetch: number;
|
|
7
|
+
}
|
|
8
|
+
interface PagerOptions extends Partial<ReaderOptions> {
|
|
9
|
+
backward?: boolean;
|
|
10
|
+
useWorker?: boolean;
|
|
11
|
+
}
|
|
12
|
+
interface Pager extends AsyncIterable<string[]>, Iterable<string[]> {
|
|
13
|
+
next(): Promise<string[] | null>;
|
|
14
|
+
nextSync(): string[] | null;
|
|
15
|
+
close(): Promise<void>;
|
|
16
|
+
}
|
|
17
|
+
//#endregion
|
|
18
|
+
//#region src/native.d.ts
|
|
19
|
+
declare function createNativePager(filepath: string, {
|
|
20
|
+
pageSize,
|
|
21
|
+
delimiter
|
|
22
|
+
}?: {
|
|
23
|
+
pageSize?: number | undefined;
|
|
24
|
+
delimiter?: string | undefined;
|
|
25
|
+
}): Pager;
|
|
26
|
+
//#endregion
|
|
27
|
+
export { ReaderOptions as i, Pager as n, PagerOptions as r, createNativePager as t };
|
package/dist/native.cjs
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
let node_module = require("node:module");
|
|
3
|
+
let node_os = require("node:os");
|
|
4
|
+
//#region src/native.ts
|
|
5
|
+
const require$1 = (0, node_module.createRequire)(require("url").pathToFileURL(__filename).href);
|
|
6
|
+
function isMusl() {
|
|
7
|
+
if ((0, node_os.platform)() !== "linux") return false;
|
|
8
|
+
try {
|
|
9
|
+
return !(process.report?.getReport?.())?.header?.glibcVersionRuntime;
|
|
10
|
+
} catch {
|
|
11
|
+
return false;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
function getPackageName() {
|
|
15
|
+
const p = (0, node_os.platform)();
|
|
16
|
+
const a = (0, node_os.arch)();
|
|
17
|
+
switch (p) {
|
|
18
|
+
case "darwin":
|
|
19
|
+
case "win32": return `@devmor-j/readline-pager-${p}-${a}`;
|
|
20
|
+
case "linux": return `@devmor-j/readline-pager-${p}-${isMusl() ? "musl-" : ""}${a}`;
|
|
21
|
+
default: throw new Error(`Unsupported platform: ${p}/${a}`);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
function loadNativeAddon() {
|
|
25
|
+
try {
|
|
26
|
+
return require$1(getPackageName());
|
|
27
|
+
} catch {
|
|
28
|
+
const UNAVAILABLE = `Native addon not available for ${(0, node_os.platform)()}/${(0, node_os.arch)()}.`;
|
|
29
|
+
throw new Error(UNAVAILABLE);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
function createNativePager(filepath, { pageSize = 1e3, delimiter = "\n" } = {}) {
|
|
33
|
+
const pagerNative = loadNativeAddon();
|
|
34
|
+
let fd = null;
|
|
35
|
+
let closed = false;
|
|
36
|
+
const init = () => {
|
|
37
|
+
fd = pagerNative.open(filepath, pageSize, delimiter);
|
|
38
|
+
};
|
|
39
|
+
const next = async () => {
|
|
40
|
+
if (closed || !fd) return null;
|
|
41
|
+
const data = await pagerNative.next(fd);
|
|
42
|
+
if (!data) return null;
|
|
43
|
+
return data.toString("utf8").split(delimiter);
|
|
44
|
+
};
|
|
45
|
+
const nextSync = () => {
|
|
46
|
+
if (closed || !fd) return null;
|
|
47
|
+
const data = pagerNative.nextSync(fd);
|
|
48
|
+
if (!data) return null;
|
|
49
|
+
return data.toString("utf8").split(delimiter);
|
|
50
|
+
};
|
|
51
|
+
const close = async () => {
|
|
52
|
+
if (!closed || fd) {
|
|
53
|
+
closed = true;
|
|
54
|
+
await pagerNative.close(fd);
|
|
55
|
+
fd = null;
|
|
56
|
+
}
|
|
57
|
+
};
|
|
58
|
+
function tryClose() {
|
|
59
|
+
close().catch(() => {});
|
|
60
|
+
}
|
|
61
|
+
init();
|
|
62
|
+
return {
|
|
63
|
+
next,
|
|
64
|
+
nextSync,
|
|
65
|
+
close,
|
|
66
|
+
async *[Symbol.asyncIterator]() {
|
|
67
|
+
try {
|
|
68
|
+
while (true) {
|
|
69
|
+
const p = await next();
|
|
70
|
+
if (!p) break;
|
|
71
|
+
yield p;
|
|
72
|
+
}
|
|
73
|
+
} finally {
|
|
74
|
+
tryClose();
|
|
75
|
+
}
|
|
76
|
+
},
|
|
77
|
+
*[Symbol.iterator]() {
|
|
78
|
+
try {
|
|
79
|
+
while (true) {
|
|
80
|
+
const p = nextSync();
|
|
81
|
+
if (!p) break;
|
|
82
|
+
yield p;
|
|
83
|
+
}
|
|
84
|
+
} finally {
|
|
85
|
+
tryClose();
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
//#endregion
|
|
91
|
+
exports.createNativePager = createNativePager;
|
package/dist/native.mjs
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { createRequire } from "node:module";
|
|
2
|
+
import { arch, platform } from "node:os";
|
|
3
|
+
//#region src/native.ts
|
|
4
|
+
const require = createRequire(import.meta.url);
|
|
5
|
+
function isMusl() {
|
|
6
|
+
if (platform() !== "linux") return false;
|
|
7
|
+
try {
|
|
8
|
+
return !(process.report?.getReport?.())?.header?.glibcVersionRuntime;
|
|
9
|
+
} catch {
|
|
10
|
+
return false;
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
function getPackageName() {
|
|
14
|
+
const p = platform();
|
|
15
|
+
const a = arch();
|
|
16
|
+
switch (p) {
|
|
17
|
+
case "darwin":
|
|
18
|
+
case "win32": return `@devmor-j/readline-pager-${p}-${a}`;
|
|
19
|
+
case "linux": return `@devmor-j/readline-pager-${p}-${isMusl() ? "musl-" : ""}${a}`;
|
|
20
|
+
default: throw new Error(`Unsupported platform: ${p}/${a}`);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
function loadNativeAddon() {
|
|
24
|
+
try {
|
|
25
|
+
return require(getPackageName());
|
|
26
|
+
} catch {
|
|
27
|
+
const UNAVAILABLE = `Native addon not available for ${platform()}/${arch()}.`;
|
|
28
|
+
throw new Error(UNAVAILABLE);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
function createNativePager(filepath, { pageSize = 1e3, delimiter = "\n" } = {}) {
|
|
32
|
+
const pagerNative = loadNativeAddon();
|
|
33
|
+
let fd = null;
|
|
34
|
+
let closed = false;
|
|
35
|
+
const init = () => {
|
|
36
|
+
fd = pagerNative.open(filepath, pageSize, delimiter);
|
|
37
|
+
};
|
|
38
|
+
const next = async () => {
|
|
39
|
+
if (closed || !fd) return null;
|
|
40
|
+
const data = await pagerNative.next(fd);
|
|
41
|
+
if (!data) return null;
|
|
42
|
+
return data.toString("utf8").split(delimiter);
|
|
43
|
+
};
|
|
44
|
+
const nextSync = () => {
|
|
45
|
+
if (closed || !fd) return null;
|
|
46
|
+
const data = pagerNative.nextSync(fd);
|
|
47
|
+
if (!data) return null;
|
|
48
|
+
return data.toString("utf8").split(delimiter);
|
|
49
|
+
};
|
|
50
|
+
const close = async () => {
|
|
51
|
+
if (!closed || fd) {
|
|
52
|
+
closed = true;
|
|
53
|
+
await pagerNative.close(fd);
|
|
54
|
+
fd = null;
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
function tryClose() {
|
|
58
|
+
close().catch(() => {});
|
|
59
|
+
}
|
|
60
|
+
init();
|
|
61
|
+
return {
|
|
62
|
+
next,
|
|
63
|
+
nextSync,
|
|
64
|
+
close,
|
|
65
|
+
async *[Symbol.asyncIterator]() {
|
|
66
|
+
try {
|
|
67
|
+
while (true) {
|
|
68
|
+
const p = await next();
|
|
69
|
+
if (!p) break;
|
|
70
|
+
yield p;
|
|
71
|
+
}
|
|
72
|
+
} finally {
|
|
73
|
+
tryClose();
|
|
74
|
+
}
|
|
75
|
+
},
|
|
76
|
+
*[Symbol.iterator]() {
|
|
77
|
+
try {
|
|
78
|
+
while (true) {
|
|
79
|
+
const p = nextSync();
|
|
80
|
+
if (!p) break;
|
|
81
|
+
yield p;
|
|
82
|
+
}
|
|
83
|
+
} finally {
|
|
84
|
+
tryClose();
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
//#endregion
|
|
90
|
+
export { createNativePager };
|
package/dist/worker.cjs
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
let node_fs_promises = require("node:fs/promises");
|
|
2
2
|
let node_worker_threads = require("node:worker_threads");
|
|
3
3
|
//#region src/worker.ts
|
|
4
|
-
const { filepath,
|
|
4
|
+
const { filepath, options } = node_worker_threads.workerData;
|
|
5
|
+
const { chunkSize, pageSize, delimiter } = options;
|
|
5
6
|
(async () => {
|
|
6
7
|
const fd = await (0, node_fs_promises.open)(filepath, "r");
|
|
7
8
|
const { size } = await fd.stat();
|
package/dist/worker.mjs
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { open } from "node:fs/promises";
|
|
2
2
|
import { parentPort, workerData } from "node:worker_threads";
|
|
3
3
|
//#region src/worker.ts
|
|
4
|
-
const { filepath,
|
|
4
|
+
const { filepath, options } = workerData;
|
|
5
|
+
const { chunkSize, pageSize, delimiter } = options;
|
|
5
6
|
(async () => {
|
|
6
7
|
const fd = await open(filepath, "r");
|
|
7
8
|
const { size } = await fd.stat();
|
package/package.json
CHANGED
|
@@ -1,25 +1,33 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "readline-pager",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.9",
|
|
4
4
|
"scripts": {
|
|
5
5
|
"build:js": "tsdown",
|
|
6
|
-
"build:native": "
|
|
6
|
+
"build:native": "node-gyp rebuild",
|
|
7
7
|
"build": "npm run build:native && npm run build:js",
|
|
8
|
+
"pkg:native": "node scripts/package-native.ts",
|
|
8
9
|
"pretest": "npm run build",
|
|
9
|
-
"test": "node --test --experimental-test-coverage test/**/*.test.ts",
|
|
10
|
-
"prepublishOnly": "npm run build",
|
|
11
|
-
"benchmark": "node test/_benchmark.ts
|
|
10
|
+
"test": "TEST_CLEANUPS=1 node --test --experimental-test-coverage test/**/*.test.ts",
|
|
11
|
+
"prepublishOnly": "npm run build && npm run test",
|
|
12
|
+
"benchmark:node": "node test/_benchmark.ts",
|
|
13
|
+
"benchmark:deno": "deno --allow-write --allow-read --allow-env test/_benchmark.ts",
|
|
14
|
+
"benchmark:bun": "bun test/_benchmark.ts"
|
|
12
15
|
},
|
|
13
16
|
"devDependencies": {
|
|
14
17
|
"@types/bun": "~1.3.11",
|
|
15
18
|
"@types/deno": "~2.5.0",
|
|
16
19
|
"@types/node": "~25.5.0",
|
|
17
20
|
"node-gyp": "~12.2.0",
|
|
18
|
-
"prebuildify": "~6.0.1",
|
|
19
21
|
"prettier": "~3.8.1",
|
|
20
22
|
"prettier-plugin-organize-imports": "~4.3.0",
|
|
21
|
-
"tsdown": "~0.21.
|
|
22
|
-
"typescript": "~
|
|
23
|
+
"tsdown": "~0.21.5",
|
|
24
|
+
"typescript": "~6.0.2"
|
|
25
|
+
},
|
|
26
|
+
"optionalDependencies": {
|
|
27
|
+
"@devmor-j/readline-pager-linux-arm64": "0.4.9",
|
|
28
|
+
"@devmor-j/readline-pager-linux-x64": "0.4.9",
|
|
29
|
+
"@devmor-j/readline-pager-linux-musl-arm64": "0.4.9",
|
|
30
|
+
"@devmor-j/readline-pager-linux-musl-x64": "0.4.9"
|
|
23
31
|
},
|
|
24
32
|
"gypfile": false,
|
|
25
33
|
"type": "module",
|
|
@@ -38,9 +46,7 @@
|
|
|
38
46
|
],
|
|
39
47
|
"keywords": [
|
|
40
48
|
"nodejs",
|
|
41
|
-
"readline",
|
|
42
49
|
"readline-pager",
|
|
43
|
-
"pager",
|
|
44
50
|
"file-reader",
|
|
45
51
|
"pagination",
|
|
46
52
|
"large-files",
|
|
@@ -49,9 +55,21 @@
|
|
|
49
55
|
"memory-efficient",
|
|
50
56
|
"high-performance",
|
|
51
57
|
"log-processing",
|
|
52
|
-
"backward-reading"
|
|
58
|
+
"backward-reading",
|
|
59
|
+
"native-addon",
|
|
60
|
+
"cpp",
|
|
61
|
+
"avx2",
|
|
62
|
+
"neon"
|
|
53
63
|
],
|
|
54
|
-
"description": "
|
|
64
|
+
"description": "High-performance paginated file reader for Node.js. Efficiently process large text files without loading them into memory.",
|
|
65
|
+
"repository": {
|
|
66
|
+
"type": "git",
|
|
67
|
+
"url": "git+https://github.com/devmor-j/readline-pager.git"
|
|
68
|
+
},
|
|
69
|
+
"homepage": "https://github.com/devmor-j/readline-pager#readme",
|
|
70
|
+
"bugs": {
|
|
71
|
+
"url": "https://github.com/devmor-j/readline-pager/issues"
|
|
72
|
+
},
|
|
55
73
|
"engines": {
|
|
56
74
|
"node": ">=18.12"
|
|
57
75
|
},
|