readline-pager 0.5.1 → 0.6.3

package/README.md

@@ -20,7 +20,7 @@
 - 🔁 Async (`for await...of`) and sync (`for...of`) iteration
 - 📄 Page-based reading with manual control (`next`, `nextSync`)
 - 🔀 Forward and backward reading support
-- 🧪 Fully typed with high test coverage (~95%)
+- 🧪 Fully typed with high test coverage (>90%)
 
 > **Important:**  
 > Performance depends heavily on the `chunkSize` option. Tune it for your storage device. A value of **64 KiB** is usually a good starting point. Increasing it may improve throughput until you reach the best value for your hardware.
@@ -68,7 +68,7 @@ let page;
 const pager = createPager("./bigfile.txt");
 while ((page = pager.nextSync()) !== null) {}
 
-// Native C++ (fastest)
+// Native C++
 for await (const page of createNativePager("./bigfile.txt")) {
 }
 ```
@@ -87,6 +87,12 @@ createPager(filepath, {
   useWorker?: boolean,  // default: false
   tryNative?: boolean,  // default: true
 });
+
+createNativePager(filepath, {
+  pageSize?: number,    // default: 1_000
+  delimiter?: string,   // default: "\n"
+  backward?: boolean,   // default: false
+});
 ```
 
 - `chunkSize` — number of bytes read per I/O operation.
@@ -98,7 +104,7 @@ createPager(filepath, {
 - `tryNative` — attempts to use the native reader, falls back to the non-native version if it fails.
 
 > **Note:**
-> `createNativePager` supports only `pageSize`, `delimiter`, and `backward` options and does **not** support multi-character delimiters.
+> `createNativePager` requires x86 AVX2 or ARM NEON CPU instruction set extensions and will throw if they are not available. It also does **not** support multi-character delimiters due to fast SIMD-based scanning.
 
 ---
 
@@ -138,7 +144,7 @@ Run the benchmark locally:
 npm run benchmark:node
 
 # or customize with args
-node test/_benchmark.ts --lines=20000 --page-size=500 --backward
+node test/benchmark.ts --lines=20000 --page-size=500 --backward
 ```
 
 > Test setup: generated text files (UUID lines), NVMe SSD, Node.js runtime.
@@ -164,7 +170,7 @@ node test/_benchmark.ts --lines=20000 --page-size=500 --backward
 Run tests:
 
 ```bash
-npm ci
+npm i
 npm test
 ```
 

package/dist/main.cjs

@@ -6,7 +6,7 @@ const require_native = require("./native.cjs");
 let node_fs = require("node:fs");
 let node_fs_promises = require("node:fs/promises");
 let node_worker_threads = require("node:worker_threads");
-//#region src/queue.ts
+//#region src/helper.ts
 function createRingBuffer(capacity) {
 	if (!Number.isFinite(capacity) || capacity <= 0) throw new RangeError("capacity must be a positive number");
 	let buf = new Array(capacity);
@@ -102,115 +102,99 @@ function createBackwardReader(filepath, options) {
 	let buffer = "";
 	let done = false;
 	let closed = false;
+	let flushed = false;
 	let startsWithDelimiter = false;
+	function consumeBuffer() {
+		let idx;
+		while ((idx = buffer.lastIndexOf(delimiter)) !== -1) {
+			const line = buffer.slice(idx + delimiter.length);
+			buffer = buffer.slice(0, idx);
+			local.push(line);
+			while (local.length >= pageSize) pageQueue.push(local.splice(0, pageSize));
+		}
+	}
+	function flushTail() {
+		if (flushed) return;
+		flushed = true;
+		if (buffer.length > 0) local.push(buffer);
+		else if (startsWithDelimiter) local.push("");
+		buffer = "";
+		while (local.length > 0) {
+			const page = local.slice(local.length - Math.min(pageSize, local.length));
+			local.length -= page.length;
+			pageQueue.push(page);
+		}
+		done = true;
+		pageQueue.wake();
+	}
 	fdSync = (0, node_fs.openSync)(filepath, "r");
 	pos = (0, node_fs.statSync)(filepath).size;
 	if (pos === 0) {
-		pageQueue.push([buffer]);
+		pageQueue.push([""]);
 		done = true;
+		flushed = true;
 		pageQueue.wake();
 	}
 	(async () => {
-		try {
-			fd = await (0, node_fs_promises.open)(filepath, "r");
-			pos = (await fd.stat()).size;
-			if (pos === 0) {
-				if (!done) {
-					pageQueue.push([buffer]);
-					done = true;
-				}
-				if (fd) {
-					await fd.close();
-					fd = null;
-				}
-				pageQueue.wake();
-				return;
+		fd = await (0, node_fs_promises.open)(filepath, "r");
+		pos = (await fd.stat()).size;
+		if (pos === 0) {
+			if (!done) {
+				pageQueue.push([""]);
+				done = true;
+				flushed = true;
 			}
-			while (!done && !closed) {
-				while (pageQueue.count < prefetch && pos > 0 && !closed) {
-					const readSize = Math.min(chunkSize, pos);
-					pos -= readSize;
-					const buf = Buffer.allocUnsafe(readSize);
-					await fd.read(buf, 0, readSize, pos);
-					buffer = buf.toString("utf8") + buffer;
-					if (pos === 0 && buffer.startsWith(delimiter)) startsWithDelimiter = true;
-					let idx;
-					while ((idx = buffer.lastIndexOf(delimiter)) !== -1) {
-						const line = buffer.slice(idx + delimiter.length);
-						buffer = buffer.slice(0, idx);
-						local.push(line);
-						while (local.length >= pageSize) {
-							const page = local.splice(0, pageSize);
-							pageQueue.push(page);
-						}
-					}
-				}
-				if (pos === 0 && !done) {
-					if (buffer.length > 0) local.push(buffer);
-					else if (startsWithDelimiter) local.push("");
-					buffer = "";
-					while (local.length > 0 && !closed) {
-						const page = local.slice(local.length - Math.min(pageSize, local.length));
-						local.length -= page.length;
-						pageQueue.push(page);
-					}
-					done = true;
-					if (fd) {
-						await fd.close();
-						fd = null;
-					}
-					pageQueue.wake();
-					break;
-				}
-				if (!done && !closed) await new Promise((r) => setImmediate(r));
+			if (fd) {
+				try {
+					await fd.close();
+				} catch {}
+				fd = null;
 			}
-		} catch {
-			done = true;
 			pageQueue.wake();
-			try {
+			return;
+		}
+		while (!done && !closed) {
+			while (pageQueue.count < prefetch && pos > 0 && !closed) {
+				const readSize = Math.min(chunkSize, pos);
+				pos -= readSize;
+				const buf = Buffer.allocUnsafe(readSize);
+				const { bytesRead } = await fd.read(buf, 0, readSize, pos);
+				buffer = buf.toString("utf8", 0, bytesRead) + buffer;
+				if (pos === 0 && buffer.startsWith(delimiter)) startsWithDelimiter = true;
+				consumeBuffer();
+			}
+			if (pos === 0 && !flushed) {
+				flushTail();
 				if (fd) {
-					await fd.close();
+					try {
+						await fd.close();
+					} catch {}
 					fd = null;
 				}
-			} catch {}
+				break;
+			}
+			if (!done && !closed) await new Promise((r) => setImmediate(r));
 		}
 	})();
 	function fillSync() {
-		if (done || closed) return;
-		if (fdSync === null) return;
+		if (done || closed || !fdSync) return;
 		while (pageQueue.count < prefetch && pos > 0 && !closed) {
 			const readSize = Math.min(chunkSize, pos);
 			pos -= readSize;
 			const buf = Buffer.allocUnsafe(readSize);
-			(0, node_fs.readSync)(fdSync, buf, 0, readSize, pos);
-			buffer = buf.toString("utf8") + buffer;
+			const bytesRead = (0, node_fs.readSync)(fdSync, buf, 0, readSize, pos);
+			buffer = buf.toString("utf8", 0, bytesRead) + buffer;
 			if (pos === 0 && buffer.startsWith(delimiter)) startsWithDelimiter = true;
-			let idx;
-			while ((idx = buffer.lastIndexOf(delimiter)) !== -1) {
-				const line = buffer.slice(idx + delimiter.length);
-				buffer = buffer.slice(0, idx);
-				local.push(line);
-				while (local.length >= pageSize) {
-					const page = local.splice(0, pageSize);
-					pageQueue.push(page);
-				}
-			}
+			consumeBuffer();
 		}
-		if (pos === 0 && !done) {
-			if (buffer.length > 0) local.push(buffer);
-			else if (startsWithDelimiter) local.push("");
-			buffer = "";
-			while (local.length > 0) {
-				const page = local.slice(local.length - Math.min(pageSize, local.length));
-				local.length -= page.length;
-				pageQueue.push(page);
-			}
-			done = true;
-			if (fdSync !== null) {
-				(0, node_fs.closeSync)(fdSync);
+		if (pos === 0 && !flushed) {
+			flushTail();
+			if (fdSync) {
+				try {
+					(0, node_fs.closeSync)(fdSync);
+				} catch {}
 				fdSync = null;
 			}
-			pageQueue.wake();
 		}
 	}
 	async function next() {
@@ -223,20 +207,21 @@ function createBackwardReader(filepath, options) {
 		return pageQueue.shiftSync();
 	}
 	async function close() {
+		if (closed) return;
 		closed = true;
 		done = true;
 		pageQueue.clear();
-		if (fd) {
+		if (fdSync) {
 			try {
-				await fd.close();
+				(0, node_fs.closeSync)(fdSync);
 			} catch {}
-			fd = null;
+			fdSync = null;
 		}
-		if (fdSync !== null) {
+		if (fd) {
 			try {
-				(0, node_fs.closeSync)(fdSync);
+				await fd.close();
 			} catch {}
-			fdSync = null;
+			fd = null;
 		}
 	}
 	return {
@@ -265,14 +250,18 @@ function createBackwardReader(filepath, options) {
 				closed = true;
 				done = true;
 				pageQueue.clear();
-				try {
-					if (fdSync) (0, node_fs.closeSync)(fdSync);
-				} catch {}
-				fdSync = null;
-				try {
-					if (fd?.fd) (0, node_fs.closeSync)(fd.fd);
-				} catch {}
-				fd = null;
+				if (fdSync) {
+					try {
+						(0, node_fs.closeSync)(fdSync);
+					} catch {}
+					fdSync = null;
+				}
+				if (fd?.fd) {
+					try {
+						(0, node_fs.closeSync)(fd.fd);
+					} catch {}
+					fd = null;
+				}
 			}
 		}
 	};
@@ -291,106 +280,90 @@ function createForwardReader(filepath, options) {
 	let done = false;
 	let closed = false;
 	let flushed = false;
+	function consumeBuffer() {
+		let idx;
+		while ((idx = buffer.indexOf(delimiter)) !== -1) {
+			const line = buffer.slice(0, idx);
+			buffer = buffer.slice(idx + delimiter.length);
+			local.push(line);
+			while (local.length >= pageSize) pageQueue.push(local.splice(0, pageSize));
+		}
+	}
+	function flushTail() {
+		if (flushed) return;
+		flushed = true;
+		local.push(buffer.length > 0 ? buffer : "");
+		buffer = "";
+		while (local.length > 0) pageQueue.push(local.splice(0, pageSize));
+		done = true;
+		pageQueue.wake();
+	}
 	fdSync = (0, node_fs.openSync)(filepath, "r");
 	size = (0, node_fs.statSync)(filepath).size;
 	if (size === 0) {
-		pageQueue.push([buffer]);
+		pageQueue.push([""]);
 		done = true;
+		flushed = true;
 		pageQueue.wake();
 	}
 	(async () => {
-		try {
-			fd = await (0, node_fs_promises.open)(filepath, "r");
-			size = (await fd.stat()).size;
-			if (size === 0) {
-				if (!done) {
-					pageQueue.push([buffer]);
-					done = true;
-				}
-				if (fd) {
-					await fd.close();
-					fd = null;
-				}
-				pageQueue.wake();
-				return;
+		fd = await (0, node_fs_promises.open)(filepath, "r");
+		size = (await fd.stat()).size;
+		if (size === 0) {
+			if (!done) {
+				pageQueue.push([""]);
+				done = true;
+				flushed = true;
 			}
-			while (!done && !closed) {
-				while (pageQueue.count < prefetch && pos < size && !closed) {
-					const readSize = Math.min(chunkSize, size - pos);
-					const buf = Buffer.allocUnsafe(readSize);
-					const { bytesRead } = await fd.read(buf, 0, readSize, pos);
-					pos += bytesRead;
-					buffer = buffer + buf.toString("utf8", 0, bytesRead);
-					let idx;
-					while ((idx = buffer.indexOf(delimiter)) !== -1) {
-						const line = buffer.slice(0, idx);
-						buffer = buffer.slice(idx + delimiter.length);
-						local.push(line);
-						while (local.length >= pageSize) pageQueue.push(local.splice(0, pageSize));
-					}
-				}
-				if (pos >= size && !flushed) {
-					flushed = true;
-					local.push(buffer.length > 0 ? buffer : "");
-					buffer = "";
-					while (local.length > 0 && !closed) {
-						const page = local.slice(0, pageSize);
-						local.length -= page.length;
-						pageQueue.push(page);
-					}
-					done = true;
-					if (fd) {
-						await fd.close();
-						fd = null;
-					}
-					pageQueue.wake();
-					break;
-				}
-				if (!done && !closed) await new Promise((r) => setImmediate(r));
+			if (fd) {
+				try {
+					await fd.close();
+				} catch {}
+				fd = null;
 			}
-		} catch {
-			done = true;
 			pageQueue.wake();
-			try {
+			return;
+		}
+		while (!done && !closed) {
+			while (pageQueue.count < prefetch && pos < size && !closed) {
+				const readSize = Math.min(chunkSize, size - pos);
+				const buf = Buffer.allocUnsafe(readSize);
+				const { bytesRead } = await fd.read(buf, 0, readSize, pos);
+				pos += bytesRead;
+				buffer = buffer + buf.toString("utf8", 0, bytesRead);
+				consumeBuffer();
+			}
+			if (pos >= size && !flushed) {
+				flushTail();
 				if (fd) {
-					await fd.close();
+					try {
+						await fd.close();
+					} catch {}
 					fd = null;
 				}
-			} catch {}
+				break;
+			}
+			if (!done && !closed) await new Promise((r) => setImmediate(r));
 		}
 	})();
 	function fillSync() {
-		if (done || closed) return;
-		if (fdSync === null) return;
+		if (done || closed || !fdSync) return;
 		while (pageQueue.count < prefetch && pos < size && !closed) {
 			const readSize = Math.min(chunkSize, size - pos);
 			const buf = Buffer.allocUnsafe(readSize);
 			const bytesRead = (0, node_fs.readSync)(fdSync, buf, 0, readSize, pos);
 			pos += bytesRead;
 			buffer = buffer + buf.toString("utf8", 0, bytesRead);
-			let idx;
-			while ((idx = buffer.indexOf(delimiter)) !== -1) {
-				const line = buffer.slice(0, idx);
-				buffer = buffer.slice(idx + delimiter.length);
-				local.push(line);
-				while (local.length >= pageSize) pageQueue.push(local.splice(0, pageSize));
-			}
+			consumeBuffer();
 		}
 		if (pos >= size && !flushed) {
-			flushed = true;
-			local.push(buffer.length > 0 ? buffer : "");
-			buffer = "";
-			while (local.length > 0) {
-				const page = local.slice(0, pageSize);
-				local.length -= page.length;
-				pageQueue.push(page);
-			}
-			done = true;
-			if (fdSync !== null) {
-				(0, node_fs.closeSync)(fdSync);
+			flushTail();
+			if (fdSync) {
+				try {
+					(0, node_fs.closeSync)(fdSync);
+				} catch {}
 				fdSync = null;
 			}
-			pageQueue.wake();
 		}
 	}
 	async function next() {
@@ -403,20 +376,21 @@ function createForwardReader(filepath, options) {
 		return pageQueue.shiftSync();
 	}
 	async function close() {
+		if (closed) return;
 		closed = true;
 		done = true;
 		pageQueue.clear();
-		if (fd) {
+		if (fdSync) {
 			try {
-				await fd.close();
+				(0, node_fs.closeSync)(fdSync);
 			} catch {}
-			fd = null;
+			fdSync = null;
 		}
-		if (fdSync !== null) {
+		if (fd) {
 			try {
-				(0, node_fs.closeSync)(fdSync);
+				await fd.close();
 			} catch {}
-			fdSync = null;
+			fd = null;
 		}
 	}
 	return {
@@ -445,14 +419,18 @@ function createForwardReader(filepath, options) {
 				closed = true;
 				done = true;
 				pageQueue.clear();
-				try {
-					if (fdSync !== null) (0, node_fs.closeSync)(fdSync);
-				} catch {}
-				fdSync = null;
-				try {
-					if (fd?.fd) (0, node_fs.closeSync)(fd.fd);
-				} catch {}
-				fd = null;
+				if (fdSync) {
+					try {
+						(0, node_fs.closeSync)(fdSync);
+					} catch {}
+					fdSync = null;
+				}
+				if (fd?.fd) {
+					try {
+						(0, node_fs.closeSync)(fd.fd);
+					} catch {}
+					fd = null;
+				}
 			}
 		}
 	};
@@ -462,18 +440,23 @@ function createForwardReader(filepath, options) {
 const workerFile = new URL("./worker.mjs", require("url").pathToFileURL(__filename).href);
 function createWorkerReader(filepath, options) {
 	const { prefetch } = options;
-	const pageQueue = createRingBuffer(Math.max(2, prefetch + 1));
 	let done = false;
 	let closed = false;
+	const pageQueue = createRingBuffer(Math.max(2, prefetch + 1));
 	const worker = new node_worker_threads.Worker(new URL(workerFile, require("url").pathToFileURL(__filename).href), { workerData: {
 		filepath,
 		options
 	} });
 	worker.on("message", (msg) => {
-		if (msg.type === "page") pageQueue.push(msg.data);
-		if (msg.type === "done") {
-			done = true;
-			pageQueue.wake();
+		switch (msg.type) {
+			case "page":
+				pageQueue.push(msg.data);
+				break;
+			case "done":
+			case "error":
+				done = true;
+				pageQueue.wake();
+				break;
 		}
 	});
 	worker.on("error", () => {
@@ -493,10 +476,13 @@ function createWorkerReader(filepath, options) {
 		return pageQueue.shiftSync();
 	}
 	async function close() {
+		if (closed) return;
 		closed = true;
 		done = true;
 		pageQueue.clear();
-		await worker.terminate();
+		try {
+			await worker.terminate();
+		} catch {}
 	}
 	function tryClose() {
 		close().catch(() => {});

package/dist/main.d.cts

@@ -1,6 +1,6 @@
-import { a as NativeReaderOptions, c as ReaderOptions, i as NativeAddon, n as AddonData, o as Pager, r as AddonFD, s as PagerOptions, t as createNativePager } from "./native-46pCT8Rc.cjs";
+import { a as PagerOptions, i as Pager, n as NativeAddon, o as ReaderOptions, r as NativeReaderOptions, s as WorkerMessage, t as createNativePager } from "./native-_NmVYcF6.cjs";
 
 //#region src/main.d.ts
 declare function createPager(filepath: string, options?: PagerOptions): Pager;
 //#endregion
-export { AddonData, AddonFD, NativeAddon, NativeReaderOptions, Pager, PagerOptions, ReaderOptions, createNativePager, createPager, createPager as default };
+export { NativeAddon, NativeReaderOptions, Pager, PagerOptions, ReaderOptions, WorkerMessage, createNativePager, createPager, createPager as default };

package/dist/main.d.mts

@@ -1,6 +1,6 @@
-import { a as NativeReaderOptions, c as ReaderOptions, i as NativeAddon, n as AddonData, o as Pager, r as AddonFD, s as PagerOptions, t as createNativePager } from "./native-DeBXdY3U.mjs";
+import { a as PagerOptions, i as Pager, n as NativeAddon, o as ReaderOptions, r as NativeReaderOptions, s as WorkerMessage, t as createNativePager } from "./native-DGzYrMHK.mjs";
 
 //#region src/main.d.ts
 declare function createPager(filepath: string, options?: PagerOptions): Pager;
 //#endregion
-export { AddonData, AddonFD, NativeAddon, NativeReaderOptions, Pager, PagerOptions, ReaderOptions, createNativePager, createPager, createPager as default };
+export { NativeAddon, NativeReaderOptions, Pager, PagerOptions, ReaderOptions, WorkerMessage, createNativePager, createPager, createPager as default };