readline-pager 0.5.1 → 0.6.2

package/dist/worker.cjs

@@ -3,38 +3,50 @@ let node_worker_threads = require("node:worker_threads");
 //#region src/worker.ts
 const { filepath, options } = node_worker_threads.workerData;
 const { chunkSize, pageSize, delimiter } = options;
+const backpressure = pageSize * 8;
+const post = (msg) => {
+	if (!node_worker_threads.parentPort) process.exit(1);
+	node_worker_threads.parentPort.postMessage(msg);
+};
 (async () => {
-	const fd = await (0, node_fs_promises.open)(filepath, "r");
-	const { size } = await fd.stat();
-	let pos = 0;
-	let buffer = "";
-	const local = [];
-	while (pos < size) {
-		const readSize = Math.min(chunkSize, size - pos);
-		const buf = Buffer.allocUnsafe(readSize);
-		const { bytesRead } = await fd.read(buf, 0, readSize, pos);
-		pos += bytesRead;
-		buffer += buf.toString("utf8", 0, bytesRead);
-		let idx;
-		while ((idx = buffer.indexOf(delimiter)) !== -1) {
-			const line = buffer.slice(0, idx);
-			buffer = buffer.slice(idx + delimiter.length);
-			local.push(line);
-			while (local.length >= pageSize) node_worker_threads.parentPort?.postMessage({
-				type: "page",
-				data: local.splice(0, pageSize)
-			});
+	try {
+		const fd = await (0, node_fs_promises.open)(filepath, "r");
+		const { size } = await fd.stat();
+		let pos = 0;
+		let buffer = "";
+		const local = [];
+		while (pos < size) {
+			const readSize = Math.min(chunkSize, size - pos);
+			const buf = Buffer.allocUnsafe(readSize);
+			const { bytesRead } = await fd.read(buf, 0, readSize, pos);
+			pos += bytesRead;
+			buffer = buffer + buf.toString("utf8", 0, bytesRead);
+			let idx;
+			while ((idx = buffer.indexOf(delimiter)) !== -1) {
+				const line = buffer.slice(0, idx);
+				buffer = buffer.slice(idx + delimiter.length);
+				local.push(line);
+				if (local.length >= pageSize) {
+					post({
+						type: "page",
+						data: local.splice(0, pageSize)
+					});
+					if (local.length > backpressure) await new Promise((r) => setImmediate(r));
+				}
+			}
 		}
-	}
-	local.push(buffer);
-	while (local.length > 0) {
-		const page = local.splice(0, pageSize);
-		node_worker_threads.parentPort?.postMessage({
+		local.push(buffer);
+		while (local.length > 0) post({
 			type: "page",
-			data: page
+			data: local.splice(0, pageSize)
+		});
+		post({ type: "done" });
+		await fd.close();
+	} catch (err) {
+		post({
+			type: "error",
+			error: err
 		});
 	}
-	node_worker_threads.parentPort?.postMessage({ type: "done" });
-	await fd.close();
 })();
 //#endregion

package/dist/worker.mjs

@@ -3,39 +3,51 @@ import { parentPort, workerData } from "node:worker_threads";
 //#region src/worker.ts
 const { filepath, options } = workerData;
 const { chunkSize, pageSize, delimiter } = options;
+const backpressure = pageSize * 8;
+const post = (msg) => {
+	if (!parentPort) process.exit(1);
+	parentPort.postMessage(msg);
+};
 (async () => {
-	const fd = await open(filepath, "r");
-	const { size } = await fd.stat();
-	let pos = 0;
-	let buffer = "";
-	const local = [];
-	while (pos < size) {
-		const readSize = Math.min(chunkSize, size - pos);
-		const buf = Buffer.allocUnsafe(readSize);
-		const { bytesRead } = await fd.read(buf, 0, readSize, pos);
-		pos += bytesRead;
-		buffer += buf.toString("utf8", 0, bytesRead);
-		let idx;
-		while ((idx = buffer.indexOf(delimiter)) !== -1) {
-			const line = buffer.slice(0, idx);
-			buffer = buffer.slice(idx + delimiter.length);
-			local.push(line);
-			while (local.length >= pageSize) parentPort?.postMessage({
-				type: "page",
-				data: local.splice(0, pageSize)
-			});
+	try {
+		const fd = await open(filepath, "r");
+		const { size } = await fd.stat();
+		let pos = 0;
+		let buffer = "";
+		const local = [];
+		while (pos < size) {
+			const readSize = Math.min(chunkSize, size - pos);
+			const buf = Buffer.allocUnsafe(readSize);
+			const { bytesRead } = await fd.read(buf, 0, readSize, pos);
+			pos += bytesRead;
+			buffer = buffer + buf.toString("utf8", 0, bytesRead);
+			let idx;
+			while ((idx = buffer.indexOf(delimiter)) !== -1) {
+				const line = buffer.slice(0, idx);
+				buffer = buffer.slice(idx + delimiter.length);
+				local.push(line);
+				if (local.length >= pageSize) {
+					post({
+						type: "page",
+						data: local.splice(0, pageSize)
+					});
+					if (local.length > backpressure) await new Promise((r) => setImmediate(r));
+				}
+			}
 		}
-	}
-	local.push(buffer);
-	while (local.length > 0) {
-		const page = local.splice(0, pageSize);
-		parentPort?.postMessage({
+		local.push(buffer);
+		while (local.length > 0) post({
 			type: "page",
-			data: page
+			data: local.splice(0, pageSize)
+		});
+		post({ type: "done" });
+		await fd.close();
+	} catch (err) {
+		post({
+			type: "error",
+			error: err
 		});
 	}
-	parentPort?.postMessage({ type: "done" });
-	await fd.close();
 })();
 //#endregion
 export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "readline-pager",
-  "version": "0.5.1",
+  "version": "0.6.2",
   "scripts": {
     "build:js": "tsdown",
     "build:native": "node-gyp rebuild",
@@ -9,9 +9,9 @@
     "pretest": "npm run build",
     "test": "TEST_CLEANUPS=1 node --test --experimental-test-coverage test/**/*.test.ts",
     "prepublishOnly": "npm run build && npm run test",
-    "benchmark:node": "node test/_benchmark.ts",
-    "benchmark:deno": "deno --allow-write --allow-read --allow-env test/_benchmark.ts",
-    "benchmark:bun": "bun test/_benchmark.ts"
+    "benchmark:node": "node test/benchmark.ts",
+    "benchmark:deno": "deno --allow-write --allow-read --allow-env test/benchmark.ts",
+    "benchmark:bun": "bun test/benchmark.ts"
   },
   "devDependencies": {
     "@types/bun": "~1.3.11",
@@ -20,14 +20,14 @@
     "node-gyp": "~12.2.0",
     "prettier": "~3.8.1",
     "prettier-plugin-organize-imports": "~4.3.0",
-    "tsdown": "~0.21.5",
+    "tsdown": "~0.21.6",
     "typescript": "~6.0.2"
   },
   "optionalDependencies": {
-    "@devmor-j/readline-pager-linux-arm64": "0.5.1",
-    "@devmor-j/readline-pager-linux-musl-arm64": "0.5.1",
-    "@devmor-j/readline-pager-linux-musl-x64": "0.5.1",
-    "@devmor-j/readline-pager-linux-x64": "0.5.1"
+    "@devmor-j/readline-pager-linux-arm64": "0.6.2",
+    "@devmor-j/readline-pager-linux-musl-arm64": "0.6.2",
+    "@devmor-j/readline-pager-linux-musl-x64": "0.6.2",
+    "@devmor-j/readline-pager-linux-x64": "0.6.2"
   },
   "gypfile": false,
   "type": "module",