readline-pager 0.2.6 → 0.3.0

package/dist/main.d.mts

@@ -9,12 +9,10 @@ interface PagerOptions extends Partial<ReaderOptions> {
   backward?: boolean;
   useWorker?: boolean;
 }
-interface Pager extends AsyncIterable<string[]> {
+interface Pager extends AsyncIterable<string[]>, Iterable<string[]> {
   next(): Promise<string[] | null>;
-  close(): void;
-  readonly lineCount: number;
-  readonly firstLine: string | null;
-  readonly lastLine: string | null;
+  nextSync(): string[] | null;
+  close(): Promise<void>;
 }
 //#endregion
 //#region src/main.d.ts

package/dist/main.mjs

@@ -1,67 +1,189 @@
 import { createRequire } from "node:module";
+import { closeSync, openSync, readSync, statSync } from "node:fs";
 import { open } from "node:fs/promises";
 import { Worker } from "node:worker_threads";
-
 //#region \0rolldown/runtime.js
 var __require = /* @__PURE__ */ createRequire(import.meta.url);
-
 //#endregion
 //#region src/queue.ts
-function createPageQueue() {
-	const queue = [];
-	let resolver = null;
+function createRingBuffer(capacity) {
+	if (!Number.isFinite(capacity) || capacity <= 0) throw new RangeError("capacity must be a positive number");
+	let buf = new Array(capacity);
+	let head = 0;
+	let tail = 0;
+	let count = 0;
+	let consumerWaiter = null;
+	let producerWaiter = null;
+	function push(item) {
+		if (count === buf.length) {
+			const newCap = buf.length * 2;
+			const newBuf = new Array(newCap);
+			for (let i = 0; i < count; i++) newBuf[i] = buf[(head + i) % buf.length];
+			buf = newBuf;
+			head = 0;
+			tail = count;
+		}
+		buf[tail] = item;
+		tail++;
+		if (tail === buf.length) tail = 0;
+		count++;
+		if (consumerWaiter) {
+			const w = consumerWaiter;
+			consumerWaiter = null;
+			w();
+		}
+	}
+	function shiftSync() {
+		if (count === 0) return null;
+		const v = buf[head];
+		buf[head] = void 0;
+		head++;
+		if (head === buf.length) head = 0;
+		count--;
+		if (producerWaiter) {
+			const w = producerWaiter;
+			producerWaiter = null;
+			w();
+		}
+		return v;
+	}
+	async function shift(done = false) {
+		if (count) return shiftSync();
+		if (done) return null;
+		await new Promise((r) => {
+			consumerWaiter = r;
+		});
+		if (count) return shiftSync();
+		return null;
+	}
+	function wake() {
+		if (consumerWaiter) {
+			const w = consumerWaiter;
+			consumerWaiter = null;
+			w();
+		}
+		if (producerWaiter) {
+			const w = producerWaiter;
+			producerWaiter = null;
+			w();
+		}
+	}
+	function clear() {
+		for (let i = 0; i < buf.length; i++) buf[i] = void 0;
+		head = 0;
+		tail = 0;
+		count = 0;
+		wake();
+	}
 	return {
-		queue,
-		push(page) {
-			queue.push(page);
-			resolver?.();
-			resolver = null;
-		},
-		wake() {
-			resolver?.();
-			resolver = null;
+		push,
+		shift,
+		shiftSync,
+		wake,
+		clear,
+		get count() {
+			return count;
 		},
-		async shift(done) {
-			if (queue.length) return queue.shift();
-			if (done()) return null;
-			await new Promise((r) => resolver = r);
-			if (queue.length) return queue.shift();
-			if (done()) return null;
-			return null;
+		get capacity() {
+			return buf.length;
 		}
 	};
 }
-
 //#endregion
 //#region src/reader/backward.reader.ts
 function createBackwardReader(filepath, options) {
 	const { chunkSize, pageSize, delimiter, prefetch } = options;
-	const pageQueue = createPageQueue();
+	const pageQueue = createRingBuffer(Math.max(2, prefetch + 1));
+	const local = [];
 	let fd = null;
+	let fdSync = null;
 	let pos = 0;
 	let buffer = "";
 	let done = false;
 	let closed = false;
-	let emittedCount = 0;
-	let firstLine = null;
-	let lastLine = null;
-	const local = [];
-	async function init() {
-		if (fd) return;
-		fd = await open(filepath, "r");
-		pos = (await fd.stat()).size;
-		if (pos === 0) done = true;
+	let startsWithDelimiter = false;
+	fdSync = openSync(filepath, "r");
+	pos = statSync(filepath).size;
+	if (pos === 0) {
+		pageQueue.push([buffer]);
+		done = true;
+		pageQueue.wake();
 	}
-	async function fill() {
+	(async () => {
+		try {
+			fd = await open(filepath, "r");
+			pos = (await fd.stat()).size;
+			if (pos === 0) {
+				if (!done) {
+					pageQueue.push([buffer]);
+					done = true;
+				}
+				if (fd) {
+					await fd.close();
+					fd = null;
+				}
+				pageQueue.wake();
+				return;
+			}
+			while (!done && !closed) {
+				while (pageQueue.count < prefetch && pos > 0 && !closed) {
+					const readSize = Math.min(chunkSize, pos);
+					pos -= readSize;
+					const buf = Buffer.allocUnsafe(readSize);
+					await fd.read(buf, 0, readSize, pos);
+					buffer = buf.toString("utf8") + buffer;
+					if (pos === 0 && buffer.startsWith(delimiter)) startsWithDelimiter = true;
+					let idx;
+					while ((idx = buffer.lastIndexOf(delimiter)) !== -1) {
+						const line = buffer.slice(idx + delimiter.length);
+						buffer = buffer.slice(0, idx);
+						local.push(line);
+						while (local.length >= pageSize) {
+							const page = local.splice(0, pageSize);
+							pageQueue.push(page);
+						}
+					}
+				}
+				if (pos === 0 && !done) {
+					if (buffer.length > 0) local.push(buffer);
+					else if (startsWithDelimiter) local.push("");
+					buffer = "";
+					while (local.length > 0 && !closed) {
+						const page = local.slice(local.length - Math.min(pageSize, local.length));
+						local.length -= page.length;
+						pageQueue.push(page);
+					}
+					done = true;
+					if (fd) {
+						await fd.close();
+						fd = null;
+					}
+					pageQueue.wake();
+					break;
+				}
+				if (!done && !closed) await new Promise((r) => setImmediate(r));
+			}
+		} catch {
+			done = true;
+			pageQueue.wake();
+			try {
+				if (fd) {
+					await fd.close();
+					fd = null;
+				}
+			} catch {}
+		}
+	})();
+	function fillSync() {
 		if (done || closed) return;
-		await init();
-		if (!fd) return;
-		while (pageQueue.queue.length < prefetch && pos > 0) {
+		if (fdSync === null) return;
+		while (pageQueue.count < prefetch && pos > 0 && !closed) {
 			const readSize = Math.min(chunkSize, pos);
 			pos -= readSize;
 			const buf = Buffer.allocUnsafe(readSize);
-			await fd.read(buf, 0, readSize, pos);
-			buffer += buf.toString("utf8");
+			readSync(fdSync, buf, 0, readSize, pos);
+			buffer = buf.toString("utf8") + buffer;
+			if (pos === 0 && buffer.startsWith(delimiter)) startsWithDelimiter = true;
 			let idx;
 			while ((idx = buffer.lastIndexOf(delimiter)) !== -1) {
 				const line = buffer.slice(idx + delimiter.length);
@@ -73,90 +195,178 @@ function createBackwardReader(filepath, options) {
 				}
 			}
 		}
-		if (pos === 0) {
-			local.push(buffer);
+		if (pos === 0 && !done) {
+			if (buffer.length > 0) local.push(buffer);
+			else if (startsWithDelimiter) local.push("");
 			buffer = "";
 			while (local.length > 0) {
-				const sliceSize = Math.min(pageSize, local.length);
-				const page = local.splice(local.length - sliceSize, sliceSize);
+				const page = local.slice(local.length - Math.min(pageSize, local.length));
+				local.length -= page.length;
 				pageQueue.push(page);
 			}
 			done = true;
-			if (fd) {
-				await fd.close();
-				fd = null;
+			if (fdSync !== null) {
+				closeSync(fdSync);
+				fdSync = null;
 			}
+			pageQueue.wake();
 		}
 	}
 	async function next() {
 		if (closed) return null;
-		await fill();
-		const page = await pageQueue.shift(() => done);
-		if (!page) return null;
-		emittedCount += page.length;
-		firstLine ??= page[0];
-		lastLine = page[page.length - 1];
-		return page;
+		return await pageQueue.shift(done);
+	}
+	function nextSync() {
+		if (closed) return null;
+		fillSync();
+		return pageQueue.shiftSync();
 	}
 	async function close() {
 		closed = true;
 		done = true;
-		pageQueue.queue.length = 0;
-		if (fd) await fd.close();
+		pageQueue.clear();
+		if (fd) {
+			try {
+				await fd.close();
+			} catch {}
+			fd = null;
+		}
+		if (fdSync !== null) {
+			try {
+				closeSync(fdSync);
+			} catch {}
+			fdSync = null;
+		}
 	}
 	return {
 		next,
+		nextSync,
 		close,
-		get lineCount() {
-			return emittedCount;
-		},
-		get firstLine() {
-			return firstLine;
-		},
-		get lastLine() {
-			return lastLine;
-		},
 		async *[Symbol.asyncIterator]() {
-			while (true) {
-				const p = await next();
-				if (!p) break;
-				yield p;
+			try {
+				while (true) {
+					const p = await next();
+					if (!p) break;
+					yield p;
+				}
+			} finally {
+				await close();
+			}
+		},
+		*[Symbol.iterator]() {
+			try {
+				while (true) {
+					const p = nextSync();
+					if (!p) break;
+					yield p;
+				}
+			} finally {
+				closed = true;
+				done = true;
+				pageQueue.clear();
+				try {
+					if (fdSync) closeSync(fdSync);
+				} catch {}
+				fdSync = null;
+				try {
+					if (fd?.fd) closeSync(fd.fd);
+				} catch {}
+				fd = null;
 			}
 		}
 	};
 }
-
 //#endregion
 //#region src/reader/forward.reader.ts
 function createForwardReader(filepath, options) {
 	const { chunkSize, pageSize, delimiter, prefetch } = options;
-	const pageQueue = createPageQueue();
+	const pageQueue = createRingBuffer(Math.max(2, prefetch + 1));
+	const local = [];
 	let fd = null;
+	let fdSync = null;
 	let pos = 0;
 	let size = 0;
 	let buffer = "";
 	let done = false;
 	let closed = false;
-	let emittedCount = 0;
-	let firstLine = null;
-	let lastLine = null;
-	const local = [];
-	async function init() {
-		if (fd) return;
-		fd = await open(filepath, "r");
-		size = (await fd.stat()).size;
-		if (size === 0) done = true;
+	let flushed = false;
+	fdSync = openSync(filepath, "r");
+	size = statSync(filepath).size;
+	if (size === 0) {
+		pageQueue.push([buffer]);
+		done = true;
+		pageQueue.wake();
 	}
-	async function fill() {
+	(async () => {
+		try {
+			fd = await open(filepath, "r");
+			size = (await fd.stat()).size;
+			if (size === 0) {
+				if (!done) {
+					pageQueue.push([buffer]);
+					done = true;
+				}
+				if (fd) {
+					await fd.close();
+					fd = null;
+				}
+				pageQueue.wake();
+				return;
+			}
+			while (!done && !closed) {
+				while (pageQueue.count < prefetch && pos < size && !closed) {
+					const readSize = Math.min(chunkSize, size - pos);
+					const buf = Buffer.allocUnsafe(readSize);
+					const { bytesRead } = await fd.read(buf, 0, readSize, pos);
+					pos += bytesRead;
+					buffer = buffer + buf.toString("utf8", 0, bytesRead);
+					let idx;
+					while ((idx = buffer.indexOf(delimiter)) !== -1) {
+						const line = buffer.slice(0, idx);
+						buffer = buffer.slice(idx + delimiter.length);
+						local.push(line);
+						while (local.length >= pageSize) pageQueue.push(local.splice(0, pageSize));
+					}
+				}
+				if (pos >= size && !flushed) {
+					flushed = true;
+					local.push(buffer.length > 0 ? buffer : "");
+					buffer = "";
+					while (local.length > 0 && !closed) {
+						const page = local.slice(0, pageSize);
+						local.length -= page.length;
+						pageQueue.push(page);
+					}
+					done = true;
+					if (fd) {
+						await fd.close();
+						fd = null;
+					}
+					pageQueue.wake();
+					break;
+				}
+				if (!done && !closed) await new Promise((r) => setImmediate(r));
+			}
+		} catch {
+			done = true;
+			pageQueue.wake();
+			try {
+				if (fd) {
+					await fd.close();
+					fd = null;
+				}
+			} catch {}
+		}
+	})();
+	function fillSync() {
 		if (done || closed) return;
-		await init();
-		if (!fd) return;
-		while (pageQueue.queue.length < prefetch && pos < size) {
+		if (fdSync === null) return;
+		while (pageQueue.count < prefetch && pos < size && !closed) {
 			const readSize = Math.min(chunkSize, size - pos);
 			const buf = Buffer.allocUnsafe(readSize);
-			const { bytesRead } = await fd.read(buf, 0, readSize, pos);
+			const bytesRead = readSync(fdSync, buf, 0, readSize, pos);
 			pos += bytesRead;
-			buffer += buf.toString("utf8", 0, bytesRead);
+			buffer = buffer + buf.toString("utf8", 0, bytesRead);
 			let idx;
 			while ((idx = buffer.indexOf(delimiter)) !== -1) {
 				const line = buffer.slice(0, idx);
@@ -165,73 +375,102 @@ function createForwardReader(filepath, options) {
 				while (local.length >= pageSize) pageQueue.push(local.splice(0, pageSize));
 			}
 		}
-		if (pos >= size) {
-			const parts = buffer.length > 0 ? buffer.split(delimiter) : [""];
-			for (const line of parts) local.push(line);
+		if (pos >= size && !flushed) {
+			flushed = true;
+			local.push(buffer.length > 0 ? buffer : "");
 			buffer = "";
-			while (local.length > 0) pageQueue.push(local.splice(0, pageSize));
+			while (local.length > 0) {
+				const page = local.slice(0, pageSize);
+				local.length -= page.length;
+				pageQueue.push(page);
+			}
 			done = true;
-			if (fd) {
-				await fd.close();
-				fd = null;
+			if (fdSync !== null) {
+				closeSync(fdSync);
+				fdSync = null;
 			}
+			pageQueue.wake();
 		}
 	}
 	async function next() {
 		if (closed) return null;
-		await fill();
-		const page = await pageQueue.shift(() => done);
-		if (!page) return null;
-		emittedCount += page.length;
-		firstLine ??= page[0];
-		lastLine = page[page.length - 1];
-		return page;
+		return await pageQueue.shift(done);
+	}
+	function nextSync() {
+		if (closed) return null;
+		fillSync();
+		return pageQueue.shiftSync();
 	}
 	async function close() {
 		closed = true;
 		done = true;
-		pageQueue.queue.length = 0;
-		if (fd) await fd.close();
+		pageQueue.clear();
+		if (fd) {
+			try {
+				await fd.close();
+			} catch {}
+			fd = null;
+		}
+		if (fdSync !== null) {
+			try {
+				closeSync(fdSync);
+			} catch {}
+			fdSync = null;
+		}
 	}
 	return {
 		next,
+		nextSync,
 		close,
-		get lineCount() {
-			return emittedCount;
-		},
-		get firstLine() {
-			return firstLine;
-		},
-		get lastLine() {
-			return lastLine;
-		},
 		async *[Symbol.asyncIterator]() {
-			while (true) {
-				const p = await next();
-				if (!p) break;
-				yield p;
+			try {
+				while (true) {
+					const p = await next();
+					if (!p) break;
+					yield p;
+				}
+			} finally {
+				await close();
+			}
+		},
+		*[Symbol.iterator]() {
+			try {
+				while (true) {
+					const p = nextSync();
+					if (!p) break;
+					yield p;
+				}
+			} finally {
+				closed = true;
+				done = true;
+				pageQueue.clear();
+				try {
+					if (fdSync !== null) closeSync(fdSync);
+				} catch {}
+				fdSync = null;
+				try {
+					if (fd?.fd) closeSync(fd.fd);
+				} catch {}
+				fd = null;
 			}
 		}
 	};
 }
-
 //#endregion
 //#region src/reader/worker.reader.ts
 const workerFile = typeof import.meta !== "undefined" ? new URL("./worker.mjs", import.meta.url) : __require.resolve("./worker.cjs");
 function createWorkerReader(filepath, options) {
 	const { chunkSize, pageSize, delimiter, prefetch } = options;
+	const pageQueue = createRingBuffer(Math.max(2, prefetch + 1));
+	let done = false;
+	let closed = false;
 	const worker = new Worker(new URL(workerFile, import.meta.url), { workerData: {
 		filepath,
 		chunkSize,
 		pageSize,
-		delimiter
+		delimiter,
+		prefetch
 	} });
-	const pageQueue = createPageQueue();
-	let done = false;
-	let closed = false;
-	let emittedCount = 0;
-	let firstLine = null;
-	let lastLine = null;
 	worker.on("message", (msg) => {
 		if (msg.type === "page") pageQueue.push(msg.data);
 		if (msg.type === "done") {
@@ -239,49 +478,66 @@ function createWorkerReader(filepath, options) {
 			pageQueue.wake();
 		}
 	});
+	worker.on("error", () => {
+		done = true;
+		pageQueue.wake();
+	});
+	worker.on("exit", () => {
+		done = true;
+		pageQueue.wake();
+	});
 	async function next() {
 		if (closed) return null;
-		const page = await pageQueue.shift(() => done);
-		if (!page) return null;
-		emittedCount += page.length;
-		firstLine ??= page[0];
-		lastLine = page[page.length - 1];
-		return page;
+		return await pageQueue.shift(done);
+	}
+	function nextSync() {
+		if (closed) return null;
+		return pageQueue.shiftSync();
 	}
 	async function close() {
 		closed = true;
 		done = true;
-		worker.terminate();
+		pageQueue.clear();
+		await worker.terminate();
+	}
+	function tryClose() {
+		close().catch(() => {});
 	}
 	return {
 		next,
+		nextSync,
 		close,
-		get lineCount() {
-			return emittedCount;
-		},
-		get firstLine() {
-			return firstLine;
-		},
-		get lastLine() {
-			return lastLine;
-		},
 		async *[Symbol.asyncIterator]() {
-			while (true) {
-				const p = await next();
-				if (!p) break;
-				yield p;
+			try {
+				while (true) {
+					const p = await next();
+					if (!p) break;
+					yield p;
+				}
+			} finally {
+				tryClose();
+			}
+		},
+		*[Symbol.iterator]() {
+			try {
+				while (true) {
+					const p = nextSync();
+					if (!p) break;
+					yield p;
+				}
+			} finally {
+				tryClose();
 			}
 		}
 	};
 }
-
 //#endregion
 //#region src/main.ts
 function createPager(filepath, options = {}) {
-	const { chunkSize = 64 * 1024, pageSize = 1e3, delimiter = "\n", prefetch = 1, backward = false, useWorker = false } = options;
+	const { chunkSize = 64 * 1024, pageSize = 1e3, delimiter = "\n", prefetch = 8, backward = false, useWorker = false } = options;
 	if (!filepath) throw new Error("filepath required");
-	if (pageSize <= 0) throw new RangeError("pageSize must be > 0");
-	if (prefetch <= 0) throw new RangeError("prefetch must be >= 1");
+	if (pageSize < 1) throw new RangeError("pageSize must be >= 1");
+	if (prefetch < 1) throw new RangeError("prefetch must be >= 1");
 	if (backward && useWorker) throw new Error("backward not supported with useWorker");
 	return useWorker ? createWorkerReader(filepath, {
 		chunkSize,
@@ -300,6 +556,5 @@ function createPager(filepath, options = {}) {
 		delimiter
 	});
 }
-
 //#endregion
-export { createPager, createPager as default };
+export { createPager, createPager as default };