typescript-virtual-container 1.4.6 → 1.4.7

package/src/VirtualFileSystem/index.ts

@@ -1,3 +1,4 @@
+/** biome-ignore-all lint/style/useNamingConvention: NW ? */
 import { EventEmitter } from "node:events";
 import * as fsSync from "node:fs";
 import * as path from "node:path";
@@ -19,6 +20,7 @@ import type {
 	InternalFileNode,
 	InternalNode,
 } from "./internalTypes";
+import { appendJournalEntry, JournalOp, readJournal, truncateJournal } from "./journal";
 import { getNode, getParentDirectory, normalizePath } from "./path";
 
 // ── Persistence options ───────────────────────────────────────────────────────
@@ -46,6 +48,26 @@ export interface VfsOptions {
 	 * Required when `mode` is `"fs"`.
 	 */
 	snapshotPath?: string;
+	/**
+	 * Interval in milliseconds between automatic checkpoints in `"fs"` mode.
+	 * Set to `0` to disable automatic flushing (manual `flushMirror()` only).
+	 * Default: 30_000 (30 seconds).
+	 */
+	flushIntervalMs?: number;
+	/**
+	 * Trigger a checkpoint after this many write operations, regardless of the
+	 * timer interval. Prevents unbounded journal growth during bulk operations
+	 * (e.g. a 15 000-file SFTP transfer). Default: 500.
+	 * Set to `0` to disable write-count flushing.
+	 */
+	flushAfterNWrites?: number;
+	/**
+	 * Files larger than this threshold (bytes) are evicted from RAM after each
+	 * `flushMirror()` and reloaded on demand from the snapshot.
+	 * Default: 65536 (64 KB). Set to `0` to disable eviction.
+	 * Only applies to `"fs"` mode.
+	 */
+	evictionThresholdBytes?: number;
 }
 
 // ── VirtualFileSystem ─────────────────────────────────────────────────────────
@@ -77,6 +99,18 @@ class VirtualFileSystem extends EventEmitter {
 	private root: InternalDirectoryNode;
 	private readonly mode: VfsPersistenceMode;
 	private readonly snapshotFile: string | null;
+	/** Path to the WAL journal file (null in memory mode). */
+	private readonly journalFile: string | null;
+	/** Eviction threshold in bytes (0 = disabled). Files above this are purged after flush. */
+	private readonly evictionThreshold: number;
+	/** Max writes between forced flushes (0 = disabled). */
+	private readonly flushAfterNWrites: number;
+	/** Pending write counter since last checkpoint. */
+	private _writesSinceFlush = 0;
+	/** NodeJS timer handle for periodic auto-flush (null = disabled or stopped). */
+	private _flushTimer: ReturnType<typeof setInterval> | null = null;
+	/** True if the VFS has unflushed changes. */
+	private _dirty = false;
 	/** Active host-directory mounts: vPath → { hostPath, readOnly } */
 	private readonly mounts = new Map<string, { hostPath: string; readOnly: boolean }>();
 	/** True when running in a browser environment (no host FS access). */
@@ -96,8 +130,24 @@ class VirtualFileSystem extends EventEmitter {
 				options.snapshotPath,
 				"vfs-snapshot.vfsb",
 			);
+			this.journalFile = path.resolve(options.snapshotPath, "vfs-journal.bin");
+			this.evictionThreshold = options.evictionThresholdBytes ?? 64 * 1024; // 64 KB default
+			this.flushAfterNWrites = options.flushAfterNWrites ?? 500;
+			const intervalMs = options.flushIntervalMs ?? 30_000;
+			if (intervalMs > 0) {
+				this._flushTimer = setInterval(() => {
+					if (this._dirty) void this._autoFlush();
+				}, intervalMs);
+				// Don't block process exit on this timer
+				if (typeof this._flushTimer === "object" && this._flushTimer !== null && "unref" in this._flushTimer) {
+					(this._flushTimer as NodeJS.Timeout).unref();
+				}
+			}
 		} else {
 			this.snapshotFile = null;
+			this.journalFile = null;
+			this.evictionThreshold = 0; // disabled in memory mode
+			this.flushAfterNWrites = 0;
 		}
 		this.root = this.makeDir("", 0o755);
 	}
@@ -147,6 +197,7 @@ class VirtualFileSystem extends EventEmitter {
 				child = this.makeDir(part, mode);
 				current.children.set(part, child);
 				this.emit("dir:create", { path: builtPath, mode });
+				this._journal({ op: JournalOp.MKDIR, path: builtPath, mode });
 			} else if (child.type !== "directory") {
 				throw new Error(
 					`Cannot create directory '${builtPath}': path is a file.`,
@@ -168,7 +219,14 @@ class VirtualFileSystem extends EventEmitter {
 	public async restoreMirror(): Promise<void> {
 		if (this.mode !== "fs" || !this.snapshotFile) return;
 
-		if (!fsSync.existsSync(this.snapshotFile)) return;
+		if (!fsSync.existsSync(this.snapshotFile)) {
+			// No snapshot yet — but replay journal if it exists (crash after writes, before first flush)
+			if (this.journalFile) {
+				const entries = readJournal(this.journalFile);
+				if (entries.length > 0) this._replayJournal(entries);
+			}
+			return;
+		}
 
 		try {
 			const raw = fsSync.readFileSync(this.snapshotFile);
@@ -184,6 +242,11 @@ class VirtualFileSystem extends EventEmitter {
 				);
 			}
 			this.emit("snapshot:restore", { path: this.snapshotFile });
+			// Replay WAL journal on top of the loaded snapshot
+			if (this.journalFile) {
+				const entries = readJournal(this.journalFile);
+				if (entries.length > 0) this._replayJournal(entries);
+			}
 		} catch (err) {
 			// Corrupt or unreadable snapshot — start fresh and warn
 			console.warn(
@@ -210,7 +273,13 @@ class VirtualFileSystem extends EventEmitter {
 		fsSync.mkdirSync(dir, { recursive: true });
 		const binary = encodeVfs(this.root);
 		fsSync.writeFileSync(this.snapshotFile, binary);
+		// Checkpoint complete — truncate the journal (entries are now in the snapshot)
+		if (this.journalFile) truncateJournal(this.journalFile);
+		this._dirty = false;
+		this._writesSinceFlush = 0;
 		this.emit("mirror:flush", { path: this.snapshotFile });
+		// Evict large files from RAM now that the snapshot is on disk
+		this.evictLargeFiles();
 	}
 
 	/** Returns the current persistence mode. */
@@ -227,6 +296,147 @@ class VirtualFileSystem extends EventEmitter {
 
 	/** Creates a directory (and any missing parents). */
 
+
+
+	// ── Auto-flush scheduler ──────────────────────────────────────────────────
+
+	/** Internal: flush triggered by timer or write-count threshold. */
+	private async _autoFlush(): Promise<void> {
+		if (!this._dirty) return;
+		await this.flushMirror();
+	}
+
+	/** Mark VFS as having unflushed writes and trigger threshold flush if needed. */
+	private _markDirty(): void {
+		this._dirty = true;
+		if (this.flushAfterNWrites > 0) {
+			this._writesSinceFlush++;
+			if (this._writesSinceFlush >= this.flushAfterNWrites) {
+				this._writesSinceFlush = 0;
+				void this._autoFlush();
+			}
+		}
+	}
+
+	/**
+	 * Stop the automatic flush timer and perform a final checkpoint.
+	 * Call this when shutting down to ensure all data is persisted.
+	 *
+	 * @example
+	 * ```ts
+	 * process.on("SIGINT", async () => {
+	 *   await shell.vfs.stopAutoFlush();
+	 *   process.exit(0);
+	 * });
+	 * ```
+	 */
+	public async stopAutoFlush(): Promise<void> {
+		if (this._flushTimer !== null) {
+			clearInterval(this._flushTimer);
+			this._flushTimer = null;
+		}
+		if (this._dirty) await this.flushMirror();
+	}
+
+	// ── WAL Journal helpers ───────────────────────────────────────────────────
+
+	/** Set to true during journal replay to suppress re-journaling. */
+	private _replayMode = false;
+
+	/** Append a journal entry if in fs mode and not replaying. */
+	private _journal(entry: Parameters<typeof appendJournalEntry>[1]): void {
+		if (this.journalFile && !this._replayMode) {
+			appendJournalEntry(this.journalFile, entry);
+			this._markDirty();
+		}
+	}
+
+	/** Replay a list of journal entries onto the in-memory tree. */
+	private _replayJournal(entries: ReturnType<typeof readJournal>): void {
+		this._replayMode = true;
+		try {
+			for (const e of entries) {
+				try {
+					if (e.op === JournalOp.WRITE) {
+						this.writeFile(e.path, e.content ?? Buffer.alloc(0), { mode: e.mode });
+					} else if (e.op === JournalOp.MKDIR) {
+						this.mkdir(e.path, e.mode);
+					} else if (e.op === JournalOp.REMOVE) {
+						if (this.exists(e.path)) this.remove(e.path, { recursive: true });
+					} else if (e.op === JournalOp.CHMOD) {
+						if (this.exists(e.path)) this.chmod(e.path, e.mode ?? 0o644);
+					} else if (e.op === JournalOp.MOVE) {
+						if (this.exists(e.path) && e.dest) this.move(e.path, e.dest);
+					} else if (e.op === JournalOp.SYMLINK) {
+						if (e.dest) this.symlink(e.dest, e.path);
+					}
+				} catch { /* ignore individual replay errors — best-effort */ }
+			}
+		} finally {
+			this._replayMode = false;
+		}
+	}
+
+
+	// ── RAM eviction ──────────────────────────────────────────────────────────
+
+	/**
+	 * Walk the in-memory tree and evict file contents that exceed
+	 * `evictionThreshold`. Called automatically after `flushMirror()`.
+	 * Safe to call at any time — evicted files are reloaded on demand.
+	 */
+	public evictLargeFiles(): void {
+		if (!this.snapshotFile || this.evictionThreshold === 0) return;
+		if (!fsSync.existsSync(this.snapshotFile)) return;
+		this._evictDir(this.root);
+	}
+
+	private _evictDir(dir: InternalDirectoryNode): void {
+		for (const node of dir.children.values()) {
+			if (node.type === "directory") {
+				this._evictDir(node);
+			} else if (!node.evicted) {
+				const rawSize = node.compressed
+					? (node.size ?? node.content.length * 2) // estimate uncompressed
+					: node.content.length;
+				if (rawSize > this.evictionThreshold) {
+					node.size    = rawSize;
+					node.content = Buffer.alloc(0); // free heap
+					node.evicted = true;
+				}
+			}
+		}
+	}
+
+	/**
+	 * Reload a single evicted file node's content from the current snapshot.
+	 * No-op if the node is not evicted.
+	 */
+	private _reloadEvicted(node: InternalFileNode, normalizedPath: string): void {
+		if (!node.evicted || !this.snapshotFile) return;
+		if (!fsSync.existsSync(this.snapshotFile)) return;
+		try {
+			// Load and parse the snapshot to find this specific node
+			const raw = fsSync.readFileSync(this.snapshotFile);
+			const tmpRoot = decodeVfs(raw);
+			const parts = normalizedPath.split("/").filter(Boolean);
+			let cur: InternalNode = tmpRoot;
+			for (const part of parts) {
+				if (cur.type !== "directory") return;
+				const next = cur.children.get(part);
+				if (!next) return;
+				cur = next;
+			}
+			if (cur.type === "file") {
+				node.content = cur.content;
+				node.compressed = cur.compressed;
+				node.evicted = undefined;
+			}
+		} catch {
+			// Snapshot unreadable — leave evicted; caller will get empty content
+		}
+	}
+
 	// ── Mount API ─────────────────────────────────────────────────────────────
 
 	/**
@@ -387,6 +597,7 @@ class VirtualFileSystem extends EventEmitter {
 		}
 
 		this.emit("file:write", { path: normalized, size: storedContent.length });
+		this._journal({ op: JournalOp.WRITE, path: normalized, content: rawContent, mode });
 	}
 
 	/**
@@ -405,6 +616,7 @@ class VirtualFileSystem extends EventEmitter {
 			throw new Error(`Cannot read '${targetPath}': not a file.`);
 		}
 		const f = node as InternalFileNode;
+		if (f.evicted) this._reloadEvicted(f, normalized);
 		const raw = f.compressed ? gunzipSync(f.content) : f.content;
 		this.emit("file:read", { path: normalized, size: raw.length });
 		return raw.toString("utf8");
@@ -423,6 +635,7 @@ class VirtualFileSystem extends EventEmitter {
 			throw new Error(`Cannot read '${targetPath}': not a file.`);
 		}
 		const f = node as InternalFileNode;
+		if (f.evicted) this._reloadEvicted(f, normalized);
 		const raw = f.compressed ? gunzipSync(f.content) : f.content;
 		this.emit("file:read", { path: normalized, size: raw.length });
 		return raw;
@@ -442,7 +655,9 @@ class VirtualFileSystem extends EventEmitter {
 
 	/** Updates mode bits on a node. */
 	public chmod(targetPath: string, mode: number): void {
-		getNode(this.root, normalizePath(targetPath)).mode = mode;
+		const normalized = normalizePath(targetPath);
+		getNode(this.root, normalized).mode = mode;
+		this._journal({ op: JournalOp.CHMOD, path: normalized, mode });
 	}
 
 	/** Returns metadata for a file or directory. */
@@ -488,7 +703,7 @@ class VirtualFileSystem extends EventEmitter {
 				createdAt: f.createdAt,
 				updatedAt: f.updatedAt,
 				compressed: f.compressed,
-				size: f.content.length,
+				size: f.evicted ? (f.size ?? 0) : f.content.length,
 			};
 		}
 		const d = node as InternalDirectoryNode;
@@ -617,6 +832,8 @@ class VirtualFileSystem extends EventEmitter {
 			updatedAt: new Date(),
 		};
 		parent.children.set(name, symNode);
+		// Journal before emit
+		this._journal({ op: JournalOp.SYMLINK, path: normalizedLink, dest: normalizedTarget });
 		this.emit("symlink:create", {
 			link: normalizedLink,
 			target: normalizedTarget,
@@ -692,6 +909,7 @@ class VirtualFileSystem extends EventEmitter {
 		);
 		parent.children.delete(name);
 		this.emit("node:remove", { path: normalized });
+		this._journal({ op: JournalOp.REMOVE, path: normalized });
 	}
 
 	/** Moves or renames a node. */
@@ -721,6 +939,7 @@ class VirtualFileSystem extends EventEmitter {
 		srcParent.children.delete(srcName);
 		node.name = destName;
 		destParent.children.set(destName, node);
+		this._journal({ op: JournalOp.MOVE, path: fromNormalized, dest: toNormalized });
 	}
 
 	// ── Snapshot serialisation ─────────────────────────────────────────────────

package/src/VirtualFileSystem/internalTypes.ts

@@ -11,6 +11,10 @@ export interface InternalFileNode extends InternalBaseNode {
 	type: "file";
 	content: Buffer;
 	compressed: boolean;
+	/** When true, content has been purged from RAM. Reloaded from snapshot on demand. */
+	evicted?: true;
+	/** Byte length of the original (uncompressed) content — preserved when evicted. */
+	size?: number;
 }
 
 export interface InternalDirectoryNode extends InternalBaseNode {

package/src/VirtualFileSystem/journal.ts

@@ -0,0 +1,163 @@
+/**
+ * journal.ts — Write-Ahead Log for VirtualFileSystem "fs" mode.
+ *
+ * Each mutating VFS operation is appended to `vfs-journal.bin` immediately.
+ * On `restoreMirror()` the base snapshot is loaded first, then the journal
+ * is replayed on top. On `flushMirror()` a new checkpoint is written and
+ * the journal is truncated.
+ *
+ * Entry format (binary, little-endian):
+ *   [1B op] [2B path_len] [path bytes] [payload per op]
+ *
+ * Op codes:
+ *   0x01 WRITE  — [4B content_len] [content bytes] [4B mode]
+ *   0x02 MKDIR  — [4B mode]
+ *   0x03 REMOVE — (no payload)
+ *   0x04 CHMOD  — [4B mode]
+ *   0x05 MOVE   — [2B dest_len] [dest bytes]
+ *   0x06 SYMLINK — [2B target_len] [target bytes]
+ */
+/** biome-ignore-all lint/style/useNamingConvention: JournalOp modes */
+
+import * as fsSync from "node:fs";
+
+export const JournalOp = {
+	WRITE:   0x01,
+	MKDIR:   0x02,
+	REMOVE:  0x03,
+	CHMOD:   0x04,
+	MOVE:    0x05,
+	SYMLINK: 0x06,
+} as const;
+
+export type JournalOp = typeof JournalOp[keyof typeof JournalOp];
+
+export interface JournalEntry {
+	op:      JournalOp;
+	path:    string;
+	content?: Buffer;
+	mode?:   number;
+	dest?:   string;   // MOVE destination, SYMLINK target
+}
+
+const ENC = "utf8" as const;
+
+function writeString2(buf: Buffer, offset: number, s: string): number {
+	const b = Buffer.from(s, ENC);
+	buf.writeUInt16LE(b.length, offset);
+	b.copy(buf, offset + 2);
+	return 2 + b.length;
+}
+
+/** Serialise one entry to a Buffer. */
+export function encodeEntry(e: JournalEntry): Buffer {
+	const pathBuf = Buffer.from(e.path, ENC);
+	let payloadLen = 0;
+
+	if (e.op === JournalOp.WRITE) {
+		payloadLen = 4 + (e.content?.length ?? 0) + 4;
+	} else if (e.op === JournalOp.MKDIR) {
+		payloadLen = 4;
+	} else if (e.op === JournalOp.REMOVE) {
+		payloadLen = 0;
+	} else if (e.op === JournalOp.CHMOD) {
+		payloadLen = 4;
+	} else if (e.op === JournalOp.MOVE || e.op === JournalOp.SYMLINK) {
+		payloadLen = 2 + Buffer.byteLength(e.dest ?? "", ENC);
+	}
+
+	const total = 1 + 2 + pathBuf.length + payloadLen;
+	const buf   = Buffer.allocUnsafe(total);
+	let off = 0;
+
+	buf.writeUInt8(e.op, off++);
+	buf.writeUInt16LE(pathBuf.length, off); off += 2;
+	pathBuf.copy(buf, off); off += pathBuf.length;
+
+	if (e.op === JournalOp.WRITE) {
+		const c = e.content ?? Buffer.alloc(0);
+		buf.writeUInt32LE(c.length, off); off += 4;
+		c.copy(buf, off); off += c.length;
+		buf.writeUInt32LE(e.mode ?? 0o644, off); off += 4;
+	} else if (e.op === JournalOp.MKDIR) {
+		buf.writeUInt32LE(e.mode ?? 0o755, off); off += 4;
+	} else if (e.op === JournalOp.CHMOD) {
+		buf.writeUInt32LE(e.mode ?? 0o644, off); off += 4;
+	} else if (e.op === JournalOp.MOVE || e.op === JournalOp.SYMLINK) {
+		off += writeString2(buf, off, e.dest ?? "");
+	}
+
+	return buf;
+}
+
+/** Parse all entries from a journal Buffer. Returns empty array on corrupt data. */
+export function decodeJournal(buf: Buffer): JournalEntry[] {
+	const entries: JournalEntry[] = [];
+	let off = 0;
+
+	try {
+		while (off < buf.length) {
+			if (off + 3 > buf.length) break;
+			const op      = buf.readUInt8(off++) as JournalOp;
+			const pathLen = buf.readUInt16LE(off); off += 2;
+			if (off + pathLen > buf.length) break;
+			const path    = buf.subarray(off, off + pathLen).toString(ENC); off += pathLen;
+
+			if (op === JournalOp.WRITE) {
+				if (off + 4 > buf.length) break;
+				const cLen = buf.readUInt32LE(off); off += 4;
+				if (off + cLen + 4 > buf.length) break;
+				const content = Buffer.from(buf.subarray(off, off + cLen)); off += cLen;
+				const mode    = buf.readUInt32LE(off); off += 4;
+				entries.push({ op, path, content, mode });
+			} else if (op === JournalOp.MKDIR) {
+				if (off + 4 > buf.length) break;
+				const mode = buf.readUInt32LE(off); off += 4;
+				entries.push({ op, path, mode });
+			} else if (op === JournalOp.REMOVE) {
+				entries.push({ op, path });
+			} else if (op === JournalOp.CHMOD) {
+				if (off + 4 > buf.length) break;
+				const mode = buf.readUInt32LE(off); off += 4;
+				entries.push({ op, path, mode });
+			} else if (op === JournalOp.MOVE || op === JournalOp.SYMLINK) {
+				if (off + 2 > buf.length) break;
+				const dLen = buf.readUInt16LE(off); off += 2;
+				if (off + dLen > buf.length) break;
+				const dest = buf.subarray(off, off + dLen).toString(ENC); off += dLen;
+				entries.push({ op, path, dest });
+			} else {
+				// Unknown op — skip rest of buffer to avoid corrupt replay
+				break;
+			}
+		}
+	} catch {
+		// Truncated or corrupt entry — return what we have
+	}
+
+	return entries;
+}
+
+/** Append a single entry to the journal file (O_APPEND, atomic write). */
+export function appendJournalEntry(journalPath: string, entry: JournalEntry): void {
+	const buf = encodeEntry(entry);
+	const fd  = fsSync.openSync(journalPath, fsSync.constants.O_WRONLY | fsSync.constants.O_CREAT | fsSync.constants.O_APPEND);
+	try {
+		fsSync.writeSync(fd, buf);
+	} finally {
+		fsSync.closeSync(fd);
+	}
+}
+
+/** Read and decode all entries from a journal file. Returns [] if file is absent/empty. */
+export function readJournal(journalPath: string): JournalEntry[] {
+	if (!fsSync.existsSync(journalPath)) return [];
+	const buf = fsSync.readFileSync(journalPath);
+	if (buf.length === 0) return [];
+	return decodeJournal(buf);
+}
+
+/** Delete the journal file (after a successful checkpoint). */
+export function truncateJournal(journalPath: string): void {
+	if (fsSync.existsSync(journalPath)) fsSync.unlinkSync(journalPath);
+}

package/src/VirtualShell/shell.ts

@@ -202,7 +202,7 @@ export function startShell(
 			cwd = result.nextCwd;
 		}
 
-		await shell.vfs.flushMirror();
+		// WAL: checkpoint handled by auto-flush timer
 		renderLine();
 	}
 
@@ -221,7 +221,7 @@ export function startShell(
 					activeSession.targetPath,
 					updatedContent,
 				);
-				await shell.vfs.flushMirror();
+				// WAL: checkpoint handled by auto-flush timer
 			} catch {
 				// If temp file does not exist, nano exited without writing.
 			}
@@ -498,7 +498,7 @@ export function startShell(
 				historyDraft = "";
 				stream.write("bye\r\n");
 				pushHistory("bye");
-				await shell.vfs.flushMirror();
+				// WAL: checkpoint handled by auto-flush timer
 				stream.write("logout\r\n");
 				stream.exit(0);
 				stream.end();
@@ -657,7 +657,7 @@ export function startShell(
 						cursorPos = 0;
 					}
 
-					await shell.vfs.flushMirror();
+					// WAL: checkpoint handled by auto-flush timer
 				}
 
 				renderLine();

package/src/commands/man.ts

@@ -4,8 +4,10 @@ const MANUAL_ALIASES: Record<string, string> = {
 	gunzip: "gzip",
 };
 
+const __dirname = import.meta.dirname;
+
 const manualCache = new Map<string, string | null>();
-const manualsBaseUrl = `${__dirname}/manuals/`;
+const manualsBaseUrl = new URL("./manuals/", import.meta.url);
 
 async function dynamicImport(specifier: string): Promise<unknown> {
 	const importer = new Function(

package/src/self-standalone.ts

@@ -62,7 +62,7 @@ function writeLastLogin(username: string, from: string): void {
 }
 
 async function flushVfs(): Promise<void> {
-	await virtualShell.vfs.flushMirror();
+	await virtualShell.vfs.stopAutoFlush();
 }
 
 function loadHistory(authUser: string): string[] {