pi-supernova 0.5.0 → 0.7.0

package/src/fs/vfs.js

@@ -1,20 +1,240 @@
 import * as fs from "node:fs/promises";
 import * as path from "node:path";
 import { isString } from "../shared/decode.js";
-import { randomUUID } from "node:crypto";
-
-const VFS_CACHE_MAX = 1024;
+import { createHash, randomUUID } from "node:crypto";
 
 // Serialize validation + replacement across Supernova transactions in this host.
 let commitTail = Promise.resolve();
 
+function textSignature(text) {
+  return { size: Buffer.byteLength(text, "utf8"), sha256: createHash("sha256").update(text, "utf8").digest("hex") };
+}
+
+function sameFileVersion(a, b) {
+  return ["dev", "ino", "size", "mtimeMs", "ctimeMs"].every(key => a[key] === b[key]);
+}
+
+async function fileSignature(target, signal, observed) {
+  const file = await fs.open(target, fs.constants.O_RDONLY | (fs.constants.O_NONBLOCK ?? 0));
+
+  try {
+    const actual = await file.stat();
+
+    if (!actual.isFile()) throw new Error("read requires a regular file: " + target);
+    if (observed && !sameFileVersion(observed, actual)) throw new Error("file changed while reading: " + target);
+    const hash = createHash("sha256");
+
+    for await (const chunk of file.createReadStream({ autoClose: false, signal })) hash.update(chunk);
+    const after = await file.stat();
+
+    if (!after.isFile() || !sameFileVersion(actual, after)) throw new Error("file changed while signing: " + target);
+
+    return { size: actual.size, sha256: hash.digest("hex") };
+  } finally {
+    await file.close();
+  }
+}
+
+// realpath() cannot resolve a missing leaf. Canonicalize its nearest existing
+// ancestor so two symlink spellings still share one commit destination.
+async function canonicalNewPath(target) {
+  let ancestor = path.dirname(target);
+
+  for (;;) {
+    try { return path.join(await fs.realpath(ancestor), path.relative(ancestor, target)); }
+    catch (error) {
+      if (error.code !== "ENOENT") throw error;
+      const parent = path.dirname(ancestor);
+
+      if (parent === ancestor) throw error;
+      ancestor = parent;
+    }
+  }
+}
+
+function sameSignature(a, b) {
+  return a === b || (a !== null && b !== null && a.size === b.size && a.sha256 === b.sha256);
+}
+
+function tooLargeRead(label, maxBytes) {
+  return new Error(label + " exceeds " + maxBytes + " bytes; use a streaming parser through bash");
+}
+
+function overlayOrThrow(overlay, maxBytes, label) {
+  if (maxBytes !== undefined && Buffer.byteLength(overlay, "utf8") > maxBytes) throw tooLargeRead(label, maxBytes);
+
+  return overlay;
+}
+
+function assertReadableFile(stat, target) {
+  if (stat.isDirectory()) throw new Error("read path is a directory, not a file: " + target);
+
+  if (!stat.isFile()) throw new Error("read requires a regular file: " + target);
+}
+
+async function readLimitedBytes(file, stat, maxBytes, label, signal) {
+  if (stat.size > maxBytes) throw tooLargeRead(label, maxBytes);
+  const chunks = [];
+  let size = 0;
+
+  for await (const chunk of file.createReadStream({ end: maxBytes, autoClose: false, signal })) {
+    size += chunk.length;
+
+    if (size > maxBytes) throw tooLargeRead(label, maxBytes);
+    chunks.push(chunk);
+  }
+
+  return Buffer.concat(chunks);
+}
+
+function remapReadError(err, target) {
+  if (err.code === "EISDIR") throw new Error("read path is a directory, not a file: " + target);
+
+  if (err.code === "ENOENT") {
+    const missing = new Error("no such file: " + target + ' (locate it with read using a directory path or source question)');
+    missing.code = "ENOENT";
+    throw missing;
+  }
+
+  throw err;
+}
+
+async function resolveExistingFile(logicalPath) {
+  const target = await fs.realpath(logicalPath);
+  const stat = await fs.stat(target);
+
+  if (!stat.isFile()) throw new Error("cannot write to a non-file: " + logicalPath);
+
+  return { target, stat };
+}
+
+async function resolveCommitTarget(logicalPath) {
+  try {
+    return await resolveExistingFile(logicalPath);
+  } catch (err) {
+    if (err.code !== "ENOENT") throw err;
+
+    return { target: await canonicalNewPath(logicalPath), stat: undefined };
+  }
+}
+
+async function collectMissingAncestors(parent) {
+  const missing = [];
+  let probe = parent;
+
+  for (;;) {
+    try { await fs.stat(probe); break; } catch (err) {
+      if (err.code !== "ENOENT") throw err;
+      missing.push(probe);
+      probe = path.dirname(probe);
+    }
+  }
+
+  return missing;
+}
+
+async function writeTemporary(entry, content, stat) {
+  await fs.writeFile(entry.temporary, content, { encoding: "utf8", flag: "wx", mode: stat ? stat.mode & 0o7777 : 0o666 });
+
+  if (stat) await fs.chmod(entry.temporary, stat.mode & 0o7777);
+}
+
+async function stageReplacement(entry, content, stat, target) {
+  const replacement = writeTemporary(entry, content, stat);
+  // These touch separate staging files. Settle both before cleanup, even
+  // on failure: Promise.all could leave a late backup after rollback.
+  const staging = [replacement];
+
+  if (stat) staging.push(fs.copyFile(target, entry.backup, fs.constants.COPYFILE_EXCL));
+  const outcomes = await Promise.allSettled(staging);
+  const failure = outcomes.find(outcome => outcome.status === "rejected");
+
+  if (failure) throw failure.reason;
+}
+
+function makeStageEntry(logicalPath, target, content, parent, stat) {
+  const token = ".supernova-" + randomUUID();
+
+  return { logicalPath, target, content, temporary: path.join(parent, token + ".new"), backup: path.join(parent, token + ".bak"), existed: !!stat, replaced: false };
+}
+
+async function recoverReplaced(staged) {
+  const recoveryErrors = [];
+
+  for (const entry of staged.toReversed()) {
+    if (!entry.replaced) continue;
+
+    try {
+      if (entry.existed) await fs.rename(entry.backup, entry.target);
+      else await fs.unlink(entry.target);
+    } catch (err) {
+      // Keep the backup if recovery fails; never delete the remaining original.
+      entry.keepBackup = true;
+      recoveryErrors.push(entry.target + ": " + err.message + " (backup: " + entry.backup + ")");
+    }
+  }
+
+  return recoveryErrors;
+}
+
+async function cleanupStaged(staged, failed, createdDirs) {
+  for (const entry of staged) {
+    // A successful rename consumed the temporary path. These are known
+    // files, so unlink avoids rm's extra type probe; missing files stay benign.
+    if (!entry.replaced) await fs.unlink(entry.temporary).catch(() => {});
+
+    if (entry.existed && !entry.keepBackup) await fs.unlink(entry.backup).catch(() => {});
+  }
+
+  if (failed) for (const dir of createdDirs.toReversed()) await fs.rmdir(dir).catch(() => {});
+}
+
+async function assertExpectedSignature(vfs, logicalPath, target, stat) {
+  if (!vfs.expected.has(logicalPath)) return;
+  const current = stat ? await fileSignature(target, vfs.signal) : null;
+
+  if (!sameSignature(current, vfs.expected.get(logicalPath))) {
+    throw new Error("write conflict: file changed since it was read: " + logicalPath + "; read it again before retrying");
+  }
+}
+
+async function installStaged(vfs, staged) {
+  for (const entry of staged) {
+    vfs.signal?.throwIfAborted();
+    await fs.rename(entry.temporary, entry.target);
+    entry.replaced = true;
+  }
+
+  for (const entry of staged) {
+    vfs.expected.set(entry.logicalPath, textSignature(entry.content));
+  }
+
+  // Canonical commit destinations must not rewrite established event paths
+  // for newly created files, whose callers supplied a logical cwd spelling.
+  if (staged.length) vfs.onNewFile?.(staged.map(entry => entry.existed ? entry.target : entry.logicalPath));
+}
+
+async function failCommit(vfs, staged, error) {
+  const recoveryErrors = await recoverReplaced(staged);
+  // Keep CAS baselines: a failed commit must not forgive conflicts on files it
+  // never touched. If recovery left disk diverging from a baseline, the next
+  // write to that path fails loudly and forces a re-read instead of silently
+  // re-capturing unknown bytes as the new truth.
+
+  if (recoveryErrors.length) vfs.mutations.recoveryFailed = true;
+
+  if (recoveryErrors.length) vfs.onNewFile?.(null);
+
+  if (recoveryErrors.length) throw new AggregateError([error, ...recoveryErrors.map(message => new Error(message))], "commit failed: " + error.message + "; recovery failed: " + recoveryErrors.join("; "));
+  throw error;
+}
+
 export class CausalVfs {
   constructor(onNewFile, validateWrite) {
     this.validateWrite = validateWrite;
-    // Last-seen original bytes for write CAS, not a read cache. read() always
-    // hits disk unless an overlay is staged. Serving cache on read would be a
-    // false-valid against editors/git between two reads.
-    this.cache = new Map();
+    // No body cache: every read hits disk (or its overlay) so observed bytes
+    // are never stale. CAS baselines in `expected` are the only retained
+    // per-file state, cleared only at external-mutation boundaries.
     this.overlays = [];
     this.expected = new Map();
     this.onNewFile = onNewFile;
@@ -28,11 +248,6 @@ export class CausalVfs {
     this.signal?.throwIfAborted();
   }
 
-  setCache(target, content) {
-    if (this.cache.size >= VFS_CACHE_MAX && !this.cache.has(target)) this.cache.delete(this.cache.keys().next().value);
-    this.cache.set(target, content);
-  }
-
   getOverlay(target) {
     for (let i = this.overlays.length - 1; i >= 0; i--) {
       if (this.overlays[i].has(target)) return this.overlays[i].get(target);
@@ -43,60 +258,54 @@ export class CausalVfs {
     return [...new Set(this.overlays.flatMap(overlay => [...overlay.keys()]))];
   }
 
-  async read(target, { preserveRead = false, maxBytes } = {}) {
+  async read(target, { preserveRead = false, maxBytes, label = "read input" } = {}) {
     const overlay = this.getOverlay(target);
 
-    if (overlay !== undefined) {
-      if (maxBytes !== undefined && Buffer.byteLength(overlay, "utf8") > maxBytes) throw new Error("JSON input exceeds " + maxBytes + " bytes; use a streaming parser through bash");
-
-      return overlay;
-    }
+    if (overlay !== undefined) return overlayOrThrow(overlay, maxBytes, label);
 
     // External editors and captured tools can change a file between any two reads.
+    // Open once with O_NONBLOCK so a FIFO or device cannot park a host I/O worker.
     try {
-      let text;
-
-      if (maxBytes === undefined) text = await fs.readFile(target, "utf8");
-      else {
-        const file = await fs.open(target, fs.constants.O_RDONLY | (fs.constants.O_NONBLOCK ?? 0));
-
-        try {
-          const stat = await file.stat();
-
-          if (!stat.isFile()) throw new Error("JSON read requires a regular file: " + target);
-          const tooLarge = () => new Error("JSON input exceeds " + maxBytes + " bytes; use a streaming parser through bash");
-
-          if (stat.size > maxBytes) throw tooLarge();
-          const chunks = [];
-          let size = 0;
-
-          for await (const chunk of file.createReadStream({ end: maxBytes, autoClose: false, signal: this.signal })) {
-            size += chunk.length;
+      const file = await fs.open(target, fs.constants.O_RDONLY | (fs.constants.O_NONBLOCK ?? 0));
+      let bytes;
+
+      try {
+        const stat = await file.stat();
+        assertReadableFile(stat, target);
+        bytes = maxBytes === undefined
+          ? await file.readFile({ signal: this.signal })
+          : await readLimitedBytes(file, stat, maxBytes, label, this.signal);
+        if (!sameFileVersion(stat, await file.stat())) throw new Error("file changed while reading: " + target);
+      } finally { await file.close(); }
+
+      // Hash the actual bytes, not a lossy UTF-8 decode/re-encode.
+      if (!preserveRead || !this.expected.has(target)) this.expected.set(target, textSignature(bytes));
+
+      return bytes.toString("utf8");
+    } catch (err) {
+      remapReadError(err, target);
+    }
+  }
 
-            if (size > maxBytes) throw tooLarge();
-            chunks.push(chunk);
-          }
+  async #diskSignature(target) {
+    let stat;
 
-          text = Buffer.concat(chunks).toString("utf8");
-        } finally { await file.close(); }
-      }
+    try { stat = await fs.stat(target); }
+    catch (error) { if (error.code !== "ENOENT") throw error; }
 
-      if (!preserveRead || !this.cache.has(target)) this.setCache(target, text);
-
-      return text;
-    } catch (err) {
-      this.cache.delete(target);
+    return stat?.isFile() ? await fileSignature(target, this.signal, stat) : null;
+  }
 
-      if (err.code === "EISDIR") throw new Error("read path is a directory, not a file: " + target);
+  async captureExpected(target) {
+    if (this.getOverlay(target) !== undefined || this.expected.has(target)) return;
 
-      if (err.code === "ENOENT") {
-        const missing = new Error("no such file: " + target + ' (locate it with read using a directory path or source question)');
-        missing.code = "ENOENT";
-        throw missing;
-      }
+    this.expected.set(target, await this.#diskSignature(target));
+  }
 
-      throw err;
-    }
+  async recordExpected(target, observed) {
+    if (this.getOverlay(target) !== undefined) return;
+    const signature = observed ? await fileSignature(target, this.signal, observed) : await this.#diskSignature(target);
+    this.expected.set(target, signature);
   }
 
   async write(target, content) {
@@ -112,14 +321,7 @@ export class CausalVfs {
 
     this.assertWritable();
 
-    if (this.getOverlay(target) === undefined) {
-      let original;
-
-      try { original = this.cache.has(target) ? this.cache.get(target) : await fs.readFile(target, "utf8"); }
-      catch (error) { if (error.code !== "ENOENT") throw error; original = null; }
-
-      this.expected.set(target, original);
-    }
+    await this.captureExpected(target);
 
     this.assertWritable();
 
@@ -159,114 +361,28 @@ export class CausalVfs {
     try {
       for (const [logicalPath, content] of writes) {
         this.signal?.throwIfAborted();
-        let target = logicalPath;
-        let stat;
-
-        try {
-          target = await fs.realpath(logicalPath);
-          stat = await fs.stat(target);
-
-          if (!stat.isFile()) throw new Error("cannot write to a non-file: " + logicalPath);
-        } catch (err) {
-          if (err.code !== "ENOENT") throw err;
-        }
-
+        const { target, stat } = await resolveCommitTarget(logicalPath);
         await this.validateWrite?.(logicalPath);
 
         if (targets.has(target)) throw new Error("conflicting write aliases: " + logicalPath);
         targets.add(target);
-
-        if (this.expected.has(logicalPath)) {
-          const current = stat ? await fs.readFile(target, "utf8") : null;
-
-          if (current !== this.expected.get(logicalPath)) {
-            throw new Error("write conflict: file changed since it was read: " + logicalPath + "; read it again before retrying");
-          }
-        }
-
+        await assertExpectedSignature(this, logicalPath, target, stat);
         const parent = path.dirname(target);
-        const missing = [];
-        let probe = parent;
-
-        for (;;) {
-          try { await fs.stat(probe); break; } catch (err) {
-            if (err.code !== "ENOENT") throw err;
-            missing.push(probe);
-            probe = path.dirname(probe);
-          }
-        }
-
+        const missing = await collectMissingAncestors(parent);
         await fs.mkdir(parent, { recursive: true });
         createdDirs.push(...missing.reverse());
-        const token = ".supernova-" + randomUUID();
-        const entry = { logicalPath, target, content, temporary: path.join(parent, token + ".new"), backup: path.join(parent, token + ".bak"), existed: !!stat, replaced: false };
+        const entry = makeStageEntry(logicalPath, target, content, parent, stat);
         staged.push(entry);
-
-        const replacement = (async () => {
-          await fs.writeFile(entry.temporary, content, { encoding: "utf8", flag: "wx", mode: stat ? stat.mode & 0o7777 : 0o666 });
-
-          if (stat) await fs.chmod(entry.temporary, stat.mode & 0o7777);
-        })();
-
-        // These touch separate staging files. Settle both before cleanup, even
-        // on failure: Promise.all could leave a late backup after rollback.
-        const staging = [replacement];
-
-        if (stat) staging.push(fs.copyFile(target, entry.backup, fs.constants.COPYFILE_EXCL));
-        const outcomes = await Promise.allSettled(staging);
-        const failure = outcomes.find(outcome => outcome.status === "rejected");
-
-        if (failure) throw failure.reason;
+        await stageReplacement(entry, content, stat, target);
       }
 
-      for (const entry of staged) {
-        this.signal?.throwIfAborted();
-        await fs.rename(entry.temporary, entry.target);
-        entry.replaced = true;
-      }
-
-      for (const entry of staged) {
-        this.setCache(entry.logicalPath, entry.content);
-        this.expected.delete(entry.logicalPath);
-      }
-
-      if (staged.length) this.onNewFile?.(staged.map(entry => entry.target));
+      await installStaged(this, staged);
       this.mutations.committed += staged.length;
     } catch (error) {
       failed = true;
-      const recoveryErrors = [];
-
-      for (const entry of staged.toReversed()) {
-        if (!entry.replaced) continue;
-
-        try {
-          if (entry.existed) await fs.rename(entry.backup, entry.target);
-          else await fs.unlink(entry.target);
-        } catch (err) {
-          // Keep the backup if recovery fails; never delete the remaining original.
-          entry.keepBackup = true;
-          recoveryErrors.push(entry.target + ": " + err.message + " (backup: " + entry.backup + ")");
-        }
-      }
-
-      this.invalidateCache();
-
-      if (recoveryErrors.length) this.mutations.recoveryFailed = true;
-
-      if (recoveryErrors.length) this.onNewFile?.(null);
-
-      if (recoveryErrors.length) throw new AggregateError([error, ...recoveryErrors.map(message => new Error(message))], "commit failed: " + error.message + "; recovery failed: " + recoveryErrors.join("; "));
-      throw error;
+      await failCommit(this, staged, error);
     } finally {
-      for (const entry of staged) {
-        // A successful rename consumed the temporary path. These are known
-        // files, so unlink avoids rm's extra type probe; missing files stay benign.
-        if (!entry.replaced) await fs.unlink(entry.temporary).catch(() => {});
-
-        if (entry.existed && !entry.keepBackup) await fs.unlink(entry.backup).catch(() => {});
-      }
-
-      if (failed) for (const dir of createdDirs.toReversed()) await fs.rmdir(dir).catch(() => {});
+      await cleanupStaged(staged, failed, createdDirs);
     }
   }
 
@@ -291,9 +407,8 @@ export class CausalVfs {
     const top = this.overlays.pop();
     this.mutations.rolledBack += top?.size ?? 0;
 
-    for (const target of top?.keys() ?? []) {
-      if (this.getOverlay(target) === undefined) this.expected.delete(target);
-    }
+    // Rolling back staged writes does not undo observations of disk. Keep the
+    // read snapshot, including for files without a surviving parent overlay.
 
     return { rolledBack: top?.size ?? 0, depth: this.overlays.length };
   }
@@ -316,7 +431,19 @@ export class CausalVfs {
     return pending.size > 0;
   }
 
-  invalidateCache() { this.cache.clear(); }
-  getCacheSize() { return this.cache.size; }
+  /** External-mutation boundary: drop CAS baselines so the next access re-observes disk. */
+  invalidateObserved() { this.expected.clear(); }
   getOverlayDepth() { return this.overlays.length; }
+  describeOverlays() {
+    let files = 0, bytes = 0;
+
+    for (const layer of this.overlays) {
+      for (const content of layer.values()) {
+        files++;
+        bytes += Buffer.byteLength(content, "utf8");
+      }
+    }
+
+    return { files, bytes };
+  }
 }