tledger 0.3.0 → 0.4.0

package/lib/token-ledger-snapshot.mjs

@@ -1,14 +1,21 @@
-import { randomUUID } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
+import { constants as fsConstants } from "node:fs";
 import {
-  chmod,
+  lstat,
   mkdir,
-  readFile,
+  open,
+  readdir,
   rename,
   rm,
-  stat,
-  writeFile,
 } from "node:fs/promises";
-import { dirname, resolve } from "node:path";
+import { homedir } from "node:os";
+import {
+  basename,
+  dirname,
+  isAbsolute,
+  relative,
+  resolve,
+} from "node:path";
 import { promisify } from "node:util";
 import {
   gunzip as gunzipCallback,
@@ -23,12 +30,34 @@ import {
   usageBuckets,
 } from "./token-ledger-usage.mjs";
 
+const DURABLE_LEDGER_BASENAME = "token-ledger-ledger.sqlite";
+const DURABLE_LEDGER_PATH_SUFFIXES = Object.freeze([
+  DURABLE_LEDGER_BASENAME,
+  `${DURABLE_LEDGER_BASENAME}.writer-lock.sqlite`,
+  `${DURABLE_LEDGER_BASENAME}-journal`,
+  `${DURABLE_LEDGER_BASENAME}-wal`,
+  `${DURABLE_LEDGER_BASENAME}-shm`,
+]);
+const PRIVATE_STATE_DIRECTORY = resolve(homedir(), ".token-ledger");
+const TEST_PRIVATE_STATE_DIRECTORY =
+  process.env.NODE_TEST_CONTEXT && process.env.TOKEN_LEDGER_TEST_STATE_ROOT
+    ? resolve(process.env.TOKEN_LEDGER_TEST_STATE_ROOT)
+    : null;
+const PRIVATE_STATE_DIRECTORIES = Object.freeze([
+  PRIVATE_STATE_DIRECTORY,
+  ...(TEST_PRIVATE_STATE_DIRECTORY ? [TEST_PRIVATE_STATE_DIRECTORY] : []),
+]);
+
 export const DEFAULT_SNAPSHOT_MAX_BYTES = 16 * 1024 * 1024;
 export const DEFAULT_SNAPSHOT_TARGET_BYTES = 12 * 1024 * 1024;
 export const DEFAULT_SNAPSHOT_MAX_JSON_BYTES = 64 * 1024 * 1024;
 export const DEFAULT_SNAPSHOT_TARGET_JSON_BYTES = 48 * 1024 * 1024;
 
 const PRECOMPACT_BUCKET_COUNT = 50_000;
+const SNAPSHOT_READ_CHUNK_BYTES = 64 * 1024;
+const SNAPSHOT_TEMP_HASH_LENGTH = 16;
+const SNAPSHOT_TEMP_UUID_PATTERN =
+  "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";
 
 const gzip = promisify(gzipCallback);
 const gunzip = promisify(gunzipCallback);
@@ -37,6 +66,123 @@ function snapshotEncoding(path) {
   return path.toLowerCase().endsWith(".gz") ? "gzip" : "json";
 }
 
+function snapshotDestinationHash(destination) {
+  return createHash("sha256")
+    .update(destination)
+    .digest("hex")
+    .slice(0, SNAPSHOT_TEMP_HASH_LENGTH);
+}
+
+function snapshotTemporaryName(destination) {
+  return `.token-ledger-${snapshotDestinationHash(destination)}-${process.pid}-${randomUUID()}.tmp`;
+}
+
+function snapshotDestinationError(destination) {
+  const error = new Error(
+    `Snapshot destination is reserved for durable ledger state: ${destination}`,
+  );
+  error.code = "ERR_SNAPSHOT_RESERVED_PATH";
+  return error;
+}
+
+function pathIsInside(directory, destination) {
+  const child = relative(directory, destination);
+  return child === "" || (!child.startsWith("..") && !isAbsolute(child));
+}
+
+function assertSnapshotDestination(destination, reservedPaths) {
+  const explicitlyReserved = new Set(
+    reservedPaths.map((path) => resolve(path)),
+  );
+  if (
+    explicitlyReserved.has(destination) ||
+    (
+      PRIVATE_STATE_DIRECTORIES.some((directory) =>
+        pathIsInside(directory, destination),
+      ) &&
+      DURABLE_LEDGER_PATH_SUFFIXES.includes(basename(destination))
+    )
+  ) {
+    throw snapshotDestinationError(destination);
+  }
+}
+
+function processIsDemonstrablyGone(pid) {
+  try {
+    process.kill(pid, 0);
+    return false;
+  } catch (error) {
+    return error?.code === "ESRCH";
+  }
+}
+
+async function removeOrphanedSnapshotCandidates(destination) {
+  const directory = dirname(destination);
+  const destinationHash = snapshotDestinationHash(destination);
+  const currentUid = process.getuid instanceof Function
+    ? process.getuid()
+    : null;
+  const candidatePattern = new RegExp(
+    `^\\.token-ledger-${destinationHash}-([1-9][0-9]*)-${SNAPSHOT_TEMP_UUID_PATTERN}\\.tmp$`,
+  );
+  let names;
+  try {
+    names = await readdir(directory);
+  } catch (error) {
+    if (["ENOENT", "ENOTDIR"].includes(error?.code)) return;
+    throw error;
+  }
+  for (const name of names) {
+    const match = name.match(candidatePattern);
+    if (!match) continue;
+    const ownerPid = Number(match[1]);
+    if (
+      !Number.isSafeInteger(ownerPid) ||
+      ownerPid === process.pid ||
+      !processIsDemonstrablyGone(ownerPid)
+    ) continue;
+    const candidate = resolve(directory, name);
+    let candidateStat;
+    try {
+      candidateStat = await lstat(candidate);
+    } catch (error) {
+      if (["ENOENT", "ENOTDIR"].includes(error?.code)) continue;
+      throw error;
+    }
+    // Only unlink an ordinary, single-link candidate. Symlinks, directories,
+    // hard links, and targets that race away are left untouched.
+    if (
+      !candidateStat.isFile() ||
+      Number(candidateStat.nlink) !== 1 ||
+      (currentUid !== null && Number(candidateStat.uid) !== currentUid)
+    ) continue;
+    let confirmedStat;
+    try {
+      confirmedStat = await lstat(candidate);
+    } catch (error) {
+      if (["ENOENT", "ENOTDIR"].includes(error?.code)) continue;
+      throw error;
+    }
+    if (
+      !confirmedStat.isFile() ||
+      Number(confirmedStat.nlink) !== 1 ||
+      Number(confirmedStat.dev) !== Number(candidateStat.dev) ||
+      Number(confirmedStat.ino) !== Number(candidateStat.ino) ||
+      (currentUid !== null && Number(confirmedStat.uid) !== currentUid)
+    ) continue;
+    try {
+      await rm(candidate, { force: true });
+    } catch (error) {
+      if (
+        ["ENOENT", "ENOTDIR", "EISDIR", "EACCES", "EPERM"].includes(
+          error?.code,
+        )
+      ) continue;
+      throw error;
+    }
+  }
+}
+
 function formatMebibytes(bytes) {
   return `${(bytes / (1024 * 1024)).toFixed(1)} MiB`;
 }
@@ -118,6 +264,90 @@ async function boundedEncoding(
   };
 }
 
+function snapshotSizeLimitError(message) {
+  const error = new Error(message);
+  error.code = "ERR_SNAPSHOT_SIZE_LIMIT";
+  return error;
+}
+
+function snapshotNotRegularFileError() {
+  const error = new Error("Snapshot input must be a regular file.");
+  error.code = "ERR_SNAPSHOT_NOT_REGULAR";
+  return error;
+}
+
+function snapshotCandidateChangedError() {
+  const error = new Error(
+    "Snapshot candidate changed before it could be published.",
+  );
+  error.code = "ERR_SNAPSHOT_CANDIDATE_CHANGED";
+  return error;
+}
+
+function isSameSnapshotCandidate(candidate, pinned) {
+  return Boolean(
+    candidate?.isFile() &&
+      Number(candidate.nlink) === 1 &&
+      Number(candidate.dev) === Number(pinned.dev) &&
+      Number(candidate.ino) === Number(pinned.ino),
+  );
+}
+
+async function readBoundedSnapshot(source, sourceLimit, encoding) {
+  const handle = await open(
+    source,
+    fsConstants.O_RDONLY | (fsConstants.O_NONBLOCK ?? 0),
+  );
+  try {
+    const sourceStats = await handle.stat();
+    if (!sourceStats.isFile()) {
+      throw snapshotNotRegularFileError();
+    }
+    if (sourceStats.size > sourceLimit) {
+      throw snapshotSizeLimitError(
+        `Snapshot input is ${formatMebibytes(sourceStats.size)}, exceeding the ${formatMebibytes(sourceLimit)} ${encoding === "gzip" ? "compressed" : "JSON"} read limit.`,
+      );
+    }
+
+    const chunks = [];
+    let bytesRead = 0;
+    while (bytesRead < sourceLimit) {
+      const chunkLength = Math.min(
+        SNAPSHOT_READ_CHUNK_BYTES,
+        sourceLimit - bytesRead,
+      );
+      const chunk = Buffer.allocUnsafe(chunkLength);
+      const result = await handle.read(
+        chunk,
+        0,
+        chunkLength,
+        bytesRead,
+      );
+      if (result.bytesRead === 0) break;
+      chunks.push(chunk.subarray(0, result.bytesRead));
+      bytesRead += result.bytesRead;
+    }
+    if (bytesRead >= sourceLimit) {
+      const extra = Buffer.allocUnsafe(1);
+      const result = await handle.read(extra, 0, 1, bytesRead);
+      if (result.bytesRead > 0) {
+        throw snapshotSizeLimitError(
+          `Snapshot input exceeds the ${formatMebibytes(sourceLimit)} ${encoding === "gzip" ? "compressed" : "JSON"} read limit.`,
+        );
+      }
+    }
+    const finalStats = await handle.stat();
+    if (finalStats.size > sourceLimit) {
+      throw snapshotSizeLimitError(
+        `Snapshot input grew beyond the ${formatMebibytes(sourceLimit)} ${encoding === "gzip" ? "compressed" : "JSON"} read limit.`,
+      );
+    }
+    return Buffer.concat(chunks, bytesRead);
+  } finally {
+    await handle.close();
+  }
+}
+
 export async function readPrivateSnapshot(
   input,
   {
@@ -134,22 +364,7 @@ export async function readPrivateSnapshot(
   }
   const encoding = snapshotEncoding(source);
   const sourceLimit = encoding === "gzip" ? maxBytes : maxJsonBytes;
-  const sourceStats = await stat(source);
-  if (sourceStats.size > sourceLimit) {
-    const error = new Error(
-      `Snapshot input is ${formatMebibytes(sourceStats.size)}, exceeding the ${formatMebibytes(sourceLimit)} ${encoding === "gzip" ? "compressed" : "JSON"} read limit.`,
-    );
-    error.code = "ERR_SNAPSHOT_SIZE_LIMIT";
-    throw error;
-  }
-  const encoded = await readFile(source);
-  if (encoded.byteLength > sourceLimit) {
-    const error = new Error(
-      `Snapshot input grew beyond the ${formatMebibytes(sourceLimit)} ${encoding === "gzip" ? "compressed" : "JSON"} read limit.`,
-    );
-    error.code = "ERR_SNAPSHOT_SIZE_LIMIT";
-    throw error;
-  }
+  const encoded = await readBoundedSnapshot(source, sourceLimit, encoding);
   let decoded;
   if (encoding === "gzip") {
     try {
@@ -167,16 +382,14 @@ export async function readPrivateSnapshot(
     decoded = encoded;
   }
   if (decoded.byteLength > maxJsonBytes) {
-    const error = new Error(
+    throw snapshotSizeLimitError(
       `Snapshot JSON representation is ${formatMebibytes(decoded.byteLength)}, exceeding the ${formatMebibytes(maxJsonBytes)} read limit.`,
     );
-    error.code = "ERR_SNAPSHOT_SIZE_LIMIT";
-    throw error;
   }
   return JSON.parse(decoded.toString("utf8"));
 }
 
-export async function writePrivateSnapshot(
+export async function stagePrivateSnapshot(
   output,
   snapshot,
   {
@@ -187,13 +400,15 @@ export async function writePrivateSnapshot(
       DEFAULT_SNAPSHOT_TARGET_JSON_BYTES,
       maxJsonBytes,
     ),
+    reservedPaths = [],
   } = {},
 ) {
   const destination = resolve(output);
+  assertSnapshotDestination(destination, reservedPaths);
   const directory = dirname(destination);
   const temporary = resolve(
     directory,
-    `.token-ledger-${process.pid}-${randomUUID()}.tmp`,
+    snapshotTemporaryName(destination),
   );
   const encoding = snapshotEncoding(destination);
   if (!Number.isSafeInteger(maxBytes) || maxBytes < 1) {
@@ -241,19 +456,85 @@ export async function writePrivateSnapshot(
   }
 
   await mkdir(directory, { recursive: true });
+  await removeOrphanedSnapshotCandidates(destination);
+  let candidateHandle = null;
+  let candidateIdentity = null;
   try {
-    await writeFile(temporary, encoded, {
-      flag: "wx",
-      mode: 0o600,
-    });
-    await chmod(temporary, 0o600);
-    await rename(temporary, destination);
-    await chmod(destination, 0o600);
-  } finally {
+    candidateHandle = await open(
+      temporary,
+      fsConstants.O_WRONLY |
+        fsConstants.O_CREAT |
+        fsConstants.O_EXCL |
+        (fsConstants.O_NOFOLLOW ?? 0),
+      0o600,
+    );
+    await candidateHandle.writeFile(encoded);
+    await candidateHandle.chmod(0o600);
+    await candidateHandle.sync();
+    candidateIdentity = await candidateHandle.stat();
+    if (!candidateIdentity.isFile() || Number(candidateIdentity.nlink) !== 1) {
+      throw snapshotCandidateChangedError();
+    }
+  } catch (error) {
+    await candidateHandle?.close();
     await rm(temporary, { force: true });
+    throw error;
   }
 
+  let published = false;
+  let discarded = false;
+  const publish = async () => {
+    if (published) return;
+    if (discarded) {
+      throw new Error("Cannot publish a discarded snapshot candidate.");
+    }
+    let currentIdentity;
+    try {
+      currentIdentity = await lstat(temporary);
+    } catch (error) {
+      if (["ENOENT", "ENOTDIR"].includes(error?.code)) {
+        throw snapshotCandidateChangedError();
+      }
+      throw error;
+    }
+    if (!isSameSnapshotCandidate(currentIdentity, candidateIdentity)) {
+      throw snapshotCandidateChangedError();
+    }
+    await rename(temporary, destination);
+    const publishedIdentity = await lstat(destination);
+    if (!isSameSnapshotCandidate(publishedIdentity, candidateIdentity)) {
+      // A competing writer may have replaced the destination between the
+      // rename and this verification. Never remove an inode we do not own;
+      // the candidate handle still pins our original file and can be closed
+      // without touching the competing replacement.
+      await candidateHandle?.close();
+      candidateHandle = null;
+      throw snapshotCandidateChangedError();
+    }
+    await candidateHandle.close();
+    candidateHandle = null;
+    published = true;
+  };
+  const discard = async () => {
+    if (published || discarded) return;
+    discarded = true;
+    await candidateHandle?.close();
+    candidateHandle = null;
+    let currentIdentity;
+    try {
+      currentIdentity = await lstat(temporary);
+    } catch (error) {
+      if (["ENOENT", "ENOTDIR"].includes(error?.code)) return;
+      throw error;
+    }
+    if (isSameSnapshotCandidate(currentIdentity, candidateIdentity)) {
+      await rm(temporary, { force: true });
+    }
+  };
+
   return {
+    publish,
+    discard,
     encoding,
     bytesWritten: encoded.byteLength,
     jsonBytes: serialized.byteLength,
@@ -265,3 +546,23 @@ export async function writePrivateSnapshot(
     snapshot: storedSnapshot,
   };
 }
+
+export async function writePrivateSnapshot(output, snapshot, options = {}) {
+  const staged = await stagePrivateSnapshot(output, snapshot, options);
+  try {
+    await staged.publish();
+    return {
+      encoding: staged.encoding,
+      bytesWritten: staged.bytesWritten,
+      jsonBytes: staged.jsonBytes,
+      maxBytes: staged.maxBytes,
+      targetBytes: staged.targetBytes,
+      maxJsonBytes: staged.maxJsonBytes,
+      targetJsonBytes: staged.targetJsonBytes,
+      adaptiveResolutionSeconds: staged.adaptiveResolutionSeconds,
+      snapshot: staged.snapshot,
+    };
+  } finally {
+    await staged.discard();
+  }
+}

package/lib/token-ledger-terminal-text.mjs

@@ -0,0 +1,11 @@
+const OSC_SEQUENCE =
+  /(?:\u001b\]|\u009d)[\s\S]*?(?:\u0007|\u001b\\|\u009c|$)/g;
+const CSI_SEQUENCE = /(?:\u001b\[|\u009b)[0-?]*[ -/]*[@-~]/g;
+const CONTROL_CHARACTERS = /[\u0000-\u001f\u007f-\u009f]/g;
+
+export function sanitizeTerminalText(value) {
+  return String(value ?? "")
+    .replace(OSC_SEQUENCE, "")
+    .replace(CSI_SEQUENCE, "")
+    .replace(CONTROL_CHARACTERS, " ");
+}