stitchkit 0.104.2 → 0.105.0

package/dist/entrypoints/application/diagnostic-journal.js

@@ -1,24 +1,37 @@
 import {
   DiagnosticJournalFrameSchema2,
   createDiagnosticJournalReadResultSchema2,
+  DIAGNOSTIC_JOURNAL_LISTED_QUARANTINE_LIMIT,
   DiagnosticJournalRecoveryStatusSchema2,
   DiagnosticJournalRecoveryError2,
   DiagnosticJournalLimitsSchema2,
   DiagnosticJournalLockPolicySchema2,
+  DiagnosticJournalStartupScanSchema2,
+  DiagnosticJournalStartupRefusalPolicySchema2,
   DiagnosticJournalStatusSchema2,
   DiagnosticJournalSubmitResultSchema2,
   DiagnosticJournalWaitResultSchema2,
   DiagnosticJournalCloseResultSchema2,
   parseDiagnosticJournalMode
-} from "../../index-z94wht4y.js";
+} from "../../index-dky0dcmf.js";
 import {
   createBoundedChannel2
 } from "../../index-w0eqdhyd.js";
 import {
   attemptExclusiveLock
-} from "../../index-za6j7nwy.js";
-import"../../index-5csmzesf.js";
+} from "../../index-dsnjzb62.js";
+import"../../index-azx12jgv.js";
 import"../../index-v5ee8ath.js";
+import {
+  BoundedFileReadError,
+  openRegularFile
+} from "../../index-m616f0yz.js";
+import {
+  closeAfter
+} from "../../index-b56nnscm.js";
+import {
+  assertPositiveSafeInteger
+} from "../../index-0s4xebbg.js";
 import {
   isRecord
 } from "../../index-9jfdm5tg.js";
@@ -27,12 +40,116 @@ import {
 import { randomUUID } from "node:crypto";
 
 // src/application/diagnostic-journal-manager.ts
-import { z as z3 } from "zod";
+import { z as z5 } from "zod";
 
 // src/application/diagnostic-journal-storage.ts
-import { constants as constants2 } from "node:fs";
-import { lstat as lstat2, open as open2, readdir, realpath, rename, unlink } from "node:fs/promises";
-import { basename, dirname, isAbsolute as isAbsolute2, resolve as resolve2 } from "node:path";
+import { constants } from "node:fs";
+import { lstat as lstat2, open, realpath, rename as rename2, unlink } from "node:fs/promises";
+import { basename, dirname, isAbsolute as isAbsolute2, resolve as resolve3 } from "node:path";
+
+// src/application/diagnostic-journal-generations.ts
+import { lstat, readdir } from "node:fs/promises";
+import { resolve } from "node:path";
+
+// src/application/diagnostic-journal-quarantine.ts
+import { rename } from "node:fs/promises";
+function createStartupRefusalHandler(policy, epoch) {
+  return async (refusal) => {
+    if (policy === "fail")
+      throw new DiagnosticJournalRecoveryError2(refusal);
+    const quarantinedAs = `${refusal.file}.quarantined-${epoch}`;
+    try {
+      await rename(refusal.file, quarantinedAs);
+    } catch (error) {
+      throw new DiagnosticJournalRecoveryError2({
+        ...refusal,
+        quarantineFailed: true,
+        cause: new AggregateError(refusal.cause === undefined ? [error] : [refusal.cause, error], "Diagnostic journal could not move a refused file aside")
+      });
+    }
+    const code = refusal.reason === "unreadable" && isRecord(refusal.cause) && typeof refusal.cause.code === "string" ? refusal.cause.code : undefined;
+    return {
+      file: refusal.file,
+      quarantinedAs,
+      reason: refusal.reason,
+      ...code !== undefined && { code }
+    };
+  };
+}
+function quarantinedOrigin(name, prefix) {
+  if (!name.startsWith(prefix))
+    return;
+  const match = /^(?:(\d+)\.)?quarantined-[\s\S]+$/.exec(name.slice(prefix.length));
+  if (!match)
+    return;
+  const index = match[1];
+  return index === undefined ? prefix.slice(0, -1) : `${prefix}${index}`;
+}
+function listQuarantinedFiles(movedNow, movedEarlier) {
+  const total = movedNow.length + movedEarlier.length;
+  if (total === 0)
+    return {};
+  const limit = DIAGNOSTIC_JOURNAL_LISTED_QUARANTINE_LIMIT;
+  const quarantined = [
+    ...movedNow.slice(0, limit),
+    ...movedEarlier.slice(0, Math.max(0, limit - movedNow.length))
+  ];
+  const unlisted = total - quarantined.length;
+  return { quarantined, ...unlisted > 0 && { quarantinedUnlisted: unlisted } };
+}
+
+// src/application/diagnostic-journal-generations.ts
+function generation(path, index) {
+  return `${path}.${index}`;
+}
+function generationIndex(name, prefix) {
+  if (!name.startsWith(prefix))
+    return;
+  const suffix = name.slice(prefix.length);
+  if (!/^\d+$/.test(suffix))
+    return;
+  const value = Number(suffix);
+  return Number.isSafeInteger(value) && value > 0 ? value : undefined;
+}
+async function listDiagnosticJournalGenerations(parent, prefix, maxFiles, refuse) {
+  const expired = [];
+  const quarantined = [];
+  const quarantinedEarlier = [];
+  let existing = 0;
+  for (const name of (await readdir(parent)).sort()) {
+    const origin = quarantinedOrigin(name, prefix);
+    if (origin !== undefined) {
+      quarantinedEarlier.push({
+        file: resolve(parent, origin),
+        quarantinedAs: resolve(parent, name)
+      });
+      continue;
+    }
+    const index = generationIndex(name, prefix);
+    if (index === undefined)
+      continue;
+    const path = resolve(parent, name);
+    const info = await lstat(path);
+    if (info.isSymbolicLink() || !info.isFile()) {
+      quarantined.push(await refuse({ file: path, reason: "not-a-regular-file" }));
+    } else if (index >= maxFiles) {
+      expired.push(path);
+    } else {
+      existing += 1;
+    }
+  }
+  return { expired, existing, quarantined, quarantinedEarlier };
+}
+async function retainedDiagnosticJournalGenerations(parent, prefix, maxFiles) {
+  const paths = [];
+  for (const name of await readdir(parent)) {
+    const index = generationIndex(name, prefix);
+    if (index !== undefined && index < maxFiles) {
+      paths.push({ index, path: resolve(parent, name) });
+    }
+  }
+  return paths.sort((left, right) => right.index - left.index).map((file) => file.path);
+}
 
 // src/application/diagnostic-journal-lock.ts
 async function acquireDiagnosticJournalLock(lockPath, mode, policy, declaredIdentity) {
@@ -52,13 +169,28 @@ async function acquireDiagnosticJournalLock(lockPath, mode, policy, declaredIden
 }
 
 // src/application/diagnostic-journal-recovery.ts
-import { z as z2 } from "zod";
+import { z as z4 } from "zod";
 
 // src/application/diagnostic-journal-reader.ts
-import { constants } from "node:fs";
-import { lstat, open } from "node:fs/promises";
-import { isAbsolute, resolve } from "node:path";
+import { isAbsolute, resolve as resolve2 } from "node:path";
+import { z as z2 } from "zod";
+
+// src/application/diagnostic-journal-open.ts
 import { z } from "zod";
+async function openJournalFile(path) {
+  try {
+    const { handle, before } = await openRegularFile(path, { rejectSymlinks: true });
+    return { handle, size: z.number().int().nonnegative().parse(before.size) };
+  } catch (error) {
+    if (error instanceof BoundedFileReadError && (error.code === "FILE_UNSAFE_LINK" || error.code === "FILE_NOT_REGULAR"))
+      throw new TypeError("Diagnostic journal reader requires a regular file, never a symlink", {
+        cause: error
+      });
+    throw error;
+  }
+}
+
+// src/application/diagnostic-journal-reader.ts
 function decodeLine(schema, location, body) {
   const anomaly = (reason, skippedBytes = location.bytes) => schema.parse({
     type: "anomaly",
@@ -102,35 +234,19 @@ function decodeLine(schema, location, body) {
   }
   return location.terminated ? [parsed.data] : [anomaly("unterminated-line", 0), parsed.data];
 }
-async function openRegularFile(path) {
-  const before = await lstat(path);
-  if (before.isSymbolicLink() || !before.isFile()) {
-    throw new TypeError("Diagnostic journal reader requires a regular file, never a symlink");
-  }
-  const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
-  try {
-    const info = await handle.stat();
-    if (!info.isFile())
-      throw new TypeError("Diagnostic journal reader requires a regular file");
-    return { handle, size: z.number().int().nonnegative().parse(info.size) };
-  } catch (error) {
-    await handle.close();
-    throw error;
-  }
-}
 async function* readDiagnosticJournal(config) {
-  const maxLineBytes = z.number().int().positive().parse(config.maxLineBytes);
+  const maxLineBytes = z2.number().int().positive().parse(config.maxLineBytes);
   if (config.paths.length === 0)
     throw new TypeError("Diagnostic journal reader needs at least one path");
   for (const path of config.paths) {
-    if (!isAbsolute(path) || resolve(path) !== path) {
+    if (!isAbsolute(path) || resolve2(path) !== path) {
       throw new TypeError("Diagnostic journal reader paths must be normalized and absolute");
     }
   }
   const schema = createDiagnosticJournalReadResultSchema2(config.eventSchema);
   for (const file of config.paths) {
     config.signal?.throwIfAborted();
-    const { handle, size } = await openRegularFile(file);
+    const { handle, size } = await openJournalFile(file);
     try {
       const chunk = Buffer.alloc(64 * 1024);
       let offset = 0;
@@ -193,26 +309,98 @@ async function* readDiagnosticJournal(config) {
   }
 }
 
+// src/application/diagnostic-journal-tail.ts
+import { z as z3 } from "zod";
+var SCAN_CHUNK_BYTES = 1024 * 1024;
+var jsonSchema = z3.json();
+var resultSchema = createDiagnosticJournalReadResultSchema2(jsonSchema);
+async function* readDiagnosticJournalTails(paths, maxLineBytes) {
+  for (const file of paths) {
+    const { handle, size } = await openJournalFile(file);
+    const last = await closeAfter(handle, async () => {
+      if (size === 0)
+        return;
+      const chunk = Buffer.alloc(SCAN_CHUNK_BYTES);
+      let lineFeeds = 0;
+      let lastFeed = -1;
+      let previousFeed = -1;
+      for (let offset = 0;offset < size; ) {
+        const { bytesRead } = await handle.read(chunk, 0, Math.min(chunk.length, size - offset), offset);
+        if (bytesRead === 0)
+          throw new Error(`Diagnostic journal file was truncated during read: ${file}`);
+        const block = chunk.subarray(0, bytesRead);
+        for (let at = block.indexOf(10);at !== -1; at = block.indexOf(10, at + 1)) {
+          lineFeeds += 1;
+          previousFeed = lastFeed;
+          lastFeed = offset + at;
+        }
+        offset += bytesRead;
+      }
+      const terminated = lastFeed === size - 1;
+      const start = (terminated ? previousFeed : lastFeed) + 1;
+      const length = (terminated ? size - 1 : size) - start;
+      const location = {
+        file,
+        offset: start,
+        line: terminated ? lineFeeds : lineFeeds + 1,
+        bytes: length + (terminated ? 1 : 0),
+        terminated,
+        position: "tail"
+      };
+      if (length > maxLineBytes)
+        return { location, body: undefined };
+      const body = Buffer.alloc(length);
+      for (let filled = 0;filled < length; ) {
+        const { bytesRead } = await handle.read(body, filled, length - filled, start + filled);
+        if (bytesRead === 0)
+          throw new Error(`Diagnostic journal file was truncated during read: ${file}`);
+        filled += bytesRead;
+      }
+      return { location, body };
+    });
+    if (last)
+      yield* decodeLine(resultSchema, last.location, last.body);
+  }
+}
+
 // src/application/diagnostic-journal-recovery.ts
-async function inspectDiagnosticJournalRecovery(paths, maxLineBytes) {
+function readFile(file, maxLineBytes, scan) {
+  return scan === "full" ? readDiagnosticJournal({ paths: [file], maxLineBytes, eventSchema: z4.json() }) : readDiagnosticJournalTails([file], maxLineBytes);
+}
+async function inspectDiagnosticJournalRecovery(inspection) {
+  let filesChecked = 0;
   let anomalies = 0;
   let skippedBytes = 0;
   let firstAnomaly;
   let lastAnomaly;
-  for await (const result of readDiagnosticJournal({
-    paths,
-    maxLineBytes,
-    eventSchema: z2.json()
-  })) {
-    if (result.type !== "anomaly")
+  for (const file of inspection.paths) {
+    let fileAnomalies = 0;
+    let fileSkipped = 0;
+    let fileFirst;
+    let fileLast;
+    try {
+      for await (const result of readFile(file, inspection.maxLineBytes, inspection.scan)) {
+        if (result.type !== "anomaly")
+          continue;
+        fileAnomalies += 1;
+        fileSkipped += result.anomaly.skippedBytes;
+        fileFirst ??= result.anomaly;
+        fileLast = result.anomaly;
+      }
+    } catch (error) {
+      if (!inspection.onUnreadable)
+        throw error;
+      await inspection.onUnreadable(file, error);
       continue;
-    anomalies += 1;
-    skippedBytes += result.anomaly.skippedBytes;
-    firstAnomaly ??= result.anomaly;
-    lastAnomaly = result.anomaly;
+    }
+    filesChecked += 1;
+    anomalies += fileAnomalies;
+    skippedBytes += fileSkipped;
+    firstAnomaly ??= fileFirst;
+    lastAnomaly = fileLast ?? lastAnomaly;
   }
   return DiagnosticJournalRecoveryStatusSchema2.parse({
-    filesChecked: paths.length,
+    filesChecked,
     anomalies,
     skippedBytes,
     ...firstAnomaly && { firstAnomaly },
@@ -250,7 +438,7 @@ async function moveIfPresent(from, to) {
     if (info.isSymbolicLink() || !info.isFile()) {
       throw new Error("Diagnostic journal generations must be regular files");
     }
-    await rename(from, to);
+    await rename2(from, to);
     return true;
   } catch (error) {
     if (isMissing(error))
@@ -258,24 +446,12 @@ async function moveIfPresent(from, to) {
     throw error;
   }
 }
-function generation(path, index) {
-  return `${path}.${index}`;
-}
-function generationIndex(name, prefix) {
-  if (!name.startsWith(prefix))
-    return;
-  const suffix = name.slice(prefix.length);
-  if (!/^\d+$/.test(suffix))
-    return;
-  const value = Number(suffix);
-  return Number.isSafeInteger(value) && value > 0 ? value : undefined;
-}
 async function createRotatingDiagnosticJournalStorage(config) {
-  if (!isAbsolute2(config.path) || resolve2(config.path) !== config.path) {
+  if (!isAbsolute2(config.path) || resolve3(config.path) !== config.path) {
     throw new TypeError("Diagnostic journal path must be normalized and absolute");
   }
   const parent = await realpath(dirname(config.path));
-  const journalPath = resolve2(parent, basename(config.path));
+  const journalPath = resolve3(parent, basename(config.path));
   const lockPath = `${journalPath}.lock`;
   const { lock, reclaimedStale } = await acquireDiagnosticJournalLock(lockPath, config.mode, config.lock, config.machineIdentity);
   let handle;
@@ -286,7 +462,7 @@ async function createRotatingDiagnosticJournalStorage(config) {
   let recovery;
   let closed = false;
   const openCurrent = async () => {
-    handle = await open2(journalPath, constants2.O_APPEND | constants2.O_CREAT | constants2.O_RDWR | constants2.O_NOFOLLOW, config.mode);
+    handle = await open(journalPath, constants.O_APPEND | constants.O_CREAT | constants.O_RDWR | constants.O_NOFOLLOW, config.mode);
     const info = await handle.stat();
     if (!info.isFile()) {
       await handle.close();
@@ -315,52 +491,65 @@ async function createRotatingDiagnosticJournalStorage(config) {
       throw new DiagnosticJournalStorageError("rotation", "Diagnostic journal rotation failed", { cause: error });
     }
   };
+  const refuse = createStartupRefusalHandler(config.onStartupRefusal, config.epoch);
   try {
-    const names = await readdir(parent);
     const prefix = `${basename(journalPath)}.`;
-    const expiredGenerations = [];
-    let existingGenerations = 0;
-    for (const name of names) {
-      const index = generationIndex(name, prefix);
-      if (index === undefined)
-        continue;
-      const path = resolve2(parent, name);
-      const info = await lstat2(path);
-      if (info.isSymbolicLink() || !info.isFile()) {
-        throw new Error("Diagnostic journal generations must be regular files");
-      }
-      if (index >= config.maxFiles) {
-        expiredGenerations.push(path);
-      } else {
-        existingGenerations += 1;
-      }
-    }
-    retainedFiles = Math.min(config.maxFiles, existingGenerations + 1);
+    const listing = await listDiagnosticJournalGenerations(parent, prefix, config.maxFiles, refuse);
+    const quarantined = [...listing.quarantined];
+    retainedFiles = Math.min(config.maxFiles, listing.existing + 1);
     await openCurrent();
     if (currentFileBytes > 0) {
       const tail = new Uint8Array(1);
       const read = await handle?.read(tail, 0, 1, currentFileBytes - 1);
       if (read?.bytesRead === 1 && tail[0] !== 10) {
+        partialTails += 1;
         if (config.maxFiles === 1) {
-          throw new DiagnosticJournalRecoveryError2(await inspectDiagnosticJournalRecovery([journalPath], config.maxFileBytes));
+          const recovery = config.onStartupRefusal === "fail" ? await inspectDiagnosticJournalRecovery({
+            paths: [journalPath],
+            maxLineBytes: config.maxFileBytes,
+            scan: config.scan
+          }) : undefined;
+          await handle?.close();
+          handle = undefined;
+          quarantined.push(await refuse({
+            file: journalPath,
+            reason: "torn-without-retention-slot",
+            ...recovery && { recovery }
+          }));
+          await openCurrent();
+        } else {
+          await rotate();
         }
-        partialTails += 1;
-        await rotate();
       }
     }
-    for (const path of expiredGenerations)
+    for (const path of listing.expired)
       await removeIfPresent(path);
-    const paths = [];
-    for (const name of await readdir(parent)) {
-      const index = generationIndex(name, prefix);
-      if (index !== undefined && index < config.maxFiles) {
-        paths.push({ index, path: resolve2(parent, name) });
+    const retained = await retainedDiagnosticJournalGenerations(parent, prefix, config.maxFiles);
+    const inspected = await inspectDiagnosticJournalRecovery({
+      paths: [...retained, journalPath],
+      maxLineBytes: config.maxFileBytes,
+      scan: config.scan,
+      onUnreadable: async (file, error) => {
+        const active = file === journalPath;
+        if (active) {
+          await handle?.close();
+          handle = undefined;
+        }
+        quarantined.push(await refuse({
+          file,
+          reason: error instanceof TypeError ? "not-a-regular-file" : "unreadable",
+          cause: error
+        }));
+        if (active)
+          await openCurrent();
+        else
+          retainedFiles -= 1;
       }
+    });
+    const listed = listQuarantinedFiles(quarantined, listing.quarantinedEarlier);
+    if (inspected.anomalies > 0 || listed.quarantined) {
+      recovery = { ...inspected, ...listed };
     }
-    paths.sort((left, right) => right.index - left.index);
-    const inspected = await inspectDiagnosticJournalRecovery([...paths.map((file) => file.path), journalPath], config.maxFileBytes);
-    if (inspected.anomalies > 0)
-      recovery = inspected;
   } catch (error) {
     await handle?.close().catch(() => {
       return;
@@ -403,33 +592,34 @@ async function createRotatingDiagnosticJournalStorage(config) {
       if (closed)
         return;
       closed = true;
-      let failure;
-      try {
-        await handle?.close();
-      } catch (error) {
-        failure = error;
-      }
-      try {
-        await lock.release();
-      } catch (error) {
-        failure ??= error;
-      }
-      if (failure !== undefined) {
-        throw new DiagnosticJournalStorageError("close", "Diagnostic journal close failed", {
-          cause: failure
-        });
-      }
+      await closeBoth(handle, lock);
     }
   };
 }
+async function closeBoth(handle, lock) {
+  let failure;
+  try {
+    await handle?.close();
+  } catch (error) {
+    failure = error;
+  }
+  try {
+    await lock.release();
+  } catch (error) {
+    failure ??= error;
+  }
+  if (failure !== undefined) {
+    throw new DiagnosticJournalStorageError("close", "Diagnostic journal close failed", {
+      cause: failure
+    });
+  }
+}
 
 // src/application/diagnostic-journal-manager.ts
 function waitBudget(options) {
   if (options.timeoutMs === undefined)
     return;
-  if (!Number.isSafeInteger(options.timeoutMs) || options.timeoutMs <= 0) {
-    throw new TypeError("Diagnostic journal timeoutMs must be a positive safe integer");
-  }
+  assertPositiveSafeInteger("Diagnostic journal timeoutMs", options.timeoutMs);
   return options.timeoutMs;
 }
 var NO_REFUSALS = {
@@ -600,7 +790,7 @@ function createDiagnosticJournalManager(config, storage) {
       const parsed = config.eventSchema.safeParse(event);
       if (!parsed.success)
         return refuse("invalid");
-      const json = z3.json().safeParse(parsed.data);
+      const json = z5.json().safeParse(parsed.data);
       if (!json.success)
         return refuse("invalid");
       const payload = encoder.encode(JSON.stringify(json.data));
@@ -666,17 +856,21 @@ function createDiagnosticJournalManager(config, storage) {
 async function createDiagnosticJournal(config) {
   const limits = DiagnosticJournalLimitsSchema2.parse(config.limits);
   const lock = DiagnosticJournalLockPolicySchema2.parse(config.lock ?? "refuse");
+  const epoch = randomUUID();
   const storage = await createRotatingDiagnosticJournalStorage({
     path: config.path,
     maxFileBytes: limits.maxFileBytes,
     maxFiles: limits.maxFiles,
     mode: parseDiagnosticJournalMode(config.mode),
     lock,
+    scan: DiagnosticJournalStartupScanSchema2.parse(config.startupScan ?? "tails"),
+    onStartupRefusal: DiagnosticJournalStartupRefusalPolicySchema2.parse(config.onStartupRefusal ?? "quarantine"),
+    epoch,
     ...config.machineIdentity !== undefined && { machineIdentity: config.machineIdentity }
   });
   return createDiagnosticJournalManager({
     eventSchema: config.eventSchema,
-    epoch: randomUUID(),
+    epoch,
     limits,
     lock: { policy: lock, reclaimedStale: storage.reclaimedStale },
     ...config.onFailure && { onFailure: config.onFailure }