salidium 0.4.1 → 0.5.0

package/bundle/salidium.mjs

@@ -13,6 +13,7 @@ import {
   existsSync as existsSync21,
   mkdirSync as mkdirSync12,
   openSync as openSync7,
+  readFileSync as readFileSync13,
   statfsSync as statfsSync3,
   statSync as statSync15
 } from "node:fs";
@@ -21,6 +22,7 @@ import { arch, homedir as homedir5, platform as platform2, release } from "node:
 import { join as join23, resolve as resolve6, sep as sep3 } from "node:path";
 import { createInterface } from "node:readline/promises";
 import { setTimeout as sleep } from "node:timers/promises";
+import { parentPort } from "node:worker_threads";
 
 // ../core/dist/claims/markdown.js
 var FENCE = /(^|\n)\s*(```|~~~)/;
@@ -18989,6 +18991,30 @@ var OperationsStoreMeasurementSchema = external_exports.object({
   retention: RetentionDaysSchema.nullable(),
   lastIngestAt: CanonicalTimestampSchema.nullable()
 }).strict();
+var StorageCompositionPartSchema = external_exports.object({
+  key: external_exports.enum(["sessions", "checkpoints", "provenance", "structure", "reusable"]),
+  bytes: external_exports.number().int().nonnegative()
+}).strict();
+var StorageCompositionProjectSchema = external_exports.object({
+  /** Repository root, else working directory, else empty when the session recorded neither. */
+  path: external_exports.string(),
+  sessions: external_exports.number().int().nonnegative(),
+  bytes: external_exports.number().int().nonnegative()
+}).strict();
+var StorageCompositionSchema = external_exports.object({
+  contractVersion: external_exports.literal(OPERATIONS_CONTRACT_VERSION),
+  state: external_exports.enum(["absent", "running", "ready", "failed"]),
+  computedAt: CanonicalTimestampSchema.nullable(),
+  elapsedMs: external_exports.number().int().nonnegative().nullable(),
+  /** The database file at the moment of measurement, from page count and page size. */
+  fileBytes: external_exports.number().int().nonnegative().nullable(),
+  sessions: external_exports.number().int().nonnegative().nullable(),
+  parts: external_exports.array(StorageCompositionPartSchema),
+  projects: external_exports.array(StorageCompositionProjectSchema),
+  /** Projects measured but not listed, so a truncated list can say so rather than imply totality. */
+  projectsOmitted: external_exports.number().int().nonnegative(),
+  failure: external_exports.string().max(500).nullable()
+}).strict();
 var MaintenancePhaseSchema = external_exports.enum([
   "idle",
   "pause",
@@ -26912,6 +26938,87 @@ var SqliteStore = class {
     });
     return true;
   }
+  /*
+   * What the store is made of, measured rather than estimated.
+   *
+   * Every part here is one pass over a table. The events pass is the expensive one: `length(json)`
+   * does not read a blob's payload, but it does read the header of every row, so on a store with
+   * 1.3 million events that is a ten second scan and the reason this never runs on the health path
+   * or on a timer. The daemon runs it on a worker with its own read-only connection for the same
+   * reason it runs usage backfill there: a ten second synchronous scan on the main loop is ten
+   * seconds of unanswered hooks.
+   *
+   * `structure` is deliberately a subtraction. Naming what each index costs needs `dbstat`, which
+   * is a further nine to fifteen seconds because it walks every page in the file, and the answer
+   * it adds is "your indexes are large" rather than anything a reader can act on. The difference
+   * is honest, arrives free, and is labelled as a remainder wherever it is shown.
+   */
+  storageComposition(projectLimit = 12) {
+    const started = Date.now();
+    const scalar = (sql) => {
+      const row = this.db.prepare(sql).get();
+      const value2 = row ? Object.values(row)[0] : 0;
+      return typeof value2 === "number" ? value2 : Number(value2 ?? 0);
+    };
+    const pageSize = scalar("PRAGMA page_size");
+    const fileBytes = scalar("PRAGMA page_count") * pageSize;
+    const reusable = scalar("PRAGMA freelist_count") * pageSize;
+    const sessions = this.db.prepare(`SELECT id, COALESCE(NULLIF(repo_root, ''), NULLIF(cwd, ''), '') AS project FROM sessions`).all();
+    const projectOf = new Map(sessions.map((row) => [row.id, row.project]));
+    const perProject = /* @__PURE__ */ new Map();
+    for (const row of sessions) {
+      const entry2 = perProject.get(row.project) ?? { sessions: 0, bytes: 0 };
+      entry2.sessions += 1;
+      perProject.set(row.project, entry2);
+    }
+    let sessionBytes = 0;
+    let checkpointBytes = 0;
+    const accumulate = (sql, into) => {
+      for (const row of this.db.prepare(sql).all()) {
+        const bytes = Number(row.b ?? 0);
+        into(bytes);
+        const project = projectOf.get(row.id);
+        if (project === void 0)
+          continue;
+        const entry2 = perProject.get(project);
+        if (entry2)
+          entry2.bytes += bytes;
+      }
+    };
+    accumulate("SELECT session_id AS id, SUM(length(json)) AS b FROM events GROUP BY session_id", (b) => {
+      sessionBytes += b;
+    });
+    accumulate("SELECT session_id AS id, SUM(length(json)) AS b FROM changes GROUP BY session_id", (b) => {
+      sessionBytes += b;
+    });
+    accumulate("SELECT session_id AS id, SUM(length(state_json)) AS b FROM checkpoints GROUP BY session_id", (b) => {
+      checkpointBytes += b;
+    });
+    const provenanceBytes = scalar(`SELECT COALESCE(SUM(length(session_id) + length(event_id) + length(path)
+         + length(record_hash) + length(captured_at) + length(COALESCE(origin, ''))), 0) AS b
+       FROM raw_record_fingerprints`);
+    const named = sessionBytes + checkpointBytes + provenanceBytes + reusable;
+    const ranked = [...perProject.entries()].map(([path, value2]) => ({ path, sessions: value2.sessions, bytes: value2.bytes })).sort((a, b) => b.bytes - a.bytes || b.sessions - a.sessions);
+    return {
+      contractVersion: OPERATIONS_CONTRACT_VERSION,
+      state: "ready",
+      computedAt: new Date(started).toISOString(),
+      elapsedMs: Date.now() - started,
+      fileBytes,
+      sessions: sessions.length,
+      parts: [
+        { key: "sessions", bytes: sessionBytes },
+        { key: "checkpoints", bytes: checkpointBytes },
+        { key: "provenance", bytes: provenanceBytes },
+        // A store measured while it is being written to can name more than the file holds.
+        { key: "structure", bytes: Math.max(fileBytes - named, 0) },
+        { key: "reusable", bytes: reusable }
+      ],
+      projects: ranked.slice(0, projectLimit),
+      projectsOmitted: Math.max(ranked.length - projectLimit, 0),
+      failure: null
+    };
+  }
   /** Offline space reclamation after bounded cleanup batches. */
   compact() {
     if (!this.usageBackfillProgress().complete)
@@ -28756,6 +28863,13 @@ function createHttpServer(deps) {
     }
     if (req.method === "POST" && url2.pathname === "/api/operations/maintenance/drain" && deps.operations)
       return json2(res, 200, deps.operations.drainQueue());
+    if (url2.pathname === "/api/operations/storage" && deps.operations) {
+      if (req.method === "GET")
+        return json2(res, 200, deps.operations.storageComposition());
+      if (req.method === "POST")
+        return json2(res, 200, deps.operations.analyzeStorage());
+      return json2(res, 405, { error: "method not allowed" });
+    }
     const acknowledge = /^\/api\/operations\/alerts\/([^/]+)\/acknowledge$/.exec(url2.pathname);
     if (req.method === "POST" && acknowledge?.[1] && deps.operations) {
       try {
@@ -29495,6 +29609,67 @@ async function startDaemon(overrides = {}) {
     refreshHealthSampling?.();
     return effective2;
   };
+  let composition = {
+    contractVersion: OPERATIONS_CONTRACT_VERSION,
+    state: "absent",
+    computedAt: null,
+    elapsedMs: null,
+    fileBytes: null,
+    sessions: null,
+    parts: [],
+    projects: [],
+    projectsOmitted: 0,
+    failure: null
+  };
+  let compositionWorker;
+  const analyzeStorage = () => {
+    if (compositionWorker)
+      return composition;
+    const sibling = new URL("./storage/compositionWorker.js", import.meta.url);
+    const worker = existsSync16(fileURLToPath(sibling)) ? new Worker(sibling, { workerData: paths.db }) : (() => {
+      const runtime = process.argv[1];
+      if (!runtime)
+        return void 0;
+      return new Worker(resolve4(runtime), {
+        argv: ["__storage-composition", paths.db]
+      });
+    })();
+    if (!worker) {
+      composition = {
+        ...composition,
+        state: "failed",
+        failure: "Salidium could not locate a runtime to measure the store with."
+      };
+      return composition;
+    }
+    compositionWorker = worker;
+    worker.unref();
+    composition = { ...composition, state: "running", failure: null };
+    worker.once("message", (message) => {
+      composition = message;
+    });
+    worker.once("error", (error51) => {
+      log.warn("storage composition worker failed", { err: String(error51) });
+      composition = { ...composition, state: "failed", failure: String(error51).slice(0, 500) };
+    });
+    worker.once("exit", (code) => {
+      if (compositionWorker === worker)
+        compositionWorker = void 0;
+      if (code !== 0 && composition.state === "running")
+        composition = {
+          ...composition,
+          state: "failed",
+          failure: `The measurement stopped with status ${code}.`
+        };
+      else if (composition.state === "running")
+        composition = {
+          ...composition,
+          state: "failed",
+          failure: "The measurement produced no result."
+        };
+    });
+    return composition;
+  };
   let port = config2.port;
   const server = createHttpServer({
     registry: registry2,
@@ -29530,7 +29705,9 @@ async function startDaemon(overrides = {}) {
         return inspectQueue(config2.home, { entryLimit: limit });
       },
       drainQueue: () => runQueueDrainMaintenance(config2.home, () => hooks.drainSpool()),
-      acknowledgeAlert: (id) => acknowledgeLocalAlert(config2.home, id)
+      acknowledgeAlert: (id) => acknowledgeLocalAlert(config2.home, id),
+      storageComposition: () => composition,
+      analyzeStorage
     },
     settings: {
       explainer: explainerSettings,
@@ -29621,6 +29798,8 @@ async function startDaemon(overrides = {}) {
       clearTimeout(usageBackfillRetryTimer);
     if (usageBackfillWorker)
       void usageBackfillWorker.terminate();
+    if (compositionWorker)
+      void compositionWorker.terminate();
     if (healthTimer)
       clearTimeout(healthTimer);
     if (hookTrustTimer)
@@ -31567,6 +31746,15 @@ async function runFirstRunOnboarding(context, io, options = {}) {
   };
 }
 
+// src/pauseOnRun.ts
+function clearsPauseOnRun(command, argument, argv) {
+  if (argv.includes("--no-resume")) return false;
+  if (["pause", "resume", "stop", "service"].includes(command)) return false;
+  if (command.startsWith("__")) return false;
+  if (command === "storage" && argument === "optimize") return false;
+  return true;
+}
+
 // src/render.ts
 var H = "\u2500";
 var CONTROL = /[\u0000-\u001F\u007F-\u009F]/g;
@@ -31797,6 +31985,7 @@ Usage:
   salidium retention apply      Apply one cleanup batch now (daemon must be stopped)
   salidium retention compact    Return reusable SQLite pages to the OS (daemon must be stopped)
   salidium storage              Show the lossless storage layout and page size
+  salidium storage composition  Measure what is using the space, by part and by project
   salidium storage optimize     Coordinate, copy, verify, and install the compact layout
   salidium pin [session]        Exempt a session from automatic retention
   salidium unpin [session]      Remove the retention exemption
@@ -31818,7 +32007,9 @@ Environment:
 
 Native Windows imports transcript history but does not install the POSIX live-hook relay.
 Ordinary commands resume an expired or manual pause. pause, stop, service commands, and coordinated
-storage optimize do not; resume changes collection state explicitly.
+storage optimize do not; resume changes collection state explicitly. Add --no-resume to any command
+to leave a pause in place, for a caller that has already seen the daemon answer and so cannot be
+recovering a marker its dead owner left behind.
 `;
 var require3 = createRequire2(import.meta.url);
 var VERSION2 = (() => {
@@ -31838,11 +32029,10 @@ async function main(argv) {
   const jsonOutput = argv.includes("--json");
   const quiet = argv.includes("--quiet");
   const positional = argv.filter(
-    (value2) => !["--yes", "-y", "--no-open", "--json", "--quiet"].includes(value2)
+    (value2) => !["--yes", "-y", "--no-open", "--json", "--quiet", "--no-resume"].includes(value2)
   );
   const [cmd = "up", arg, ...args] = positional;
-  if (!["pause", "resume", "stop", "service", "__usage-backfill"].includes(cmd) && !(cmd === "storage" && arg === "optimize"))
-    await implicitlyResumeCollection();
+  if (clearsPauseOnRun(cmd, arg, argv)) await implicitlyResumeCollection();
   switch (cmd) {
     case "help":
     case "--help":
@@ -31885,6 +32075,24 @@ async function main(argv) {
         store.close();
       }
     }
+    /*
+     * The other private worker entrypoint. Measuring what the store is made of reads the header of
+     * every stored event, which is ten seconds on a three gigabyte store, and `node:sqlite` is
+     * synchronous: run on the daemon's loop that is ten seconds of hooks going unanswered and
+     * spooling to disk. It runs here instead, on its own event loop and its own connection, and
+     * posts one message back.
+     */
+    case "__storage-composition": {
+      if (!arg || !resolve6(arg).startsWith(`${resolve6(salidiumHome)}${sep3}`))
+        throw new Error("storage composition store must be inside the Salidium state directory");
+      const store = new SqliteStore(resolve6(arg), { concurrentWriter: true });
+      try {
+        parentPort?.postMessage(store.storageComposition());
+        return 0;
+      } finally {
+        store.close();
+      }
+    }
     case "start": {
       const running = await ensureDaemon();
       const explanations = await currentExplanationState(running, "reachable");
@@ -32521,10 +32729,54 @@ Change with: salidium explanations off|when-done|each-reply
 `);
         return 1;
       }
-      if (arg !== void 0 && arg !== "optimize") {
-        process.stderr.write("storage accepts only `optimize`\n");
+      if (arg !== void 0 && arg !== "optimize" && arg !== "composition") {
+        process.stderr.write("storage accepts only `composition` or `optimize`\n");
         return 2;
       }
+      if (arg === "composition") {
+        const store = new SqliteStore(db, { concurrentWriter: true });
+        try {
+          const measured = store.storageComposition();
+          if (jsonOutput) {
+            process.stdout.write(`${JSON.stringify(measured)}
+`);
+            return 0;
+          }
+          const label = {
+            sessions: "Recorded sessions",
+            checkpoints: "Replay checkpoints",
+            provenance: "Provenance records",
+            structure: "Indexes and internal structure",
+            reusable: "Reusable space"
+          };
+          process.stdout.write(`On this Mac: ${formatBytes(measured.fileBytes ?? 0)}
+`);
+          for (const part of measured.parts)
+            process.stdout.write(
+              `  ${(label[part.key] ?? part.key).padEnd(30)} ${formatBytes(part.bytes)}
+`
+            );
+          process.stdout.write(
+            `
+${measured.sessions ?? 0} sessions across ${measured.projects.length + measured.projectsOmitted} projects.
+`
+          );
+          for (const project of measured.projects)
+            process.stdout.write(
+              `  ${formatBytes(project.bytes).padStart(9)}  ${String(project.sessions).padStart(4)} ${project.sessions === 1 ? "session " : "sessions"}  ${project.path || "No project recorded"}
+`
+            );
+          if (measured.projectsOmitted > 0)
+            process.stdout.write(`  ${measured.projectsOmitted} more not listed.
+`);
+          process.stdout.write(
+            "\nIndexes and internal structure is the remainder after the named parts, not a separate measurement.\n"
+          );
+          return 0;
+        } finally {
+          store.close();
+        }
+      }
       if (arg === void 0) {
         const layout = inspectStoreLayout(db);
         if (quiet) return layout.optimized ? 0 : 3;
@@ -33579,11 +33831,22 @@ async function ensureDaemon() {
   for (let i = 0; i < 100; i++) {
     await sleep(100);
     if (childFailure)
-      throw new Error(`daemon ${childFailure} before it became ready; see ${paths.startupLogFile}`);
+      throw new Error(
+        `daemon ${childFailure} before it became ready: ${startupFailureReason(paths.startupLogFile)}`
+      );
     const d = readDaemonJson(salidiumHome);
     if (d && d.pid === child.pid && await alive(d)) return d;
   }
-  throw new Error(`daemon did not start; see ${paths.startupLogFile}`);
+  throw new Error(`daemon did not start: ${startupFailureReason(paths.startupLogFile)}`);
+}
+function startupFailureReason(logFile) {
+  try {
+    const lines = readFileSync13(logFile, "utf8").split("\n").map((line) => line.trim()).filter(Boolean);
+    const last = lines.at(-1);
+    if (last) return `${last.slice(0, 200)} (see ${logFile})`;
+  } catch {
+  }
+  return `see ${logFile}`;
 }
 function compareSemver(left, right) {
   const parse3 = (value2) => {