sliftutils 1.4.60 → 1.4.61

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.4.60",
+    "version": "1.4.61",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/BulkDatabase2/BulkDatabaseBase.ts

@@ -658,17 +658,22 @@ export class BulkDatabaseBase<T extends { key: string }> {
         // expanded console.group so a merge takes one collapsible block, not a
         // screenful (and never the per-file name dump it used to spew).
         const steps: string[] = [];
-        steps.push(`read ${inputs.length} input file(s), ${fmtBytes(inTotal)}`);
+        // Each step records the time spent since the previous step in blue at the front, so callers can
+        // just describe what they did and not bother measuring/formatting elapsed times themselves.
+        let lastStepMs = mergeStartMs;
+        const log = (line: string) => {
+            const now = Date.now();
+            steps.push(`${blue(formatTime(now - lastStepMs))} ${line}`);
+            lastStepMs = now;
+        };
+        log(`${magenta("read")}: ${inputs.length} input file(s), ${fmtBytes(inTotal)}`);
 
         const newNames: string[] = [];
         const mergeResult = await runPlannedMerge({
             sources: readers,
             sourceNames: readerNames,
             collectionName: this.name,
-            log: line => {
-                console.log(line);
-                steps.push(line);
-            },
+            log,
             writeFile: async (data) => {
                 const fname = newFileName(timestamp);
                 await storage.set(fname, encodeCompressedBlocks(data));
@@ -688,7 +693,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
 
         const outputs = await Promise.all(outNames.map(async n => ({ name: n, size: (await storage.getInfo(n).catch(() => undefined))?.size ?? 0 })));
         const outTotal = outputs.reduce((a, f) => a + f.size, 0);
-        steps.push(`wrote ${outputs.length} output file(s), ${fmtBytes(outTotal)}${carriedDeletes ? `, ${carriedDeletes} tombstones carried` : ""}`);
+        log(`${magenta("wrote")}: ${outputs.length} output file(s), ${fmtBytes(outTotal)}${carriedDeletes ? `, ${carriedDeletes} tombstones carried` : ""}`);
 
         console.group(`${blue(this.name)} ${magenta("merge")}: ${fmtBytes(inTotal)} → ${fmtBytes(outTotal)} (${inputs.length}→${outputs.length} files) in ${formatTime(Date.now() - mergeStartMs)}`);
         for (const line of steps) console.log(line);

package/storage/BulkDatabase2/BulkDatabaseMerge.ts

@@ -1,5 +1,5 @@
-import { formatNumber, formatTime } from "socket-function/src/formatting/format";
-import { blue, red } from "socket-function/src/formatting/logColors";
+import { formatNumber } from "socket-function/src/formatting/format";
+import { blue, magenta } from "socket-function/src/formatting/logColors";
 import {
     BaseBulkDatabaseReader,
     ColumnIndex,
@@ -98,8 +98,6 @@ export async function runPlannedMerge(config: {
     const log = config.log ?? (line => console.log(`${blue(config.collectionName)} ${line}`));
 
     // ─────────────────────────────────────────── Phase 1: plan ───────────────────────────────────────────
-    const planStart = Date.now();
-
     // Aggregate keyTimes + deleteTimes across all sources (max per key).
     const deleteTime = new Map<string, number>();
     const keyTime = new Map<string, number>();
@@ -222,8 +220,7 @@ export async function runPlannedMerge(config: {
     // Build per-file plans: offsets/types/copy-runs per column.
     const plans = fileKeyRanges.map(range => buildOutputPlan(range, liveKeys, cellsPerKey, allColumns, keyTime));
 
-    const planTime = Date.now() - planStart;
-    log(`mapping done — ${formatNumber(liveKeys.length)} live keys, ${plans.length} output file(s), ${formatNumber(carriedDeletes.size)} tombstones carried, in ${red(formatTime(planTime))}`);
+    log(`${magenta("plan")}: ${formatNumber(liveKeys.length)} live keys, ${plans.length} output file(s), ${formatNumber(carriedDeletes.size)} tombstones carried`);
 
     // ───────────────────────────────────────── Phase 2: execute ──────────────────────────────────────────
     // Group output files into batches that fit within targetBatchBytes so we read inputs once per batch
@@ -244,17 +241,15 @@ export async function runPlannedMerge(config: {
     }
 
     const outputs: PlannedMergeOutput[] = [];
-    const execStart = Date.now();
     for (let bi = 0; bi < batches.length; bi++) {
         const batch = batches[bi];
         const batchBytes = batch.reduce((a, p) => a + p.estimatedFileBytes, 0);
-        log(`batch ${bi + 1}/${batches.length}: ${batch.length} output file(s), ~${fmtBytes(batchBytes)} budget`);
+        log(`${magenta("batch")} ${bi + 1}/${batches.length}: ${batch.length} output file(s), ~${fmtBytes(batchBytes)} budget`);
         const batchOutputs = await executeBatch(batch, indexesPerSource, config.sources, config.sourceNames, config.writeFile, log);
         outputs.push(...batchOutputs);
     }
-    const execTime = Date.now() - execStart;
     const writtenBytes = outputs.reduce((a, o) => a + o.size, 0);
-    log(`execute done — ${outputs.length} file(s), ${fmtBytes(writtenBytes)} written, in ${red(formatTime(execTime))} (plan + execute: ${red(formatTime(Date.now() - planStart))})`);
+    log(`${magenta("execute")}: ${outputs.length} file(s), ${fmtBytes(writtenBytes)} written`);
 
     return { outputs, carriedDeletes };
 }
@@ -387,14 +382,12 @@ async function executeBatch(
     const outputs: PlannedMergeOutput[] = [];
     for (let fi = 0; fi < plans.length; fi++) {
         const plan = plans[fi];
-        const start = Date.now();
         const valueColumns = blobsPerFile[fi].map(b => ({ name: b.name, blob: b.blob }));
         const fileBuf = assemblePlannedFile({ valueColumns, keys: plan.keys, times: plan.times });
         const { name, size } = await writeFile(fileBuf);
-        const elapsed = Date.now() - start;
         const sourcesNamed = new Map<string, number>();
         for (const [si, n] of plan.sourceCounts) sourcesNamed.set(sourceNames[si], n);
-        log(`output: ${formatNumber(plan.keys.length)} rows from ${plan.sourceCounts.size} input(s), ${fmtBytes(size)} in ${formatTime(elapsed)}`);
+        log(`${magenta("output")}: ${formatNumber(plan.keys.length)} rows from ${plan.sourceCounts.size} input(s), ${fmtBytes(size)}`);
         outputs.push({ name, minKey: plan.minKey, maxKey: plan.maxKey, rowCount: plan.keys.length, size, sources: sourcesNamed });
     }
     return outputs;