sliftutils 1.4.57 → 1.4.58

package/index.d.ts

@@ -1231,6 +1231,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseMerge" {
         collectionName: string;
         targetFileBytes?: number;
         targetBatchBytes?: number;
+        log?: (line: string) => void;
         writeFile: (data: Buffer) => Promise<{
             name: string;
             size: number;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.4.57",
+    "version": "1.4.58",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/BulkDatabase2/BulkDatabaseBase.ts

@@ -13,7 +13,7 @@ import {
 import { runPlannedMerge } from "./BulkDatabaseMerge";
 import { blockCache, encodeCompressedBlocks } from "./blockCache";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
-import { blue } from "socket-function/src/formatting/logColors";
+import { blue, magenta } from "socket-function/src/formatting/logColors";
 import { STREAM_EXTENSION, frameDeletes, frameRows, streamReaderFromEntries } from "./streamLog";
 import { broadcast as syncBroadcast, broadcastSeal as syncBroadcastSeal, connect as syncConnect, isSyncSupported, RemoteWrite } from "./syncClient";
 import { DELETED } from "./WriteOverlay";
@@ -654,14 +654,18 @@ export class BulkDatabaseBase<T extends { key: string }> {
         ];
         const inTotal = inputs.reduce((a, f) => a + f.size, 0);
         const mergeStartMs = Date.now();
-        console.log(`${blue(this.name)} merge: reading ${inputs.length} files (${fmtBytes(inTotal)}) at ${new Date(mergeStartMs).toISOString()}`);
-        for (const f of inputs) console.log(`    in  ${f.name}  ${fmtBytes(f.size)}`);
+        // Collect each step instead of logging it live; emitted at the end as one
+        // expanded console.group so a merge takes one collapsible block, not a
+        // screenful (and never the per-file name dump it used to spew).
+        const steps: string[] = [];
+        steps.push(`read ${inputs.length} input file(s), ${fmtBytes(inTotal)}`);
 
         const newNames: string[] = [];
         const mergeResult = await runPlannedMerge({
             sources: readers,
             sourceNames: readerNames,
             collectionName: this.name,
+            log: line => steps.push(line),
             writeFile: async (data) => {
                 const fname = newFileName(timestamp);
                 await storage.set(fname, encodeCompressedBlocks(data));
@@ -681,8 +685,11 @@ export class BulkDatabaseBase<T extends { key: string }> {
 
         const outputs = await Promise.all(outNames.map(async n => ({ name: n, size: (await storage.getInfo(n).catch(() => undefined))?.size ?? 0 })));
         const outTotal = outputs.reduce((a, f) => a + f.size, 0);
-        console.log(`${blue(this.name)} merge: wrote ${outputs.length} files (${fmtBytes(outTotal)}, from ${fmtBytes(inTotal)})${carriedDeletes ? `, ${carriedDeletes} tombstones carried` : ""} at ${new Date().toISOString()} (took ${formatTime(Date.now() - mergeStartMs)})`);
-        for (const f of outputs) console.log(`    out ${f.name}  ${fmtBytes(f.size)}`);
+        steps.push(`wrote ${outputs.length} output file(s), ${fmtBytes(outTotal)}${carriedDeletes ? `, ${carriedDeletes} tombstones carried` : ""}`);
+
+        console.group(`${blue(this.name)} ${magenta("merge")}: ${fmtBytes(inTotal)} → ${fmtBytes(outTotal)} (${inputs.length}→${outputs.length} files) in ${formatTime(Date.now() - mergeStartMs)}`);
+        for (const line of steps) console.log(line);
+        console.groupEnd();
 
         const remove = async (name: string) => { try { await storage.remove(name); } catch { /* already gone */ } };
         for (const f of consumedBulk) await remove(f.fileName);
@@ -746,7 +753,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
                 const sizes = await Promise.all(streamFiles.map(async f => { try { return (await storage.getInfo(f.fileName))?.size ?? 0; } catch { return 0; } }));
                 const totalStreamBytes = sizes.reduce((a, b) => a + b, 0);
                 if (totalStreamBytes > bulkDatabase2Timing.streamFoldHardLimitBytes) {
-                    console.log(`${blue(this.name)} stream tier ${fmtBytes(totalStreamBytes)} over hard limit ${fmtBytes(bulkDatabase2Timing.streamFoldHardLimitBytes)} — folding all streams now`);
+                    console.log(`${blue(this.name)} ${magenta("fold")} stream tier ${fmtBytes(totalStreamBytes)} over hard limit ${fmtBytes(bulkDatabase2Timing.streamFoldHardLimitBytes)} — folding all streams now`);
                     if (await this.mergeFileSet([], streamFiles, false, true)) merged = true;
                 }
             }

package/storage/BulkDatabase2/BulkDatabaseMerge.d.ts

@@ -38,6 +38,7 @@ export declare function runPlannedMerge(config: {
     collectionName: string;
     targetFileBytes?: number;
     targetBatchBytes?: number;
+    log?: (line: string) => void;
     writeFile: (data: Buffer) => Promise<{
         name: string;
         size: number;

package/storage/BulkDatabase2/BulkDatabaseMerge.ts

@@ -88,10 +88,14 @@ export async function runPlannedMerge(config: {
     collectionName: string;
     targetFileBytes?: number;
     targetBatchBytes?: number;
+    // Sink for step lines. When omitted, falls back to a blue-prefixed
+    // console.log so standalone callers still get identifiable output.
+    log?: (line: string) => void;
     writeFile: (data: Buffer) => Promise<{ name: string; size: number }>;
 }): Promise<{ outputs: PlannedMergeOutput[]; carriedDeletes: Map<string, number> }> {
     const targetFileBytes = config.targetFileBytes ?? TARGET_FILE_BYTES;
     const targetBatchBytes = config.targetBatchBytes ?? DEFAULT_OUTPUT_BATCH_BYTES;
+    const log = config.log ?? (line => console.log(`${blue(config.collectionName)} ${line}`));
 
     // ─────────────────────────────────────────── Phase 1: plan ───────────────────────────────────────────
     const planStart = Date.now();
@@ -219,7 +223,7 @@ export async function runPlannedMerge(config: {
     const plans = fileKeyRanges.map(range => buildOutputPlan(range, liveKeys, cellsPerKey, allColumns, keyTime));
 
     const planTime = Date.now() - planStart;
-    console.log(`${blue(config.collectionName)} planned merge: mapping done — ${formatNumber(liveKeys.length)} live keys, ${plans.length} output file(s), ${formatNumber(carriedDeletes.size)} tombstones carried, in ${red(formatTime(planTime))}`);
+    log(`mapping done — ${formatNumber(liveKeys.length)} live keys, ${plans.length} output file(s), ${formatNumber(carriedDeletes.size)} tombstones carried, in ${red(formatTime(planTime))}`);
 
     // ───────────────────────────────────────── Phase 2: execute ──────────────────────────────────────────
     // Group output files into batches that fit within targetBatchBytes so we read inputs once per batch
@@ -244,13 +248,13 @@ export async function runPlannedMerge(config: {
     for (let bi = 0; bi < batches.length; bi++) {
         const batch = batches[bi];
         const batchBytes = batch.reduce((a, p) => a + p.estimatedFileBytes, 0);
-        console.log(`${blue(config.collectionName)} merge batch ${bi + 1}/${batches.length}: ${batch.length} output file(s), ~${fmtBytes(batchBytes)} budget`);
-        const batchOutputs = await executeBatch(batch, indexesPerSource, config.sources, config.sourceNames, config.collectionName, config.writeFile);
+        log(`batch ${bi + 1}/${batches.length}: ${batch.length} output file(s), ~${fmtBytes(batchBytes)} budget`);
+        const batchOutputs = await executeBatch(batch, indexesPerSource, config.sources, config.sourceNames, config.writeFile, log);
         outputs.push(...batchOutputs);
     }
     const execTime = Date.now() - execStart;
     const writtenBytes = outputs.reduce((a, o) => a + o.size, 0);
-    console.log(`${blue(config.collectionName)} planned merge: done — ${outputs.length} file(s), ${fmtBytes(writtenBytes)} written, in ${red(formatTime(execTime))} (planning + execute: ${red(formatTime(Date.now() - planStart))})`);
+    log(`execute done — ${outputs.length} file(s), ${fmtBytes(writtenBytes)} written, in ${red(formatTime(execTime))} (plan + execute: ${red(formatTime(Date.now() - planStart))})`);
 
     return { outputs, carriedDeletes };
 }
@@ -334,8 +338,8 @@ async function executeBatch(
     indexesPerSource: Map<string, ColumnIndex>[],
     sources: BaseBulkDatabaseReader[],
     sourceNames: string[],
-    collectionName: string,
     writeFile: (data: Buffer) => Promise<{ name: string; size: number }>,
+    log: (line: string) => void,
 ): Promise<PlannedMergeOutput[]> {
     // Allocate one big buffer per (output file, column) — offsets + types are written immediately from
     // the plan, the data section is filled below by the per-source copy loop.
@@ -390,8 +394,7 @@ async function executeBatch(
         const elapsed = Date.now() - start;
         const sourcesNamed = new Map<string, number>();
         for (const [si, n] of plan.sourceCounts) sourcesNamed.set(sourceNames[si], n);
-        const srcText = [...sourcesNamed.entries()].sort((a, b) => b[1] - a[1]).map(([s, n]) => `${s}:${formatNumber(n)}`).join(", ") || "—";
-        console.log(`    ${blue(collectionName)} output ${name}: ${formatNumber(plan.keys.length)} rows [${plan.minKey} .. ${plan.maxKey}] from {${srcText}}, ${fmtBytes(size)} in ${formatTime(elapsed)}`);
+        log(`output: ${formatNumber(plan.keys.length)} rows from ${plan.sourceCounts.size} input(s), ${fmtBytes(size)} in ${formatTime(elapsed)}`);
         outputs.push({ name, minKey: plan.minKey, maxKey: plan.maxKey, rowCount: plan.keys.length, size, sources: sourcesNamed });
     }
     return outputs;