sliftutils 1.4.47 → 1.4.48

package/index.d.ts

@@ -1141,7 +1141,13 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseFormat" {
     export declare const EMPTY_BUFFER: Buffer;
     export declare const ABSENT: unique symbol;
     export declare const TARGET_FILE_BYTES: number;
-    export declare function buildFileBuffer(rows: Record<string, unknown>[], times: number[], targetBytes?: number): Buffer[];
+    export interface BuiltFile {
+        buffer: Buffer;
+        minKey: string;
+        maxKey: string;
+        rowCount: number;
+    }
+    export declare function buildFileBuffer(rows: Record<string, unknown>[], times: number[], targetBytes?: number): BuiltFile[];
     export type BaseBulkDatabaseReader = {
         rowCount: number;
         totalBytes: number;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.4.47",
+    "version": "1.4.48",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/BulkDatabase2/BulkDatabaseBase.ts

@@ -929,9 +929,9 @@ export class BulkDatabaseBase<T extends { key: string }> {
         const timestamp = nextFileTime();
         const now = Date.now();
         const times = rows.map(() => now);
-        for (const buffer of buildFileBuffer(rows, times)) {
+        for (const built of buildFileBuffer(rows, times)) {
             const name = newFileName(timestamp);
-            await storage.set(name, encodeCompressedBlocks(buffer));
+            await storage.set(name, encodeCompressedBlocks(built.buffer));
         }
         this.resetReader();
         void this.maybeMerge();
@@ -1270,10 +1270,19 @@ export class BulkDatabaseBase<T extends { key: string }> {
         // Write all outputs BEFORE deleting any input, so a throw mid-write just leaves duplicates.
         const newNames: string[] = [];
         if (rows.length) {
-            for (const buffer of buildFileBuffer(rows, times)) {
+            const built = buildFileBuffer(rows, times);
+            // When the result is big enough to split across several files, log each sub-write (its key
+            // range + start/finish), since each storage.set can be large/slow and we want to see progress.
+            const split = built.length > 1;
+            if (split) console.log(`${blue(this.name)} merge: output split into ${built.length} files (> ${fmtBytes(TARGET_FILE_BYTES)} each)`);
+            for (let i = 0; i < built.length; i++) {
+                const part = built[i];
                 const name = newFileName(timestamp);
-                await storage.set(name, encodeCompressedBlocks(buffer));
+                const subStart = Date.now();
+                if (split) console.log(`    [${i + 1}/${built.length}] writing ${formatNumber(part.rowCount)} rows [${part.minKey} .. ${part.maxKey}] → ${name} at ${new Date(subStart).toISOString()}`);
+                await storage.set(name, encodeCompressedBlocks(part.buffer));
                 newNames.push(name);
+                if (split) console.log(`    [${i + 1}/${built.length}] wrote ${name} (${fmtBytes((await storage.getInfo(name).catch(() => undefined))?.size ?? 0)}) in ${formatTime(Date.now() - subStart)}`);
             }
         }
         // Carry surviving tombstones forward only if older files exist outside this merge that they still

package/storage/BulkDatabase2/BulkDatabaseFormat.d.ts

@@ -4,7 +4,13 @@ export declare const KEY_COLUMN = "key";
 export declare const EMPTY_BUFFER: Buffer;
 export declare const ABSENT: unique symbol;
 export declare const TARGET_FILE_BYTES: number;
-export declare function buildFileBuffer(rows: Record<string, unknown>[], times: number[], targetBytes?: number): Buffer[];
+export interface BuiltFile {
+    buffer: Buffer;
+    minKey: string;
+    maxKey: string;
+    rowCount: number;
+}
+export declare function buildFileBuffer(rows: Record<string, unknown>[], times: number[], targetBytes?: number): BuiltFile[];
 export type BaseBulkDatabaseReader = {
     rowCount: number;
     totalBytes: number;

package/storage/BulkDatabase2/BulkDatabaseFormat.ts

@@ -226,14 +226,25 @@ function buildOneFile(rows: Record<string, unknown>[], times: number[]): Buffer
     return Buffer.concat([lengthPrefix, headerBuf, ...blobs]);
 }
 
-// Returns one complete, independent file buffer per chunk of rows. Rows are first sorted by key, then
+// One complete, independent file: the encoded buffer plus its key range + row count (the caller logs the
+// range when a merge splits across several files).
+export interface BuiltFile { buffer: Buffer; minKey: string; maxKey: string; rowCount: number; }
+
+// Returns one complete, independent file per chunk of rows. Rows are first sorted by key, then
 // partitioned into key-contiguous chunks of ~targetBytes each — so every returned file is key-sorted
 // (tight minKey/maxKey for the read-skip + merge planner) and stays near the target size, and no
 // single column blob / Buffer.concat approaches the ~2GB limit. The chunks have disjoint key ranges,
-// so the caller just writes each as its own file. A normal-sized write returns a single buffer.
+// so the caller just writes each as its own file. A normal-sized write returns a single file.
 // `times[i]` is row i's write-time, stored per row so reads resolve a key to its latest value by time.
-export function buildFileBuffer(rows: Record<string, unknown>[], times: number[], targetBytes = TARGET_FILE_BYTES): Buffer[] {
-    if (rows.length === 0) return [buildOneFile([], [])];
+export function buildFileBuffer(rows: Record<string, unknown>[], times: number[], targetBytes = TARGET_FILE_BYTES): BuiltFile[] {
+    // A chunk is already key-sorted, so its first/last row are its min/max key.
+    const make = (rs: Record<string, unknown>[], ts: number[]): BuiltFile => ({
+        buffer: buildOneFile(rs, ts),
+        minKey: rs.length ? (rs[0][KEY_COLUMN] as string) : "",
+        maxKey: rs.length ? (rs[rs.length - 1][KEY_COLUMN] as string) : "",
+        rowCount: rs.length,
+    });
+    if (rows.length === 0) return [make([], [])];
     // Sort rows + their times together by key so each output file is key-contiguous.
     const order = rows.map((_, i) => i).sort((a, b) => {
         const ka = rows[a][KEY_COLUMN] as string;
@@ -242,19 +253,19 @@ export function buildFileBuffer(rows: Record<string, unknown>[], times: number[]
     });
     const sortedRows = order.map(i => rows[i]);
     const sortedTimes = order.map(i => times[i]);
-    const result: Buffer[] = [];
+    const result: BuiltFile[] = [];
     let chunkStart = 0;
     let chunkBytes = 0;
     for (let i = 0; i < sortedRows.length; i++) {
         const rowBytes = estimateRowBytes(sortedRows[i]);
         if (i > chunkStart && chunkBytes + rowBytes > targetBytes) {
-            result.push(buildOneFile(sortedRows.slice(chunkStart, i), sortedTimes.slice(chunkStart, i)));
+            result.push(make(sortedRows.slice(chunkStart, i), sortedTimes.slice(chunkStart, i)));
             chunkStart = i;
             chunkBytes = 0;
         }
         chunkBytes += rowBytes;
     }
-    result.push(buildOneFile(sortedRows.slice(chunkStart), sortedTimes.slice(chunkStart)));
+    result.push(make(sortedRows.slice(chunkStart), sortedTimes.slice(chunkStart)));
     return result;
 }