sliftutils 1.4.74 → 1.4.76

package/index.d.ts

@@ -1547,7 +1547,7 @@ declare module "sliftutils/storage/BulkDatabase2/mergeMarkers" {
     }): Promise<void>;
     export declare function readDeleteMarkers(storage: FileStorage, allNames: string[]): Promise<DeleteMarker[]>;
     export declare function markerExclusions(markers: DeleteMarker[]): Set<string>;
-    export declare function processDeleteMarkers(storage: FileStorage, markers: DeleteMarker[], allNames: string[]): Promise<void>;
+    export declare function processDeleteMarkers(name: string, storage: FileStorage, markers: DeleteMarker[], allNames: string[]): Promise<void>;
 
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.4.74",
+    "version": "1.4.76",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/BulkDatabase2/BulkDatabaseBase.ts

@@ -573,7 +573,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
         const storage = await this.storage();
         const names = await storage.getKeys();
         const markers = await readDeleteMarkers(storage, names);
-        if (markers.length) await processDeleteMarkers(storage, markers, names);
+        if (markers.length) await processDeleteMarkers(this.name, storage, markers, names);
     });
 
     private async writeBulkFile(rows: Record<string, unknown>[]): Promise<void> {

package/storage/BulkDatabase2/BulkDatabaseMerge.ts

@@ -396,7 +396,7 @@ async function executeBatch(
                 }
             }
         }
-        log(`${magenta("input")} ${sourceNames[si]}: ${fmtBytes(bytesRead)} read`);
+        log(`${magenta("input")} ${si + 1}/${sources.length} ${sourceNames[si]}: ${fmtBytes(bytesRead)} read`);
     }
 
     // Assemble + write each output file in this batch.
@@ -408,7 +408,7 @@ async function executeBatch(
         const { name, size } = await writeFile(fileBuf);
         const sourcesNamed = new Map<string, number>();
         for (const [si, n] of plan.sourceCounts) sourcesNamed.set(sourceNames[si], n);
-        log(`${magenta("output")}: ${formatNumber(plan.keys.length)} rows from ${plan.sourceCounts.size} input(s), ${fmtBytes(size)}`);
+        log(`${magenta("output")} ${fi + 1}/${plans.length}: ${formatNumber(plan.keys.length)} rows from ${plan.sourceCounts.size} input(s), ${fmtBytes(size)}`);
         outputs.push({ name, minKey: plan.minKey, maxKey: plan.maxKey, rowCount: plan.keys.length, size, sources: sourcesNamed });
     }
     return outputs;

package/storage/BulkDatabase2/mergeMarkers.d.ts

@@ -12,4 +12,4 @@ export declare function writeDeleteMarker(storage: FileStorage, config: {
 }): Promise<void>;
 export declare function readDeleteMarkers(storage: FileStorage, allNames: string[]): Promise<DeleteMarker[]>;
 export declare function markerExclusions(markers: DeleteMarker[]): Set<string>;
-export declare function processDeleteMarkers(storage: FileStorage, markers: DeleteMarker[], allNames: string[]): Promise<void>;
+export declare function processDeleteMarkers(name: string, storage: FileStorage, markers: DeleteMarker[], allNames: string[]): Promise<void>;

package/storage/BulkDatabase2/mergeMarkers.ts

@@ -15,6 +15,7 @@
 //      the marker itself (kept around briefly so a concurrent reader still excludes the files consistently).
 
 import type { FileStorage } from "../FileFolderAPI";
+import { blue, magenta } from "socket-function/src/formatting/logColors";
 
 const MARKER_PREFIX = ".delete-marker_";
 const MARKER_GRACE_MS = 5 * 60 * 1000;
@@ -67,7 +68,7 @@ export function markerExclusions(markers: DeleteMarker[]): Set<string> {
 
 // Act on each marker (see file header). Idempotent and best-effort: every removal is guarded, so two
 // processes (or two reads) running this at once just race to the same already-correct end state.
-export async function processDeleteMarkers(storage: FileStorage, markers: DeleteMarker[], allNames: string[]): Promise<void> {
+export async function processDeleteMarkers(name: string, storage: FileStorage, markers: DeleteMarker[], allNames: string[]): Promise<void> {
     const present = new Set(allNames);
     const now = Date.now();
     for (const marker of markers) {
@@ -81,9 +82,12 @@ export async function processDeleteMarkers(storage: FileStorage, markers: Delete
         }
         const replacedExists = marker.replacedBy.length > 0 && marker.replacedBy.every(n => present.has(n));
         if (replacedExists || now - marker.time > MARKER_GRACE_MS) {
+            const reason = replacedExists ? `replacements landed` : `marker aged ${Math.round((now - marker.time) / 1000)}s`;
+            let deleted = 0;
             for (const n of stillPresent) {
-                try { await storage.remove(n); } catch { /* already gone */ }
+                try { await storage.remove(n); deleted++; } catch { /* already gone */ }
             }
+            if (deleted) console.log(`${blue(name)} ${magenta("retire")}: deleted ${deleted} merged-away input file(s) (${reason})`);
         }
     }
 }