sliftutils 1.4.60 → 1.4.62

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.4.60",
+    "version": "1.4.62",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/spec.txt

@@ -1,11 +1,5 @@
 
 
 
-Compact face_frames3
 
-
-
-When detecting faces in the JavaScript code and the Python code. Let's make it so that we only store the face frames for the top 10 characters. We can show the store the embeddings for all of them, But for the face frames collection only store the top 10 faces. The other ones will just be blank, and that should be fine. Also, let's make sure when we're storing them the max dimension is 128 pixels and the quality is only 80. 
-
-
-Then go verify on the other site fixing up the bad data. It should be quite fast because there's so much overlap, but we will find out. 
+Create a companion large data storage thing.  which updates the data less frequently, Where we reference it with unique IDs so we don't have to merge as much and it doesn't have to be as strict. This should allow our large databases to be merged faster, even though they are pretty fast to merge at the moment, but they probably will get slower if we make them 10 times bigger. 

package/storage/BulkDatabase2/BulkDatabaseBase.ts

@@ -658,17 +658,25 @@ export class BulkDatabaseBase<T extends { key: string }> {
         // expanded console.group so a merge takes one collapsible block, not a
         // screenful (and never the per-file name dump it used to spew).
         const steps: string[] = [];
-        steps.push(`read ${inputs.length} input file(s), ${fmtBytes(inTotal)}`);
+        // Each step records the time spent since the previous step in blue at the front, so callers can
+        // just describe what they did and not bother measuring/formatting elapsed times themselves.
+        let lastStepMs = mergeStartMs;
+        const log = (line: string) => {
+            const now = Date.now();
+            let lineFormatted = `${blue(formatTime(now - lastStepMs))} ${line}`;
+            // DO NOT REMOVE THIS LOG! Obviously, we should be logging as we run, or else we don't get progress. Why would we stop getting progress? Progress is almost more important than showing a summary at the end...
+            console.log(lineFormatted);
+            steps.push(lineFormatted);
+            lastStepMs = now;
+        };
+        log(`${magenta("read")}: ${inputs.length} input file(s), ${fmtBytes(inTotal)}`);
 
         const newNames: string[] = [];
         const mergeResult = await runPlannedMerge({
             sources: readers,
             sourceNames: readerNames,
             collectionName: this.name,
-            log: line => {
-                console.log(line);
-                steps.push(line);
-            },
+            log,
             writeFile: async (data) => {
                 const fname = newFileName(timestamp);
                 await storage.set(fname, encodeCompressedBlocks(data));
@@ -688,7 +696,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
 
         const outputs = await Promise.all(outNames.map(async n => ({ name: n, size: (await storage.getInfo(n).catch(() => undefined))?.size ?? 0 })));
         const outTotal = outputs.reduce((a, f) => a + f.size, 0);
-        steps.push(`wrote ${outputs.length} output file(s), ${fmtBytes(outTotal)}${carriedDeletes ? `, ${carriedDeletes} tombstones carried` : ""}`);
+        log(`${magenta("wrote")}: ${outputs.length} output file(s), ${fmtBytes(outTotal)}${carriedDeletes ? `, ${carriedDeletes} tombstones carried` : ""}`);
 
         console.group(`${blue(this.name)} ${magenta("merge")}: ${fmtBytes(inTotal)} → ${fmtBytes(outTotal)} (${inputs.length}→${outputs.length} files) in ${formatTime(Date.now() - mergeStartMs)}`);
         for (const line of steps) console.log(line);

package/storage/BulkDatabase2/BulkDatabaseMerge.ts

@@ -1,5 +1,5 @@
-import { formatNumber, formatTime } from "socket-function/src/formatting/format";
-import { blue, red } from "socket-function/src/formatting/logColors";
+import { formatNumber } from "socket-function/src/formatting/format";
+import { blue, magenta } from "socket-function/src/formatting/logColors";
 import {
     BaseBulkDatabaseReader,
     ColumnIndex,
@@ -98,8 +98,6 @@ export async function runPlannedMerge(config: {
     const log = config.log ?? (line => console.log(`${blue(config.collectionName)} ${line}`));
 
     // ─────────────────────────────────────────── Phase 1: plan ───────────────────────────────────────────
-    const planStart = Date.now();
-
     // Aggregate keyTimes + deleteTimes across all sources (max per key).
     const deleteTime = new Map<string, number>();
     const keyTime = new Map<string, number>();
@@ -222,8 +220,7 @@ export async function runPlannedMerge(config: {
     // Build per-file plans: offsets/types/copy-runs per column.
     const plans = fileKeyRanges.map(range => buildOutputPlan(range, liveKeys, cellsPerKey, allColumns, keyTime));
 
-    const planTime = Date.now() - planStart;
-    log(`mapping done — ${formatNumber(liveKeys.length)} live keys, ${plans.length} output file(s), ${formatNumber(carriedDeletes.size)} tombstones carried, in ${red(formatTime(planTime))}`);
+    log(`${magenta("plan")}: ${formatNumber(liveKeys.length)} live keys, ${plans.length} output file(s), ${formatNumber(carriedDeletes.size)} tombstones carried`);
 
     // ───────────────────────────────────────── Phase 2: execute ──────────────────────────────────────────
     // Group output files into batches that fit within targetBatchBytes so we read inputs once per batch
@@ -244,17 +241,15 @@ export async function runPlannedMerge(config: {
     }
 
     const outputs: PlannedMergeOutput[] = [];
-    const execStart = Date.now();
     for (let bi = 0; bi < batches.length; bi++) {
         const batch = batches[bi];
         const batchBytes = batch.reduce((a, p) => a + p.estimatedFileBytes, 0);
-        log(`batch ${bi + 1}/${batches.length}: ${batch.length} output file(s), ~${fmtBytes(batchBytes)} budget`);
+        log(`${magenta("batch")} ${bi + 1}/${batches.length}: ${batch.length} output file(s), ~${fmtBytes(batchBytes)} budget`);
         const batchOutputs = await executeBatch(batch, indexesPerSource, config.sources, config.sourceNames, config.writeFile, log);
         outputs.push(...batchOutputs);
     }
-    const execTime = Date.now() - execStart;
     const writtenBytes = outputs.reduce((a, o) => a + o.size, 0);
-    log(`execute done — ${outputs.length} file(s), ${fmtBytes(writtenBytes)} written, in ${red(formatTime(execTime))} (plan + execute: ${red(formatTime(Date.now() - planStart))})`);
+    log(`${magenta("execute")}: ${outputs.length} file(s), ${fmtBytes(writtenBytes)} written`);
 
     return { outputs, carriedDeletes };
 }
@@ -387,14 +382,12 @@ async function executeBatch(
     const outputs: PlannedMergeOutput[] = [];
     for (let fi = 0; fi < plans.length; fi++) {
         const plan = plans[fi];
-        const start = Date.now();
         const valueColumns = blobsPerFile[fi].map(b => ({ name: b.name, blob: b.blob }));
         const fileBuf = assemblePlannedFile({ valueColumns, keys: plan.keys, times: plan.times });
         const { name, size } = await writeFile(fileBuf);
-        const elapsed = Date.now() - start;
         const sourcesNamed = new Map<string, number>();
         for (const [si, n] of plan.sourceCounts) sourcesNamed.set(sourceNames[si], n);
-        log(`output: ${formatNumber(plan.keys.length)} rows from ${plan.sourceCounts.size} input(s), ${fmtBytes(size)} in ${formatTime(elapsed)}`);
+        log(`${magenta("output")}: ${formatNumber(plan.keys.length)} rows from ${plan.sourceCounts.size} input(s), ${fmtBytes(size)}`);
         outputs.push({ name, minKey: plan.minKey, maxKey: plan.maxKey, rowCount: plan.keys.length, size, sources: sourcesNamed });
     }
     return outputs;