sliftutils 1.4.59 → 1.4.61

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.4.59",
+    "version": "1.4.61",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/spec.txt

@@ -1,7 +1,11 @@
-TODO: Take the half finished apiServer code in vidfrontend, finish the todos, and move it here.
-    - Having an alternate version, where we run a server that does watching is nice, it's heavier and does require setting up certificates. but it makes hot reloading faster. And... It can be used to run an actual site that's public.  
 
-ALSO, go fix our typings in socket-function as well
 
-TODO:
-- vscode extension as well
+
+Compact face_frames3
+
+
+
+When detecting faces in the JavaScript code and the Python code. Let's make it so that we only store the face frames for the top 10 characters. We can show the store the embeddings for all of them, But for the face frames collection only store the top 10 faces. The other ones will just be blank, and that should be fine. Also, let's make sure when we're storing them the max dimension is 128 pixels and the quality is only 80. 
+
+
+Then go verify on the other site fixing up the bad data. It should be quite fast because there's so much overlap, but we will find out. 

package/storage/BulkDatabase2/BulkDatabaseBase.ts

@@ -658,14 +658,22 @@ export class BulkDatabaseBase<T extends { key: string }> {
         // expanded console.group so a merge takes one collapsible block, not a
         // screenful (and never the per-file name dump it used to spew).
         const steps: string[] = [];
-        steps.push(`read ${inputs.length} input file(s), ${fmtBytes(inTotal)}`);
+        // Each step records the time spent since the previous step in blue at the front, so callers can
+        // just describe what they did and not bother measuring/formatting elapsed times themselves.
+        let lastStepMs = mergeStartMs;
+        const log = (line: string) => {
+            const now = Date.now();
+            steps.push(`${blue(formatTime(now - lastStepMs))} ${line}`);
+            lastStepMs = now;
+        };
+        log(`${magenta("read")}: ${inputs.length} input file(s), ${fmtBytes(inTotal)}`);
 
         const newNames: string[] = [];
         const mergeResult = await runPlannedMerge({
             sources: readers,
             sourceNames: readerNames,
             collectionName: this.name,
-            log: line => steps.push(line),
+            log,
             writeFile: async (data) => {
                 const fname = newFileName(timestamp);
                 await storage.set(fname, encodeCompressedBlocks(data));
@@ -685,7 +693,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
 
         const outputs = await Promise.all(outNames.map(async n => ({ name: n, size: (await storage.getInfo(n).catch(() => undefined))?.size ?? 0 })));
         const outTotal = outputs.reduce((a, f) => a + f.size, 0);
-        steps.push(`wrote ${outputs.length} output file(s), ${fmtBytes(outTotal)}${carriedDeletes ? `, ${carriedDeletes} tombstones carried` : ""}`);
+        log(`${magenta("wrote")}: ${outputs.length} output file(s), ${fmtBytes(outTotal)}${carriedDeletes ? `, ${carriedDeletes} tombstones carried` : ""}`);
 
         console.group(`${blue(this.name)} ${magenta("merge")}: ${fmtBytes(inTotal)} → ${fmtBytes(outTotal)} (${inputs.length}→${outputs.length} files) in ${formatTime(Date.now() - mergeStartMs)}`);
         for (const line of steps) console.log(line);

package/storage/BulkDatabase2/BulkDatabaseMerge.ts

@@ -1,5 +1,5 @@
-import { formatNumber, formatTime } from "socket-function/src/formatting/format";
-import { blue, red } from "socket-function/src/formatting/logColors";
+import { formatNumber } from "socket-function/src/formatting/format";
+import { blue, magenta } from "socket-function/src/formatting/logColors";
 import {
     BaseBulkDatabaseReader,
     ColumnIndex,
@@ -98,8 +98,6 @@ export async function runPlannedMerge(config: {
     const log = config.log ?? (line => console.log(`${blue(config.collectionName)} ${line}`));
 
     // ─────────────────────────────────────────── Phase 1: plan ───────────────────────────────────────────
-    const planStart = Date.now();
-
     // Aggregate keyTimes + deleteTimes across all sources (max per key).
     const deleteTime = new Map<string, number>();
     const keyTime = new Map<string, number>();
@@ -222,8 +220,7 @@ export async function runPlannedMerge(config: {
     // Build per-file plans: offsets/types/copy-runs per column.
     const plans = fileKeyRanges.map(range => buildOutputPlan(range, liveKeys, cellsPerKey, allColumns, keyTime));
 
-    const planTime = Date.now() - planStart;
-    log(`mapping done — ${formatNumber(liveKeys.length)} live keys, ${plans.length} output file(s), ${formatNumber(carriedDeletes.size)} tombstones carried, in ${red(formatTime(planTime))}`);
+    log(`${magenta("plan")}: ${formatNumber(liveKeys.length)} live keys, ${plans.length} output file(s), ${formatNumber(carriedDeletes.size)} tombstones carried`);
 
     // ───────────────────────────────────────── Phase 2: execute ──────────────────────────────────────────
     // Group output files into batches that fit within targetBatchBytes so we read inputs once per batch
@@ -244,17 +241,15 @@ export async function runPlannedMerge(config: {
     }
 
     const outputs: PlannedMergeOutput[] = [];
-    const execStart = Date.now();
     for (let bi = 0; bi < batches.length; bi++) {
         const batch = batches[bi];
         const batchBytes = batch.reduce((a, p) => a + p.estimatedFileBytes, 0);
-        log(`batch ${bi + 1}/${batches.length}: ${batch.length} output file(s), ~${fmtBytes(batchBytes)} budget`);
+        log(`${magenta("batch")} ${bi + 1}/${batches.length}: ${batch.length} output file(s), ~${fmtBytes(batchBytes)} budget`);
         const batchOutputs = await executeBatch(batch, indexesPerSource, config.sources, config.sourceNames, config.writeFile, log);
         outputs.push(...batchOutputs);
     }
-    const execTime = Date.now() - execStart;
     const writtenBytes = outputs.reduce((a, o) => a + o.size, 0);
-    log(`execute done — ${outputs.length} file(s), ${fmtBytes(writtenBytes)} written, in ${red(formatTime(execTime))} (plan + execute: ${red(formatTime(Date.now() - planStart))})`);
+    log(`${magenta("execute")}: ${outputs.length} file(s), ${fmtBytes(writtenBytes)} written`);
 
     return { outputs, carriedDeletes };
 }
@@ -387,14 +382,12 @@ async function executeBatch(
     const outputs: PlannedMergeOutput[] = [];
     for (let fi = 0; fi < plans.length; fi++) {
         const plan = plans[fi];
-        const start = Date.now();
         const valueColumns = blobsPerFile[fi].map(b => ({ name: b.name, blob: b.blob }));
         const fileBuf = assemblePlannedFile({ valueColumns, keys: plan.keys, times: plan.times });
         const { name, size } = await writeFile(fileBuf);
-        const elapsed = Date.now() - start;
         const sourcesNamed = new Map<string, number>();
         for (const [si, n] of plan.sourceCounts) sourcesNamed.set(sourceNames[si], n);
-        log(`output: ${formatNumber(plan.keys.length)} rows from ${plan.sourceCounts.size} input(s), ${fmtBytes(size)} in ${formatTime(elapsed)}`);
+        log(`${magenta("output")}: ${formatNumber(plan.keys.length)} rows from ${plan.sourceCounts.size} input(s), ${fmtBytes(size)}`);
         outputs.push({ name, minKey: plan.minKey, maxKey: plan.maxKey, rowCount: plan.keys.length, size, sources: sourcesNamed });
     }
     return outputs;