sliftutils 1.4.61 → 1.4.63

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.4.61",
+    "version": "1.4.63",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/spec.txt

@@ -1,11 +1,5 @@
 
 
 
-Compact face_frames3
 
-
-
-When detecting faces in the JavaScript code and the Python code. Let's make it so that we only store the face frames for the top 10 characters. We can show the store the embeddings for all of them, But for the face frames collection only store the top 10 faces. The other ones will just be blank, and that should be fine. Also, let's make sure when we're storing them the max dimension is 128 pixels and the quality is only 80. 
-
-
-Then go verify on the other site fixing up the bad data. It should be quite fast because there's so much overlap, but we will find out. 
+Create a companion large data storage thing.  which updates the data less frequently, Where we reference it with unique IDs so we don't have to merge as much and it doesn't have to be as strict. This should allow our large databases to be merged faster, even though they are pretty fast to merge at the moment, but they probably will get slower if we make them 10 times bigger. 

package/storage/BulkDatabase2/BulkDatabaseBase.ts

@@ -663,7 +663,10 @@ export class BulkDatabaseBase<T extends { key: string }> {
         let lastStepMs = mergeStartMs;
         const log = (line: string) => {
             const now = Date.now();
-            steps.push(`${blue(formatTime(now - lastStepMs))} ${line}`);
+            let lineFormatted = `${blue(formatTime(now - lastStepMs))} ${line}`;
+            // DO NOT REMOVE THIS LOG! Obviously, we should be logging as we run, or else we don't get progress. Why would we stop getting progress? Progress is almost more important than showing a summary at the end...
+            console.log(lineFormatted);
+            steps.push(lineFormatted);
             lastStepMs = now;
         };
         log(`${magenta("read")}: ${inputs.length} input file(s), ${fmtBytes(inTotal)}`);

package/storage/BulkDatabase2/BulkDatabaseMerge.ts

@@ -359,6 +359,7 @@ async function executeBatch(
         }
         if (!contributes) continue;
 
+        let bytesRead = 0;
         for (let fi = 0; fi < plans.length; fi++) {
             const plan = plans[fi];
             for (let ci = 0; ci < plan.columns.length; ci++) {
@@ -373,9 +374,11 @@ async function executeBatch(
                         throw new Error(`Expected ${run.byteLength} bytes from source #${si} (${sourceNames[si]}) column ${col.name} rows [${run.sourceStartRow}, ${run.sourceEndRow}), got ${bytes.length}`);
                     }
                     bytes.copy(target.blob, target.dataStart + run.outputByteStart);
+                    bytesRead += bytes.length;
                 }
             }
         }
+        log(`${magenta("input")} ${sourceNames[si]}: ${fmtBytes(bytesRead)} read`);
     }
 
     // Assemble + write each output file in this batch.