sliftutils 1.4.81 → 1.4.82

package/storage/embeddingBench.ts

@@ -0,0 +1,152 @@
+import fs from "fs";
+import { ensureFaceData } from "./faceFramesData";
+import { encodeEmbedding, averageEmbeddings, embeddingToFloat32, releaseFloat32, getCloseness, StoredEmbedding, EmbeddingFormat } from "./embeddingFormats";
+
+// Runs a generic k-means parameterized by an arbitrary getCloseness, so we can race the closeness
+// implementations (all exported from embeddingFormats) on a real clustering workload — plus a variant that
+// decodes every embedding to float32 once and uses plain float dots. Push + run with: typenode storage/embeddingBench.ts
+
+const DIM = 512;
+const FILE_NAME = "faceEntry2.f32";
+const MODEL = "buffalo_l";
+const FORMAT: EmbeddingFormat = "q8g8_2048";
+const CONFIG = { format: FORMAT, model: MODEL };
+
+async function loadEmbeddings(count: number): Promise<StoredEmbedding[]> {
+    const byteLength = count * DIM * 4;
+    const filePath = await ensureFaceData(FILE_NAME, byteLength);
+    const buffer = Buffer.alloc(byteLength);
+    const handle = fs.openSync(filePath, "r");
+    fs.readSync(handle, buffer, 0, byteLength, 0);
+    fs.closeSync(handle);
+    const floats = new Float32Array(buffer.buffer, buffer.byteOffset, count * DIM);
+    const out: StoredEmbedding[] = [];
+    for (let index = 0; index < count; index++) {
+        out.push(encodeEmbedding({ input: floats.subarray(index * DIM, (index + 1) * DIM), format: FORMAT, model: MODEL }));
+    }
+    return out;
+}
+
+// Plain k-means: assign each member to the nearest centroid via `closeness`, recompute centroids as the
+// (re-encoded) mean. The closeness function is the only thing that varies.
+function kmeans(members: StoredEmbedding[], clusterCount: number, iterations: number, closeness: (a: StoredEmbedding, b: StoredEmbedding) => number): number {
+    let centroids: StoredEmbedding[] = [];
+    const seedStride = members.length / clusterCount;
+    for (let index = 0; index < clusterCount; index++) {
+        centroids.push(members[Math.floor(index * seedStride)]);
+    }
+    for (let iteration = 0; iteration < iterations; iteration++) {
+        const groups: StoredEmbedding[][] = [];
+        for (let index = 0; index < centroids.length; index++) {
+            groups.push([]);
+        }
+        for (const member of members) {
+            let bestIndex = 0;
+            let bestCloseness = -Infinity;
+            for (let centroidIndex = 0; centroidIndex < centroids.length; centroidIndex++) {
+                const value = closeness(member, centroids[centroidIndex]);
+                if (value > bestCloseness) {
+                    bestCloseness = value;
+                    bestIndex = centroidIndex;
+                }
+            }
+            groups[bestIndex].push(member);
+        }
+        const next: StoredEmbedding[] = [];
+        for (const group of groups) {
+            if (group.length) {
+                next.push(averageEmbeddings(group, CONFIG));
+            }
+        }
+        centroids = next;
+    }
+    return centroids.length;
+}
+
+// The same k-means but every embedding is decoded to a float32 array ONCE up front; centroids stay float
+// arrays and assignment is a plain float dot. No per-comparison decode, no centroid re-encode.
+function kmeansFloat32Once(members: StoredEmbedding[], clusterCount: number, iterations: number): number {
+    const memberFloats: Float32Array[] = [];
+    for (const member of members) {
+        memberFloats.push(embeddingToFloat32(member, true));
+    }
+    let centroids: Float32Array[] = [];
+    const seedStride = members.length / clusterCount;
+    for (let index = 0; index < clusterCount; index++) {
+        centroids.push(memberFloats[Math.floor(index * seedStride)]);
+    }
+    for (let iteration = 0; iteration < iterations; iteration++) {
+        const groups: number[][] = [];
+        for (let index = 0; index < centroids.length; index++) {
+            groups.push([]);
+        }
+        for (let memberIndex = 0; memberIndex < memberFloats.length; memberIndex++) {
+            const memberFloat = memberFloats[memberIndex];
+            let bestIndex = 0;
+            let bestDot = -Infinity;
+            for (let centroidIndex = 0; centroidIndex < centroids.length; centroidIndex++) {
+                const centroidFloat = centroids[centroidIndex];
+                const length = Math.min(memberFloat.length, centroidFloat.length);
+                let dot = 0;
+                for (let dim = 0; dim < length; dim++) {
+                    dot += memberFloat[dim] * centroidFloat[dim];
+                }
+                if (dot > bestDot) {
+                    bestDot = dot;
+                    bestIndex = centroidIndex;
+                }
+            }
+            groups[bestIndex].push(memberIndex);
+        }
+        const next: Float32Array[] = [];
+        for (const group of groups) {
+            if (!group.length) {
+                continue;
+            }
+            const length = memberFloats[group[0]].length;
+            const sum = new Float32Array(length);
+            for (const memberIndex of group) {
+                const memberFloat = memberFloats[memberIndex];
+                for (let dim = 0; dim < length; dim++) {
+                    sum[dim] += memberFloat[dim];
+                }
+            }
+            let norm = 0;
+            for (let dim = 0; dim < length; dim++) {
+                norm += sum[dim] * sum[dim];
+            }
+            const magnitude = Math.sqrt(norm) || 1;
+            for (let dim = 0; dim < length; dim++) {
+                sum[dim] /= magnitude;
+            }
+            next.push(sum);
+        }
+        centroids = next;
+    }
+    for (const memberFloat of memberFloats) {
+        releaseFloat32(memberFloat);
+    }
+    return centroids.length;
+}
+
+function time(name: string, run: () => number): number {
+    const start = Date.now();
+    const clusters = run();
+    const seconds = (Date.now() - start) / 1000;
+    console.log(`  ${name.padEnd(10)} ${seconds.toFixed(3)}s   (${clusters} clusters)`);
+    return seconds;
+}
+
+async function main() {
+    const members = await loadEmbeddings(5000);
+    const clusterCount = 40;
+    const iterations = 4;
+    console.log(`k-means: ${members.length} members, ${clusterCount} clusters, ${iterations} iterations\n`);
+
+    kmeans(members.slice(0, 500), 8, 1, getCloseness); // warm up the JIT
+
+    time("getCloseness", () => kmeans(members, clusterCount, iterations, getCloseness));
+    time("f32-once", () => kmeansFloat32Once(members, clusterCount, iterations));
+}
+
+main();

package/storage/embeddingFormats.d.ts

@@ -14,6 +14,10 @@ export type StoredEmbedding = {
     data: Uint8Array;
     scales: Uint8Array;
 };
+export declare function embeddingLength(input: Float32Array | StoredEmbedding): number;
+export declare function releaseFloat32(buffer: Float32Array): void;
+export declare function embeddingToFloat32(input: Float32Array | StoredEmbedding, usePool?: boolean): Float32Array;
+export declare const getCloseness: (a: Float32Array | StoredEmbedding, b: Float32Array | StoredEmbedding) => number;
 export declare function encodeEmbedding(config: {
     input: Float32Array | StoredEmbedding;
     format: EmbeddingFormat;
@@ -26,4 +30,3 @@ export declare function averageEmbeddings(embeddings: StoredEmbedding[], config:
     model: string;
 }): StoredEmbedding;
 export declare function hashEmbedding(stored: StoredEmbedding): string;
-export declare const getCloseness: (embedding1: Float32Array | StoredEmbedding, embedding2: Float32Array | StoredEmbedding) => number;

package/storage/embeddingFormats.ts

@@ -64,25 +64,25 @@ const f32Scratch = new Float32Array(1);
 const i32Scratch = new Int32Array(f32Scratch.buffer);
 function floatToHalf(value: number): number {
     f32Scratch[0] = value;
-    let x = i32Scratch[0];
-    let bits = (x >> 16) & 0x8000;
-    let m = (x >> 12) & 0x07ff;
-    let e = (x >> 23) & 0xff;
-    if (e < 103) return bits;
-    if (e > 142) {
+    let bitsIn = i32Scratch[0];
+    let bits = (bitsIn >> 16) & 0x8000;
+    let mantissa = (bitsIn >> 12) & 0x07ff;
+    let exponent = (bitsIn >> 23) & 0xff;
+    if (exponent < 103) return bits;
+    if (exponent > 142) {
         bits |= 0x7c00;
-        if (e !== 255) {
-            bits |= (x & 0x007fffff);
+        if (exponent !== 255) {
+            bits |= (bitsIn & 0x007fffff);
         }
         return bits;
     }
-    if (e < 113) {
-        m |= 0x0800;
-        bits |= (m >> (114 - e)) + ((m >> (113 - e)) & 1);
+    if (exponent < 113) {
+        mantissa |= 0x0800;
+        bits |= (mantissa >> (114 - exponent)) + ((mantissa >> (113 - exponent)) & 1);
         return bits;
     }
-    bits |= ((e - 112) << 10) | (m >> 1);
-    bits += m & 1;
+    bits |= ((exponent - 112) << 10) | (mantissa >> 1);
+    bits += mantissa & 1;
     return bits;
 }
 function halfToFloat(half: number): number {
@@ -110,8 +110,8 @@ function encodeFloat16(values: Float32Array): Uint8Array {
     }
     let out = new Uint8Array(values.length * 2);
     let view = new DataView(out.buffer);
-    for (let i = 0; i < values.length; i++) {
-        view.setUint16(i * 2, floatToHalf(values[i]), true);
+    for (let index = 0; index < values.length; index++) {
+        view.setUint16(index * 2, floatToHalf(values[index]), true);
     }
     return out;
 }
@@ -126,58 +126,121 @@ function asFloat16Indexable(bytes: Uint8Array): { readonly length: number;[index
     }
     let halves = new Uint16Array(bytes.buffer, bytes.byteOffset, count);
     let out = new Float32Array(count);
-    for (let i = 0; i < count; i++) {
-        out[i] = halfToFloat(halves[i]);
+    for (let index = 0; index < count; index++) {
+        out[index] = halfToFloat(halves[index]);
     }
     return out;
 }
 
-function normalizeInPlace(vector: Float32Array): Float32Array {
-    let sum = 0;
-    for (let i = 0; i < vector.length; i++) {
-        sum += vector[i] * vector[i];
-    }
-    let magnitude = Math.sqrt(sum);
-    if (!magnitude) return vector;
-    for (let i = 0; i < vector.length; i++) {
-        vector[i] /= magnitude;
-    }
-    return vector;
-}
-
-function embeddingLength(input: Float32Array | StoredEmbedding): number {
+export function embeddingLength(input: Float32Array | StoredEmbedding): number {
     if (input instanceof Float32Array) return input.length;
     if (input.kind === "float32") return input.values.length;
     return input.data.length;
 }
 
-// Decode any accepted input to a fresh, mutable Float32Array of exactly `length` dimensions
-// (truncating). Always allocates, so callers can normalize it in place without touching stored data.
-function decodeToLength(input: Float32Array | StoredEmbedding, length: number): Float32Array {
-    let out = new Float32Array(length);
-    if (input instanceof Float32Array) {
-        for (let i = 0; i < length; i++) {
-            out[i] = input[i];
+// A pool of reusable Float32 buffers keyed by length. Most embeddings share a size, so internal hot paths
+// borrow a buffer (embeddingToFloat32 with usePool) and return it (releaseFloat32) to avoid per-op
+// allocations and GC thrashing. A borrowed buffer holds stale data; whoever borrows it overwrites it fully.
+//
+// Each free buffer records when it was last released; an hourly sweep drops any that have sat idle for over
+// an hour, so the runtime can free buffers a burst of work allocated but isn't actually reusing anymore.
+const POOL_MAX_IDLE_MS = 60 * 60 * 1000;
+type PooledFloat32 = { buffer: Float32Array; releasedAt: number };
+const float32Pool = new Map<number, PooledFloat32[]>();
+function acquireFloat32(length: number): Float32Array {
+    let free = float32Pool.get(length);
+    if (free && free.length) {
+        return free.pop()!.buffer;
+    }
+    return new Float32Array(length);
+}
+export function releaseFloat32(buffer: Float32Array): void {
+    let free = float32Pool.get(buffer.length);
+    if (!free) {
+        free = [];
+        float32Pool.set(buffer.length, free);
+    }
+    free.push({ buffer, releasedAt: Date.now() });
+}
+function sweepFloat32Pool(): void {
+    let cutoff = Date.now() - POOL_MAX_IDLE_MS;
+    for (let free of float32Pool.values()) {
+        let writeIndex = 0;
+        for (let readIndex = 0; readIndex < free.length; readIndex++) {
+            if (free[readIndex].releasedAt > cutoff) {
+                free[writeIndex++] = free[readIndex];
+            }
         }
+        free.length = writeIndex;
+    }
+}
+const poolSweepTimer = setInterval(sweepFloat32Pool, POOL_MAX_IDLE_MS) as unknown as { unref?: () => void };
+if (typeof poolSweepTimer.unref === "function") {
+    poolSweepTimer.unref();
+}
+
+// Decode to a Float32Array (full length). Stored values are already unit-normalized (encodeEmbedding does
+// that), so this only de-quantizes. By default it allocates a fresh array (a float input is returned as-is,
+// no copy). With usePool it borrows a pooled buffer the caller must releaseFloat32 — used by hot internal
+// paths so repeated decodes don't allocate.
+export function embeddingToFloat32(input: Float32Array | StoredEmbedding, usePool = false): Float32Array {
+    if (input instanceof Float32Array) {
+        if (!usePool) return input;
+        let out = acquireFloat32(input.length);
+        out.set(input);
         return out;
     }
     if (input.kind === "float32") {
-        let values = input.values;
-        for (let i = 0; i < length; i++) {
-            out[i] = values[i];
-        }
+        if (!usePool) return input.values;
+        let out = acquireFloat32(input.values.length);
+        out.set(input.values);
         return out;
     }
     let int8 = asInt8(input.data);
     let scales = asFloat16Indexable(input.scales);
     let groupSize = input.groupSize;
-    for (let i = 0; i < length; i++) {
-        out[i] = int8[i] * scales[Math.floor(i / groupSize)];
+    let length = int8.length;
+    let out = usePool ? acquireFloat32(length) : new Float32Array(length);
+    let group = 0;
+    for (let start = 0; start < length; start += groupSize) {
+        let scale = scales[group++];
+        let end = Math.min(start + groupSize, length);
+        for (let index = start; index < end; index++) {
+            out[index] = int8[index] * scale;
+        }
     }
     return out;
 }
 
-// Convert a raw float32 vector (or re-encode an existing StoredEmbedding) into the requested format.
+function closenessFromDot(dot: number): number {
+    // Inputs are unit vectors, so euclidean^2 = 2 - 2*dot; clamp away tiny negatives from float error.
+    return 1 - Math.sqrt(Math.max(0, 2 - 2 * dot));
+}
+
+function dotFloat32(a: Float32Array, b: Float32Array): number {
+    let length = Math.min(a.length, b.length);
+    let dot = 0;
+    for (let index = 0; index < length; index++) {
+        dot += a[index] * b[index];
+    }
+    return dot;
+}
+
+// Closeness = 1 - euclidean distance of two unit vectors. Always decode both to (pooled) float32 and dot:
+// simple, and once the decodes are pooled it's the fastest per-call path. Batch callers (k-means) decode
+// every vector once themselves instead of going through here.
+export const getCloseness = measureWrap(function getCloseness(a: Float32Array | StoredEmbedding, b: Float32Array | StoredEmbedding): number {
+    let va = embeddingToFloat32(a, true);
+    let vb = embeddingToFloat32(b, true);
+    let result = closenessFromDot(dotFloat32(va, vb));
+    releaseFloat32(va);
+    releaseFloat32(vb);
+    return result;
+});
+
+// Encode a vector (or re-encode an embedding) into the requested format. This is THE place we normalize:
+// the (truncated) vector is made unit length here, so anything that's been encoded is reliably normalized
+// and the comparison/decoding paths never have to re-normalize.
 export function encodeEmbedding(config: {
     input: Float32Array | StoredEmbedding;
     format: EmbeddingFormat;
@@ -185,46 +248,57 @@ export function encodeEmbedding(config: {
 }): StoredEmbedding {
     let { input, format, model } = config;
     let formatConfig = FORMAT_CONFIGS[format];
-    let length = embeddingLength(input);
-    if (formatConfig.truncation !== undefined) {
-        length = Math.min(formatConfig.truncation, length);
+    let decoded = embeddingToFloat32(input, true);
+    let length = decoded.length;
+    if (formatConfig.truncation !== undefined && formatConfig.truncation < length) {
+        length = formatConfig.truncation;
+    }
+    let norm = 0;
+    for (let index = 0; index < length; index++) {
+        norm += decoded[index] * decoded[index];
+    }
+    let magnitude = Math.sqrt(norm) || 1;
+    for (let index = 0; index < length; index++) {
+        decoded[index] /= magnitude;
     }
-    // Truncate to a fresh array, then renormalize so the shortened vector is unit length again.
-    let vector = normalizeInPlace(decodeToLength(input, length));
 
+    let result: StoredEmbedding;
     if (!formatConfig.quant) {
-        return { kind: "float32", model, values: vector };
-    }
-    let { type, groupSize } = formatConfig.quant;
-    let groupCount = Math.ceil(length / groupSize);
-    let int8 = new Int8Array(length);
-    let scales = new Float32Array(groupCount);
-    for (let g = 0; g < groupCount; g++) {
-        let start = g * groupSize;
-        let end = Math.min(start + groupSize, length);
-        let maxAbs = 0;
-        for (let i = start; i < end; i++) {
-            let magnitude = Math.abs(vector[i]);
-            if (magnitude > maxAbs) maxAbs = magnitude;
-        }
-        let scale = maxAbs / INT8_MAX;
-        scales[g] = scale;
-        if (!scale) continue;
-        for (let i = start; i < end; i++) {
-            let quantized = Math.round(vector[i] / scale);
-            if (quantized > INT8_MAX) quantized = INT8_MAX;
-            if (quantized < -INT8_MAX) quantized = -INT8_MAX;
-            int8[i] = quantized;
+        result = { kind: "float32", model, values: new Float32Array(decoded.subarray(0, length)) };
+    } else {
+        let { type, groupSize } = formatConfig.quant;
+        let groupCount = Math.ceil(length / groupSize);
+        let int8 = new Int8Array(length);
+        let scales = new Float32Array(groupCount);
+        for (let group = 0; group < groupCount; group++) {
+            let start = group * groupSize;
+            let end = Math.min(start + groupSize, length);
+            let maxAbs = 0;
+            for (let index = start; index < end; index++) {
+                let absValue = Math.abs(decoded[index]);
+                if (absValue > maxAbs) maxAbs = absValue;
+            }
+            let scale = maxAbs / INT8_MAX;
+            scales[group] = scale;
+            if (!scale) continue;
+            for (let index = start; index < end; index++) {
+                let quantized = Math.round(decoded[index] / scale);
+                if (quantized > INT8_MAX) quantized = INT8_MAX;
+                if (quantized < -INT8_MAX) quantized = -INT8_MAX;
+                int8[index] = quantized;
+            }
         }
+        result = {
+            kind: "quant",
+            model,
+            type,
+            groupSize,
+            data: new Uint8Array(int8.buffer, int8.byteOffset, int8.byteLength),
+            scales: encodeFloat16(scales),
+        };
     }
-    return {
-        kind: "quant",
-        model,
-        type,
-        groupSize,
-        data: new Uint8Array(int8.buffer, int8.byteOffset, int8.byteLength),
-        scales: encodeFloat16(scales),
-    };
+    releaseFloat32(decoded);
+    return result;
 }
 
 type SerializedHeader =
@@ -273,52 +347,40 @@ export function deserializeStoredEmbedding(base64: string): StoredEmbedding {
     };
 }
 
-// Mean of several embeddings (decoded to their common length), re-encoded into the requested format
-// (which renormalizes). Used to build and split IVF cell centroids.
+// Mean of several embeddings (encodeEmbedding normalizes the result to a unit centroid). Borrows a pooled
+// accumulator.
 export function averageEmbeddings(embeddings: StoredEmbedding[], config: { format: EmbeddingFormat; model: string }): StoredEmbedding {
     let length = Infinity;
     for (let embedding of embeddings) {
         length = Math.min(length, embeddingLength(embedding));
     }
-    let sum = new Float32Array(length);
+    let sum = acquireFloat32(length);
+    sum.fill(0);
     for (let embedding of embeddings) {
-        let decoded = decodeToLength(embedding, length);
+        let vector = embeddingToFloat32(embedding, true);
         for (let dimension = 0; dimension < length; dimension++) {
-            sum[dimension] += decoded[dimension];
+            sum[dimension] += vector[dimension];
         }
+        releaseFloat32(vector);
     }
-    for (let dimension = 0; dimension < length; dimension++) {
-        sum[dimension] /= embeddings.length;
-    }
-    return encodeEmbedding({ input: sum, format: config.format, model: config.model });
+    let result = encodeEmbedding({ input: sum.subarray(0, length), format: config.format, model: config.model });
+    releaseFloat32(sum);
+    return result;
 }
 
-// A stable 16-byte (base64) content hash of an embedding, for use as a cell id derived from its centroid.
+// A stable 16-byte (base64) content hash of an embedding's values, for use as a cell id derived from its
+// centroid. Iterates via the accessor (the value bits are hashed, so any format hashes consistently).
 export function hashEmbedding(stored: StoredEmbedding): string {
-    let serialized = serializeStoredEmbedding(stored);
+    let vector = embeddingToFloat32(stored, true);
+    let length = vector.length;
     let lanes = new Uint32Array([2166136261, 2654435761, 40503, 3266489917]);
-    for (let charIndex = 0; charIndex < serialized.length; charIndex++) {
-        let code = serialized.charCodeAt(charIndex);
+    for (let index = 0; index < length; index++) {
+        f32Scratch[0] = vector[index];
+        let bits = i32Scratch[0];
         for (let lane = 0; lane < lanes.length; lane++) {
-            lanes[lane] = Math.imul(lanes[lane] ^ (code + lane * 131 + charIndex), 16777619);
+            lanes[lane] = Math.imul(lanes[lane] ^ (bits + lane * 131 + index), 16777619);
         }
     }
+    releaseFloat32(vector);
     return Buffer.from(lanes.buffer).toString("base64");
 }
-
-// Compare two embeddings in any format / truncation. Both are decoded, truncated to their common
-// length, and unit-normalized, then scored as 1 minus their euclidean distance.
-export const getCloseness = measureWrap(function getCloseness(
-    embedding1: Float32Array | StoredEmbedding,
-    embedding2: Float32Array | StoredEmbedding,
-): number {
-    let length = Math.min(embeddingLength(embedding1), embeddingLength(embedding2));
-    let vector1 = normalizeInPlace(decodeToLength(embedding1, length));
-    let vector2 = normalizeInPlace(decodeToLength(embedding2, length));
-    let sum = 0;
-    for (let i = 0; i < length; i++) {
-        let diff = vector1[i] - vector2[i];
-        sum += diff * diff;
-    }
-    return 1 - Math.sqrt(sum);
-});

package/storage/faceFramesData.d.ts

@@ -0,0 +1 @@
+export declare function ensureFaceData(fileName: string, byteLength: number): Promise<string>;

package/storage/faceFramesData.ts

@@ -0,0 +1,49 @@
+import fs from "fs";
+import os from "os";
+import path from "path";
+import http from "http";
+
+// Resolves a face-embedding test file to a local path that holds at least the requested number of bytes. On
+// the machine that owns the data it just uses the original file. Anywhere else it downloads the missing prefix
+// from the faceFramesServer over a range request and caches it in the temp dir, so re-runs only fetch what
+// they haven't already. Point it at a different server with FACEFRAMES_URL.
+const SOURCE_DIR = "/root/claude-work/face-embeddings";
+const SERVER_BASE = process.env.FACEFRAMES_URL || "http://65.109.93.113:8799";
+
+function downloadInto(fileName: string, start: number, endExclusive: number, cachePath: string): Promise<void> {
+    return new Promise((resolve, reject) => {
+        const url = `${SERVER_BASE}/${fileName}`;
+        const options = { headers: { Range: `bytes=${start}-${endExclusive - 1}` } };
+        const request = http.get(url, options, response => {
+            if (response.statusCode !== 206 && response.statusCode !== 200) {
+                response.resume();
+                reject(new Error(`${url} returned ${response.statusCode}`));
+                return;
+            }
+            const out = fs.createWriteStream(cachePath, start ? { flags: "r+", start } : { flags: "w" });
+            response.pipe(out);
+            out.on("error", reject);
+            response.on("error", reject);
+            out.on("finish", resolve);
+        });
+        request.on("error", reject);
+    });
+}
+
+export async function ensureFaceData(fileName: string, byteLength: number): Promise<string> {
+    const local = path.join(SOURCE_DIR, fileName);
+    if (fs.existsSync(local) && fs.statSync(local).size >= byteLength) {
+        return local;
+    }
+    const cachePath = path.join(os.tmpdir(), fileName);
+    let cachedBytes = 0;
+    if (fs.existsSync(cachePath)) {
+        cachedBytes = fs.statSync(cachePath).size;
+    }
+    if (cachedBytes < byteLength) {
+        const megabytes = ((byteLength - cachedBytes) / 1e6).toFixed(0);
+        console.log(`downloading ${megabytes}MB of ${fileName} from ${SERVER_BASE} -> ${cachePath}`);
+        await downloadInto(fileName, cachedBytes, byteLength, cachePath);
+    }
+    return cachePath;
+}

package/storage/faceFramesServer.d.ts

@@ -0,0 +1 @@
+export {};

package/storage/faceFramesServer.ts

@@ -0,0 +1,56 @@
+import http from "http";
+import fs from "fs";
+import path from "path";
+
+// Throwaway HTTP server that serves the face-embedding test files so the benchmarks can be run from another
+// machine (which downloads + caches a prefix via faceFramesData). Supports range requests, so a client only
+// pulls the bytes it needs out of the 7.7GB full file. Meant to be left running in the background; it dies on
+// reboot and that is fine. Run with: typenode storage/faceFramesServer.ts
+const DATA_DIR = "/root/claude-work/face-embeddings";
+const PORT = Number(process.env.PORT) || 8799;
+
+function resolveFile(requestPath: string): string | undefined {
+    const name = path.basename(decodeURIComponent(requestPath));
+    if (!name.endsWith(".f32")) {
+        return undefined;
+    }
+    const full = path.join(DATA_DIR, name);
+    if (!fs.existsSync(full)) {
+        return undefined;
+    }
+    return full;
+}
+
+const server = http.createServer((request, response) => {
+    const filePath = resolveFile(request.url || "");
+    if (!filePath) {
+        response.writeHead(404);
+        response.end("not found");
+        return;
+    }
+    const total = fs.statSync(filePath).size;
+    const rangeHeader = request.headers.range;
+    const rangeMatch = rangeHeader ? /bytes=(\d+)-(\d*)/.exec(rangeHeader) : null;
+    if (rangeMatch) {
+        const start = Number(rangeMatch[1]);
+        const end = rangeMatch[2] ? Number(rangeMatch[2]) : total - 1;
+        response.writeHead(206, {
+            "Content-Type": "application/octet-stream",
+            "Accept-Ranges": "bytes",
+            "Content-Range": `bytes ${start}-${end}/${total}`,
+            "Content-Length": end - start + 1,
+        });
+        fs.createReadStream(filePath, { start, end }).pipe(response);
+        return;
+    }
+    response.writeHead(200, {
+        "Content-Type": "application/octet-stream",
+        "Accept-Ranges": "bytes",
+        "Content-Length": total,
+    });
+    fs.createReadStream(filePath).pipe(response);
+});
+
+server.listen(PORT, () => {
+    console.log(`serving ${DATA_DIR}/*.f32 on port ${PORT}`);
+});

package/storage/proxydatabase/dist/Database.ts.cache

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true , configurable: true});
+//exports.namespaceDatabase = void 0;
+function namespaceDatabase(database, into) {
+    return {
+        readData: (deref) => database.readData(root => deref(into(root))),
+        writeData: (deref, newValue) => database.writeData(root => deref(into(root)), newValue),
+        deleteData: (deref) => database.deleteData(root => deref(into(root))),
+    };
+}
+exports.namespaceDatabase = namespaceDatabase;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiRGF0YWJhc2UuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyJEYXRhYmFzZS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOzs7QUFPQSxTQUFnQixpQkFBaUIsQ0FDN0IsUUFBd0IsRUFDeEIsSUFBeUI7SUFFekIsT0FBTztRQUNILFFBQVEsRUFBRSxDQUFDLEtBQUssRUFBRSxFQUFFLENBQUMsUUFBUSxDQUFDLFFBQVEsQ0FBQyxJQUFJLENBQUMsRUFBRSxDQUFDLEtBQUssQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQztRQUNqRSxTQUFTLEVBQUUsQ0FBQyxLQUFLLEVBQUUsUUFBUSxFQUFFLEVBQUUsQ0FBQyxRQUFRLENBQUMsU0FBUyxDQUFDLElBQUksQ0FBQyxFQUFFLENBQUMsS0FBSyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQyxFQUFFLFFBQVEsQ0FBQztRQUN2RixVQUFVLEVBQUUsQ0FBQyxLQUFLLEVBQUUsRUFBRSxDQUFDLFFBQVEsQ0FBQyxVQUFVLENBQUMsSUFBSSxDQUFDLEVBQUUsQ0FBQyxLQUFLLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUM7S0FDeEUsQ0FBQztBQUNOLENBQUM7QUFURCw4Q0FTQyIsInNvdXJjZXNDb250ZW50IjpbIi8vIHVuZGVmaW5lZCBmcm9tIGEgcmVhZCBtZWFucyBcIm5vdCBzeW5jZWQgeWV0XCIsIE5PVCBcImFic2VudFwiIOKAlCBzdG9yYWdlIGluaXRpYWxpemVzIG9uIGl0cyBvd24uIFNvIHJlYWQgZXZlcnl0aGluZyBhIHN0YWdlIG5lZWRzIGluIG9uZSByZWFkRGF0YSBhbmQgYmFpbCBpZiB0aGUgd2hvbGUgcmVzdWx0IGlzIHVuZGVmaW5lZDsgdGhlIGRhdGFiYXNlIG5ldmVyIHJldHVybnMgYW4gaW5kaXZpZHVhbCB1bnN5bmNlZCB2YWx1ZSBpbnNpZGUgYW4gb3RoZXJ3aXNlLXN1Y2Nlc3NmdWwgcmVhZC5cbmV4cG9ydCBpbnRlcmZhY2UgRGF0YWJhc2U8Um9vdD4ge1xuICAgIHJlYWREYXRhOiA8VmFsdWU+KGRlcmVmOiAocm9vdDogUm9vdCkgPT4gVmFsdWUpID0+IFZhbHVlIHwgdW5kZWZpbmVkO1xuICAgIHdyaXRlRGF0YTogPFZhbHVlPihkZXJlZjogKHJvb3Q6IFJvb3QpID0+IFZhbHVlLCBuZXdWYWx1ZTogVmFsdWUpID0+IHZvaWQ7XG4gICAgZGVsZXRlRGF0YTogKGRlcmVmOiAocm9vdDogUm9vdCkgPT4gdW5rbm93bikgPT4gdm9pZDtcbn1cblxuZXhwb3J0IGZ1bmN0aW9uIG5hbWVzcGFjZURhdGFiYXNlPFJvb3QsIFN1Yj4oXG4gICAgZGF0YWJhc2U6IERhdGFiYXNlPFJvb3Q+LFxuICAgIGludG86IChyb290OiBSb290KSA9PiBTdWIsXG4pOiBEYXRhYmFzZTxTdWI+IHtcbiAgICByZXR1cm4ge1xuICAgICAgICByZWFkRGF0YTogKGRlcmVmKSA9PiBkYXRhYmFzZS5yZWFkRGF0YShyb290ID0+IGRlcmVmKGludG8ocm9vdCkpKSxcbiAgICAgICAgd3JpdGVEYXRhOiAoZGVyZWYsIG5ld1ZhbHVlKSA9PiBkYXRhYmFzZS53cml0ZURhdGEocm9vdCA9PiBkZXJlZihpbnRvKHJvb3QpKSwgbmV3VmFsdWUpLFxuICAgICAgICBkZWxldGVEYXRhOiAoZGVyZWYpID0+IGRhdGFiYXNlLmRlbGV0ZURhdGEocm9vdCA9PiBkZXJlZihpbnRvKHJvb3QpKSksXG4gICAgfTtcbn1cbiJdfQ==
+ /* _JS_SOURCE_HASH = "93a620d26b6a59665b5f7dab689d9739693a4990e1c4781abcddc6cda24bda11"; */