sliftutils 1.7.13 → 1.7.14

package/storage/remoteStorage/storageServerState.ts

@@ -11,6 +11,7 @@ import {
     WRITE_PAST_WINDOW_GRACE, STORAGE_WRONG_VALID_WINDOW, STORAGE_WRONG_ROUTE, FULL_ROUTE,
 } from "../IArchives";
 import { ROUTING_FILE, parseRoutingData, parseHostedUrl, buildFileUrl, getConfigVersion, getRoute, routeContains, routeIntersection } from "./remoteConfig";
+import { applyDeployRemap, getFlushDeadline, onTakeoverEvent } from "./deployTakeover";
 import { createApiArchives } from "./createArchives";
 import type { IStorage } from "../IStorage";
 import type { AccessRequest, TrustRecord } from "./storageController";
@@ -157,6 +158,13 @@ function getBucketFolder(account: string, bucketName: string): string {
     return path.join(getStorageServerConfig().folder, "buckets2", account, bucketName);
 }
 
+// Ports we are listening on beyond the main config port - a deploy-takeover successor serves on a
+// temporary alternate port, and must recognize alternate-port source URLs as itself
+const extraListenPorts = new Set<number>();
+export function addExtraListenPort(port: number): void {
+    extraListenPorts.add(port);
+}
+
 function findSelfIndexes(routing: RemoteConfig, account: string, bucketName: string): number[] {
     let { domain, port } = getStorageServerConfig();
     let indexes: number[] = [];
@@ -164,7 +172,7 @@ function findSelfIndexes(routing: RemoteConfig, account: string, bucketName: str
         let source = routing.sources[i];
         if (typeof source === "string" || source.type !== "remote") continue;
         let parsed = parseHostedUrl(source.url);
-        if (parsed.address === domain && parsed.port === port && parsed.account === account && parsed.bucketName === bucketName) {
+        if (parsed.address === domain && (parsed.port === port || extraListenPorts.has(parsed.port)) && parsed.account === account && parsed.bucketName === bucketName) {
             indexes.push(i);
         }
     }
@@ -223,12 +231,17 @@ function scheduleWindowBoundaryRebuild(loaded: LoadedBucket): void {
 
 function buildBucket(account: string, bucketName: string, routing: RemoteConfig): LoadedBucket {
     let folder = getBucketFolder(account, bucketName);
-    let selfIndexes = findSelfIndexes(routing, account, bucketName);
-    let selfEntries = selfIndexes.map(i => routing.sources[i] as HostedConfig);
+    // The deploy-takeover remap is an INTERPRETATION overlay: everything behavioral (self entries,
+    // windows, downstream sources) uses the remapped view, while loaded.routing/routingJSON stay
+    // the STORED config - so version guards, change detection, and synchronization never see (or
+    // persist) the remap
+    let effective = applyDeployRemap(routing);
+    let selfIndexes = findSelfIndexes(effective, account, bucketName);
+    let selfEntries = selfIndexes.map(i => effective.sources[i] as HostedConfig);
     let self = selectEntryAt(selfEntries, Date.now());
     let selfIndex = -1;
     if (self) {
-        selfIndex = routing.sources.indexOf(self);
+        selfIndex = effective.sources.indexOf(self);
     }
     let store: IBucketStore;
     if (self?.rawDisk) {
@@ -251,8 +264,8 @@ function buildBucket(account: string, bucketName: string, routing: RemoteConfig)
         if (selfIndex === -1) {
             console.log(`This server is not in the routing config for bucket ${account}/${bucketName}; keeping its data on disk but no longer synchronizing it`);
         } else {
-            for (let i = selfIndex + 1; i < routing.sources.length; i++) {
-                let source = routing.sources[i];
+            for (let i = selfIndex + 1; i < effective.sources.length; i++) {
+                let source = effective.sources[i];
                 if (typeof source === "string" || ownIndexes.has(i)) continue;
                 // Disjoint-shard sources never talk to each other; a partial overlap syncs only
                 // the intersection (scans ignore the rest, writes only send matching keys)
@@ -271,6 +284,7 @@ function buildBucket(account: string, bucketName: string, routing: RemoteConfig)
                 void scheduleRoutingReload(account, bucketName);
             },
             readerDiskLimit: self?.readerDiskLimit,
+            getFlushDeadline,
         });
     }
     let loaded: LoadedBucket = { account, bucketName, routing, routingJSON: JSON.stringify(routing), selfEntries, self, store };
@@ -421,7 +435,9 @@ export function getBucketConfig(bucket: LoadedBucket): ArchivesConfig {
     let progress = bucket.store.getSyncProgress?.();
     return {
         supportsChangesAfter: !!bucket.store.getChangesAfter,
-        remoteConfig: bucket.routing,
+        // The remapped (interpretation) view: getConfig is how clients learn of an in-progress
+        // deploy takeover, since the remap is never written into the routing file itself
+        remoteConfig: applyDeployRemap(bucket.routing),
         index: progress?.index,
         indexSources: progress?.sources,
         readerDiskLimit: progress?.readerDiskLimit,
@@ -429,6 +445,36 @@ export function getBucketConfig(bucket: LoadedBucket): ArchivesConfig {
     };
 }
 
+export async function rebuildAllLoadedBuckets(): Promise<void> {
+    for (let key of [...buckets.keys()]) {
+        let slash = key.indexOf("/");
+        await scheduleRoutingReload(key.slice(0, slash), key.slice(slash + 1), { force: true });
+    }
+}
+
+export async function rescanAllLoadedBucketDisks(): Promise<void> {
+    for (let loadedPromise of [...buckets.values()]) {
+        let loaded = await loadedPromise.catch(() => undefined);
+        if (!loaded) continue;
+        let store = loaded.store;
+        if (!(store instanceof BlobStore)) continue;
+        try {
+            await store.rescanBase();
+        } catch (e) {
+            console.error(`Deploy switchover disk rescan failed for bucket ${loaded.account}/${loaded.bucketName}: ${(e as Error).stack ?? e}`);
+        }
+    }
+}
+
+onTakeoverEvent(event => {
+    if (event === "remapChanged") {
+        void rebuildAllLoadedBuckets();
+    }
+    if (event === "diskScan") {
+        void rescanAllLoadedBucketDisks();
+    }
+});
+
 export type ServerBucketInfo = {
     bucketName: string;
     // Loaded in memory (created or accessed since startup), so its synchronization is running
@@ -512,8 +558,9 @@ class ArchivesLocalBucket implements IArchives {
         if (!bucket) return undefined;
         return await bucket.store.get2(fileName, config);
     }
-    public async set(fileName: string, data: Buffer, config?: { lastModified?: number }): Promise<void> {
+    public async set(fileName: string, data: Buffer, config?: { lastModified?: number }): Promise<string> {
         await writeBucketFile(this.account, this.bucketName, fileName, data, config);
+        return fileName;
     }
     public async del(fileName: string): Promise<void> {
         await deleteBucketFile(this.account, this.bucketName, fileName);

package/yarn.lock

@@ -1944,10 +1944,10 @@ slash@^3.0.0:
   resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634"
   integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==
 
-socket-function@^1.2.17:
-  version "1.2.17"
-  resolved "https://registry.yarnpkg.com/socket-function/-/socket-function-1.2.17.tgz#d6f28be00253d4a0a73dab6f0981c9af8e750a49"
-  integrity sha512-HGQ8YmGwS5xLG03v9koa8l4PIPVIDmi177a/sdfa6KSGRkaxTWwinfakun4rddZNnlxb7Iu6NFSUl0TfTyv7/w==
+socket-function@^1.2.20:
+  version "1.2.20"
+  resolved "https://registry.yarnpkg.com/socket-function/-/socket-function-1.2.20.tgz#8cd141cd84504a320583e09749ed91cee293db45"
+  integrity sha512-aLHs7S9MKb6WY5pjtNG+sfISVIWIv7toeX7Zt5K6f1R/sVFLb46IBElZ0Hf81LHenVYwDNsKN0uAX/fkBBIqlw==
   dependencies:
     "@types/pako" "^2.0.3"
     "@types/ws" "^8.5.3"

package/storage/dist/faceFramesData.ts.cache

@@ -1,55 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule && mod.default) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true , configurable: true});
-//exports.ensureFaceData = void 0;
-const fs_1 = __importDefault(require("fs"));
-const os_1 = __importDefault(require("os"));
-const path_1 = __importDefault(require("path"));
-const http_1 = __importDefault(require("http"));
-// Resolves a face-embedding test file to a local path that holds at least the requested number of bytes. On
-// the machine that owns the data it just uses the original file. Anywhere else it downloads the missing prefix
-// from the faceFramesServer over a range request and caches it in the temp dir, so re-runs only fetch what
-// they haven't already. Point it at a different server with FACEFRAMES_URL.
-const SOURCE_DIR = "/root/claude-work/face-embeddings";
-const SERVER_BASE = process.env.FACEFRAMES_URL || "http://65.109.93.113:8799";
-function downloadInto(fileName, start, endExclusive, cachePath) {
-    return new Promise((resolve, reject) => {
-        const url = `${SERVER_BASE}/${fileName}`;
-        const options = { headers: { Range: `bytes=${start}-${endExclusive - 1}` } };
-        const request = http_1.default.get(url, options, response => {
-            if (response.statusCode !== 206 && response.statusCode !== 200) {
-                response.resume();
-                reject(new Error(`${url} returned ${response.statusCode}`));
-                return;
-            }
-            const out = fs_1.default.createWriteStream(cachePath, start ? { flags: "r+", start } : { flags: "w" });
-            response.pipe(out);
-            out.on("error", reject);
-            response.on("error", reject);
-            out.on("finish", resolve);
-        });
-        request.on("error", reject);
-    });
-}
-async function ensureFaceData(fileName, byteLength) {
-    const local = path_1.default.join(SOURCE_DIR, fileName);
-    if (fs_1.default.existsSync(local) && fs_1.default.statSync(local).size >= byteLength) {
-        return local;
-    }
-    const cachePath = path_1.default.join(os_1.default.tmpdir(), fileName);
-    let cachedBytes = 0;
-    if (fs_1.default.existsSync(cachePath)) {
-        cachedBytes = fs_1.default.statSync(cachePath).size;
-    }
-    if (cachedBytes < byteLength) {
-        const megabytes = ((byteLength - cachedBytes) / 1e6).toFixed(0);
-        console.log(`downloading ${megabytes}MB of ${fileName} from ${SERVER_BASE} -> ${cachePath}`);
-        await downloadInto(fileName, cachedBytes, byteLength, cachePath);
-    }
-    return cachePath;
-}
-exports.ensureFaceData = ensureFaceData;
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmFjZUZyYW1lc0RhdGEuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyJmYWNlRnJhbWVzRGF0YS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOzs7Ozs7QUFBQSw0Q0FBb0I7QUFDcEIsNENBQW9CO0FBQ3BCLGdEQUF3QjtBQUN4QixnREFBd0I7QUFFeEIsNEdBQTRHO0FBQzVHLCtHQUErRztBQUMvRywyR0FBMkc7QUFDM0csNEVBQTRFO0FBQzVFLE1BQU0sVUFBVSxHQUFHLG1DQUFtQyxDQUFDO0FBQ3ZELE1BQU0sV0FBVyxHQUFHLE9BQU8sQ0FBQyxHQUFHLENBQUMsY0FBYyxJQUFJLDJCQUEyQixDQUFDO0FBRTlFLFNBQVMsWUFBWSxDQUFDLFFBQWdCLEVBQUUsS0FBYSxFQUFFLFlBQW9CLEVBQUUsU0FBaUI7SUFDMUYsT0FBTyxJQUFJLE9BQU8sQ0FBQyxDQUFDLE9BQU8sRUFBRSxNQUFNLEVBQUUsRUFBRTtRQUNuQyxNQUFNLEdBQUcsR0FBRyxHQUFHLFdBQVcsSUFBSSxRQUFRLEVBQUUsQ0FBQztRQUN6QyxNQUFNLE9BQU8sR0FBRyxFQUFFLE9BQU8sRUFBRSxFQUFFLEtBQUssRUFBRSxTQUFTLEtBQUssSUFBSSxZQUFZLEdBQUcsQ0FBQyxFQUFFLEVBQUUsRUFBRSxDQUFDO1FBQzdFLE1BQU0sT0FBTyxHQUFHLGNBQUksQ0FBQyxHQUFHLENBQUMsR0FBRyxFQUFFLE9BQU8sRUFBRSxRQUFRLENBQUMsRUFBRTtZQUM5QyxJQUFJLFFBQVEsQ0FBQyxVQUFVLEtBQUssR0FBRyxJQUFJLFFBQVEsQ0FBQyxVQUFVLEtBQUssR0FBRyxFQUFFLENBQUM7Z0JBQzdELFFBQVEsQ0FBQyxNQUFNLEVBQUUsQ0FBQztnQkFDbEIsTUFBTSxDQUFDLElBQUksS0FBSyxDQUFDLEdBQUcsR0FBRyxhQUFhLFFBQVEsQ0FBQyxVQUFVLEVBQUUsQ0FBQyxDQUFDLENBQUM7Z0JBQzVELE9BQU87WUFDWCxDQUFDO1lBQ0QsTUFBTSxHQUFHLEdBQUcsWUFBRSxDQUFDLGlCQUFpQixDQUFDLFNBQVMsRUFBRSxLQUFLLENBQUMsQ0FBQyxDQUFDLEVBQUUsS0FBSyxFQUFFLElBQUksRUFBRSxLQUFLLEVBQUUsQ0FBQyxDQUFDLENBQUMsRUFBRSxLQUFLLEVBQUUsR0FBRyxFQUFFLENBQUMsQ0FBQztZQUM3RixRQUFRLENBQUMsSUFBSSxDQUFDLEdBQUcsQ0FBQyxDQUFDO1lBQ25CLEdBQUcsQ0FBQyxFQUFFLENBQUMsT0FBTyxFQUFFLE1BQU0sQ0FBQyxDQUFDO1lBQ3hCLFFBQVEsQ0FBQyxFQUFFLENBQUMsT0FBTyxFQUFFLE1BQU0sQ0FBQyxDQUFDO1lBQzdCLEdBQUcsQ0FBQyxFQUFFLENBQUMsUUFBUSxFQUFFLE9BQU8sQ0FBQyxDQUFDO1FBQzlCLENBQUMsQ0FBQyxDQUFDO1FBQ0gsT0FBTyxDQUFDLEVBQUUsQ0FBQyxPQUFPLEVBQUUsTUFBTSxDQUFDLENBQUM7SUFDaEMsQ0FBQyxDQUFDLENBQUM7QUFDUCxDQUFDO0FBRU0sS0FBSyxVQUFVLGNBQWMsQ0FBQyxRQUFnQixFQUFFLFVBQWtCO0lBQ3JFLE1BQU0sS0FBSyxHQUFHLGNBQUksQ0FBQyxJQUFJLENBQUMsVUFBVSxFQUFFLFFBQVEsQ0FBQyxDQUFDO0lBQzlDLElBQUksWUFBRSxDQUFDLFVBQVUsQ0FBQyxLQUFLLENBQUMsSUFBSSxZQUFFLENBQUMsUUFBUSxDQUFDLEtBQUssQ0FBQyxDQUFDLElBQUksSUFBSSxVQUFVLEVBQUUsQ0FBQztRQUNoRSxPQUFPLEtBQUssQ0FBQztJQUNqQixDQUFDO0lBQ0QsTUFBTSxTQUFTLEdBQUcsY0FBSSxDQUFDLElBQUksQ0FBQyxZQUFFLENBQUMsTUFBTSxFQUFFLEVBQUUsUUFBUSxDQUFDLENBQUM7SUFDbkQsSUFBSSxXQUFXLEdBQUcsQ0FBQyxDQUFDO0lBQ3BCLElBQUksWUFBRSxDQUFDLFVBQVUsQ0FBQyxTQUFTLENBQUMsRUFBRSxDQUFDO1FBQzNCLFdBQVcsR0FBRyxZQUFFLENBQUMsUUFBUSxDQUFDLFNBQVMsQ0FBQyxDQUFDLElBQUksQ0FBQztJQUM5QyxDQUFDO0lBQ0QsSUFBSSxXQUFXLEdBQUcsVUFBVSxFQUFFLENBQUM7UUFDM0IsTUFBTSxTQUFTLEdBQUcsQ0FBQyxDQUFDLFVBQVUsR0FBRyxXQUFXLENBQUMsR0FBRyxHQUFHLENBQUMsQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFDLENBQUM7UUFDaEUsT0FBTyxDQUFDLEdBQUcsQ0FBQyxlQUFlLFNBQVMsU0FBUyxRQUFRLFNBQVMsV0FBVyxPQUFPLFNBQVMsRUFBRSxDQUFDLENBQUM7UUFDN0YsTUFBTSxZQUFZLENBQUMsUUFBUSxFQUFFLFdBQVcsRUFBRSxVQUFVLEVBQUUsU0FBUyxDQUFDLENBQUM7SUFDckUsQ0FBQztJQUNELE9BQU8sU0FBUyxDQUFDO0FBQ3JCLENBQUM7QUFoQkQsd0NBZ0JDIiwic291cmNlc0NvbnRlbnQiOlsiaW1wb3J0IGZzIGZyb20gXCJmc1wiO1xuaW1wb3J0IG9zIGZyb20gXCJvc1wiO1xuaW1wb3J0IHBhdGggZnJvbSBcInBhdGhcIjtcbmltcG9ydCBodHRwIGZyb20gXCJodHRwXCI7XG5cbi8vIFJlc29sdmVzIGEgZmFjZS1lbWJlZGRpbmcgdGVzdCBmaWxlIHRvIGEgbG9jYWwgcGF0aCB0aGF0IGhvbGRzIGF0IGxlYXN0IHRoZSByZXF1ZXN0ZWQgbnVtYmVyIG9mIGJ5dGVzLiBPblxuLy8gdGhlIG1hY2hpbmUgdGhhdCBvd25zIHRoZSBkYXRhIGl0IGp1c3QgdXNlcyB0aGUgb3JpZ2luYWwgZmlsZS4gQW55d2hlcmUgZWxzZSBpdCBkb3dubG9hZHMgdGhlIG1pc3NpbmcgcHJlZml4XG4vLyBmcm9tIHRoZSBmYWNlRnJhbWVzU2VydmVyIG92ZXIgYSByYW5nZSByZXF1ZXN0IGFuZCBjYWNoZXMgaXQgaW4gdGhlIHRlbXAgZGlyLCBzbyByZS1ydW5zIG9ubHkgZmV0Y2ggd2hhdFxuLy8gdGhleSBoYXZlbid0IGFscmVhZHkuIFBvaW50IGl0IGF0IGEgZGlmZmVyZW50IHNlcnZlciB3aXRoIEZBQ0VGUkFNRVNfVVJMLlxuY29uc3QgU09VUkNFX0RJUiA9IFwiL3Jvb3QvY2xhdWRlLXdvcmsvZmFjZS1lbWJlZGRpbmdzXCI7XG5jb25zdCBTRVJWRVJfQkFTRSA9IHByb2Nlc3MuZW52LkZBQ0VGUkFNRVNfVVJMIHx8IFwiaHR0cDovLzY1LjEwOS45My4xMTM6ODc5OVwiO1xuXG5mdW5jdGlvbiBkb3dubG9hZEludG8oZmlsZU5hbWU6IHN0cmluZywgc3RhcnQ6IG51bWJlciwgZW5kRXhjbHVzaXZlOiBudW1iZXIsIGNhY2hlUGF0aDogc3RyaW5nKTogUHJvbWlzZTx2b2lkPiB7XG4gICAgcmV0dXJuIG5ldyBQcm9taXNlKChyZXNvbHZlLCByZWplY3QpID0+IHtcbiAgICAgICAgY29uc3QgdXJsID0gYCR7U0VSVkVSX0JBU0V9LyR7ZmlsZU5hbWV9YDtcbiAgICAgICAgY29uc3Qgb3B0aW9ucyA9IHsgaGVhZGVyczogeyBSYW5nZTogYGJ5dGVzPSR7c3RhcnR9LSR7ZW5kRXhjbHVzaXZlIC0gMX1gIH0gfTtcbiAgICAgICAgY29uc3QgcmVxdWVzdCA9IGh0dHAuZ2V0KHVybCwgb3B0aW9ucywgcmVzcG9uc2UgPT4ge1xuICAgICAgICAgICAgaWYgKHJlc3BvbnNlLnN0YXR1c0NvZGUgIT09IDIwNiAmJiByZXNwb25zZS5zdGF0dXNDb2RlICE9PSAyMDApIHtcbiAgICAgICAgICAgICAgICByZXNwb25zZS5yZXN1bWUoKTtcbiAgICAgICAgICAgICAgICByZWplY3QobmV3IEVycm9yKGAke3VybH0gcmV0dXJuZWQgJHtyZXNwb25zZS5zdGF0dXNDb2RlfWApKTtcbiAgICAgICAgICAgICAgICByZXR1cm47XG4gICAgICAgICAgICB9XG4gICAgICAgICAgICBjb25zdCBvdXQgPSBmcy5jcmVhdGVXcml0ZVN0cmVhbShjYWNoZVBhdGgsIHN0YXJ0ID8geyBmbGFnczogXCJyK1wiLCBzdGFydCB9IDogeyBmbGFnczogXCJ3XCIgfSk7XG4gICAgICAgICAgICByZXNwb25zZS5waXBlKG91dCk7XG4gICAgICAgICAgICBvdXQub24oXCJlcnJvclwiLCByZWplY3QpO1xuICAgICAgICAgICAgcmVzcG9uc2Uub24oXCJlcnJvclwiLCByZWplY3QpO1xuICAgICAgICAgICAgb3V0Lm9uKFwiZmluaXNoXCIsIHJlc29sdmUpO1xuICAgICAgICB9KTtcbiAgICAgICAgcmVxdWVzdC5vbihcImVycm9yXCIsIHJlamVjdCk7XG4gICAgfSk7XG59XG5cbmV4cG9ydCBhc3luYyBmdW5jdGlvbiBlbnN1cmVGYWNlRGF0YShmaWxlTmFtZTogc3RyaW5nLCBieXRlTGVuZ3RoOiBudW1iZXIpOiBQcm9taXNlPHN0cmluZz4ge1xuICAgIGNvbnN0IGxvY2FsID0gcGF0aC5qb2luKFNPVVJDRV9ESVIsIGZpbGVOYW1lKTtcbiAgICBpZiAoZnMuZXhpc3RzU3luYyhsb2NhbCkgJiYgZnMuc3RhdFN5bmMobG9jYWwpLnNpemUgPj0gYnl0ZUxlbmd0aCkge1xuICAgICAgICByZXR1cm4gbG9jYWw7XG4gICAgfVxuICAgIGNvbnN0IGNhY2hlUGF0aCA9IHBhdGguam9pbihvcy50bXBkaXIoKSwgZmlsZU5hbWUpO1xuICAgIGxldCBjYWNoZWRCeXRlcyA9IDA7XG4gICAgaWYgKGZzLmV4aXN0c1N5bmMoY2FjaGVQYXRoKSkge1xuICAgICAgICBjYWNoZWRCeXRlcyA9IGZzLnN0YXRTeW5jKGNhY2hlUGF0aCkuc2l6ZTtcbiAgICB9XG4gICAgaWYgKGNhY2hlZEJ5dGVzIDwgYnl0ZUxlbmd0aCkge1xuICAgICAgICBjb25zdCBtZWdhYnl0ZXMgPSAoKGJ5dGVMZW5ndGggLSBjYWNoZWRCeXRlcykgLyAxZTYpLnRvRml4ZWQoMCk7XG4gICAgICAgIGNvbnNvbGUubG9nKGBkb3dubG9hZGluZyAke21lZ2FieXRlc31NQiBvZiAke2ZpbGVOYW1lfSBmcm9tICR7U0VSVkVSX0JBU0V9IC0+ICR7Y2FjaGVQYXRofWApO1xuICAgICAgICBhd2FpdCBkb3dubG9hZEludG8oZmlsZU5hbWUsIGNhY2hlZEJ5dGVzLCBieXRlTGVuZ3RoLCBjYWNoZVBhdGgpO1xuICAgIH1cbiAgICByZXR1cm4gY2FjaGVQYXRoO1xufVxuIl19
- /* _JS_SOURCE_HASH = "623e4d719ca42b7e83579fd05b7d111f3158958917d536b1ac30e1e3373ad1ce"; */

package/storage/embeddingBench.d.ts

@@ -1 +0,0 @@
-export {};

package/storage/embeddingBench.ts

@@ -1,152 +0,0 @@
-import fs from "fs";
-import { ensureFaceData } from "./faceFramesData";
-import { encodeEmbedding, averageEmbeddings, embeddingToFloat32, releaseFloat32, getCloseness, StoredEmbedding, EmbeddingFormat } from "./embeddingFormats";
-
-// Runs a generic k-means parameterized by an arbitrary getCloseness, so we can race the closeness
-// implementations (all exported from embeddingFormats) on a real clustering workload — plus a variant that
-// decodes every embedding to float32 once and uses plain float dots. Push + run with: typenode storage/embeddingBench.ts
-
-const DIM = 512;
-const FILE_NAME = "faceEntry2.f32";
-const MODEL = "buffalo_l";
-const FORMAT: EmbeddingFormat = "q8g8_2048";
-const CONFIG = { format: FORMAT, model: MODEL };
-
-async function loadEmbeddings(count: number): Promise<StoredEmbedding[]> {
-    const byteLength = count * DIM * 4;
-    const filePath = await ensureFaceData(FILE_NAME, byteLength);
-    const buffer = Buffer.alloc(byteLength);
-    const handle = fs.openSync(filePath, "r");
-    fs.readSync(handle, buffer, 0, byteLength, 0);
-    fs.closeSync(handle);
-    const floats = new Float32Array(buffer.buffer, buffer.byteOffset, count * DIM);
-    const out: StoredEmbedding[] = [];
-    for (let index = 0; index < count; index++) {
-        out.push(encodeEmbedding({ input: floats.subarray(index * DIM, (index + 1) * DIM), format: FORMAT, model: MODEL }));
-    }
-    return out;
-}
-
-// Plain k-means: assign each member to the nearest centroid via `closeness`, recompute centroids as the
-// (re-encoded) mean. The closeness function is the only thing that varies.
-function kmeans(members: StoredEmbedding[], clusterCount: number, iterations: number, closeness: (a: StoredEmbedding, b: StoredEmbedding) => number): number {
-    let centroids: StoredEmbedding[] = [];
-    const seedStride = members.length / clusterCount;
-    for (let index = 0; index < clusterCount; index++) {
-        centroids.push(members[Math.floor(index * seedStride)]);
-    }
-    for (let iteration = 0; iteration < iterations; iteration++) {
-        const groups: StoredEmbedding[][] = [];
-        for (let index = 0; index < centroids.length; index++) {
-            groups.push([]);
-        }
-        for (const member of members) {
-            let bestIndex = 0;
-            let bestCloseness = -Infinity;
-            for (let centroidIndex = 0; centroidIndex < centroids.length; centroidIndex++) {
-                const value = closeness(member, centroids[centroidIndex]);
-                if (value > bestCloseness) {
-                    bestCloseness = value;
-                    bestIndex = centroidIndex;
-                }
-            }
-            groups[bestIndex].push(member);
-        }
-        const next: StoredEmbedding[] = [];
-        for (const group of groups) {
-            if (group.length) {
-                next.push(averageEmbeddings(group, CONFIG));
-            }
-        }
-        centroids = next;
-    }
-    return centroids.length;
-}
-
-// The same k-means but every embedding is decoded to a float32 array ONCE up front; centroids stay float
-// arrays and assignment is a plain float dot. No per-comparison decode, no centroid re-encode.
-function kmeansFloat32Once(members: StoredEmbedding[], clusterCount: number, iterations: number): number {
-    const memberFloats: Float32Array[] = [];
-    for (const member of members) {
-        memberFloats.push(embeddingToFloat32(member, true));
-    }
-    let centroids: Float32Array[] = [];
-    const seedStride = members.length / clusterCount;
-    for (let index = 0; index < clusterCount; index++) {
-        centroids.push(memberFloats[Math.floor(index * seedStride)]);
-    }
-    for (let iteration = 0; iteration < iterations; iteration++) {
-        const groups: number[][] = [];
-        for (let index = 0; index < centroids.length; index++) {
-            groups.push([]);
-        }
-        for (let memberIndex = 0; memberIndex < memberFloats.length; memberIndex++) {
-            const memberFloat = memberFloats[memberIndex];
-            let bestIndex = 0;
-            let bestDot = -Infinity;
-            for (let centroidIndex = 0; centroidIndex < centroids.length; centroidIndex++) {
-                const centroidFloat = centroids[centroidIndex];
-                const length = Math.min(memberFloat.length, centroidFloat.length);
-                let dot = 0;
-                for (let dim = 0; dim < length; dim++) {
-                    dot += memberFloat[dim] * centroidFloat[dim];
-                }
-                if (dot > bestDot) {
-                    bestDot = dot;
-                    bestIndex = centroidIndex;
-                }
-            }
-            groups[bestIndex].push(memberIndex);
-        }
-        const next: Float32Array[] = [];
-        for (const group of groups) {
-            if (!group.length) {
-                continue;
-            }
-            const length = memberFloats[group[0]].length;
-            const sum = new Float32Array(length);
-            for (const memberIndex of group) {
-                const memberFloat = memberFloats[memberIndex];
-                for (let dim = 0; dim < length; dim++) {
-                    sum[dim] += memberFloat[dim];
-                }
-            }
-            let norm = 0;
-            for (let dim = 0; dim < length; dim++) {
-                norm += sum[dim] * sum[dim];
-            }
-            const magnitude = Math.sqrt(norm) || 1;
-            for (let dim = 0; dim < length; dim++) {
-                sum[dim] /= magnitude;
-            }
-            next.push(sum);
-        }
-        centroids = next;
-    }
-    for (const memberFloat of memberFloats) {
-        releaseFloat32(memberFloat);
-    }
-    return centroids.length;
-}
-
-function time(name: string, run: () => number): number {
-    const start = Date.now();
-    const clusters = run();
-    const seconds = (Date.now() - start) / 1000;
-    console.log(`  ${name.padEnd(10)} ${seconds.toFixed(3)}s   (${clusters} clusters)`);
-    return seconds;
-}
-
-async function main() {
-    const members = await loadEmbeddings(5000);
-    const clusterCount = 40;
-    const iterations = 4;
-    console.log(`k-means: ${members.length} members, ${clusterCount} clusters, ${iterations} iterations\n`);
-
-    kmeans(members.slice(0, 500), 8, 1, getCloseness); // warm up the JIT
-
-    time("getCloseness", () => kmeans(members, clusterCount, iterations, getCloseness));
-    time("f32-once", () => kmeansFloat32Once(members, clusterCount, iterations));
-}
-
-main();

package/storage/faceFramesData.d.ts

@@ -1 +0,0 @@
-export declare function ensureFaceData(fileName: string, byteLength: number): Promise<string>;

package/storage/faceFramesData.ts

@@ -1,49 +0,0 @@
-import fs from "fs";
-import os from "os";
-import path from "path";
-import http from "http";
-
-// Resolves a face-embedding test file to a local path that holds at least the requested number of bytes. On
-// the machine that owns the data it just uses the original file. Anywhere else it downloads the missing prefix
-// from the faceFramesServer over a range request and caches it in the temp dir, so re-runs only fetch what
-// they haven't already. Point it at a different server with FACEFRAMES_URL.
-const SOURCE_DIR = "/root/claude-work/face-embeddings";
-const SERVER_BASE = process.env.FACEFRAMES_URL || "http://65.109.93.113:8799";
-
-function downloadInto(fileName: string, start: number, endExclusive: number, cachePath: string): Promise<void> {
-    return new Promise((resolve, reject) => {
-        const url = `${SERVER_BASE}/${fileName}`;
-        const options = { headers: { Range: `bytes=${start}-${endExclusive - 1}` } };
-        const request = http.get(url, options, response => {
-            if (response.statusCode !== 206 && response.statusCode !== 200) {
-                response.resume();
-                reject(new Error(`${url} returned ${response.statusCode}`));
-                return;
-            }
-            const out = fs.createWriteStream(cachePath, start ? { flags: "r+", start } : { flags: "w" });
-            response.pipe(out);
-            out.on("error", reject);
-            response.on("error", reject);
-            out.on("finish", resolve);
-        });
-        request.on("error", reject);
-    });
-}
-
-export async function ensureFaceData(fileName: string, byteLength: number): Promise<string> {
-    const local = path.join(SOURCE_DIR, fileName);
-    if (fs.existsSync(local) && fs.statSync(local).size >= byteLength) {
-        return local;
-    }
-    const cachePath = path.join(os.tmpdir(), fileName);
-    let cachedBytes = 0;
-    if (fs.existsSync(cachePath)) {
-        cachedBytes = fs.statSync(cachePath).size;
-    }
-    if (cachedBytes < byteLength) {
-        const megabytes = ((byteLength - cachedBytes) / 1e6).toFixed(0);
-        console.log(`downloading ${megabytes}MB of ${fileName} from ${SERVER_BASE} -> ${cachePath}`);
-        await downloadInto(fileName, cachedBytes, byteLength, cachePath);
-    }
-    return cachePath;
-}

package/storage/faceFramesServer.d.ts

@@ -1 +0,0 @@
-export {};

package/storage/faceFramesServer.ts

@@ -1,56 +0,0 @@
-import http from "http";
-import fs from "fs";
-import path from "path";
-
-// Throwaway HTTP server that serves the face-embedding test files so the benchmarks can be run from another
-// machine (which downloads + caches a prefix via faceFramesData). Supports range requests, so a client only
-// pulls the bytes it needs out of the 7.7GB full file. Meant to be left running in the background; it dies on
-// reboot and that is fine. Run with: typenode storage/faceFramesServer.ts
-const DATA_DIR = "/root/claude-work/face-embeddings";
-const PORT = Number(process.env.PORT) || 8799;
-
-function resolveFile(requestPath: string): string | undefined {
-    const name = path.basename(decodeURIComponent(requestPath));
-    if (!name.endsWith(".f32")) {
-        return undefined;
-    }
-    const full = path.join(DATA_DIR, name);
-    if (!fs.existsSync(full)) {
-        return undefined;
-    }
-    return full;
-}
-
-const server = http.createServer((request, response) => {
-    const filePath = resolveFile(request.url || "");
-    if (!filePath) {
-        response.writeHead(404);
-        response.end("not found");
-        return;
-    }
-    const total = fs.statSync(filePath).size;
-    const rangeHeader = request.headers.range;
-    const rangeMatch = rangeHeader ? /bytes=(\d+)-(\d*)/.exec(rangeHeader) : null;
-    if (rangeMatch) {
-        const start = Number(rangeMatch[1]);
-        const end = rangeMatch[2] ? Number(rangeMatch[2]) : total - 1;
-        response.writeHead(206, {
-            "Content-Type": "application/octet-stream",
-            "Accept-Ranges": "bytes",
-            "Content-Range": `bytes ${start}-${end}/${total}`,
-            "Content-Length": end - start + 1,
-        });
-        fs.createReadStream(filePath, { start, end }).pipe(response);
-        return;
-    }
-    response.writeHead(200, {
-        "Content-Type": "application/octet-stream",
-        "Accept-Ranges": "bytes",
-        "Content-Length": total,
-    });
-    fs.createReadStream(filePath).pipe(response);
-});
-
-server.listen(PORT, () => {
-    console.log(`serving ${DATA_DIR}/*.f32 on port ${PORT}`);
-});

package/storage/proxydatabase/ivfDbCheck.d.ts

@@ -1 +0,0 @@
-export {};

package/storage/proxydatabase/ivfDbCheck.ts

@@ -1,85 +0,0 @@
-import fs from "fs";
-import { InMemoryDatabase } from "./inMemoryDatabase";
-import { IvfEmbeddingRoot, IvfConfig, EmbeddingInput, insertEmbeddings, searchEmbeddings } from "./ivfEmbeddingDatabase";
-import { encodeEmbedding, EmbeddingFormat } from "../embeddingFormats";
-import { ensureFaceData } from "../faceFramesData";
-
-// Exercises the tiered IVF embedding database through the in-memory Database wrapper, which counts every
-// read/write/delete call and the bytes through each, across dataset sizes and load patterns, with timing.
-
-const DIM = 512;
-const FILE_NAME = "faceFrames2_full.f32";
-const MODEL = "buffalo_l";
-const FORMAT: EmbeddingFormat = "q8g8_2048";
-
-async function loadFloats(count: number): Promise<Float32Array> {
-    const byteLength = count * DIM * 4;
-    const filePath = await ensureFaceData(FILE_NAME, byteLength);
-    const buffer = Buffer.alloc(byteLength);
-    const handle = fs.openSync(filePath, "r");
-    fs.readSync(handle, buffer, 0, byteLength, 0);
-    fs.closeSync(handle);
-    return new Float32Array(buffer.buffer, buffer.byteOffset, count * DIM);
-}
-
-async function buildInputs(count: number): Promise<EmbeddingInput[]> {
-    const floats = await loadFloats(count);
-    const inputs: EmbeddingInput[] = [];
-    for (let index = 0; index < count; index++) {
-        const slice = floats.subarray(index * DIM, (index + 1) * DIM);
-        inputs.push({ ref: "e" + index, embedding: encodeEmbedding({ input: slice, format: FORMAT, model: MODEL }) });
-    }
-    return inputs;
-}
-
-function freshDatabase(config: IvfConfig): InMemoryDatabase<IvfEmbeddingRoot> {
-    const root: IvfEmbeddingRoot = { config, count: 0, flat: {}, byRef: {}, steps: {}, centroids: {}, cells: {} };
-    return new InMemoryDatabase<IvfEmbeddingRoot>(root);
-}
-
-function pad(value: string | number, width: number): string {
-    return String(value).padStart(width);
-}
-
-async function main() {
-    const config: IvfConfig = { model: MODEL, format: FORMAT, cellTargetSize: 128 };
-    const runs: { size: number; batch: number }[] = [
-        { size: 10000, batch: 1 },
-        { size: 10000, batch: 100 },
-        { size: 100000, batch: 1000 },
-    ];
-
-    let maxSize = 0;
-    for (const run of runs) {
-        if (run.size > maxSize) {
-            maxSize = run.size;
-        }
-    }
-    const loadStart = Date.now();
-    const allInputs = await buildInputs(maxSize);
-    console.log(`loaded + encoded ${maxSize} embeddings in ${((Date.now() - loadStart) / 1000).toFixed(1)}s\n`);
-
-    console.log(`${pad("added", 7)} ${pad("batch", 5)} | ${pad("secs", 7)} ${pad("reads", 8)} ${pad("writes", 8)} ${pad("deletes", 8)} ${pad("readMB", 9)} ${pad("writeMB", 8)} | recall`);
-    for (const run of runs) {
-        const database = freshDatabase(config);
-        const start = Date.now();
-        for (let offset = 0; offset < run.size; offset += run.batch) {
-            insertEmbeddings(database, allInputs.slice(offset, offset + run.batch));
-        }
-        const seconds = (Date.now() - start) / 1000;
-
-        let found = 0;
-        const sampleCount = 100;
-        for (let index = 0; index < sampleCount; index++) {
-            const query = allInputs[Math.floor(index * run.size / sampleCount)];
-            const hits = searchEmbeddings(database, query.embedding, { probeBudget: 512, resultCount: 1 });
-            if (hits && hits.length && hits[0].ref === query.ref) {
-                found++;
-            }
-        }
-        const recall = (found / sampleCount).toFixed(2);
-        console.log(`${pad(run.size, 7)} ${pad(run.batch, 5)} | ${pad(seconds.toFixed(1), 7)} ${pad(database.readCalls, 8)} ${pad(database.writeCalls, 8)} ${pad(database.deleteCalls, 8)} ${pad((database.bytesRead / 1e6).toFixed(0), 9)} ${pad((database.bytesWritten / 1e6).toFixed(0), 8)} | ${recall}`);
-    }
-}
-
-void main();