sliftutils 1.7.12 → 1.7.14

package/storage/remoteStorage/storageServer.ts

@@ -6,7 +6,8 @@ import { getExternalIP } from "socket-function/src/networking";
 import { RequireController } from "socket-function/require/RequireController";
 import { hostServer } from "../../misc/https/hostServer";
 import { RemoteStorageController } from "./storageController";
-import { setStorageServerConfig, setWritesRejectedReason } from "./storageServerState";
+import { setStorageServerConfig, setWritesRejectedReason, addExtraListenPort } from "./storageServerState";
+import { initDeployTakeover, registerAltPort, getMainPortAcquireDelay } from "./deployTakeover";
 import { parseStorageUrl } from "./ArchivesRemote";
 // Import browser code, so it is allowed to be required by the client
 import "./accessPage";
@@ -118,9 +119,21 @@ export async function hostStorageServer(config: HostStorageServerConfig): Promis
     }, DISK_SPACE_CHECK_INTERVAL_MS);
     (interval as { unref?: () => void }).unref?.();
 
+    // Zero-downtime deploys: only enabled when a deploy timeline exists - otherwise a busy port is
+    // a real misconfiguration and must throw, not silently listen elsewhere
+    let hasDeployTimeline = await initDeployTakeover({ domain, mainPort: port, storageFolder: path.resolve(folder) });
+
     await hostServer({
         domain,
         port,
         setDNSRecord: true,
+        portFallback: hasDeployTimeline && {
+            getAcquireDelay: getMainPortAcquireDelay,
+            onListening: (listeningPort, isMainPort) => {
+                if (isMainPort) return;
+                addExtraListenPort(listeningPort);
+                void registerAltPort(listeningPort).catch((e: Error) => console.error(`Registering alternate port ${listeningPort} failed: ${e.stack ?? e}`));
+            },
+        } || undefined,
     });
 }

package/storage/remoteStorage/storageServerState.d.ts

@@ -1,7 +1,7 @@
 /// <reference types="node" />
 /// <reference types="node" />
 import { IBucketStore } from "./blobStore";
-import { RemoteConfig, HostedConfig, IArchives } from "../IArchives";
+import { RemoteConfig, HostedConfig, IArchives, ArchivesConfig } from "../IArchives";
 import type { IStorage } from "../IStorage";
 import type { AccessRequest, TrustRecord } from "./storageController";
 export type StorageServerConfig = {
@@ -34,10 +34,25 @@ export type LoadedBucket = {
     self: HostedConfig | undefined;
     store: IBucketStore;
 };
+export declare function addExtraListenPort(port: number): void;
 export declare function getLoadedBucket(account: string, bucketName: string): Promise<LoadedBucket | undefined>;
 export declare function assertMutable(bucket: LoadedBucket, filePath: string, writeTime: number): Promise<void>;
 export declare function writeBucketFile(account: string, bucketName: string, filePath: string, data: Buffer, config?: {
     lastModified?: number;
 }): Promise<void>;
+export declare function getBucketConfig(bucket: LoadedBucket): ArchivesConfig;
+export declare function rebuildAllLoadedBuckets(): Promise<void>;
+export declare function rescanAllLoadedBucketDisks(): Promise<void>;
+export type ServerBucketInfo = {
+    bucketName: string;
+    active: boolean;
+    config?: ArchivesConfig;
+    error?: string;
+};
+/** Every bucket the account has on this server, active or not, each with its configuration.
+ *  Inactive buckets are inspected straight from disk WITHOUT loading them - loading would start
+ *  their synchronization, and old invalid buckets must stay inert (their parse error is reported
+ *  instead). */
+export declare function listAccountBuckets(account: string): Promise<ServerBucketInfo[]>;
 export declare function deleteBucketFile(account: string, bucketName: string, filePath: string): Promise<void>;
 export declare function getLocalArchives(account: string, bucketName: string): IArchives;

package/storage/remoteStorage/storageServerState.ts

@@ -1,4 +1,5 @@
 import path from "path";
+import fs from "fs";
 import { getFileStorageNested2 } from "../FileFolderAPI";
 import { TransactionStorage } from "../TransactionStorage";
 import { JSONStorage } from "../JSONStorage";
@@ -10,6 +11,7 @@ import {
     WRITE_PAST_WINDOW_GRACE, STORAGE_WRONG_VALID_WINDOW, STORAGE_WRONG_ROUTE, FULL_ROUTE,
 } from "../IArchives";
 import { ROUTING_FILE, parseRoutingData, parseHostedUrl, buildFileUrl, getConfigVersion, getRoute, routeContains, routeIntersection } from "./remoteConfig";
+import { applyDeployRemap, getFlushDeadline, onTakeoverEvent } from "./deployTakeover";
 import { createApiArchives } from "./createArchives";
 import type { IStorage } from "../IStorage";
 import type { AccessRequest, TrustRecord } from "./storageController";
@@ -156,6 +158,13 @@ function getBucketFolder(account: string, bucketName: string): string {
     return path.join(getStorageServerConfig().folder, "buckets2", account, bucketName);
 }
 
+// Ports we are listening on beyond the main config port - a deploy-takeover successor serves on a
+// temporary alternate port, and must recognize alternate-port source URLs as itself
+const extraListenPorts = new Set<number>();
+export function addExtraListenPort(port: number): void {
+    extraListenPorts.add(port);
+}
+
 function findSelfIndexes(routing: RemoteConfig, account: string, bucketName: string): number[] {
     let { domain, port } = getStorageServerConfig();
     let indexes: number[] = [];
@@ -163,7 +172,7 @@ function findSelfIndexes(routing: RemoteConfig, account: string, bucketName: str
         let source = routing.sources[i];
         if (typeof source === "string" || source.type !== "remote") continue;
         let parsed = parseHostedUrl(source.url);
-        if (parsed.address === domain && parsed.port === port && parsed.account === account && parsed.bucketName === bucketName) {
+        if (parsed.address === domain && (parsed.port === port || extraListenPorts.has(parsed.port)) && parsed.account === account && parsed.bucketName === bucketName) {
             indexes.push(i);
         }
     }
@@ -222,12 +231,17 @@ function scheduleWindowBoundaryRebuild(loaded: LoadedBucket): void {
 
 function buildBucket(account: string, bucketName: string, routing: RemoteConfig): LoadedBucket {
     let folder = getBucketFolder(account, bucketName);
-    let selfIndexes = findSelfIndexes(routing, account, bucketName);
-    let selfEntries = selfIndexes.map(i => routing.sources[i] as HostedConfig);
+    // The deploy-takeover remap is an INTERPRETATION overlay: everything behavioral (self entries,
+    // windows, downstream sources) uses the remapped view, while loaded.routing/routingJSON stay
+    // the STORED config - so version guards, change detection, and synchronization never see (or
+    // persist) the remap
+    let effective = applyDeployRemap(routing);
+    let selfIndexes = findSelfIndexes(effective, account, bucketName);
+    let selfEntries = selfIndexes.map(i => effective.sources[i] as HostedConfig);
     let self = selectEntryAt(selfEntries, Date.now());
     let selfIndex = -1;
     if (self) {
-        selfIndex = routing.sources.indexOf(self);
+        selfIndex = effective.sources.indexOf(self);
     }
     let store: IBucketStore;
     if (self?.rawDisk) {
@@ -250,8 +264,8 @@ function buildBucket(account: string, bucketName: string, routing: RemoteConfig)
         if (selfIndex === -1) {
             console.log(`This server is not in the routing config for bucket ${account}/${bucketName}; keeping its data on disk but no longer synchronizing it`);
         } else {
-            for (let i = selfIndex + 1; i < routing.sources.length; i++) {
-                let source = routing.sources[i];
+            for (let i = selfIndex + 1; i < effective.sources.length; i++) {
+                let source = effective.sources[i];
                 if (typeof source === "string" || ownIndexes.has(i)) continue;
                 // Disjoint-shard sources never talk to each other; a partial overlap syncs only
                 // the intersection (scans ignore the rest, writes only send matching keys)
@@ -270,6 +284,7 @@ function buildBucket(account: string, bucketName: string, routing: RemoteConfig)
                 void scheduleRoutingReload(account, bucketName);
             },
             readerDiskLimit: self?.readerDiskLimit,
+            getFlushDeadline,
         });
     }
     let loaded: LoadedBucket = { account, bucketName, routing, routingJSON: JSON.stringify(routing), selfEntries, self, store };
@@ -416,6 +431,99 @@ export async function writeBucketFile(account: string, bucketName: string, fileP
     await loaded.store.set(filePath, data, { ...getWriteConfig(loaded, writeTime, route), lastModified: writeTime });
 }
 
+export function getBucketConfig(bucket: LoadedBucket): ArchivesConfig {
+    let progress = bucket.store.getSyncProgress?.();
+    return {
+        supportsChangesAfter: !!bucket.store.getChangesAfter,
+        // The remapped (interpretation) view: getConfig is how clients learn of an in-progress
+        // deploy takeover, since the remap is never written into the routing file itself
+        remoteConfig: applyDeployRemap(bucket.routing),
+        index: progress?.index,
+        indexSources: progress?.sources,
+        readerDiskLimit: progress?.readerDiskLimit,
+        syncing: progress?.syncing,
+    };
+}
+
+export async function rebuildAllLoadedBuckets(): Promise<void> {
+    for (let key of [...buckets.keys()]) {
+        let slash = key.indexOf("/");
+        await scheduleRoutingReload(key.slice(0, slash), key.slice(slash + 1), { force: true });
+    }
+}
+
+export async function rescanAllLoadedBucketDisks(): Promise<void> {
+    for (let loadedPromise of [...buckets.values()]) {
+        let loaded = await loadedPromise.catch(() => undefined);
+        if (!loaded) continue;
+        let store = loaded.store;
+        if (!(store instanceof BlobStore)) continue;
+        try {
+            await store.rescanBase();
+        } catch (e) {
+            console.error(`Deploy switchover disk rescan failed for bucket ${loaded.account}/${loaded.bucketName}: ${(e as Error).stack ?? e}`);
+        }
+    }
+}
+
+onTakeoverEvent(event => {
+    if (event === "remapChanged") {
+        void rebuildAllLoadedBuckets();
+    }
+    if (event === "diskScan") {
+        void rescanAllLoadedBucketDisks();
+    }
+});
+
+export type ServerBucketInfo = {
+    bucketName: string;
+    // Loaded in memory (created or accessed since startup), so its synchronization is running
+    active: boolean;
+    config?: ArchivesConfig;
+    error?: string;
+};
+
+/** Every bucket the account has on this server, active or not, each with its configuration.
+ *  Inactive buckets are inspected straight from disk WITHOUT loading them - loading would start
+ *  their synchronization, and old invalid buckets must stay inert (their parse error is reported
+ *  instead). */
+export async function listAccountBuckets(account: string): Promise<ServerBucketInfo[]> {
+    let accountFolder = path.join(getStorageServerConfig().folder, "buckets2", account);
+    let names: string[];
+    try {
+        names = await fs.promises.readdir(accountFolder);
+    } catch {
+        return [];
+    }
+    let results: ServerBucketInfo[] = [];
+    for (let bucketName of names) {
+        let loadedPromise = buckets.get(`${account}/${bucketName}`);
+        if (loadedPromise) {
+            try {
+                let loaded = await loadedPromise;
+                if (loaded) {
+                    results.push({ bucketName, active: true, config: getBucketConfig(loaded) });
+                    continue;
+                }
+            } catch (e) {
+                results.push({ bucketName, active: true, error: String((e as Error).stack ?? e).slice(0, 500) });
+                continue;
+            }
+        }
+        try {
+            let data = await new ArchivesDisk(getBucketFolder(account, bucketName)).get(ROUTING_FILE);
+            if (!data) {
+                results.push({ bucketName, active: false, error: `No routing file (${ROUTING_FILE})` });
+                continue;
+            }
+            results.push({ bucketName, active: false, config: { remoteConfig: parseRoutingData(data) } });
+        } catch (e) {
+            results.push({ bucketName, active: false, error: String((e as Error).stack ?? e).slice(0, 500) });
+        }
+    }
+    return results;
+}
+
 export async function deleteBucketFile(account: string, bucketName: string, filePath: string): Promise<void> {
     if (filePath === ROUTING_FILE) {
         throw new Error(`The routing config ${JSON.stringify(ROUTING_FILE)} cannot be deleted (overwrite it to change the bucket's configuration)`);
@@ -450,8 +558,9 @@ class ArchivesLocalBucket implements IArchives {
         if (!bucket) return undefined;
         return await bucket.store.get2(fileName, config);
     }
-    public async set(fileName: string, data: Buffer, config?: { lastModified?: number }): Promise<void> {
+    public async set(fileName: string, data: Buffer, config?: { lastModified?: number }): Promise<string> {
         await writeBucketFile(this.account, this.bucketName, fileName, data, config);
+        return fileName;
     }
     public async del(fileName: string): Promise<void> {
         await deleteBucketFile(this.account, this.bucketName, fileName);
@@ -498,15 +607,8 @@ class ArchivesLocalBucket implements IArchives {
     public async getConfig(): Promise<ArchivesConfig> {
         let bucket = await this.getBucket();
         // Missing buckets say true, matching what they become once created (the default store type)
-        let progress = bucket?.store.getSyncProgress?.();
-        return {
-            supportsChangesAfter: !bucket || !!bucket.store.getChangesAfter,
-            remoteConfig: bucket?.routing,
-            index: progress?.index,
-            indexSources: progress?.sources,
-            readerDiskLimit: progress?.readerDiskLimit,
-            syncing: progress?.syncing,
-        };
+        if (!bucket) return { supportsChangesAfter: true };
+        return getBucketConfig(bucket);
     }
     public async hasWriteAccess(): Promise<boolean> {
         return true;

package/yarn.lock

@@ -1944,10 +1944,10 @@ slash@^3.0.0:
   resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634"
   integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==
 
-socket-function@^1.2.16:
-  version "1.2.16"
-  resolved "https://registry.yarnpkg.com/socket-function/-/socket-function-1.2.16.tgz#6ad7fc1971202d9768582b0212f086675fba61b5"
-  integrity sha512-o3GATr5Fa+496StsPWJIb0yytY/XRMZjJQ4cgb1LOQIfFv7uJUJ3NZtcldMeN/kOmqI0hBaMvhRLC3VHke9awA==
+socket-function@^1.2.20:
+  version "1.2.20"
+  resolved "https://registry.yarnpkg.com/socket-function/-/socket-function-1.2.20.tgz#8cd141cd84504a320583e09749ed91cee293db45"
+  integrity sha512-aLHs7S9MKb6WY5pjtNG+sfISVIWIv7toeX7Zt5K6f1R/sVFLb46IBElZ0Hf81LHenVYwDNsKN0uAX/fkBBIqlw==
   dependencies:
     "@types/pako" "^2.0.3"
     "@types/ws" "^8.5.3"

package/storage/dist/faceFramesData.ts.cache

@@ -1,55 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule && mod.default) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true , configurable: true});
-//exports.ensureFaceData = void 0;
-const fs_1 = __importDefault(require("fs"));
-const os_1 = __importDefault(require("os"));
-const path_1 = __importDefault(require("path"));
-const http_1 = __importDefault(require("http"));
-// Resolves a face-embedding test file to a local path that holds at least the requested number of bytes. On
-// the machine that owns the data it just uses the original file. Anywhere else it downloads the missing prefix
-// from the faceFramesServer over a range request and caches it in the temp dir, so re-runs only fetch what
-// they haven't already. Point it at a different server with FACEFRAMES_URL.
-const SOURCE_DIR = "/root/claude-work/face-embeddings";
-const SERVER_BASE = process.env.FACEFRAMES_URL || "http://65.109.93.113:8799";
-function downloadInto(fileName, start, endExclusive, cachePath) {
-    return new Promise((resolve, reject) => {
-        const url = `${SERVER_BASE}/${fileName}`;
-        const options = { headers: { Range: `bytes=${start}-${endExclusive - 1}` } };
-        const request = http_1.default.get(url, options, response => {
-            if (response.statusCode !== 206 && response.statusCode !== 200) {
-                response.resume();
-                reject(new Error(`${url} returned ${response.statusCode}`));
-                return;
-            }
-            const out = fs_1.default.createWriteStream(cachePath, start ? { flags: "r+", start } : { flags: "w" });
-            response.pipe(out);
-            out.on("error", reject);
-            response.on("error", reject);
-            out.on("finish", resolve);
-        });
-        request.on("error", reject);
-    });
-}
-async function ensureFaceData(fileName, byteLength) {
-    const local = path_1.default.join(SOURCE_DIR, fileName);
-    if (fs_1.default.existsSync(local) && fs_1.default.statSync(local).size >= byteLength) {
-        return local;
-    }
-    const cachePath = path_1.default.join(os_1.default.tmpdir(), fileName);
-    let cachedBytes = 0;
-    if (fs_1.default.existsSync(cachePath)) {
-        cachedBytes = fs_1.default.statSync(cachePath).size;
-    }
-    if (cachedBytes < byteLength) {
-        const megabytes = ((byteLength - cachedBytes) / 1e6).toFixed(0);
-        console.log(`downloading ${megabytes}MB of ${fileName} from ${SERVER_BASE} -> ${cachePath}`);
-        await downloadInto(fileName, cachedBytes, byteLength, cachePath);
-    }
-    return cachePath;
-}
-exports.ensureFaceData = ensureFaceData;
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmFjZUZyYW1lc0RhdGEuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyJmYWNlRnJhbWVzRGF0YS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOzs7Ozs7QUFBQSw0Q0FBb0I7QUFDcEIsNENBQW9CO0FBQ3BCLGdEQUF3QjtBQUN4QixnREFBd0I7QUFFeEIsNEdBQTRHO0FBQzVHLCtHQUErRztBQUMvRywyR0FBMkc7QUFDM0csNEVBQTRFO0FBQzVFLE1BQU0sVUFBVSxHQUFHLG1DQUFtQyxDQUFDO0FBQ3ZELE1BQU0sV0FBVyxHQUFHLE9BQU8sQ0FBQyxHQUFHLENBQUMsY0FBYyxJQUFJLDJCQUEyQixDQUFDO0FBRTlFLFNBQVMsWUFBWSxDQUFDLFFBQWdCLEVBQUUsS0FBYSxFQUFFLFlBQW9CLEVBQUUsU0FBaUI7SUFDMUYsT0FBTyxJQUFJLE9BQU8sQ0FBQyxDQUFDLE9BQU8sRUFBRSxNQUFNLEVBQUUsRUFBRTtRQUNuQyxNQUFNLEdBQUcsR0FBRyxHQUFHLFdBQVcsSUFBSSxRQUFRLEVBQUUsQ0FBQztRQUN6QyxNQUFNLE9BQU8sR0FBRyxFQUFFLE9BQU8sRUFBRSxFQUFFLEtBQUssRUFBRSxTQUFTLEtBQUssSUFBSSxZQUFZLEdBQUcsQ0FBQyxFQUFFLEVBQUUsRUFBRSxDQUFDO1FBQzdFLE1BQU0sT0FBTyxHQUFHLGNBQUksQ0FBQyxHQUFHLENBQUMsR0FBRyxFQUFFLE9BQU8sRUFBRSxRQUFRLENBQUMsRUFBRTtZQUM5QyxJQUFJLFFBQVEsQ0FBQyxVQUFVLEtBQUssR0FBRyxJQUFJLFFBQVEsQ0FBQyxVQUFVLEtBQUssR0FBRyxFQUFFLENBQUM7Z0JBQzdELFFBQVEsQ0FBQyxNQUFNLEVBQUUsQ0FBQztnQkFDbEIsTUFBTSxDQUFDLElBQUksS0FBSyxDQUFDLEdBQUcsR0FBRyxhQUFhLFFBQVEsQ0FBQyxVQUFVLEVBQUUsQ0FBQyxDQUFDLENBQUM7Z0JBQzVELE9BQU87WUFDWCxDQUFDO1lBQ0QsTUFBTSxHQUFHLEdBQUcsWUFBRSxDQUFDLGlCQUFpQixDQUFDLFNBQVMsRUFBRSxLQUFLLENBQUMsQ0FBQyxDQUFDLEVBQUUsS0FBSyxFQUFFLElBQUksRUFBRSxLQUFLLEVBQUUsQ0FBQyxDQUFDLENBQUMsRUFBRSxLQUFLLEVBQUUsR0FBRyxFQUFFLENBQUMsQ0FBQztZQUM3RixRQUFRLENBQUMsSUFBSSxDQUFDLEdBQUcsQ0FBQyxDQUFDO1lBQ25CLEdBQUcsQ0FBQyxFQUFFLENBQUMsT0FBTyxFQUFFLE1BQU0sQ0FBQyxDQUFDO1lBQ3hCLFFBQVEsQ0FBQyxFQUFFLENBQUMsT0FBTyxFQUFFLE1BQU0sQ0FBQyxDQUFDO1lBQzdCLEdBQUcsQ0FBQyxFQUFFLENBQUMsUUFBUSxFQUFFLE9BQU8sQ0FBQyxDQUFDO1FBQzlCLENBQUMsQ0FBQyxDQUFDO1FBQ0gsT0FBTyxDQUFDLEVBQUUsQ0FBQyxPQUFPLEVBQUUsTUFBTSxDQUFDLENBQUM7SUFDaEMsQ0FBQyxDQUFDLENBQUM7QUFDUCxDQUFDO0FBRU0sS0FBSyxVQUFVLGNBQWMsQ0FBQyxRQUFnQixFQUFFLFVBQWtCO0lBQ3JFLE1BQU0sS0FBSyxHQUFHLGNBQUksQ0FBQyxJQUFJLENBQUMsVUFBVSxFQUFFLFFBQVEsQ0FBQyxDQUFDO0lBQzlDLElBQUksWUFBRSxDQUFDLFVBQVUsQ0FBQyxLQUFLLENBQUMsSUFBSSxZQUFFLENBQUMsUUFBUSxDQUFDLEtBQUssQ0FBQyxDQUFDLElBQUksSUFBSSxVQUFVLEVBQUUsQ0FBQztRQUNoRSxPQUFPLEtBQUssQ0FBQztJQUNqQixDQUFDO0lBQ0QsTUFBTSxTQUFTLEdBQUcsY0FBSSxDQUFDLElBQUksQ0FBQyxZQUFFLENBQUMsTUFBTSxFQUFFLEVBQUUsUUFBUSxDQUFDLENBQUM7SUFDbkQsSUFBSSxXQUFXLEdBQUcsQ0FBQyxDQUFDO0lBQ3BCLElBQUksWUFBRSxDQUFDLFVBQVUsQ0FBQyxTQUFTLENBQUMsRUFBRSxDQUFDO1FBQzNCLFdBQVcsR0FBRyxZQUFFLENBQUMsUUFBUSxDQUFDLFNBQVMsQ0FBQyxDQUFDLElBQUksQ0FBQztJQUM5QyxDQUFDO0lBQ0QsSUFBSSxXQUFXLEdBQUcsVUFBVSxFQUFFLENBQUM7UUFDM0IsTUFBTSxTQUFTLEdBQUcsQ0FBQyxDQUFDLFVBQVUsR0FBRyxXQUFXLENBQUMsR0FBRyxHQUFHLENBQUMsQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFDLENBQUM7UUFDaEUsT0FBTyxDQUFDLEdBQUcsQ0FBQyxlQUFlLFNBQVMsU0FBUyxRQUFRLFNBQVMsV0FBVyxPQUFPLFNBQVMsRUFBRSxDQUFDLENBQUM7UUFDN0YsTUFBTSxZQUFZLENBQUMsUUFBUSxFQUFFLFdBQVcsRUFBRSxVQUFVLEVBQUUsU0FBUyxDQUFDLENBQUM7SUFDckUsQ0FBQztJQUNELE9BQU8sU0FBUyxDQUFDO0FBQ3JCLENBQUM7QUFoQkQsd0NBZ0JDIiwic291cmNlc0NvbnRlbnQiOlsiaW1wb3J0IGZzIGZyb20gXCJmc1wiO1xuaW1wb3J0IG9zIGZyb20gXCJvc1wiO1xuaW1wb3J0IHBhdGggZnJvbSBcInBhdGhcIjtcbmltcG9ydCBodHRwIGZyb20gXCJodHRwXCI7XG5cbi8vIFJlc29sdmVzIGEgZmFjZS1lbWJlZGRpbmcgdGVzdCBmaWxlIHRvIGEgbG9jYWwgcGF0aCB0aGF0IGhvbGRzIGF0IGxlYXN0IHRoZSByZXF1ZXN0ZWQgbnVtYmVyIG9mIGJ5dGVzLiBPblxuLy8gdGhlIG1hY2hpbmUgdGhhdCBvd25zIHRoZSBkYXRhIGl0IGp1c3QgdXNlcyB0aGUgb3JpZ2luYWwgZmlsZS4gQW55d2hlcmUgZWxzZSBpdCBkb3dubG9hZHMgdGhlIG1pc3NpbmcgcHJlZml4XG4vLyBmcm9tIHRoZSBmYWNlRnJhbWVzU2VydmVyIG92ZXIgYSByYW5nZSByZXF1ZXN0IGFuZCBjYWNoZXMgaXQgaW4gdGhlIHRlbXAgZGlyLCBzbyByZS1ydW5zIG9ubHkgZmV0Y2ggd2hhdFxuLy8gdGhleSBoYXZlbid0IGFscmVhZHkuIFBvaW50IGl0IGF0IGEgZGlmZmVyZW50IHNlcnZlciB3aXRoIEZBQ0VGUkFNRVNfVVJMLlxuY29uc3QgU09VUkNFX0RJUiA9IFwiL3Jvb3QvY2xhdWRlLXdvcmsvZmFjZS1lbWJlZGRpbmdzXCI7XG5jb25zdCBTRVJWRVJfQkFTRSA9IHByb2Nlc3MuZW52LkZBQ0VGUkFNRVNfVVJMIHx8IFwiaHR0cDovLzY1LjEwOS45My4xMTM6ODc5OVwiO1xuXG5mdW5jdGlvbiBkb3dubG9hZEludG8oZmlsZU5hbWU6IHN0cmluZywgc3RhcnQ6IG51bWJlciwgZW5kRXhjbHVzaXZlOiBudW1iZXIsIGNhY2hlUGF0aDogc3RyaW5nKTogUHJvbWlzZTx2b2lkPiB7XG4gICAgcmV0dXJuIG5ldyBQcm9taXNlKChyZXNvbHZlLCByZWplY3QpID0+IHtcbiAgICAgICAgY29uc3QgdXJsID0gYCR7U0VSVkVSX0JBU0V9LyR7ZmlsZU5hbWV9YDtcbiAgICAgICAgY29uc3Qgb3B0aW9ucyA9IHsgaGVhZGVyczogeyBSYW5nZTogYGJ5dGVzPSR7c3RhcnR9LSR7ZW5kRXhjbHVzaXZlIC0gMX1gIH0gfTtcbiAgICAgICAgY29uc3QgcmVxdWVzdCA9IGh0dHAuZ2V0KHVybCwgb3B0aW9ucywgcmVzcG9uc2UgPT4ge1xuICAgICAgICAgICAgaWYgKHJlc3BvbnNlLnN0YXR1c0NvZGUgIT09IDIwNiAmJiByZXNwb25zZS5zdGF0dXNDb2RlICE9PSAyMDApIHtcbiAgICAgICAgICAgICAgICByZXNwb25zZS5yZXN1bWUoKTtcbiAgICAgICAgICAgICAgICByZWplY3QobmV3IEVycm9yKGAke3VybH0gcmV0dXJuZWQgJHtyZXNwb25zZS5zdGF0dXNDb2RlfWApKTtcbiAgICAgICAgICAgICAgICByZXR1cm47XG4gICAgICAgICAgICB9XG4gICAgICAgICAgICBjb25zdCBvdXQgPSBmcy5jcmVhdGVXcml0ZVN0cmVhbShjYWNoZVBhdGgsIHN0YXJ0ID8geyBmbGFnczogXCJyK1wiLCBzdGFydCB9IDogeyBmbGFnczogXCJ3XCIgfSk7XG4gICAgICAgICAgICByZXNwb25zZS5waXBlKG91dCk7XG4gICAgICAgICAgICBvdXQub24oXCJlcnJvclwiLCByZWplY3QpO1xuICAgICAgICAgICAgcmVzcG9uc2Uub24oXCJlcnJvclwiLCByZWplY3QpO1xuICAgICAgICAgICAgb3V0Lm9uKFwiZmluaXNoXCIsIHJlc29sdmUpO1xuICAgICAgICB9KTtcbiAgICAgICAgcmVxdWVzdC5vbihcImVycm9yXCIsIHJlamVjdCk7XG4gICAgfSk7XG59XG5cbmV4cG9ydCBhc3luYyBmdW5jdGlvbiBlbnN1cmVGYWNlRGF0YShmaWxlTmFtZTogc3RyaW5nLCBieXRlTGVuZ3RoOiBudW1iZXIpOiBQcm9taXNlPHN0cmluZz4ge1xuICAgIGNvbnN0IGxvY2FsID0gcGF0aC5qb2luKFNPVVJDRV9ESVIsIGZpbGVOYW1lKTtcbiAgICBpZiAoZnMuZXhpc3RzU3luYyhsb2NhbCkgJiYgZnMuc3RhdFN5bmMobG9jYWwpLnNpemUgPj0gYnl0ZUxlbmd0aCkge1xuICAgICAgICByZXR1cm4gbG9jYWw7XG4gICAgfVxuICAgIGNvbnN0IGNhY2hlUGF0aCA9IHBhdGguam9pbihvcy50bXBkaXIoKSwgZmlsZU5hbWUpO1xuICAgIGxldCBjYWNoZWRCeXRlcyA9IDA7XG4gICAgaWYgKGZzLmV4aXN0c1N5bmMoY2FjaGVQYXRoKSkge1xuICAgICAgICBjYWNoZWRCeXRlcyA9IGZzLnN0YXRTeW5jKGNhY2hlUGF0aCkuc2l6ZTtcbiAgICB9XG4gICAgaWYgKGNhY2hlZEJ5dGVzIDwgYnl0ZUxlbmd0aCkge1xuICAgICAgICBjb25zdCBtZWdhYnl0ZXMgPSAoKGJ5dGVMZW5ndGggLSBjYWNoZWRCeXRlcykgLyAxZTYpLnRvRml4ZWQoMCk7XG4gICAgICAgIGNvbnNvbGUubG9nKGBkb3dubG9hZGluZyAke21lZ2FieXRlc31NQiBvZiAke2ZpbGVOYW1lfSBmcm9tICR7U0VSVkVSX0JBU0V9IC0+ICR7Y2FjaGVQYXRofWApO1xuICAgICAgICBhd2FpdCBkb3dubG9hZEludG8oZmlsZU5hbWUsIGNhY2hlZEJ5dGVzLCBieXRlTGVuZ3RoLCBjYWNoZVBhdGgpO1xuICAgIH1cbiAgICByZXR1cm4gY2FjaGVQYXRoO1xufVxuIl19
- /* _JS_SOURCE_HASH = "623e4d719ca42b7e83579fd05b7d111f3158958917d536b1ac30e1e3373ad1ce"; */

package/storage/embeddingBench.d.ts

@@ -1 +0,0 @@
-export {};

package/storage/embeddingBench.ts

@@ -1,152 +0,0 @@
-import fs from "fs";
-import { ensureFaceData } from "./faceFramesData";
-import { encodeEmbedding, averageEmbeddings, embeddingToFloat32, releaseFloat32, getCloseness, StoredEmbedding, EmbeddingFormat } from "./embeddingFormats";
-
-// Runs a generic k-means parameterized by an arbitrary getCloseness, so we can race the closeness
-// implementations (all exported from embeddingFormats) on a real clustering workload — plus a variant that
-// decodes every embedding to float32 once and uses plain float dots. Push + run with: typenode storage/embeddingBench.ts
-
-const DIM = 512;
-const FILE_NAME = "faceEntry2.f32";
-const MODEL = "buffalo_l";
-const FORMAT: EmbeddingFormat = "q8g8_2048";
-const CONFIG = { format: FORMAT, model: MODEL };
-
-async function loadEmbeddings(count: number): Promise<StoredEmbedding[]> {
-    const byteLength = count * DIM * 4;
-    const filePath = await ensureFaceData(FILE_NAME, byteLength);
-    const buffer = Buffer.alloc(byteLength);
-    const handle = fs.openSync(filePath, "r");
-    fs.readSync(handle, buffer, 0, byteLength, 0);
-    fs.closeSync(handle);
-    const floats = new Float32Array(buffer.buffer, buffer.byteOffset, count * DIM);
-    const out: StoredEmbedding[] = [];
-    for (let index = 0; index < count; index++) {
-        out.push(encodeEmbedding({ input: floats.subarray(index * DIM, (index + 1) * DIM), format: FORMAT, model: MODEL }));
-    }
-    return out;
-}
-
-// Plain k-means: assign each member to the nearest centroid via `closeness`, recompute centroids as the
-// (re-encoded) mean. The closeness function is the only thing that varies.
-function kmeans(members: StoredEmbedding[], clusterCount: number, iterations: number, closeness: (a: StoredEmbedding, b: StoredEmbedding) => number): number {
-    let centroids: StoredEmbedding[] = [];
-    const seedStride = members.length / clusterCount;
-    for (let index = 0; index < clusterCount; index++) {
-        centroids.push(members[Math.floor(index * seedStride)]);
-    }
-    for (let iteration = 0; iteration < iterations; iteration++) {
-        const groups: StoredEmbedding[][] = [];
-        for (let index = 0; index < centroids.length; index++) {
-            groups.push([]);
-        }
-        for (const member of members) {
-            let bestIndex = 0;
-            let bestCloseness = -Infinity;
-            for (let centroidIndex = 0; centroidIndex < centroids.length; centroidIndex++) {
-                const value = closeness(member, centroids[centroidIndex]);
-                if (value > bestCloseness) {
-                    bestCloseness = value;
-                    bestIndex = centroidIndex;
-                }
-            }
-            groups[bestIndex].push(member);
-        }
-        const next: StoredEmbedding[] = [];
-        for (const group of groups) {
-            if (group.length) {
-                next.push(averageEmbeddings(group, CONFIG));
-            }
-        }
-        centroids = next;
-    }
-    return centroids.length;
-}
-
-// The same k-means but every embedding is decoded to a float32 array ONCE up front; centroids stay float
-// arrays and assignment is a plain float dot. No per-comparison decode, no centroid re-encode.
-function kmeansFloat32Once(members: StoredEmbedding[], clusterCount: number, iterations: number): number {
-    const memberFloats: Float32Array[] = [];
-    for (const member of members) {
-        memberFloats.push(embeddingToFloat32(member, true));
-    }
-    let centroids: Float32Array[] = [];
-    const seedStride = members.length / clusterCount;
-    for (let index = 0; index < clusterCount; index++) {
-        centroids.push(memberFloats[Math.floor(index * seedStride)]);
-    }
-    for (let iteration = 0; iteration < iterations; iteration++) {
-        const groups: number[][] = [];
-        for (let index = 0; index < centroids.length; index++) {
-            groups.push([]);
-        }
-        for (let memberIndex = 0; memberIndex < memberFloats.length; memberIndex++) {
-            const memberFloat = memberFloats[memberIndex];
-            let bestIndex = 0;
-            let bestDot = -Infinity;
-            for (let centroidIndex = 0; centroidIndex < centroids.length; centroidIndex++) {
-                const centroidFloat = centroids[centroidIndex];
-                const length = Math.min(memberFloat.length, centroidFloat.length);
-                let dot = 0;
-                for (let dim = 0; dim < length; dim++) {
-                    dot += memberFloat[dim] * centroidFloat[dim];
-                }
-                if (dot > bestDot) {
-                    bestDot = dot;
-                    bestIndex = centroidIndex;
-                }
-            }
-            groups[bestIndex].push(memberIndex);
-        }
-        const next: Float32Array[] = [];
-        for (const group of groups) {
-            if (!group.length) {
-                continue;
-            }
-            const length = memberFloats[group[0]].length;
-            const sum = new Float32Array(length);
-            for (const memberIndex of group) {
-                const memberFloat = memberFloats[memberIndex];
-                for (let dim = 0; dim < length; dim++) {
-                    sum[dim] += memberFloat[dim];
-                }
-            }
-            let norm = 0;
-            for (let dim = 0; dim < length; dim++) {
-                norm += sum[dim] * sum[dim];
-            }
-            const magnitude = Math.sqrt(norm) || 1;
-            for (let dim = 0; dim < length; dim++) {
-                sum[dim] /= magnitude;
-            }
-            next.push(sum);
-        }
-        centroids = next;
-    }
-    for (const memberFloat of memberFloats) {
-        releaseFloat32(memberFloat);
-    }
-    return centroids.length;
-}
-
-function time(name: string, run: () => number): number {
-    const start = Date.now();
-    const clusters = run();
-    const seconds = (Date.now() - start) / 1000;
-    console.log(`  ${name.padEnd(10)} ${seconds.toFixed(3)}s   (${clusters} clusters)`);
-    return seconds;
-}
-
-async function main() {
-    const members = await loadEmbeddings(5000);
-    const clusterCount = 40;
-    const iterations = 4;
-    console.log(`k-means: ${members.length} members, ${clusterCount} clusters, ${iterations} iterations\n`);
-
-    kmeans(members.slice(0, 500), 8, 1, getCloseness); // warm up the JIT
-
-    time("getCloseness", () => kmeans(members, clusterCount, iterations, getCloseness));
-    time("f32-once", () => kmeansFloat32Once(members, clusterCount, iterations));
-}
-
-main();

package/storage/faceFramesData.d.ts

@@ -1 +0,0 @@
-export declare function ensureFaceData(fileName: string, byteLength: number): Promise<string>;

package/storage/faceFramesData.ts

@@ -1,49 +0,0 @@
-import fs from "fs";
-import os from "os";
-import path from "path";
-import http from "http";
-
-// Resolves a face-embedding test file to a local path that holds at least the requested number of bytes. On
-// the machine that owns the data it just uses the original file. Anywhere else it downloads the missing prefix
-// from the faceFramesServer over a range request and caches it in the temp dir, so re-runs only fetch what
-// they haven't already. Point it at a different server with FACEFRAMES_URL.
-const SOURCE_DIR = "/root/claude-work/face-embeddings";
-const SERVER_BASE = process.env.FACEFRAMES_URL || "http://65.109.93.113:8799";
-
-function downloadInto(fileName: string, start: number, endExclusive: number, cachePath: string): Promise<void> {
-    return new Promise((resolve, reject) => {
-        const url = `${SERVER_BASE}/${fileName}`;
-        const options = { headers: { Range: `bytes=${start}-${endExclusive - 1}` } };
-        const request = http.get(url, options, response => {
-            if (response.statusCode !== 206 && response.statusCode !== 200) {
-                response.resume();
-                reject(new Error(`${url} returned ${response.statusCode}`));
-                return;
-            }
-            const out = fs.createWriteStream(cachePath, start ? { flags: "r+", start } : { flags: "w" });
-            response.pipe(out);
-            out.on("error", reject);
-            response.on("error", reject);
-            out.on("finish", resolve);
-        });
-        request.on("error", reject);
-    });
-}
-
-export async function ensureFaceData(fileName: string, byteLength: number): Promise<string> {
-    const local = path.join(SOURCE_DIR, fileName);
-    if (fs.existsSync(local) && fs.statSync(local).size >= byteLength) {
-        return local;
-    }
-    const cachePath = path.join(os.tmpdir(), fileName);
-    let cachedBytes = 0;
-    if (fs.existsSync(cachePath)) {
-        cachedBytes = fs.statSync(cachePath).size;
-    }
-    if (cachedBytes < byteLength) {
-        const megabytes = ((byteLength - cachedBytes) / 1e6).toFixed(0);
-        console.log(`downloading ${megabytes}MB of ${fileName} from ${SERVER_BASE} -> ${cachePath}`);
-        await downloadInto(fileName, cachedBytes, byteLength, cachePath);
-    }
-    return cachePath;
-}

package/storage/faceFramesServer.d.ts

@@ -1 +0,0 @@
-export {};

package/storage/faceFramesServer.ts

@@ -1,56 +0,0 @@
-import http from "http";
-import fs from "fs";
-import path from "path";
-
-// Throwaway HTTP server that serves the face-embedding test files so the benchmarks can be run from another
-// machine (which downloads + caches a prefix via faceFramesData). Supports range requests, so a client only
-// pulls the bytes it needs out of the 7.7GB full file. Meant to be left running in the background; it dies on
-// reboot and that is fine. Run with: typenode storage/faceFramesServer.ts
-const DATA_DIR = "/root/claude-work/face-embeddings";
-const PORT = Number(process.env.PORT) || 8799;
-
-function resolveFile(requestPath: string): string | undefined {
-    const name = path.basename(decodeURIComponent(requestPath));
-    if (!name.endsWith(".f32")) {
-        return undefined;
-    }
-    const full = path.join(DATA_DIR, name);
-    if (!fs.existsSync(full)) {
-        return undefined;
-    }
-    return full;
-}
-
-const server = http.createServer((request, response) => {
-    const filePath = resolveFile(request.url || "");
-    if (!filePath) {
-        response.writeHead(404);
-        response.end("not found");
-        return;
-    }
-    const total = fs.statSync(filePath).size;
-    const rangeHeader = request.headers.range;
-    const rangeMatch = rangeHeader ? /bytes=(\d+)-(\d*)/.exec(rangeHeader) : null;
-    if (rangeMatch) {
-        const start = Number(rangeMatch[1]);
-        const end = rangeMatch[2] ? Number(rangeMatch[2]) : total - 1;
-        response.writeHead(206, {
-            "Content-Type": "application/octet-stream",
-            "Accept-Ranges": "bytes",
-            "Content-Range": `bytes ${start}-${end}/${total}`,
-            "Content-Length": end - start + 1,
-        });
-        fs.createReadStream(filePath, { start, end }).pipe(response);
-        return;
-    }
-    response.writeHead(200, {
-        "Content-Type": "application/octet-stream",
-        "Accept-Ranges": "bytes",
-        "Content-Length": total,
-    });
-    fs.createReadStream(filePath).pipe(response);
-});
-
-server.listen(PORT, () => {
-    console.log(`serving ${DATA_DIR}/*.f32 on port ${PORT}`);
-});

package/storage/proxydatabase/ivfDbCheck.d.ts

@@ -1 +0,0 @@
-export {};