sliftutils 1.7.9 → 1.7.11

package/spec.txt

@@ -1,60 +1,5 @@
 
-Searching
-    1) We want to create something that's database agnostic
-        - So I think we need like multiple phases for reading. And for applying it? We need to be able to know all of our reads ahead of time, which we should be able to roughly know. Or at least we'll have multiple stages. That should be fine. I guess we could do it so it actually understands our database. Like it reads, and then if it's not there, it returns. That might be fine too. 
 
-interface Database<Root> {
-    readData: <T>(fnc: (root: Root) => T) => T | undefined;
-    writeData: <T>(fnc: (root: Root) => T, newValue: T) => void;
-}
-I guess this works. We need multiple stages, so it's gonna have to read in the index, In parallel, and then once it has the full index, then it can do the next step in parallel, etc. 
 
 
-remaining
-    1) The code book creation. We kind of do want to recreate it sometimes. Maybe we can have something that does a full expensive recreation at certain early on points?
-    2) The searching there's a few different stages. Like, we want to search more IVF cells, but then even within the cells, we kind of want to download more of the final embeddings.  because the codes aren't gonna to be perfect. So we have to do both at the same time?
 
-parts
-    p8g8 format (q8), so we can store 2048 dimensions is a bit over 2KB
-    IVF cell size 32? PQ index reduces by 32X, and if IVF cells reduce by 32X... then 500K 2KB values (1GB) would still just be... 1MB IVF index, so... that's good.
-    0) In IVF, sample 32, then keep sampling more, up to 128 cells
-    1) Pq 64 index (byte values each, break into 64 groups, always... should work well, For face vectors, they're already really optimized. They have so few dimensions, and so we don't compress it by that much. But for text vectors, they're less compressed, so we compress it more. So it should work in all cases)
-    2) Retraining index on a subset
-        - Or... just don't. Freeze after a certain point, and... it might be fine? HMM... Although no, we could allow retraining. It should be cheap enough. 
-    3) Split into transaction logs, so we can read the PQ index efficiently, but still update it
-        UGH... I mean basically you would just be replicating some of what we actually store on disk, but in our database, which is stupid, but I think we'll probably do it anyway. Basically, just a transaction log. We have to read everything every time, but it should help the speed a little bit...
-            - And I think it's fair because it's us saying, hey, we're going to read all this data every time, which the database can't know, so it can't optimize for that, but we can know, and then we can optimize for it
-    4) IVF the pq index to vastly decrease the initial access time
-    5.0) automatically split cells when they get big enough (2X their initial size)
-    5) occasionally regenerate the entire IVF.
-    5.1) Have something to store a subset of the data codes?
-    5.1) Have something that stores some of the data randomly so we can easily access a subset. We won't represent deletions in it. That's fine. This will allow us to rebalance the IVF fairly efficiently. 
-        - And maybe even slowly move stuff over to the new IVF as we access stuff, we see if it's in the new IVF, and if not, we move it?
-        - 10% of the data? And... maybe just the pq index values?
-        - Should make retraining the IVF ten times faster and shouldn't affect the recall by too much, especially if we have more cells
-        - AND, Maybe we can actually sample less the more data we have. Ramping up to 10%. 
-unknown
-    exactly how we're going to update the IVF. 
-    IVF pq index, so we can read part of it
-        mini transaction logs in each cell?
-
-    BUT... how do we update IVF
-
-
-
-
-1) Converting once is faster, so we should investigate if we can have a pool of memory. If we can reuse it, then the allocations are free. 
-
-1) Fix embedding formats. It's really, really stupid. decodeToLength was just absolutely retarded. 
-
-
-
-
-
-
-0) Make sure the AI stops trying to stop when a read didn't work because it doesn't fucking matter. I don't think there's a single place that's doing it where it actually matters. 
-    - I mean, I guess there are a few places. Like when we're rebalancing, that's one place.
-    - I guess some of the returns are valid, but I guess we have to check all of them. 
-
-1) Make sure the AI gets rid of all those four each's. I mean, what the fuck? 
-1)  Ensure the AI efficiently switches from the hard-coded stuff to the regular stuff and correctly regenerates the codebook. 

package/storage/ArchivesDisk.d.ts

@@ -0,0 +1,67 @@
+/// <reference types="node" />
+/// <reference types="node" />
+import { IArchives, ArchiveFileInfo, ArchivesConfig } from "./IArchives";
+export declare class ArchivesDisk implements IArchives {
+    private folder;
+    constructor(folder: string);
+    private filesDir;
+    private uploadsDir;
+    private handles;
+    private largeUploads;
+    private nextLargeUploadId;
+    init: {
+        (): Promise<void>;
+        reset(): void;
+        set(newValue: Promise<void>): void;
+    };
+    getDebugName(): string;
+    getConfig(): Promise<ArchivesConfig>;
+    hasWriteAccess(): Promise<boolean>;
+    private filePath;
+    set(key: string, data: Buffer, config?: {
+        lastModified?: number;
+    }): Promise<void>;
+    del(key: string): Promise<void>;
+    get(key: string, config?: {
+        range?: {
+            start: number;
+            end: number;
+        };
+    }): Promise<Buffer | undefined>;
+    get2(key: string, config?: {
+        range?: {
+            start: number;
+            end: number;
+        };
+    }): Promise<{
+        data: Buffer;
+        writeTime: number;
+        size: number;
+    } | undefined>;
+    getInfo(key: string): Promise<{
+        writeTime: number;
+        size: number;
+    } | undefined>;
+    find(prefix: string, config?: {
+        shallow?: boolean;
+        type: "files" | "folders";
+    }): Promise<string[]>;
+    findInfo(prefix: string, config?: {
+        shallow?: boolean;
+        type?: "files" | "folders";
+    }): Promise<ArchiveFileInfo[]>;
+    private collectFiles;
+    setLargeFile(config: {
+        path: string;
+        getNextData(): Promise<Buffer | undefined>;
+    }): Promise<void>;
+    startLargeUpload(): Promise<string>;
+    appendLargeUpload(id: string, data: Buffer): Promise<void>;
+    finishLargeUpload(id: string, key: string): Promise<void>;
+    cancelLargeUpload(id: string): Promise<void>;
+    getURL(path: string): Promise<string>;
+}
+export declare function applyFindInfoShape(files: ArchiveFileInfo[], prefix: string, config?: {
+    shallow?: boolean;
+    type?: "files" | "folders";
+}): ArchiveFileInfo[];

package/storage/ArchivesDisk.ts

@@ -0,0 +1,369 @@
+import fs from "fs";
+import path from "path";
+import { lazy } from "socket-function/src/caching";
+import { runInfinitePoll } from "socket-function/src/batching";
+import { sort, binarySearchBasic } from "socket-function/src/misc";
+import { IArchives, ArchiveFileInfo, ArchivesConfig, assertValidLastModified } from "./IArchives";
+
+// The base file-system IArchives: storage is one-to-one with the file system, every key is exactly
+// one real file under <folder>/files, so the file system itself is the index. File handles are
+// cached and reused, and closed once idle (see FileHandleCache). All operations on a file run in
+// serial, so they can't collide with each other or with handle closing. Used as the disk
+// synchronization source of BlobStore (see remoteStorage/blobStore.ts).
+
+const HANDLE_IDLE_TIMEOUT = 1000 * 60;
+const HANDLE_SWEEP_INTERVAL = 1000 * 15;
+
+type HandleEntry = {
+    filePath: string;
+    handle: fs.promises.FileHandle;
+    lastUse: number;
+};
+
+// Caches open file handles, closing them once idle for HANDLE_IDLE_TIMEOUT. Instead of one
+// setTimeout per handle, a list sorted by last use is swept periodically (entries are moved via
+// binary search on access, and lastUse values only increase, so touched entries append at the
+// end). Also serializes operations per file: each operation only starts once the previous one on
+// the same file finished, and a handle is never closed while an operation is pending on it.
+class FileHandleCache {
+    private entries = new Map<string, HandleEntry>();
+    // Sorted by lastUse ascending (least recently used first)
+    private lru: HandleEntry[] = [];
+    private pending = new Map<string, Promise<void>>();
+
+    constructor() {
+        runInfinitePoll(HANDLE_SWEEP_INTERVAL, () => this.sweep());
+    }
+
+    // Runs fnc after every previously scheduled operation on filePath has finished
+    public run<T>(filePath: string, fnc: () => Promise<T>): Promise<T> {
+        let prev = this.pending.get(filePath) || Promise.resolve();
+        let result = prev.then(fnc);
+        // The pending chain must never reject, or one failed operation would poison all later ones
+        let last = result.then(() => { }, () => { });
+        this.pending.set(filePath, last);
+        void last.then(() => {
+            // If we're still the last pending operation on this file, clear ourselves
+            if (this.pending.get(filePath) === last) {
+                this.pending.delete(filePath);
+            }
+        });
+        return result;
+    }
+
+    // Only call inside run() for the same filePath (so opens can't collide with closes)
+    public async getHandle(filePath: string, flags: number): Promise<fs.promises.FileHandle> {
+        let entry = this.entries.get(filePath);
+        if (entry) {
+            this.removeFromLRU(entry);
+            entry.lastUse = Date.now();
+            this.lru.push(entry);
+            return entry.handle;
+        }
+        let handle = await fs.promises.open(filePath, flags);
+        entry = { filePath, handle, lastUse: Date.now() };
+        this.entries.set(filePath, entry);
+        this.lru.push(entry);
+        return handle;
+    }
+
+    // Only call inside run() for the same filePath
+    public async closeNow(filePath: string): Promise<void> {
+        let entry = this.entries.get(filePath);
+        if (!entry) return;
+        this.entries.delete(filePath);
+        this.removeFromLRU(entry);
+        await entry.handle.close();
+    }
+
+    private removeFromLRU(entry: HandleEntry) {
+        let index = binarySearchBasic(this.lru, x => x.lastUse, entry.lastUse);
+        if (index < 0) return;
+        // Multiple entries can share a lastUse, so scan for the exact one
+        while (index < this.lru.length && this.lru[index].lastUse === entry.lastUse) {
+            if (this.lru[index] === entry) {
+                this.lru.splice(index, 1);
+                return;
+            }
+            index++;
+        }
+    }
+
+    private async sweep(): Promise<void> {
+        let cutoff = Date.now() - HANDLE_IDLE_TIMEOUT;
+        let index = 0;
+        while (index < this.lru.length && this.lru[index].lastUse <= cutoff) {
+            let entry = this.lru[index];
+            // Can't close a handle with a pending operation; it'll be swept after it finishes
+            if (this.pending.has(entry.filePath)) {
+                index++;
+                continue;
+            }
+            this.lru.splice(index, 1);
+            this.entries.delete(entry.filePath);
+            await entry.handle.close();
+        }
+    }
+}
+
+export class ArchivesDisk implements IArchives {
+    constructor(private folder: string) { }
+
+    private filesDir = path.join(this.folder, "files");
+    private uploadsDir = path.join(this.folder, "uploads");
+    private handles = new FileHandleCache();
+    private largeUploads = new Map<string, { tmpPath: string }>();
+    private nextLargeUploadId = 1;
+
+    public init = lazy(async () => {
+        await fs.promises.mkdir(this.filesDir, { recursive: true });
+        await fs.promises.mkdir(this.uploadsDir, { recursive: true });
+        // Uploads don't survive restarts (the uploader streams into them), so old ones are garbage
+        for (let file of await fs.promises.readdir(this.uploadsDir)) {
+            await fs.promises.unlink(path.join(this.uploadsDir, file));
+        }
+    });
+
+    public getDebugName() {
+        return `disk/${this.folder}`;
+    }
+
+    public async getConfig(): Promise<ArchivesConfig> {
+        return {};
+    }
+
+    public async hasWriteAccess(): Promise<boolean> {
+        return true;
+    }
+
+    private filePath(key: string): string {
+        let result = path.join(this.filesDir, key);
+        if (!result.startsWith(this.filesDir + path.sep)) {
+            throw new Error(`Invalid key ${JSON.stringify(key.slice(0, 200))}, it escapes the store folder`);
+        }
+        return result;
+    }
+
+    public async set(key: string, data: Buffer, config?: { lastModified?: number }): Promise<void> {
+        await this.init();
+        let lastModified = config?.lastModified;
+        if (lastModified) {
+            assertValidLastModified(lastModified);
+        }
+        let filePath = this.filePath(key);
+        await this.handles.run(filePath, async () => {
+            if (lastModified) {
+                let existing = await statOrUndefined(filePath);
+                // An older write never overwrites a newer one (see IArchives.set)
+                if (existing && lastModified < existing.mtimeMs) return;
+            }
+            await fs.promises.mkdir(path.dirname(filePath), { recursive: true });
+            let handle = await this.handles.getHandle(filePath, fs.constants.O_RDWR | fs.constants.O_CREAT);
+            await handle.truncate(0);
+            await handle.write(data, 0, data.length, 0);
+            if (lastModified) {
+                await handle.utimes(new Date(lastModified), new Date(lastModified));
+            }
+        });
+    }
+
+    public async del(key: string): Promise<void> {
+        await this.init();
+        let filePath = this.filePath(key);
+        await this.handles.run(filePath, async () => {
+            await this.handles.closeNow(filePath);
+            try {
+                await fs.promises.unlink(filePath);
+            } catch (e: any) {
+                if (e.code !== "ENOENT") throw e;
+            }
+        });
+    }
+
+    public async get(key: string, config?: { range?: { start: number; end: number } }): Promise<Buffer | undefined> {
+        let result = await this.get2(key, config);
+        return result && result.data || undefined;
+    }
+
+    public async get2(key: string, config?: { range?: { start: number; end: number } }): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
+        await this.init();
+        let range = config?.range;
+        let filePath = this.filePath(key);
+        return await this.handles.run(filePath, async () => {
+            let handle: fs.promises.FileHandle;
+            try {
+                handle = await this.handles.getHandle(filePath, fs.constants.O_RDWR);
+            } catch (e: any) {
+                if (e.code === "ENOENT") return undefined;
+                throw e;
+            }
+            let stats = await handle.stat();
+            let size = stats.size;
+            let start = range && Math.min(range.start, size) || 0;
+            let end = range && Math.min(range.end, size) || size;
+            if (end <= start) return { data: Buffer.alloc(0), writeTime: stats.mtimeMs, size };
+            let buffer = Buffer.alloc(end - start);
+            let { bytesRead } = await handle.read(buffer, 0, buffer.length, start);
+            if (bytesRead !== buffer.length) {
+                throw new Error(`Expected ${buffer.length} bytes at ${filePath}:${start}, read ${bytesRead}`);
+            }
+            return { data: buffer, writeTime: stats.mtimeMs, size };
+        });
+    }
+
+    public async getInfo(key: string): Promise<{ writeTime: number; size: number } | undefined> {
+        await this.init();
+        let filePath = this.filePath(key);
+        return await this.handles.run(filePath, async () => {
+            let stats = await statOrUndefined(filePath);
+            if (!stats || !stats.isFile()) return undefined;
+            return { writeTime: stats.mtimeMs, size: stats.size };
+        });
+    }
+
+    public async find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]> {
+        return (await this.findInfo(prefix, config)).map(x => x.path);
+    }
+
+    public async findInfo(prefix: string, config?: { shallow?: boolean; type?: "files" | "folders" }): Promise<ArchiveFileInfo[]> {
+        await this.init();
+        let infos = new Map<string, ArchiveFileInfo>();
+        await this.collectFiles("", prefix, infos);
+        let files = Array.from(infos.values());
+        files = applyFindInfoShape(files, prefix, config);
+        sort(files, x => x.path);
+        return files;
+    }
+
+    // relDir is "" or ends with "/". Only descends into directories that can still match the prefix.
+    private async collectFiles(relDir: string, prefix: string, infos: Map<string, ArchiveFileInfo>): Promise<void> {
+        let entries: fs.Dirent[];
+        try {
+            entries = await fs.promises.readdir(path.join(this.filesDir, relDir), { withFileTypes: true });
+        } catch (e: any) {
+            if (e.code === "ENOENT") return;
+            throw e;
+        }
+        for (let entry of entries) {
+            let relPath = relDir + entry.name;
+            if (entry.isDirectory()) {
+                let dirPath = relPath + "/";
+                if (dirPath.startsWith(prefix) || prefix.startsWith(dirPath)) {
+                    await this.collectFiles(dirPath, prefix, infos);
+                }
+                continue;
+            }
+            if (!entry.isFile()) continue;
+            if (!relPath.startsWith(prefix)) continue;
+            let stats = await statOrUndefined(path.join(this.filesDir, relPath));
+            // Deleted while we were walking
+            if (!stats) continue;
+            infos.set(relPath, { path: relPath, createTime: stats.mtimeMs, size: stats.size });
+        }
+    }
+
+    public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined> }): Promise<void> {
+        let id = await this.startLargeUpload();
+        try {
+            while (true) {
+                let data = await config.getNextData();
+                if (!data) break;
+                await this.appendLargeUpload(id, data);
+            }
+            await this.finishLargeUpload(id, config.path);
+        } catch (e) {
+            await this.cancelLargeUpload(id);
+            throw e;
+        }
+    }
+
+    // Large files stream into their own file under uploads/, then move into place on finish. No
+    // handle is opened until the first append actually happens.
+    public async startLargeUpload(): Promise<string> {
+        await this.init();
+        let id = `${Date.now()}_${this.nextLargeUploadId++}`;
+        this.largeUploads.set(id, { tmpPath: path.join(this.uploadsDir, `upload_${id}.tmp`) });
+        return id;
+    }
+    public async appendLargeUpload(id: string, data: Buffer): Promise<void> {
+        let upload = this.largeUploads.get(id);
+        if (!upload) throw new Error(`Unknown large upload ${id}`);
+        const tmpPath = upload.tmpPath;
+        await this.handles.run(tmpPath, async () => {
+            let handle = await this.handles.getHandle(tmpPath, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_APPEND);
+            await handle.write(data, 0, data.length);
+        });
+    }
+    public async finishLargeUpload(id: string, key: string): Promise<void> {
+        let upload = this.largeUploads.get(id);
+        if (!upload) throw new Error(`Unknown large upload ${id}`);
+        const tmpPath = upload.tmpPath;
+        this.largeUploads.delete(id);
+        let filePath = this.filePath(key);
+        await this.handles.run(tmpPath, () => this.handles.closeNow(tmpPath));
+        await this.handles.run(filePath, async () => {
+            // Close any cached handle to the file we're replacing, so later reads reopen the new file
+            await this.handles.closeNow(filePath);
+            await fs.promises.mkdir(path.dirname(filePath), { recursive: true });
+            try {
+                await fs.promises.rename(tmpPath, filePath);
+            } catch (e: any) {
+                if (e.code !== "ENOENT") throw e;
+                // Nothing was ever appended, so the upload file was never created
+                await fs.promises.writeFile(filePath, Buffer.alloc(0));
+            }
+        });
+    }
+    public async cancelLargeUpload(id: string): Promise<void> {
+        let upload = this.largeUploads.get(id);
+        if (!upload) return;
+        const tmpPath = upload.tmpPath;
+        this.largeUploads.delete(id);
+        await this.handles.run(tmpPath, async () => {
+            await this.handles.closeNow(tmpPath);
+            try {
+                await fs.promises.unlink(tmpPath);
+            } catch (e: any) {
+                if (e.code !== "ENOENT") throw e;
+            }
+        });
+    }
+
+    public async getURL(path: string): Promise<string> {
+        throw new Error(`getURL is not supported for disk archives (${this.getDebugName()}, path ${path})`);
+    }
+}
+
+async function statOrUndefined(filePath: string): Promise<fs.Stats | undefined> {
+    try {
+        return await fs.promises.stat(filePath);
+    } catch (e: any) {
+        if (e.code === "ENOENT") return undefined;
+        throw e;
+    }
+}
+
+// The folders/shallow post-processing shared by findInfo implementations that list flat files
+// (used by ArchivesDisk on the raw disk walk, and by BlobStore on its index).
+export function applyFindInfoShape(files: ArchiveFileInfo[], prefix: string, config?: { shallow?: boolean; type?: "files" | "folders" }): ArchiveFileInfo[] {
+    if (config?.type === "folders") {
+        let folders = new Map<string, ArchiveFileInfo>();
+        for (let file of files) {
+            let rest = file.path.slice(prefix.length);
+            let restParts = rest.split("/");
+            if (restParts.length < 2) continue;
+            let folder: string;
+            if (config.shallow) {
+                folder = prefix + restParts[0];
+            } else {
+                folder = file.path.split("/").slice(0, -1).join("/");
+            }
+            folders.set(folder, { path: folder, createTime: file.createTime, size: file.size });
+        }
+        return Array.from(folders.values());
+    }
+    if (config?.shallow) {
+        return files.filter(file => !file.path.slice(prefix.length).includes("/"));
+    }
+    return files;
+}