querysub 0.2.0

package/src/-a-archives/archiveCache.ts

@@ -0,0 +1,913 @@
+import { getStorageDir, getSubFolder } from "../fs";
+import { Archives } from "./archives";
+import { diskEscapeFileName, getArchivesLocal } from "./archivesDisk";
+import fs from "fs";
+
+import diskusage from "diskusage";
+import { binarySearchBasic, binarySearchIndex, list, nextId, sort, timeInHour, timeInMinute } from "socket-function/src/misc";
+import { cache, lazy } from "socket-function/src/caching";
+import { delay, runInSerial, runInfinitePoll } from "socket-function/src/batching";
+import { sha256 } from "js-sha256";
+import { devDebugbreak } from "../config";
+import child_process from "child_process";
+import os from "os";
+import fsext, { flock } from "fs-ext";
+import debugbreak from "debugbreak";
+import readline from "readline";
+import { getPPID } from "../misc/getParentProcessId";
+import { Args } from "socket-function/src/types";
+import { getArchivesBackblaze } from "./archivesBackBlaze";
+import { logErrors } from "../errors";
+import { formatNumber, formatTime } from "socket-function/src/formatting/format";
+import { SizeLimiter } from "../diagnostics/SizeLimiter";
+
+const SIZE_LIMIT = new SizeLimiter({
+    diskRoot: getStorageDir(),
+    maxBytes: 1024 * 1024 * 1024 * 50,
+    // Anything less than this and we can't even load enough weights models for a single task
+    minBytes: 1024 * 1024 * 1024 * 8,
+    maxDiskFraction: 0.3,
+    maxTotalDiskFraction: 0.95,
+    maxFiles: 1000 * 25,
+});
+
+const UPDATE_METRICS_INTERVAL = timeInMinute * 30;
+const cacheArchives2 = getSubFolder("cache");
+
+const LOCK_SUFFIX = ".lock";
+const TEMP_SUFFIX = ".tmp";
+const TEMP_THRESHOLD = timeInHour * 3;
+
+const LARGE_FILE_CHUNK = 1024 * 1024 * 32;
+const CACHE_SUFFIX = ".cache";
+export function getArchiveCachePath(archives: Archives, key: string): string {
+    // Remove all "/backblaze/" values from the path, as they make equivalent paths
+    //  hash differently, which we don't want.
+    let fullFileName = (archives.getDebugName() + "/" + key).replaceAll("//", "/").replaceAll("backblaze/", "/");
+    let hash = sha256(fullFileName);
+    let name = fullFileName.replace(/[<>:"\/\\|?*\x00-\x1F]/g, "_");
+    // Remove spaces, as I think they are causing some issues if they are at the end
+    name = name.replaceAll(" ", "");
+    return cacheArchives2 + hash.slice(0, 16) + "." + name.slice(0, 128) + CACHE_SUFFIX;
+}
+
+const getDiskMetricsBase = async () => {
+    ensureDiskMetricsUpdated();
+
+    let fileSizes: {
+        path: string;
+        time: number;
+        bytes: number;
+    }[] = [];
+
+    let usedCacheBytes = 0;
+    let usedCacheFiles = 0;
+    let cacheFiles = await fs.promises.readdir(cacheArchives2);
+    for (let file of cacheFiles) {
+        if (file.endsWith(LOCK_SUFFIX)) {
+            let base = file.slice(0, -LOCK_SUFFIX.length);
+            if (!fs.existsSync(base) && !await isLocked(cacheArchives2 + base)) {
+                try {
+                    // NOTE: This races, it might not be locked when we check, but a lock
+                    //  might be added right here. But... what else can we do? We need
+                    //  to cleanup unused locks at some point...
+                    await fs.promises.unlink(cacheArchives2 + file);
+                } catch { }
+            }
+        } else if (file.endsWith(CACHE_SUFFIX)) {
+            let info: fs.Stats | undefined;
+            try {
+                info = await fs.promises.stat(file);
+            } catch { }
+            if (!info) continue;
+            fileSizes.push({
+                path: file,
+                time: info.atimeMs,
+                bytes: info.size,
+            });
+            usedCacheBytes += info.size;
+            usedCacheFiles++;
+        } else {
+            // TEMP files, and... any files?
+            // If it's too old, delete it
+            let stat = await fs.promises.stat(cacheArchives2 + file);
+            let threshold = Date.now() - TEMP_THRESHOLD;
+            if (stat.mtimeMs < threshold) {
+                try {
+                    await fs.promises.unlink(cacheArchives2 + file);
+                } catch { }
+            }
+        }
+    }
+
+    let { remove, availableBytes, availableFiles } = await SIZE_LIMIT.limit(fileSizes);
+    for (let file of remove) {
+        try {
+            await fs.promises.unlink(cacheArchives2 + file.path);
+        } catch { }
+    }
+
+    function updateAccessTime(path: string) {
+        let index = fileSizes.findIndex(x => x.path === path);
+        if (index < 0) return;
+        let obj = fileSizes[index];
+        obj.time = Date.now();
+    }
+
+    function removeFile(path: string) {
+        let index = fileSizes.findIndex(x => x.path === path);
+        if (index < 0) return false;
+        let obj = fileSizes[index];
+        availableBytes += obj.bytes;
+        availableFiles++;
+        fileSizes.splice(index, 1);
+        return true;
+    }
+
+    // We require our input to be a temp file, so our update can be atomic
+    async function addCacheFile(archives: Archives, key: string, sourceTempFile: string): Promise<void> {
+        let path = getArchiveCachePath(archives, key);
+        let hasFile = fileSizes.some(x => x.path === path);
+        if (hasFile) {
+            removeFile(path);
+        }
+        let sourceTempFileStat = await fs.promises.stat(sourceTempFile);
+        availableBytes -= sourceTempFileStat.size;
+        availableFiles--;
+        while ((availableBytes < 0 || availableFiles < 0) && fileSizes.length > 0) {
+            let leastRecentlyAccessed = fileSizes[0];
+            for (let file of fileSizes) {
+                if (file.time < leastRecentlyAccessed.time) {
+                    leastRecentlyAccessed = file;
+                }
+            }
+            let size = 0;
+            try {
+                size = (await fs.promises.stat(leastRecentlyAccessed.path)).size;
+            } catch { }
+            if (await deleteCacheFile(leastRecentlyAccessed.path)) {
+                removeFile(leastRecentlyAccessed.path);
+            }
+        }
+        fileSizes.push({
+            path,
+            bytes: sourceTempFileStat.size,
+            time: Date.now(),
+        });
+        availableBytes -= sourceTempFileStat.size;
+        availableFiles--;
+        try {
+            await fs.promises.rename(sourceTempFile, path);
+        } catch (e) {
+            let destExists = fs.existsSync(path);
+            if (!destExists) {
+                console.error("Error renaming file", sourceTempFile, path, e);
+            }
+        }
+    }
+    async function getCacheFile(
+        archives: Archives,
+        key: string,
+        config?: { range?: { start: number; end: number } }
+    ): Promise<Buffer | undefined> {
+        let path = getArchiveCachePath(archives, key);
+        let buffer: Buffer | undefined;
+        let handle: fs.promises.FileHandle | undefined;
+        try {
+            handle = await fs.promises.open(path, "r");
+            let stats = await handle.stat();
+            let start = config?.range?.start ?? 0;
+            let end = config?.range?.end ?? stats.size;
+            let readBuffer = Buffer.alloc(end - start);
+            let read = await handle.read(readBuffer, 0, end - start, start);
+            buffer = readBuffer.slice(0, read.bytesRead);
+        }
+        catch { }
+        finally {
+            if (handle) {
+                await handle.close();
+            }
+        }
+        if (buffer) {
+            updateAccessTime(path);
+        }
+        return buffer;
+    }
+    async function getPathAndLock(archives: Archives, key: string): Promise<string> {
+        let path = getArchiveCachePath(archives, key);
+        await lockFile(path);
+        updateAccessTime(path);
+        return path;
+    }
+    async function delCacheFile(archives: Archives, key: string): Promise<void> {
+        let path = getArchiveCachePath(archives, key);
+        if (removeFile(path)) {
+            try {
+                await fs.promises.unlink(path);
+            } catch { }
+        }
+    }
+
+    // Keep track of an in memory access list, to make freeing and space estimation faster
+    //  (that way we can find least recently accessed files without having to scan the disk
+    //      every time... and it will be accurate, as long as we don't have multiple processes)
+    return {
+        addCacheFile: runInSerial(addCacheFile),
+        getCacheFile,
+        delCacheFile,
+        getPathAndLock,
+    };
+};
+let curSeqNum = 1;
+const getDiskMetricSeqNum = cache((seqNum: number) => getDiskMetricsBase());
+async function getDiskMetrics() {
+    return getDiskMetricSeqNum(curSeqNum);
+}
+
+const ensureDiskMetricsUpdated = lazy(() => {
+    runInfinitePoll(UPDATE_METRICS_INTERVAL, async () => {
+        await getDiskMetricSeqNum(curSeqNum + 1);
+        curSeqNum++;
+        getDiskMetricSeqNum.clear(curSeqNum - 1);
+    });
+});
+
+export type LockFncs = {
+    // Get a path to the file locally, locking it so it won't be garbage collected until lockRegion completes
+    //  - If the file isn't in the archives, returns undefined, and doesn't lock it
+    getPathAndLock(fileName: string): Promise<string | undefined>;
+    // Gets it, and only checks the cache, not the archives. Faster, but means it might be deleted in
+    //  the archives and we will still use it locally.
+    getPathAndLockFast(fileName: string): Promise<string | undefined>;
+    getPathAndLockCacheOnly(fileName: string): Promise<string | undefined>;
+    // MOVES the sourceFileName to the archives (so sourceFileName will be deleted after this runs),
+    //  and locks the archives file until lockRegion completes
+    moveFileAndLock(config: {
+        archivesFilename: string;
+        sourceFileName: string;
+        onlyCache?: boolean;
+    }): Promise<string>;
+};
+
+let cacheArchivesSymbol = Symbol("cacheArchives");
+/** IMPORTANT! The cache assumes the files contents immutable, and they will only be created
+ *      and deleted, never mutated.
+ */
+export function wrapArchivesWithCache(archives: Archives): Archives & {
+    // NOTE: lockRegion / path based functions are preferred for external accesses, as they ensure files
+    //  won't be garbage collected, and uses paths, which will be required for external processes.
+    //  - Locks only protect the local cache. The values can still be deleted explicitly.
+    lockRegion<T>(
+        code: (
+            fncs: LockFncs
+        ) => Promise<T>
+    ): Promise<T>;
+    debugGetPath(key: string): string;
+} {
+    if (cacheArchivesSymbol in archives) {
+        return archives as any;
+    }
+    async function setLargeFile(config: Args<Archives["setLargeFile"]>[0]) {
+        const tempPath = cacheArchives2 + nextId() + TEMP_SUFFIX;
+        let handle: fs.promises.FileHandle | undefined;
+        try {
+            handle = await fs.promises.open(tempPath, "w");
+            let pos = 0;
+            while (true) {
+                let data = await config.getNextData();
+                if (!data?.length) break;
+                await handle.write(data, 0, data.length, pos);
+                pos += data.length;
+            }
+        } finally {
+            if (handle) {
+                try {
+                    await handle.close();
+                } catch { }
+            }
+        }
+
+        let metrics = await getDiskMetrics();
+        await metrics.addCacheFile(archives, config.path, tempPath);
+        let cachePath = getArchiveCachePath(archives, config.path);
+
+        await lockFile(cachePath);
+        let pos = 0;
+        let cacheHandle: fs.promises.FileHandle = await fs.promises.open(cachePath, "r");
+        let data = Buffer.alloc(LARGE_FILE_CHUNK);
+        async function getNextData(): Promise<Buffer | undefined> {
+            try {
+                let read = await cacheHandle.read(data, 0, LARGE_FILE_CHUNK, pos);
+                if (read.bytesRead === 0) return undefined;
+                let curData = data;
+                if (read.bytesRead < LARGE_FILE_CHUNK) {
+                    curData = curData.slice(0, read.bytesRead);
+                }
+                pos += read.bytesRead;
+                return curData;
+            } catch {
+                return undefined;
+            }
+        }
+        try {
+            await archives.setLargeFile({
+                ...config,
+                path: config.path,
+                getNextData,
+            });
+        } finally {
+            await cacheHandle.close();
+            await unlockFile(cachePath);
+        }
+    }
+    function createGetPathAndLock(locked: string[]) {
+        return async function getPathAndLock(fileName: string) {
+            const info = await archives.getInfo(fileName);
+            if (!info) return undefined;
+
+            let path = getArchiveCachePath(archives, fileName);
+            await lockFile(path);
+            locked.push(path);
+
+            try {
+                let cacheStat = await fs.promises.stat(path);
+                // NOTE: We check the size, and not just the existence, in case the file
+                //  is partially populated? Even though this isn't a real thing, as we rename
+                //  when we copy it...
+                if (cacheStat.size === info.size) {
+                    return path;
+                }
+            } catch {
+                // It doesn't exist, so read it in
+            }
+
+            let startRead = Date.now();
+
+            let readPos = 0;
+            let getNextDataBase = async (): Promise<Buffer | undefined> => {
+                if (readPos >= info.size) return undefined;
+                let curPos = readPos;
+                readPos += LARGE_FILE_CHUNK;
+                let end = Math.min(readPos, info.size);
+                let data = await archives.get(fileName, { range: { start: curPos, end } });
+                if (!data?.length) return undefined;
+                return data;
+            };
+
+            let nextWriteIndex = 0;
+            let buffers: (Buffer | { error: Error } | null | undefined | (() => void))[] = [];
+
+            async function getThread() {
+                while (true) {
+                    let index = nextWriteIndex++;
+                    let next: typeof buffers[number];
+                    try {
+                        next = await getNextDataBase();
+                    } catch (e: any) {
+                        next = { error: e };
+                    }
+                    let prev = buffers[index];
+                    buffers[index] = next || null;
+                    if (typeof prev === "function") {
+                        prev();
+                    }
+                    if (!next || "error" in next) break;
+                }
+            }
+            // Read in parallel, for faster read times, and so we can read from backblaze
+            //  while writing to disk.
+            void list(8).map(getThread);
+
+            let nextReadIndex = 0;
+            let getNextData = async (): Promise<Buffer | undefined> => {
+                let index = nextReadIndex++;
+                if (buffers[index] === null) return undefined;
+                if (!buffers[index]) {
+                    await new Promise<void>(resolve => {
+                        buffers[index] = resolve;
+                    });
+                }
+                let result = buffers[index] as any;
+                // Clear it, so we don't store all buffers in memory
+                buffers[index] = undefined;
+                return result;
+            };
+
+            let size = 0;
+            const tempPath = cacheArchives2 + nextId() + TEMP_SUFFIX;
+            let handle: fs.promises.FileHandle | undefined;
+            try {
+                handle = await fs.promises.open(tempPath, "w");
+                while (true) {
+                    let data = await getNextData();
+                    if (!data?.length) break;
+                    await handle.write(data, 0, data.length, size);
+                    size += data.length;
+                }
+            } finally {
+                if (handle) {
+                    try {
+                        await handle.close();
+                    } catch { }
+                }
+            }
+            let totalRead = Date.now() - startRead;
+            console.log(`Read ${formatNumber(info.size)}B at ${formatNumber(info.size / totalRead)}B/s into cache (${fileName})`);
+
+            let metrics = await getDiskMetrics();
+            await metrics.addCacheFile(archives, fileName, tempPath);
+
+            return path;
+        };
+    }
+    function createGetPathAndLockFast(locked: string[]) {
+        return async function getPathAndLockFast(fileName: string) {
+            let path = getArchiveCachePath(archives, fileName);
+            await lockFile(path);
+            locked.push(path);
+            try {
+                // If it exists in the cache, just return the path. Otherwise, we might have to read it in
+                await fs.promises.stat(path);
+                return path;
+            } catch {
+                return createGetPathAndLock(locked)(fileName);
+            }
+        };
+    }
+    function createMovePathFromFileAndLock(locked: string[]) {
+        return async function movePathFromFileAndLock(config: {
+            archivesFilename: string;
+            sourceFileName: string;
+            onlyCache?: boolean;
+        }): Promise<string> {
+            let { archivesFilename, sourceFileName } = config;
+            if (!config.onlyCache) {
+                // NOTE: While we COULD use a rename to quickly move the file... we have to at least copy
+                //  it to the underlying archives, so... moving it piece by piece is fine...
+
+                let handle: fs.promises.FileHandle = await fs.promises.open(sourceFileName, "r");
+                let data = Buffer.alloc(LARGE_FILE_CHUNK);
+                let pos = 0;
+                async function getNextData(): Promise<Buffer | undefined> {
+                    try {
+                        let read = await handle.read(data, 0, LARGE_FILE_CHUNK, pos);
+                        if (read.bytesRead === 0) return undefined;
+                        if (read.bytesRead < LARGE_FILE_CHUNK) {
+                            data = data.slice(0, read.bytesRead);
+                        }
+                        pos += read.bytesRead;
+                        return data;
+                    } catch {
+                        return undefined;
+                    }
+                }
+                try {
+                    await setLargeFile({
+                        path: archivesFilename,
+                        getNextData,
+                    });
+                    await handle.close();
+                    handle = undefined as any;
+                    await fs.promises.unlink(sourceFileName);
+                } finally {
+                    if (handle) {
+                        await handle.close();
+                    }
+                }
+            } else {
+                await fs.promises.rename(sourceFileName, getArchiveCachePath(archives, archivesFilename));
+            }
+            let path = getArchiveCachePath(archives, archivesFilename);
+            await lockFile(path);
+            locked.push(path);
+            return path;
+        };
+    }
+    function createGetPathAndLockCacheOnly(locked: string[]) {
+        return async function getPathAndLockCacheOnly(fileName: string) {
+            let path = getArchiveCachePath(archives, fileName);
+            await lockFile(path);
+            locked.push(path);
+            try {
+                await fs.promises.stat(path);
+                return path;
+            } catch {
+                return undefined;
+            }
+        };
+    }
+    function debugGetPath(key: string) {
+        return getArchiveCachePath(archives, key);
+    }
+    return {
+        [cacheArchivesSymbol]: true,
+        getDebugName: () => archives.getDebugName(),
+        debugGetPath,
+
+        get: async (fileName: string, config) => {
+            let metrics = await getDiskMetrics();
+            // Get info to at least see if it is removed or not. This makes sure we don't get in
+            //  a bad state, at least for immutable files (for mutable files we don't have an easy
+            //  way to check)
+            //  - TODO: Set hash in file metadata (maybe for all archive writes?), and use this to
+            //      compare it against our cache file (which can have the hash in the file name).
+            let info = await archives.getInfo(fileName);
+            if (!info) {
+                // If it is gone remotely, remove it from the cache, to save space.
+                await metrics.delCacheFile(archives, fileName);
+                return;
+            }
+            let buffer = await metrics.getCacheFile(archives, fileName, config);
+            if (buffer) return buffer;
+            let result = await archives.get(fileName);
+            if (result) {
+                const tempPath = cacheArchives2 + nextId() + TEMP_SUFFIX;
+                await fs.promises.writeFile(tempPath, result);
+                await metrics.addCacheFile(archives, fileName, tempPath);
+                let range = config?.range;
+                if (range) {
+                    result = result.slice(range.start, range.end);
+                }
+            }
+            return result;
+        },
+        set: async (fileName: string, data: Buffer) => {
+            let metrics = await getDiskMetrics();
+            const tempPath = cacheArchives2 + nextId() + TEMP_SUFFIX;
+            await fs.promises.writeFile(tempPath, data);
+            await metrics.addCacheFile(archives, fileName, tempPath);
+            await archives.set(fileName, data);
+        },
+        setLargeFile,
+        del: async (fileName: string) => {
+            let metrics = await getDiskMetrics();
+            await metrics.delCacheFile(archives, fileName);
+            await archives.del(fileName);
+        },
+        getInfo: (fileName: string) => archives.getInfo(fileName),
+        find: async (prefix: string, config) => archives.find(prefix, config),
+        findInfo: async (prefix: string, config) => archives.findInfo(prefix, config),
+        enableLogging: () => archives.enableLogging(),
+        move: async (config) => {
+            // Delete the file from ourselves, even if it is a self move, as not having
+            //  values is in the cache is the safest thing, but having extra values
+            //  breaks a lot of code (because then delete doesn't really delete!)
+            let metrics = await getDiskMetrics();
+            await metrics.delCacheFile(archives, config.path);
+            // IMPORTANT. Unwrap, otherwise move will get a get, which might cause our value to be cached again
+            if (cacheArchivesSymbol in config.target) {
+                config = { ...config };
+                config.target = config.target.getBaseArchives!()!.archives;
+            }
+            await archives.move(config);
+        },
+        copy: config => archives.copy(config),
+        assertPathValid: (path: string) => archives.assertPathValid(path),
+        getBaseArchives: () => ({ archives: archives, parentPath: "" }),
+
+        async lockRegion(code) {
+            let locked: string[] = [];
+            let fncs = {
+                getPathAndLock: createGetPathAndLock(locked),
+                getPathAndLockFast: createGetPathAndLockFast(locked),
+                moveFileAndLock: createMovePathFromFileAndLock(locked),
+                getPathAndLockCacheOnly: createGetPathAndLockCacheOnly(locked),
+            };
+            try {
+                return await code(fncs);
+            } finally {
+                for (let path of locked) {
+                    await unlockFile(path);
+                }
+            }
+        },
+    };
+}
+
+
+function lockFilePath(path: string): string {
+    return path + LOCK_SUFFIX;
+}
+
+function getUniqueId() {
+    return process.pid + " " + process.ppid;
+}
+
+let lockCache = new Map<string, number>();
+async function lockFile(path: string): Promise<void> {
+    // NOTE: If we don't stop ourself from locking it multiple times, a single process
+    //  could use up the fill lock file limit, which would break the system.
+    let prevCount = lockCache.get(path);
+    if (prevCount) {
+        lockCache.set(path, prevCount + 1);
+        return;
+    }
+    if (!prevCount) {
+        const lockPath = lockFilePath(path);
+        // NOTE: Locking is taking WAY too long, so... we're not going to wait. This should still
+        //  be mostly fine, due to how we maintain transactions files.
+        lockRetryLoop(async function appendFile() {
+            await fs.promises.appendFile(lockPath, "lock " + getUniqueId() + "\n");
+        }).catch(e => console.error("Error appending file lock for", path, e));
+    }
+    prevCount = lockCache.get(path) || 0;
+    lockCache.set(path, prevCount + 1);
+}
+
+async function unlockFile(path: string): Promise<void> {
+    let prevCount = lockCache.get(path);
+    if (!prevCount) {
+        console.warn(`Unlocking a file that wasn't locked: ${path}`);
+    } else {
+        prevCount--;
+        lockCache.set(path, prevCount);
+        if (prevCount <= 0) {
+            lockCache.delete(path);
+        }
+    }
+    if (!prevCount) {
+        await isLocked(path, { type: "unlock", lock: "lock " + getUniqueId() });
+    }
+}
+
+async function lockRetryLoop<T>(code: () => Promise<T>): Promise<T> {
+    while (true) {
+        try {
+            return await code();
+        } catch (error: any) {
+            if (error.code === "EBUSY") {
+                console.log("Lock file busy, retrying in 1 second");
+                await new Promise(r => setTimeout(r, 1000));
+                continue;
+            }
+            throw error;
+        }
+    }
+}
+
+async function getHandle(path: string) {
+    while (true) {
+        try {
+            return await fs.promises.open(path, "r+");
+        } catch (error: any) {
+            if (error.code === "ENOENT") {
+                try {
+                    return await fs.promises.open(path, "wx+");
+                } catch (error: any) {
+                    if (error.code === "EEXIST") {
+                        continue;
+                    }
+                    throw error;
+                }
+            }
+            throw error;
+        }
+    }
+}
+
+
+/** NOTE: This inherently has a race condition, as anything you do based on isLocked
+ *      could be invalidated by the time you do it. But... this should be good enough
+ *      for our use case...
+ */
+async function isLocked(path: string, operation?: {
+    type: "delete";
+} | {
+    type: "unlock";
+    lock: string;
+}): Promise<number> {
+    const lockPath = lockFilePath(path);
+
+    async function filterToValidState(locks: string[]): Promise<string[]> {
+        let existingLocks: string[] = [];
+        for (let lock of locks) {
+            if (lock.startsWith("lock ")) {
+                let [_, pid, ppid] = lock.split(" ");
+                const invalid = (
+                    !await isProcessAlive(+pid)
+                    || !await isProcessAlive(+ppid)
+                    || await getPPID(+pid) !== +ppid
+                );
+                await getPPID(+pid);
+                if (!invalid) {
+                    existingLocks.push(lock);
+                }
+            } else {
+                continue;
+            }
+        }
+        return Array.from(existingLocks);
+    }
+
+    return await lockRetryLoop(async function isLocked() {
+        let handle = await getHandle(lockPath);
+        try {
+            await new Promise<void>((r, e) => flock(handle.fd, "ex", err => err ? e(err) : r()));
+            let contents = (await handle.readFile()).toString();
+            let locks = contents.toString().replaceAll("\0", "").trim().split("\n").filter(x => x);
+            let validLocks = await filterToValidState(locks);
+            if (operation?.type === "unlock") {
+                let index = validLocks.indexOf(operation.lock);
+                if (index >= 0) {
+                    validLocks.splice(index, 1);
+                }
+            }
+            if (validLocks.length !== locks.length) {
+                //console.log(`Lock count changed from ${locks.length} to ${validLocks.length}`);
+                let newContents = Buffer.from(validLocks.map(x => x + "\n").join(""));
+                if (newContents.length < contents.length) {
+                    newContents = Buffer.concat([newContents, Buffer.alloc(contents.length - newContents.length)]);
+                }
+                await handle.write(newContents, 0, newContents.length, 0);
+            } else {
+                //console.log(`Lock count unchanged: ${validLocks.length}`);
+            }
+            if (operation?.type === "delete") {
+                if (validLocks.length === 0) {
+                    await fs.promises.unlink(path);
+                }
+            }
+            return validLocks.length;
+        } catch (error: any) {
+            if (error.code === "ENOENT") {
+                return 0; // Lock file doesn't exist
+            }
+            throw error; // Unexpected error, rethrow
+        } finally {
+            await handle.close();
+        }
+    });
+}
+async function execPromise(command: string, args: string[]): Promise<string> {
+    return new Promise((resolve, reject) => {
+        child_process.execFile(command, args, (error, stdout, stderr) => {
+            if (error) {
+                reject(error);
+                return;
+            }
+            resolve(stdout);
+        });
+    });
+}
+
+
+async function isProcessAlive(pid: number): Promise<boolean> {
+    try {
+        process.kill(pid, 0);
+        return true;
+    } catch (error) {
+        return false;
+    }
+}
+
+
+async function atomicRead(path: string) {
+    while (true) {
+        let stat0 = fs.statSync(path);
+        let contents = fs.readFileSync(path);
+        let stat1 = fs.statSync(path);
+        if (stat0.mtimeMs === stat1.mtimeMs) {
+            return contents;
+        }
+    }
+};
+
+async function lockCacheFile(archives: Archives, key: string): Promise<void> {
+    await lockFile(getArchiveCachePath(archives, key));
+}
+async function unlockCacheFile(archives: Archives, key: string): Promise<void> {
+    await unlockFile(getArchiveCachePath(archives, key));
+}
+// Returns true if it was deleted
+async function deleteCacheFile(path: string): Promise<boolean> {
+    let cannotDelete = await isLocked(path, { type: "delete" });
+    return !cannotDelete;
+}
+
+// for i in {1..100}; do yarn typenode src/-b-archives/archiveCache.ts & done
+
+async function testLocks() {
+    let testFiles = ["a", "b", "c"].map(x => cacheArchives2 + x);
+    function getAFile() {
+        let file = testFiles[Math.floor(Math.random() * testFiles.length)];
+        console.log(file);
+        return file;
+    }
+
+    function assert(condition: unknown, message: string) {
+        if (!condition) {
+            throw new Error(message);
+        }
+    }
+
+    async function test1() {
+        let file = getAFile();
+        await lockFile(file);
+        let locked = await isLocked(file);
+        assert(locked, "File should be locked");
+        await new Promise(r => setTimeout(r, 100));
+        await unlockFile(file);
+        console.log(await fs.promises.readFile(file + LOCK_SUFFIX, "utf8"));
+    }
+    async function test2() {
+        let file = getAFile();
+        let notDeleted = await isLocked(file, { type: "delete" });
+        if (notDeleted) {
+            console.log("File was not deleted, as it was locked");
+        } else {
+            console.log("File was deleted");
+        }
+    }
+    async function test3() {
+        let count = 10;
+        let file = getAFile();
+        console.log(`Lock count before: ${await isLocked(file)}`);
+        for (let i = 0; i < count; i++) {
+            await lockFile(file);
+        }
+        for (let i = 0; i < count; i++) {
+            await unlockFile(file);
+        }
+        console.log(`Lock count after: ${await isLocked(file)}`);
+    }
+    async function test4() {
+        let file = getAFile();
+        await lockFile(file);
+        let notDeleted = await isLocked(file, { type: "delete" });
+        assert(notDeleted, "File should not be deleted if it is locked");
+        await unlockFile(file);
+    }
+
+
+    async function runRandomTest() {
+        let testFn = [test1, test2, test3, test4][Math.floor(Math.random() * 4)];
+        console.log(`Running test: ${testFn.name}`);
+        await testFn();
+    }
+    for (let i = 0; i < 1000; i++) {
+        await runRandomTest();
+    }
+}
+//testLocks().catch(console.error).finally(() => process.exit(0));
+
+
+async function testLargeFiles() {
+    const largeFile = "E:/downloads/Weird.Science.1985.EXTENDED.1080p.BluRay.H264.AAC-RARBG/weirdscience.mp4";
+    let test = wrapArchivesWithCache(getArchivesBackblaze("querysub.com-testbucket"));
+
+    /*
+    let cacheHandle: fs.promises.FileHandle = await fs.promises.open(largeFile, "r");
+    let data = Buffer.alloc(LARGE_FILE_CHUNK);
+    let pos = 0;
+    async function getNextData(): Promise<Buffer | undefined> {
+        try {
+            let read = await cacheHandle.read(data, 0, LARGE_FILE_CHUNK, pos);
+            if (read.bytesRead === 0) return undefined;
+            if (read.bytesRead < LARGE_FILE_CHUNK) {
+                data = data.slice(0, read.bytesRead);
+            }
+            pos += read.bytesRead;
+            return data;
+        } catch {
+            return undefined;
+        }
+    }
+
+    await test.setLargeFile({
+        path: "test.mp4",
+        getNextData,
+    });
+    */
+
+
+    await test.lockRegion(async fncs => {
+        let newPath = await fncs.getPathAndLock("copy.mp4");
+        console.log({ newPath });
+    });
+
+
+    /*
+    logErrors((async () => {
+        while (true) {
+            let first100Bytes = await test.get("copy.mp4", { range: { start: 0, end: 100 } });
+            console.log({ first100Bytes });
+            await delay(5000);
+        }
+    })());
+
+    {
+        let pos = 0;
+        async function getNextData(): Promise<Buffer | undefined> {
+            let data = await test.get("test.mp4", { range: { start: pos, end: pos + LARGE_FILE_CHUNK } });
+            if (!data?.length) return undefined;
+            pos += data.length;
+            return data;
+        }
+        await test.setLargeFile({
+            path: "copy.mp4",
+            getNextData,
+        });
+    }
+    */
+}
+//testLargeFiles().catch(console.error).finally(() => process.exit(0));