querysub 0.433.0 → 0.436.0

package/src/-a-archives/archivesDisk.ts

@@ -1,455 +1,455 @@
-import { isNode } from "socket-function/src/misc";
-import os from "os";
-import { fsExistsAsync, getSubFolder } from "../fs";
-import fs from "fs";
-import { blue, red, yellow } from "socket-function/src/formatting/logColors";
-import { measureFnc } from "socket-function/src/profiling/measure";
-import { delay } from "socket-function/src/batching";
-import debugbreak from "debugbreak";
-import { Archives, getArchives } from "./archives";
-import { cache, lazy } from "socket-function/src/caching";
-import { isDefined } from "../misc";
-
-export let storageDisabled = false;
-/** Useful for tests */
-export function disableStorage() {
-    storageDisabled = true;
-}
-
-let maxFileNameLength = 260;
-let maxFilePartLength = 255;
-
-class ArchivesDisk {
-    public LAG = 0;
-    // Must end with "/"
-    public LOCAL_ARCHIVE_FOLDER = "";
-
-    private logging = false;
-    public enableLogging() {
-        this.logging = true;
-    }
-    private log(text: string) {
-        if (!this.logging) return;
-        console.log(text);
-    }
-
-    public getDebugName() {
-        return "disk/" + this.LOCAL_ARCHIVE_FOLDER;
-    }
-
-    private initFinished = false;
-    // TODO: Now that init isn't async, call it directly in validate, instead of the callers to validate
-    private init = lazy(() => {
-        if (isNode() && process.platform === "win32") {
-            try {
-                let longPart = "LONG_FILE_PATH_TEST_" + Array(210 - "LONG_FILE_PATH_TEST_".length).fill("_").join("");
-                let testPath = this.LOCAL_ARCHIVE_FOLDER + longPart + "/" + longPart + "/" + longPart;
-                fs.mkdirSync(testPath, { recursive: true });
-                maxFileNameLength = 600;
-            } catch (e) {
-                console.log(e);
-                console.error(red(`Failed to set long file name. Google "windows enable long paths", and enable long paths to allow long file names.`));
-                // TODO: We don't NEED to exit, but for now we will, because we can easily
-                //  setup our machine properly.
-                console.log(`Exitting now`);
-                process.exit();
-            }
-        } else {
-            // 4096 for linux, but... 600 handles a lot more cases (backblaze, etc)
-            maxFileNameLength = 600;
-        }
-        this.initFinished = true;
-    });
-
-    private async ensureDirsExist(path: string) {
-        let fileNameParts = path.split("/");
-        // Don't create the drive (and also only add up to the last path, via slicing (0, i)
-        for (let i = 1; i < fileNameParts.length; i++) {
-            let dir = this.LOCAL_ARCHIVE_FOLDER + fileNameParts.slice(0, i).join("/");
-            try {
-                if (!await fsExistsAsync(dir)) {
-                    await fs.promises.mkdir(dir, {
-                        recursive: true,
-                    });
-                }
-            } catch { }
-        }
-    }
-
-    // We don't time the file system writes as it's almost never blocking, it's not stopping our processing. And if it is stopping our processing, then we actually want to know the parent function which is calling it, and then we want it to just void it instead of awaiting it (And the only way to know the parent caller is to not time these functions).
-    public async set(fileName: string, data: Buffer): Promise<void> {
-        await this.init();
-
-        this.log(blue(`Setting file ${fileName} = ${data.length} bytes`));
-        if (storageDisabled) return;
-        fileName = escapeFileName(fileName);
-        await this.simulateLag();
-
-        await this.ensureDirsExist(fileName);
-
-        await fs.promises.writeFile(this.LOCAL_ARCHIVE_FOLDER + fileName, data);
-    }
-    public async append(fileName: string, data: Buffer): Promise<void> {
-        await this.init();
-
-        this.log(blue(`Appending to file ${fileName} += ${data.length} bytes`));
-        if (storageDisabled) return;
-        fileName = escapeFileName(fileName);
-        await this.simulateLag();
-
-        await this.ensureDirsExist(fileName);
-
-        await fs.promises.appendFile(this.LOCAL_ARCHIVE_FOLDER + fileName, data);
-    }
-    public async del(fileName: string): Promise<void> {
-        await this.init();
-        this.log(blue(`Deleting file ${fileName}`));
-        if (storageDisabled) return;
-        fileName = escapeFileName(fileName);
-        await this.simulateLag();
-        if (!await fsExistsAsync(this.LOCAL_ARCHIVE_FOLDER + fileName)) return;
-        try {
-            await fs.promises.rm(this.LOCAL_ARCHIVE_FOLDER + fileName, { recursive: true });
-        } catch { }
-        try {
-            let dir = fileName;
-            while (dir.length > 0) {
-                dir = dir.replaceAll("\\", "/").split("/").slice(0, -1).join("/");
-                if (dir.endsWith(":")) break;
-                let files = await fs.promises.readdir(this.LOCAL_ARCHIVE_FOLDER + dir);
-                if (files.length > 0) break;
-                await fs.promises.rm(this.LOCAL_ARCHIVE_FOLDER + dir, { recursive: true });
-            }
-        } catch { }
-    }
-
-    public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined>; }): Promise<void> {
-        await this.init();
-        let { path } = config;
-        path = escapeFileName(path);
-        await this.ensureDirsExist(path);
-        let handle: fs.promises.FileHandle | undefined;
-        try {
-            handle = await fs.promises.open(this.LOCAL_ARCHIVE_FOLDER + path, "w");
-            let pos = 0;
-            while (true) {
-                let data = await config.getNextData();
-                if (!data?.length) break;
-                await handle.write(data, 0, data.length, pos);
-                pos += data.length;
-            }
-        } finally {
-            if (handle) {
-                try {
-                    await handle.close();
-                } catch { }
-            }
-        }
-    }
-
-    @measureFnc
-    public async gcDir(dir: string) {
-        await this.init();
-        if (!dir.startsWith(this.LOCAL_ARCHIVE_FOLDER)) {
-            dir = this.LOCAL_ARCHIVE_FOLDER + dir;
-        }
-        if (!dir.endsWith("/")) dir += "/";
-        let files: string[] = [];
-        try {
-            files = await fs.promises.readdir(dir);
-        } catch { }
-        for (let file of files) {
-            let isDir = false;
-            let path = dir + file + "/";
-            try {
-                let stat = await fs.promises.stat(path);
-                if (stat.isDirectory()) {
-                    isDir = true;
-                }
-            } catch { }
-            if (isDir) {
-                await this.gcDir(path);
-            }
-        }
-        try {
-            files = await fs.promises.readdir(dir);
-        } catch { }
-        if (files.length === 0) {
-            try {
-                await fs.promises.rm(dir, { recursive: true });
-            } catch { }
-        }
-    }
-
-    /**
-     * NOTE: Generally we load data into memory and keep it there, so a streaming endpoint (instead of a buffer),
-     *      wouldn't be that useful (and would be a lot slower, as streaming data is a lot slower than reading
-     *      it all at once).
-     * @fsErrorRetryCount Retries count for file system error (not user errors). Ex, too many open files, etc.
-     */
-    // NOTE: I've commented out the gate measuring so we can determine what is taking the time. We might want to remove all the timing from this file, as if we find any timing this file is slow, it doesn't help us optimize at all, because we then need to figure out what is calling it. 
-    //@measureFnc
-    public async get(fileNameInput: string, config?: { range?: { start: number; end: number; }; retryCount?: number }): Promise<Buffer | undefined> {
-        await this.init();
-        this.log(blue(`Start read file ${fileNameInput}`));
-        if (storageDisabled) return undefined;
-        let fileName = escapeFileName(fileNameInput);
-        await this.simulateLag();
-        let handle: fs.promises.FileHandle | undefined;
-        let retryCount = config?.retryCount ?? 3;
-        config = config ?? {};
-        try {
-            handle = await fs.promises.open(this.LOCAL_ARCHIVE_FOLDER + fileName, "r");
-            let stats = await handle.stat();
-            let start = config.range?.start ?? 0;
-            let end = config.range?.end ?? stats.size;
-            let buffer = Buffer.alloc(end - start);
-            let read = await handle.read(buffer, 0, end - start, start);
-
-            let stats2 = await handle.stat();
-            if (stats.mtimeMs !== stats2.mtimeMs) {
-                if (retryCount > 0) {
-                    // Wait for the file to stop changing
-                    await delay(10);
-                    return this.get(fileNameInput, { ...config, retryCount: retryCount - 1 });
-                }
-                throw new Error(`File modified changed while reading file ${fileName}`);
-            }
-
-            // Make sure we only take as much data as we want in case we are given extra data. 
-            return buffer.slice(0, end - start);
-        } catch (e: any) {
-            if (e.code === "EMFILE") {
-                if (retryCount > 0) {
-                    // Wait for some files to close.
-                    await delay(10);
-                    return this.get(fileNameInput, { ...config, retryCount: retryCount - 1 });
-                } else {
-                    console.log(`Fixable error reading file, but we hit the retry limit. Returning undefined, even though the file likely exists. ${fileName}`, e);
-                }
-            }
-            return undefined;
-        } finally {
-            if (handle) {
-                try {
-                    await handle.close();
-                } catch { }
-            }
-        }
-    }
-
-    @measureFnc
-    public async find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]> {
-        await this.init();
-        prefix = escapeFileName(prefix);
-        this.log(blue(`findFileNames ${prefix}`));
-        if (storageDisabled) return [];
-        await this.simulateLag();
-        let fileNames: string[] = [];
-        let folderNames: string[] = [];
-        async function readDir(dir: string, depthToRead: number) {
-            if (!await fsExistsAsync(dir)) {
-                return;
-            }
-            try {
-                let curFileNames = await fs.promises.readdir(dir, {
-                    withFileTypes: true
-                });
-                await Promise.all(curFileNames.map(async fileObj => {
-                    let fileName = fileObj.name;
-                    // try/catch so nested directory errors don't break everything
-                    //  (I guess if the root directory is deleted we will still through though...)
-                    try {
-                        if (fileObj.isDirectory()) {
-                            folderNames.push(dir + fileName);
-                            await readDir(dir + fileName + "/", depthToRead - 1);
-                        } else {
-                            fileNames.push(dir + fileName);
-                        }
-                    } catch { }
-                }));
-            } catch { }
-        }
-
-        let rootDir = "";
-        let pathParts = prefix.split("/");
-        if (pathParts.length > 1) {
-            rootDir = pathParts.slice(0, -1).join("/") + "/";
-        }
-        // We don't know if the prefix they gave us is the prefix of folders or if it is a folder itself. So we have to cut off the end and then read to a depth of 2 to make this work.
-        await readDir(this.LOCAL_ARCHIVE_FOLDER + rootDir, config?.shallow ? 2 : Number.MAX_SAFE_INTEGER);
-
-        let results = config?.type === "folders" ? folderNames : fileNames;
-
-        results = results.map(name => name.slice(this.LOCAL_ARCHIVE_FOLDER.length));
-        results = results.filter(name => name.startsWith(prefix));
-
-        if (config?.shallow) {
-            let targetDepth = prefix.split("/").length;
-            results = results.filter(name => name.split("/").length === targetDepth);
-        }
-
-        results = results.map(name => unescapeFileName(name));
-
-        return results;
-    }
-    @measureFnc
-    public async findInfo(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<{ path: string; createTime: number; size: number; }[]> {
-        // TODO: Isn't there any api to read file names and infos at the same time?
-        let files = await this.find(prefix, config);
-        return (await Promise.all(files.map(async file => {
-            try {
-                let stats = await fs.promises.stat(this.LOCAL_ARCHIVE_FOLDER + escapeFileName(file));
-                return { path: file, createTime: stats.ctimeMs, size: stats.size };
-            } catch {
-                return undefined;
-            }
-        }))).filter(isDefined);
-    }
-
-
-    @measureFnc
-    public async getInfo(pathInput: string): Promise<{
-        writeTime: number;
-        size: number;
-    } | undefined> {
-        let path = escapeFileName(pathInput);
-        try {
-            let stats = await fs.promises.stat(this.LOCAL_ARCHIVE_FOLDER + path);
-            return {
-                writeTime: stats.mtimeMs,
-                size: stats.size,
-            };
-        } catch {
-            return undefined;
-        }
-    }
-
-    private simulateLag() {
-        if (this.LAG === 0) return;
-        return delay(this.LAG);
-    }
-
-    public async assertPathValid(path: string) {
-        await this.init();
-        escapeFileName(this.LOCAL_ARCHIVE_FOLDER + path);
-    }
-
-    @measureFnc
-    public async move(config: {
-        path: string;
-        target: Archives;
-        targetPath: string;
-    }) {
-        let { path, target, targetPath } = config;
-        while (true) {
-            let targetUnwrapped = target.getBaseArchives?.();
-            if (!targetUnwrapped) break;
-            target = targetUnwrapped.archives;
-            targetPath = targetUnwrapped.parentPath + targetPath;
-        }
-
-        if (target instanceof ArchivesDisk) {
-            path = escapeFileName(path);
-            targetPath = escapeFileName(targetPath);
-            await this.ensureDirsExist(path);
-            await target.ensureDirsExist(targetPath);
-            await fs.promises.rename(
-                this.LOCAL_ARCHIVE_FOLDER + path,
-                target.LOCAL_ARCHIVE_FOLDER + targetPath
-            );
-        } else {
-            let data = await this.get(path);
-            if (!data) throw new Error(`File not found to move: ${path}`);
-            await target.set(targetPath, data);
-            await this.del(path);
-        }
-    }
-
-    @measureFnc
-    public async copy(config: {
-        path: string;
-        target: Archives;
-        targetPath: string;
-    }) {
-        let { path, target, targetPath } = config;
-        while (true) {
-            let targetUnwrapped = target.getBaseArchives?.();
-            if (!targetUnwrapped) break;
-            target = targetUnwrapped.archives;
-            targetPath = targetUnwrapped.parentPath + targetPath;
-        }
-
-        if (target instanceof ArchivesDisk) {
-            path = escapeFileName(path);
-            targetPath = escapeFileName(targetPath);
-            await this.ensureDirsExist(path);
-            await target.ensureDirsExist(targetPath);
-            await fs.promises.copyFile(
-                this.LOCAL_ARCHIVE_FOLDER + path,
-                target.LOCAL_ARCHIVE_FOLDER + targetPath
-            );
-        } else {
-            let data = await this.get(path);
-            if (!data) throw new Error(`File not found to move: ${path}`);
-            await target.set(targetPath, data);
-        }
-    }
-}
-
-export const diskEscapeFileName = escapeFileName;
-
-function escapeFileName(fileName: string): string {
-    fileName = fileName.replaceAll("_", "__");
-    if (fileName[1] === ":" && fileName[2] === "/") {
-        fileName = fileName.slice(0, 2) + fileName.slice(2).replaceAll(":", "_=");
-    } else {
-        fileName = fileName.replaceAll(":", "_=");
-    }
-    fileName = fileName.replaceAll(`"`, `_'`);
-    // TODO: Better escaping for file system paths
-    // TODO: Support longer filenames in some way? (backblaze can support longer names, but...
-    //  we're always going to want to support just using the local filesystem, as that is always
-    //  preferred during development).
-    if (fileName.length > maxFileNameLength) {
-        throw new Error(`File name too long, must be at most ${maxFileNameLength} characters long, was ${fileName.length}, file name ${JSON.stringify(fileName)}`);
-    }
-    fileName = fileName.replaceAll("\\", "/");
-    let parts = fileName.split("/");
-    for (let part of parts) {
-        if (part.length > maxFilePartLength) {
-            throw new Error(`File name part too long, must be at most ${maxFilePartLength} characters long, was ${part.length}, file name ${JSON.stringify(fileName)}`);
-        }
-    }
-    if (fileName.startsWith("/")) {
-        fileName = fileName.slice("/".length);
-    }
-    return fileName;
-}
-function unescapeFileName(fileName: string): string {
-    fileName = fileName.replaceAll(`_'`, `"`);
-    fileName = fileName.replaceAll("_=", ":");
-    fileName = fileName.replaceAll("__", "_");
-    return fileName;
-}
-
-export const getArchivesLocal = cache((domain: string): Archives => {
-
-    const archivesLocal = new ArchivesDisk();
-
-    if (isNode()) {
-        archivesLocal.LOCAL_ARCHIVE_FOLDER = getSubFolder(domain).replaceAll("\\", "/");
-    }
-
-    return archivesLocal;
-});
-
-export const getArchivesHome = cache((domain: string): Archives => {
-    const archivesLocal = new ArchivesDisk();
-
-    if (isNode()) {
-        archivesLocal.LOCAL_ARCHIVE_FOLDER = os.homedir() + "/querysub/" + domain + "/";
-    }
-
-    return archivesLocal;
+import { isNode } from "socket-function/src/misc";
+import os from "os";
+import { fsExistsAsync, getSubFolder } from "../fs";
+import fs from "fs";
+import { blue, red, yellow } from "socket-function/src/formatting/logColors";
+import { measureFnc } from "socket-function/src/profiling/measure";
+import { delay } from "socket-function/src/batching";
+import debugbreak from "debugbreak";
+import { Archives, getArchives } from "./archives";
+import { cache, lazy } from "socket-function/src/caching";
+import { isDefined } from "../misc";
+
+export let storageDisabled = false;
+/** Useful for tests */
+export function disableStorage() {
+    storageDisabled = true;
+}
+
+let maxFileNameLength = 260;
+let maxFilePartLength = 255;
+
+class ArchivesDisk {
+    public LAG = 0;
+    // Must end with "/"
+    public LOCAL_ARCHIVE_FOLDER = "";
+
+    private logging = false;
+    public enableLogging() {
+        this.logging = true;
+    }
+    private log(text: string) {
+        if (!this.logging) return;
+        console.log(text);
+    }
+
+    public getDebugName() {
+        return "disk/" + this.LOCAL_ARCHIVE_FOLDER;
+    }
+
+    private initFinished = false;
+    // TODO: Now that init isn't async, call it directly in validate, instead of the callers to validate
+    private init = lazy(() => {
+        if (isNode() && process.platform === "win32") {
+            try {
+                let longPart = "LONG_FILE_PATH_TEST_" + Array(210 - "LONG_FILE_PATH_TEST_".length).fill("_").join("");
+                let testPath = this.LOCAL_ARCHIVE_FOLDER + longPart + "/" + longPart + "/" + longPart;
+                fs.mkdirSync(testPath, { recursive: true });
+                maxFileNameLength = 600;
+            } catch (e) {
+                console.log(e);
+                console.error(red(`Failed to set long file name. Google "windows enable long paths", and enable long paths to allow long file names.`));
+                // TODO: We don't NEED to exit, but for now we will, because we can easily
+                //  setup our machine properly.
+                console.log(`Exitting now`);
+                process.exit();
+            }
+        } else {
+            // 4096 for linux, but... 600 handles a lot more cases (backblaze, etc)
+            maxFileNameLength = 600;
+        }
+        this.initFinished = true;
+    });
+
+    private async ensureDirsExist(path: string) {
+        let fileNameParts = path.split("/");
+        // Don't create the drive (and also only add up to the last path, via slicing (0, i)
+        for (let i = 1; i < fileNameParts.length; i++) {
+            let dir = this.LOCAL_ARCHIVE_FOLDER + fileNameParts.slice(0, i).join("/");
+            try {
+                if (!await fsExistsAsync(dir)) {
+                    await fs.promises.mkdir(dir, {
+                        recursive: true,
+                    });
+                }
+            } catch { }
+        }
+    }
+
+    // We don't time the file system writes as it's almost never blocking, it's not stopping our processing. And if it is stopping our processing, then we actually want to know the parent function which is calling it, and then we want it to just void it instead of awaiting it (And the only way to know the parent caller is to not time these functions).
+    public async set(fileName: string, data: Buffer): Promise<void> {
+        await this.init();
+
+        this.log(blue(`Setting file ${fileName} = ${data.length} bytes`));
+        if (storageDisabled) return;
+        fileName = escapeFileName(fileName);
+        await this.simulateLag();
+
+        await this.ensureDirsExist(fileName);
+
+        await fs.promises.writeFile(this.LOCAL_ARCHIVE_FOLDER + fileName, data);
+    }
+    public async append(fileName: string, data: Buffer): Promise<void> {
+        await this.init();
+
+        this.log(blue(`Appending to file ${fileName} += ${data.length} bytes`));
+        if (storageDisabled) return;
+        fileName = escapeFileName(fileName);
+        await this.simulateLag();
+
+        await this.ensureDirsExist(fileName);
+
+        await fs.promises.appendFile(this.LOCAL_ARCHIVE_FOLDER + fileName, data);
+    }
+    public async del(fileName: string): Promise<void> {
+        await this.init();
+        this.log(blue(`Deleting file ${fileName}`));
+        if (storageDisabled) return;
+        fileName = escapeFileName(fileName);
+        await this.simulateLag();
+        if (!await fsExistsAsync(this.LOCAL_ARCHIVE_FOLDER + fileName)) return;
+        try {
+            await fs.promises.rm(this.LOCAL_ARCHIVE_FOLDER + fileName, { recursive: true });
+        } catch { }
+        try {
+            let dir = fileName;
+            while (dir.length > 0) {
+                dir = dir.replaceAll("\\", "/").split("/").slice(0, -1).join("/");
+                if (dir.endsWith(":")) break;
+                let files = await fs.promises.readdir(this.LOCAL_ARCHIVE_FOLDER + dir);
+                if (files.length > 0) break;
+                await fs.promises.rm(this.LOCAL_ARCHIVE_FOLDER + dir, { recursive: true });
+            }
+        } catch { }
+    }
+
+    public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined>; }): Promise<void> {
+        await this.init();
+        let { path } = config;
+        path = escapeFileName(path);
+        await this.ensureDirsExist(path);
+        let handle: fs.promises.FileHandle | undefined;
+        try {
+            handle = await fs.promises.open(this.LOCAL_ARCHIVE_FOLDER + path, "w");
+            let pos = 0;
+            while (true) {
+                let data = await config.getNextData();
+                if (!data?.length) break;
+                await handle.write(data, 0, data.length, pos);
+                pos += data.length;
+            }
+        } finally {
+            if (handle) {
+                try {
+                    await handle.close();
+                } catch { }
+            }
+        }
+    }
+
+    @measureFnc
+    public async gcDir(dir: string) {
+        await this.init();
+        if (!dir.startsWith(this.LOCAL_ARCHIVE_FOLDER)) {
+            dir = this.LOCAL_ARCHIVE_FOLDER + dir;
+        }
+        if (!dir.endsWith("/")) dir += "/";
+        let files: string[] = [];
+        try {
+            files = await fs.promises.readdir(dir);
+        } catch { }
+        for (let file of files) {
+            let isDir = false;
+            let path = dir + file + "/";
+            try {
+                let stat = await fs.promises.stat(path);
+                if (stat.isDirectory()) {
+                    isDir = true;
+                }
+            } catch { }
+            if (isDir) {
+                await this.gcDir(path);
+            }
+        }
+        try {
+            files = await fs.promises.readdir(dir);
+        } catch { }
+        if (files.length === 0) {
+            try {
+                await fs.promises.rm(dir, { recursive: true });
+            } catch { }
+        }
+    }
+
+    /**
+     * NOTE: Generally we load data into memory and keep it there, so a streaming endpoint (instead of a buffer),
+     *      wouldn't be that useful (and would be a lot slower, as streaming data is a lot slower than reading
+     *      it all at once).
+     * @fsErrorRetryCount Retries count for file system error (not user errors). Ex, too many open files, etc.
+     */
+    // NOTE: I've commented out the gate measuring so we can determine what is taking the time. We might want to remove all the timing from this file, as if we find any timing this file is slow, it doesn't help us optimize at all, because we then need to figure out what is calling it. 
+    //@measureFnc
+    public async get(fileNameInput: string, config?: { range?: { start: number; end: number; }; retryCount?: number }): Promise<Buffer | undefined> {
+        await this.init();
+        this.log(blue(`Start read file ${fileNameInput}`));
+        if (storageDisabled) return undefined;
+        let fileName = escapeFileName(fileNameInput);
+        await this.simulateLag();
+        let handle: fs.promises.FileHandle | undefined;
+        let retryCount = config?.retryCount ?? 3;
+        config = config ?? {};
+        try {
+            handle = await fs.promises.open(this.LOCAL_ARCHIVE_FOLDER + fileName, "r");
+            let stats = await handle.stat();
+            let start = config.range?.start ?? 0;
+            let end = config.range?.end ?? stats.size;
+            let buffer = Buffer.alloc(end - start);
+            let read = await handle.read(buffer, 0, end - start, start);
+
+            let stats2 = await handle.stat();
+            if (stats.mtimeMs !== stats2.mtimeMs) {
+                if (retryCount > 0) {
+                    // Wait for the file to stop changing
+                    await delay(10);
+                    return this.get(fileNameInput, { ...config, retryCount: retryCount - 1 });
+                }
+                throw new Error(`File modified changed while reading file ${fileName}`);
+            }
+
+            // Make sure we only take as much data as we want in case we are given extra data. 
+            return buffer.slice(0, end - start);
+        } catch (e: any) {
+            if (e.code === "EMFILE") {
+                if (retryCount > 0) {
+                    // Wait for some files to close.
+                    await delay(10);
+                    return this.get(fileNameInput, { ...config, retryCount: retryCount - 1 });
+                } else {
+                    console.log(`Fixable error reading file, but we hit the retry limit. Returning undefined, even though the file likely exists. ${fileName}`, e);
+                }
+            }
+            return undefined;
+        } finally {
+            if (handle) {
+                try {
+                    await handle.close();
+                } catch { }
+            }
+        }
+    }
+
+    @measureFnc
+    public async find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]> {
+        await this.init();
+        prefix = escapeFileName(prefix);
+        this.log(blue(`findFileNames ${prefix}`));
+        if (storageDisabled) return [];
+        await this.simulateLag();
+        let fileNames: string[] = [];
+        let folderNames: string[] = [];
+        async function readDir(dir: string, depthToRead: number) {
+            if (!await fsExistsAsync(dir)) {
+                return;
+            }
+            try {
+                let curFileNames = await fs.promises.readdir(dir, {
+                    withFileTypes: true
+                });
+                await Promise.all(curFileNames.map(async fileObj => {
+                    let fileName = fileObj.name;
+                    // try/catch so nested directory errors don't break everything
+                    //  (I guess if the root directory is deleted we will still through though...)
+                    try {
+                        if (fileObj.isDirectory()) {
+                            folderNames.push(dir + fileName);
+                            await readDir(dir + fileName + "/", depthToRead - 1);
+                        } else {
+                            fileNames.push(dir + fileName);
+                        }
+                    } catch { }
+                }));
+            } catch { }
+        }
+
+        let rootDir = "";
+        let pathParts = prefix.split("/");
+        if (pathParts.length > 1) {
+            rootDir = pathParts.slice(0, -1).join("/") + "/";
+        }
+        // We don't know if the prefix they gave us is the prefix of folders or if it is a folder itself. So we have to cut off the end and then read to a depth of 2 to make this work.
+        await readDir(this.LOCAL_ARCHIVE_FOLDER + rootDir, config?.shallow ? 2 : Number.MAX_SAFE_INTEGER);
+
+        let results = config?.type === "folders" ? folderNames : fileNames;
+
+        results = results.map(name => name.slice(this.LOCAL_ARCHIVE_FOLDER.length));
+        results = results.filter(name => name.startsWith(prefix));
+
+        if (config?.shallow) {
+            let targetDepth = prefix.split("/").length;
+            results = results.filter(name => name.split("/").length === targetDepth);
+        }
+
+        results = results.map(name => unescapeFileName(name));
+
+        return results;
+    }
+    @measureFnc
+    public async findInfo(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<{ path: string; createTime: number; size: number; }[]> {
+        // TODO: Isn't there any api to read file names and infos at the same time?
+        let files = await this.find(prefix, config);
+        return (await Promise.all(files.map(async file => {
+            try {
+                let stats = await fs.promises.stat(this.LOCAL_ARCHIVE_FOLDER + escapeFileName(file));
+                return { path: file, createTime: stats.ctimeMs, size: stats.size };
+            } catch {
+                return undefined;
+            }
+        }))).filter(isDefined);
+    }
+
+
+    @measureFnc
+    public async getInfo(pathInput: string): Promise<{
+        writeTime: number;
+        size: number;
+    } | undefined> {
+        let path = escapeFileName(pathInput);
+        try {
+            let stats = await fs.promises.stat(this.LOCAL_ARCHIVE_FOLDER + path);
+            return {
+                writeTime: stats.mtimeMs,
+                size: stats.size,
+            };
+        } catch {
+            return undefined;
+        }
+    }
+
+    private simulateLag() {
+        if (this.LAG === 0) return;
+        return delay(this.LAG);
+    }
+
+    public async assertPathValid(path: string) {
+        await this.init();
+        escapeFileName(this.LOCAL_ARCHIVE_FOLDER + path);
+    }
+
+    @measureFnc
+    public async move(config: {
+        path: string;
+        target: Archives;
+        targetPath: string;
+    }) {
+        let { path, target, targetPath } = config;
+        while (true) {
+            let targetUnwrapped = target.getBaseArchives?.();
+            if (!targetUnwrapped) break;
+            target = targetUnwrapped.archives;
+            targetPath = targetUnwrapped.parentPath + targetPath;
+        }
+
+        if (target instanceof ArchivesDisk) {
+            path = escapeFileName(path);
+            targetPath = escapeFileName(targetPath);
+            await this.ensureDirsExist(path);
+            await target.ensureDirsExist(targetPath);
+            await fs.promises.rename(
+                this.LOCAL_ARCHIVE_FOLDER + path,
+                target.LOCAL_ARCHIVE_FOLDER + targetPath
+            );
+        } else {
+            let data = await this.get(path);
+            if (!data) throw new Error(`File not found to move: ${path}`);
+            await target.set(targetPath, data);
+            await this.del(path);
+        }
+    }
+
+    @measureFnc
+    public async copy(config: {
+        path: string;
+        target: Archives;
+        targetPath: string;
+    }) {
+        let { path, target, targetPath } = config;
+        while (true) {
+            let targetUnwrapped = target.getBaseArchives?.();
+            if (!targetUnwrapped) break;
+            target = targetUnwrapped.archives;
+            targetPath = targetUnwrapped.parentPath + targetPath;
+        }
+
+        if (target instanceof ArchivesDisk) {
+            path = escapeFileName(path);
+            targetPath = escapeFileName(targetPath);
+            await this.ensureDirsExist(path);
+            await target.ensureDirsExist(targetPath);
+            await fs.promises.copyFile(
+                this.LOCAL_ARCHIVE_FOLDER + path,
+                target.LOCAL_ARCHIVE_FOLDER + targetPath
+            );
+        } else {
+            let data = await this.get(path);
+            if (!data) throw new Error(`File not found to move: ${path}`);
+            await target.set(targetPath, data);
+        }
+    }
+}
+
+export const diskEscapeFileName = escapeFileName;
+
+function escapeFileName(fileName: string): string {
+    fileName = fileName.replaceAll("_", "__");
+    if (fileName[1] === ":" && fileName[2] === "/") {
+        fileName = fileName.slice(0, 2) + fileName.slice(2).replaceAll(":", "_=");
+    } else {
+        fileName = fileName.replaceAll(":", "_=");
+    }
+    fileName = fileName.replaceAll(`"`, `_'`);
+    // TODO: Better escaping for file system paths
+    // TODO: Support longer filenames in some way? (backblaze can support longer names, but...
+    //  we're always going to want to support just using the local filesystem, as that is always
+    //  preferred during development).
+    if (fileName.length > maxFileNameLength) {
+        throw new Error(`File name too long, must be at most ${maxFileNameLength} characters long, was ${fileName.length}, file name ${JSON.stringify(fileName)}`);
+    }
+    fileName = fileName.replaceAll("\\", "/");
+    let parts = fileName.split("/");
+    for (let part of parts) {
+        if (part.length > maxFilePartLength) {
+            throw new Error(`File name part too long, must be at most ${maxFilePartLength} characters long, was ${part.length}, file name ${JSON.stringify(fileName)}`);
+        }
+    }
+    if (fileName.startsWith("/")) {
+        fileName = fileName.slice("/".length);
+    }
+    return fileName;
+}
+function unescapeFileName(fileName: string): string {
+    fileName = fileName.replaceAll(`_'`, `"`);
+    fileName = fileName.replaceAll("_=", ":");
+    fileName = fileName.replaceAll("__", "_");
+    return fileName;
+}
+
+export const getArchivesLocal = cache((domain: string): Archives => {
+
+    const archivesLocal = new ArchivesDisk();
+
+    if (isNode()) {
+        archivesLocal.LOCAL_ARCHIVE_FOLDER = getSubFolder(domain).replaceAll("\\", "/");
+    }
+
+    return archivesLocal;
+});
+
+export const getArchivesHome = cache((domain: string): Archives => {
+    const archivesLocal = new ArchivesDisk();
+
+    if (isNode()) {
+        archivesLocal.LOCAL_ARCHIVE_FOLDER = os.homedir() + "/querysub/" + domain + "/";
+    }
+
+    return archivesLocal;
 });