querysub 0.2.0

package/src/-a-archives/archivesDisk.ts

@@ -0,0 +1,418 @@
+import { isNode } from "socket-function/src/misc";
+import { getSubFolder } from "../fs";
+import fs from "fs";
+import { blue, red, yellow } from "socket-function/src/formatting/logColors";
+import { measureFnc } from "socket-function/src/profiling/measure";
+import { delay } from "socket-function/src/batching";
+import debugbreak from "debugbreak";
+import { Archives, getArchives } from "./archives";
+import { cache, lazy } from "socket-function/src/caching";
+import { isDefined } from "../misc";
+
+export let storageDisabled = false;
+/** Useful for tests */
+export function disableStorage() {
+    storageDisabled = true;
+}
+
+let maxFileNameLength = 260;
+let maxFilePartLength = 255;
+
+class ArchivesDisk {
+    public LAG = 0;
+    // Must end with "/"
+    public LOCAL_ARCHIVE_FOLDER = "";
+
+    private logging = false;
+    public enableLogging() {
+        this.logging = true;
+    }
+    private log(text: string) {
+        if (!this.logging) return;
+        console.log(text);
+    }
+
+    public getDebugName() {
+        return "disk/" + this.LOCAL_ARCHIVE_FOLDER;
+    }
+
+    private initFinished = false;
+    // TODO: Now that init isn't async, call it directly in validate, instead of the callers to validate
+    private init = lazy(() => {
+        if (isNode() && process.platform === "win32") {
+            try {
+                let longPart = "LONG_FILE_PATH_TEST_" + Array(210 - "LONG_FILE_PATH_TEST_".length).fill("_").join("");
+                let testPath = this.LOCAL_ARCHIVE_FOLDER + longPart + "/" + longPart + "/" + longPart;
+                fs.mkdirSync(testPath, { recursive: true });
+                maxFileNameLength = 600;
+            } catch (e) {
+                console.log(e);
+                console.error(red(`Failed to set long file name. Google "windows enable long paths", and enable long paths to allow long file names.`));
+                // TODO: We don't NEED to exit, but for now we will, because we can easily
+                //  setup our machine properly.
+                console.log(`Exitting now`);
+                process.exit();
+            }
+        } else {
+            // 4096 for linux, but... 600 handles a lot more cases (backblaze, etc)
+            maxFileNameLength = 600;
+        }
+        this.initFinished = true;
+    });
+
+    private async ensureDirsExist(path: string) {
+        let fileNameParts = path.split("/");
+        // Don't create the drive (and also only add up to the last path, via slicing (0, i)
+        for (let i = 1; i < fileNameParts.length; i++) {
+            let dir = this.LOCAL_ARCHIVE_FOLDER + fileNameParts.slice(0, i).join("/");
+            if (!fs.existsSync(dir)) {
+                fs.mkdirSync(dir);
+            }
+        }
+    }
+
+    @measureFnc
+    public async set(fileName: string, data: Buffer): Promise<void> {
+        await this.init();
+
+        this.log(blue(`Setting file ${fileName} = ${data.length} bytes`));
+        if (storageDisabled) return;
+        fileName = escapeFileName(fileName);
+        await this.simulateLag();
+
+        await this.ensureDirsExist(fileName);
+
+        await fs.promises.writeFile(this.LOCAL_ARCHIVE_FOLDER + fileName, data);
+    }
+    @measureFnc
+    public async del(fileName: string): Promise<void> {
+        await this.init();
+        this.log(blue(`Deleting file ${fileName}`));
+        if (storageDisabled) return;
+        fileName = escapeFileName(fileName);
+        await this.simulateLag();
+        if (!fs.existsSync(this.LOCAL_ARCHIVE_FOLDER + fileName)) return;
+        try {
+            await fs.promises.unlink(this.LOCAL_ARCHIVE_FOLDER + fileName);
+        } catch { }
+        let dir = fileName.replaceAll("\\", "/").split("/").slice(0, -1).join("/");
+        await this.gcDir(dir);
+    }
+
+    public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined>; }): Promise<void> {
+        await this.init();
+        let { path } = config;
+        path = escapeFileName(path);
+        await this.ensureDirsExist(path);
+        let handle: fs.promises.FileHandle | undefined;
+        try {
+            handle = await fs.promises.open(this.LOCAL_ARCHIVE_FOLDER + path, "w");
+            let pos = 0;
+            while (true) {
+                let data = await config.getNextData();
+                if (!data?.length) break;
+                await handle.write(data, 0, data.length, pos);
+                pos += data.length;
+            }
+        } finally {
+            if (handle) {
+                try {
+                    await handle.close();
+                } catch { }
+            }
+        }
+    }
+
+    @measureFnc
+    public async gcDir(dir: string) {
+        await this.init();
+        if (!dir.startsWith(this.LOCAL_ARCHIVE_FOLDER)) {
+            dir = this.LOCAL_ARCHIVE_FOLDER + dir;
+        }
+        if (!dir.endsWith("/")) dir += "/";
+        let files: string[] = [];
+        try {
+            files = await fs.promises.readdir(dir);
+        } catch { }
+        for (let file of files) {
+            let isDir = false;
+            let path = dir + file + "/";
+            try {
+                let stat = await fs.promises.stat(path);
+                if (stat.isDirectory()) {
+                    isDir = true;
+                }
+            } catch { }
+            if (isDir) {
+                await this.gcDir(path);
+            }
+        }
+        try {
+            files = await fs.promises.readdir(dir);
+        } catch { }
+        if (files.length === 0) {
+            try {
+                await fs.promises.rmdir(dir);
+            } catch { }
+        }
+    }
+
+    /**
+     * NOTE: Generally we load data into memory and keep it there, so a streaming endpoint (instead of a buffer),
+     *      wouldn't be that useful (and would be a lot slower, as streaming data is a lot slower than reading
+     *      it all at once).
+     * @fsErrorRetryCount Retries count for file system error (not user errors). Ex, too many open files, etc.
+     */
+    @measureFnc
+    public async get(fileNameInput: string, config?: { range?: { start: number; end: number; }; retryCount?: number }): Promise<Buffer | undefined> {
+        await this.init();
+        this.log(blue(`Start read file ${fileNameInput}`));
+        if (storageDisabled) return undefined;
+        let fileName = escapeFileName(fileNameInput);
+        await this.simulateLag();
+        let handle: fs.promises.FileHandle | undefined;
+        let retryCount = config?.retryCount ?? 3;
+        config = config ?? {};
+        try {
+            handle = await fs.promises.open(this.LOCAL_ARCHIVE_FOLDER + fileName, "r");
+            let stats = await handle.stat();
+            let start = config.range?.start ?? 0;
+            let end = config.range?.end ?? stats.size;
+            let buffer = Buffer.alloc(end - start);
+            let read = await handle.read(buffer, 0, end - start, start);
+
+            let stats2 = await handle.stat();
+            if (stats.mtimeMs !== stats2.mtimeMs) {
+                if (retryCount > 0) {
+                    // Wait for the file to stop changing
+                    await delay(10);
+                    return this.get(fileNameInput, { ...config, retryCount: retryCount - 1 });
+                }
+                throw new Error(`File modified changed while reading file ${fileName}`);
+            }
+
+            return buffer.slice(0, read.bytesRead);
+        } catch (e: any) {
+            if (e.code === "EMFILE") {
+                if (retryCount > 0) {
+                    // Wait for some files to close.
+                    await delay(10);
+                    return this.get(fileNameInput, { ...config, retryCount: retryCount - 1 });
+                } else {
+                    console.log(`Fixable error reading file, but we hit the retry limit. Returning undefined, even though the file likely exists. ${fileName}`, e);
+                }
+            }
+            return undefined;
+        } finally {
+            if (handle) {
+                try {
+                    await handle.close();
+                } catch { }
+            }
+        }
+    }
+
+    @measureFnc
+    public async find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]> {
+        await this.init();
+        prefix = escapeFileName(prefix);
+        this.log(blue(`findFileNames ${prefix}`));
+        if (storageDisabled) return [];
+        await this.simulateLag();
+        let fileNames: string[] = [];
+        let folderNames: string[] = [];
+        async function readDir(dir: string) {
+            if (!fs.existsSync(dir)) {
+                return;
+            }
+            try {
+                let curFileNames = await fs.promises.readdir(dir, {
+                    withFileTypes: true
+                });
+                await Promise.all(curFileNames.map(async fileObj => {
+                    let fileName = fileObj.name;
+                    // try/catch so nested directory errors don't break everything
+                    //  (I guess if the root directory is deleted we will still through though...)
+                    try {
+                        if (fileObj.isDirectory()) {
+                            folderNames.push(dir + fileName);
+                            await readDir(dir + fileName + "/");
+                        } else {
+                            fileNames.push(dir + fileName);
+                        }
+                    } catch { }
+                }));
+            } catch { }
+        }
+
+        let rootDir = "";
+        let pathParts = prefix.split("/");
+        if (pathParts.length > 1) {
+            rootDir = pathParts.slice(0, -1).join("/") + "/";
+        }
+        await readDir(this.LOCAL_ARCHIVE_FOLDER + rootDir);
+
+        let results = config?.type === "folders" ? folderNames : fileNames;
+
+        results = results.map(name => name.slice(this.LOCAL_ARCHIVE_FOLDER.length));
+        results = results.filter(name => name.startsWith(prefix));
+
+        if (config?.shallow) {
+            let targetDepth = prefix.split("/").length;
+            results = results.filter(name => name.split("/").length === targetDepth);
+        }
+
+        results = results.map(name => unescapeFileName(name));
+
+        return results;
+    }
+    @measureFnc
+    public async findInfo(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<{ path: string; createTime: number; size: number; }[]> {
+        // TODO: Isn't there any api to read file names and infos at the same time?
+        let files = await this.find(prefix, config);
+        return (await Promise.all(files.map(async file => {
+            try {
+                let stats = await fs.promises.stat(this.LOCAL_ARCHIVE_FOLDER + escapeFileName(file));
+                return { path: file, createTime: stats.ctimeMs, size: stats.size };
+            } catch {
+                return undefined;
+            }
+        }))).filter(isDefined);
+    }
+
+
+    @measureFnc
+    public async getInfo(pathInput: string): Promise<{
+        writeTime: number;
+        size: number;
+    } | undefined> {
+        let path = escapeFileName(pathInput);
+        try {
+            let stats = await fs.promises.stat(this.LOCAL_ARCHIVE_FOLDER + path);
+            return {
+                writeTime: stats.mtimeMs,
+                size: stats.size,
+            };
+        } catch {
+            return undefined;
+        }
+    }
+
+    private simulateLag() {
+        if (this.LAG === 0) return;
+        return delay(this.LAG);
+    }
+
+    public async assertPathValid(path: string) {
+        await this.init();
+        escapeFileName(this.LOCAL_ARCHIVE_FOLDER + path);
+    }
+
+    @measureFnc
+    public async move(config: {
+        path: string;
+        target: Archives;
+        targetPath: string;
+    }) {
+        let { path, target, targetPath } = config;
+        while (true) {
+            let targetUnwrapped = target.getBaseArchives?.();
+            if (!targetUnwrapped) break;
+            target = targetUnwrapped.archives;
+            targetPath = targetUnwrapped.parentPath + targetPath;
+        }
+
+        if (target instanceof ArchivesDisk) {
+            path = escapeFileName(path);
+            targetPath = escapeFileName(targetPath);
+            await this.ensureDirsExist(path);
+            await target.ensureDirsExist(targetPath);
+            await fs.promises.rename(
+                this.LOCAL_ARCHIVE_FOLDER + path,
+                target.LOCAL_ARCHIVE_FOLDER + targetPath
+            );
+        } else {
+            let data = await this.get(path);
+            if (!data) throw new Error(`File not found to move: ${path}`);
+            await target.set(targetPath, data);
+            await this.del(path);
+        }
+    }
+
+    @measureFnc
+    public async copy(config: {
+        path: string;
+        target: Archives;
+        targetPath: string;
+    }) {
+        let { path, target, targetPath } = config;
+        while (true) {
+            let targetUnwrapped = target.getBaseArchives?.();
+            if (!targetUnwrapped) break;
+            target = targetUnwrapped.archives;
+            targetPath = targetUnwrapped.parentPath + targetPath;
+        }
+
+        if (target instanceof ArchivesDisk) {
+            path = escapeFileName(path);
+            targetPath = escapeFileName(targetPath);
+            await this.ensureDirsExist(path);
+            await target.ensureDirsExist(targetPath);
+            await fs.promises.copyFile(
+                this.LOCAL_ARCHIVE_FOLDER + path,
+                target.LOCAL_ARCHIVE_FOLDER + targetPath
+            );
+        } else {
+            let data = await this.get(path);
+            if (!data) throw new Error(`File not found to move: ${path}`);
+            await target.set(targetPath, data);
+        }
+    }
+}
+
+export const diskEscapeFileName = escapeFileName;
+
+function escapeFileName(fileName: string): string {
+    fileName = fileName.replaceAll("_", "__");
+    if (fileName[1] === ":" && fileName[2] === "/") {
+        fileName = fileName.slice(0, 2) + fileName.slice(2).replaceAll(":", "_=");
+    } else {
+        fileName = fileName.replaceAll(":", "_=");
+    }
+    fileName = fileName.replaceAll(`"`, `_'`);
+    // TODO: Better escaping for file system paths
+    // TODO: Support longer filenames in some way? (backblaze can support longer names, but...
+    //  we're always going to want to support just using the local filesystem, as that is always
+    //  preferred during development).
+    if (fileName.length > maxFileNameLength) {
+        throw new Error(`File name too long, must be at most ${maxFileNameLength} characters long, was ${fileName.length}, file name ${JSON.stringify(fileName)}`);
+    }
+    fileName = fileName.replaceAll("\\", "/");
+    let parts = fileName.split("/");
+    for (let part of parts) {
+        if (part.length > maxFilePartLength) {
+            throw new Error(`File name part too long, must be at most ${maxFilePartLength} characters long, was ${part.length}, file name ${JSON.stringify(fileName)}`);
+        }
+    }
+    if (fileName.startsWith("/")) {
+        fileName = fileName.slice("/".length);
+    }
+    return fileName;
+}
+function unescapeFileName(fileName: string): string {
+    fileName = fileName.replaceAll(`_'`, `"`);
+    fileName = fileName.replaceAll("_=", ":");
+    fileName = fileName.replaceAll("__", "_");
+    return fileName;
+}
+
+export const getArchivesLocal = cache((domain: string): Archives => {
+
+    const archivesLocal = new ArchivesDisk();
+
+    if (isNode()) {
+        archivesLocal.LOCAL_ARCHIVE_FOLDER = getSubFolder(domain).replaceAll("\\", "/");
+    }
+
+    return archivesLocal;
+});

package/src/-a-archives/copyLocalToBackblaze.ts

@@ -0,0 +1,24 @@
+import { formatNumber } from "socket-function/src/formatting/format";
+import { getDomain } from "../config";
+import { getArchivesBackblaze } from "./archivesBackBlaze";
+import { getArchivesLocal } from "./archivesDisk";
+
+async function main() {
+    const domain = getDomain();
+    let local = getArchivesLocal(domain);
+    let remote = getArchivesBackblaze(domain);
+
+    let files = await local.find("");
+    let allRemoteFiles = new Set(await remote.find(""));
+    for (let file of files) {
+        if (allRemoteFiles.has(file)) continue;
+        let buffer = await local.get(file);
+        if (!buffer) continue;
+        console.log(`Copying ${file} size ${formatNumber(buffer.length)}B`);
+        await remote.set(file, buffer);
+    }
+}
+main().catch(console.error).finally(() => process.exit(0));
+
+// archives = getArchivesLocal(domain);
+// archives = getArchivesBackblaze(domain);