querysub 0.2.0

package/src/-a-archives/archives.ts

@@ -0,0 +1,148 @@
+import { isNode } from "socket-function/src/misc";
+import { getDomain, isNoNetwork } from "../config";
+import { getStorageDir, getSubFolder } from "../fs";
+import { getArchivesBackblaze, hasBackblazePermissions } from "./archivesBackBlaze";
+import { getArchivesLocal } from "./archivesDisk";
+import { magenta } from "socket-function/src/formatting/logColors";
+import debugbreak from "debugbreak";
+import { isClient } from "../config2";
+import { wrapArchivesWithCache } from "./archiveCache";
+// https://www.backblaze.com/apidocs/b2-start-large-file
+// https://www.backblaze.com/apidocs/b2-upload-part
+// https://www.backblaze.com/apidocs/b2-finish-large-file
+
+export interface Archives {
+    getDebugName(): string;
+
+    get(path: string, config?: { range?: { start: number; end: number; }; retryCount?: number }): Promise<Buffer | undefined>;
+    set(path: string, data: Buffer): Promise<void>;
+    del(path: string): Promise<void>;
+
+    getInfo(path: string): Promise<{ writeTime: number; size: number; } | undefined>;
+
+    setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined>; }): Promise<void>;
+
+    // For example findFileNames("ips/")
+    find(
+        prefix: string,
+        config?: {
+            shallow?: boolean;
+            // Defaults to files
+            type: "files" | "folders";
+        }
+    ): Promise<string[]>;
+    findInfo(
+        prefix: string,
+        config?: {
+            shallow?: boolean;
+            // Defaults to files
+            type: "files" | "folders";
+        }
+    ): Promise<{ path: string; createTime: number; size: number; }[]>;
+
+    move(config: {
+        path: string;
+        target: Archives;
+        targetPath: string;
+    }): Promise<void>;
+    copy(config: {
+        path: string;
+        target: Archives;
+        targetPath: string;
+    }): Promise<void>;
+
+
+    enableLogging(): void;
+
+    /** Might throw due to too long of a length. Should never throw due to characters, as we can always encode special characters. */
+    assertPathValid(path: string): Promise<void>;
+
+    getBaseArchives?: () => { parentPath: string; archives: Archives; } | undefined;
+}
+
+export function nestArchives(path: string, archives: Archives): Archives {
+    if (!path) return archives;
+    if (!path.endsWith("/")) {
+        path = path + "/";
+    }
+    function stripFilePrefix(file: string) {
+        if (file.startsWith("/")) {
+            return file.slice(1);
+        }
+        return file;
+    }
+    return {
+        getDebugName: () => archives.getDebugName() + "/" + path,
+        get: (fileName: string, config) => archives.get(path + stripFilePrefix(fileName), config),
+        setLargeFile: (config) => archives.setLargeFile({ ...config, path: path + stripFilePrefix(config.path) }),
+        set: (fileName: string, data: Buffer) => archives.set(path + stripFilePrefix(fileName), data),
+        del: (fileName: string) => archives.del(path + stripFilePrefix(fileName)),
+        getInfo: (fileName: string) => archives.getInfo(path + stripFilePrefix(fileName)),
+        find: async (prefix: string, config) => archives.find(path + prefix, config).then(x => x.map(x => x.slice(path.length))),
+        findInfo: async (prefix: string, config) => archives.findInfo(path + prefix, config).then(x => x.map(x => ({ path: x.path.slice(path.length), createTime: x.createTime, size: x.size }))),
+        enableLogging: () => archives.enableLogging(),
+        move: (config) => archives.move({
+            path: path + config.path,
+            target: config.target,
+            targetPath: config.targetPath,
+        }),
+        copy: (config) => archives.copy({
+            path: path + config.path,
+            target: config.target,
+            targetPath: config.targetPath,
+        }),
+        assertPathValid: (path: string) => archives.assertPathValid(path),
+        getBaseArchives: () => ({ parentPath: path, archives }),
+    };
+}
+
+let archivesLogging = false;
+export function enableArchivesLogging() {
+    archivesLogging = true;
+}
+
+let cached = new Map<string, Archives>();
+export function getArchives(rootPath = ""): Archives {
+    let value = cached.get(rootPath);
+    if (!value) {
+        value = getArchivesBase(rootPath);
+        cached.set(rootPath, value);
+    }
+    return value;
+}
+export function getArchivesBase(rootPath = ""): Archives {
+    let archives: Archives;
+    let domain = getDomain();
+    if (isNoNetwork()) {
+        archives = getArchivesLocal(domain);
+        archives = nestArchives(rootPath, archives);
+    } else {
+        archives = getArchivesBackblaze(domain);
+        archives = nestArchives(rootPath, archives);
+        // Why did we think we should always cache? We absolutely should not. Even if we were checking
+        //  file hashes on every read (which we do not), it would still potentially fill the cache up
+        //  with large temporary files we have no intention of using (which might evict entries we
+        //  intend to use).
+        //archives = wrapArchivesWithCache(archives);
+    }
+    if (archivesLogging) {
+        archives.enableLogging();
+    }
+    return archives;
+}
+
+export function hasArchivesPermissions() {
+    if (isNoNetwork()) {
+        return true;
+    }
+    return hasBackblazePermissions();
+}
+
+if (isNode()) {
+    if (isNoNetwork()) {
+        console.log("Primary data storage: " + magenta(getStorageDir()));
+    } else {
+        console.log("Config storage path:  " + magenta(getStorageDir()));
+        console.log("Primary data storage: " + magenta(`backblaze:${getDomain()}`));
+    }
+}