querysub 0.436.0 → 0.438.0

package/src/-a-archives/archives.ts

@@ -1,203 +1,203 @@
-import { isNode } from "socket-function/src/misc";
-import { getDomain, isNoNetwork } from "../config";
-import { getStorageDir, getSubFolder } from "../fs";
-import { getArchivesBackblaze, hasBackblazePermissions } from "./archivesBackBlaze";
-import { getArchivesLocal } from "./archivesDisk";
-import { magenta } from "socket-function/src/formatting/logColors";
-import debugbreak from "debugbreak";
-import { isClient } from "../config2";
-import { wrapArchivesWithCache } from "./archiveCache";
-import { Args } from "socket-function/src/types";
-
-export interface Archives {
-    getDebugName(): string;
-
-    get(path: string, config?: {
-        range?: { start: number; end: number; };
-        retryCount?: number;
-        // Skips checking if the file exists on the remote. Only appropriate when the file name is fresh (recently obtained from find).
-        fastRead?: boolean;
-    }): Promise<Buffer | undefined>;
-    set(path: string, data: Buffer): Promise<void>;
-    del(path: string): Promise<void>;
-
-    append(path: string, data: Buffer): Promise<void>;
-
-    getInfo(path: string): Promise<{ writeTime: number; size: number; } | undefined>;
-
-    setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined>; }): Promise<void>;
-
-    // For example findFileNames("ips/")
-    /** Returns to full paths. Returns paths WILL NOT end with / */
-    find(
-        prefix: string,
-        config?: {
-            shallow?: boolean;
-            // Defaults to files
-            type: "files" | "folders";
-        }
-    ): Promise<string[]>;
-    findInfo(
-        prefix: string,
-        config?: {
-            shallow?: boolean;
-            // Defaults to files
-            type: "files" | "folders";
-        }
-    ): Promise<{ path: string; createTime: number; size: number; }[]>;
-
-    move(config: {
-        path: string;
-        target: Archives;
-        targetPath: string;
-    }): Promise<void>;
-    copy(config: {
-        path: string;
-        target: Archives;
-        targetPath: string;
-    }): Promise<void>;
-
-
-    enableLogging(): void;
-
-    /** Might throw due to too long of a length. Should never throw due to characters, as we can always encode special characters. */
-    assertPathValid(path: string): Promise<void>;
-
-    getBaseArchives?: () => { parentPath: string; archives: Archives; } | undefined;
-
-    /** Get a download URL for a file. Only implemented for some archive types (like Backblaze). */
-    getURL?: (path: string) => Promise<string>;
-
-    // NOTE: This isn't part of getURL, as it is much more efficient to call it once and reuse the authorization. This is only for backblaze as of right now.
-    // NOTE: This also gives authorization for ALL files, so... ONLY use it for internal download links. If we need external links, we should use a public folder with hash links, or... make a new endpoint.
-    getDownloadAuthorization?: (config: {
-        validDurationInSeconds: number;
-    }) => Promise<{
-        bucketId: string;
-        fileNamePrefix: string;
-        authorizationToken: string;
-    }>;
-}
-
-export function createArchivesOverride<T extends Partial<Archives>>(
-    baseArchives: Archives,
-    overrideArchives: T
-): T & Archives {
-    return {
-        getDebugName: () => baseArchives.getDebugName(),
-        get: (fileName: string, config) => baseArchives.get(fileName, config),
-        set: (fileName: string, data: Buffer) => baseArchives.set(fileName, data),
-        append: (fileName: string, data: Buffer) => baseArchives.append(fileName, data),
-        del: (fileName: string) => baseArchives.del(fileName),
-        getInfo: (fileName: string) => baseArchives.getInfo(fileName),
-        setLargeFile: (config) => baseArchives.setLargeFile(config),
-        find: async (prefix: string, config) => baseArchives.find(prefix, config),
-        findInfo: async (prefix: string, config) => baseArchives.findInfo(prefix, config),
-        enableLogging: () => baseArchives.enableLogging(),
-        move: (config) => baseArchives.move(config),
-        copy: (config) => baseArchives.copy(config),
-        assertPathValid: (path: string) => baseArchives.assertPathValid(path),
-        getBaseArchives: () => baseArchives.getBaseArchives?.() ?? ({ archives: baseArchives, parentPath: "" }),
-        getURL: baseArchives.getURL ? (filePath: string) => baseArchives.getURL!(filePath) : undefined,
-        getDownloadAuthorization: baseArchives.getDownloadAuthorization ? (config: { validDurationInSeconds: number }) => baseArchives.getDownloadAuthorization!(config) : undefined,
-        ...overrideArchives,
-    };
-}
-
-export function nestArchives(path: string, archives: Archives): Archives {
-    if (!path) return archives;
-    if (!path.endsWith("/")) {
-        path = path + "/";
-    }
-    function stripFilePrefix(file: string) {
-        if (file.startsWith("/")) {
-            return file.slice(1);
-        }
-        return file;
-    }
-    return {
-        getDebugName: () => archives.getDebugName() + "/" + path,
-        get: (fileName: string, config) => archives.get(path + stripFilePrefix(fileName), config),
-        setLargeFile: (config) => archives.setLargeFile({ ...config, path: path + stripFilePrefix(config.path) }),
-        set: (fileName: string, data: Buffer) => archives.set(path + stripFilePrefix(fileName), data),
-        append: (fileName: string, data: Buffer) => archives.append(path + stripFilePrefix(fileName), data),
-        del: (fileName: string) => archives.del(path + stripFilePrefix(fileName)),
-        getInfo: (fileName: string) => archives.getInfo(path + stripFilePrefix(fileName)),
-        find: async (prefix: string, config) => archives.find(path + prefix, config).then(x => x.map(x => x.slice(path.length))),
-        findInfo: async (prefix: string, config) => archives.findInfo(path + prefix, config).then(x => x.map(x => ({ path: x.path.slice(path.length), createTime: x.createTime, size: x.size }))),
-        enableLogging: () => archives.enableLogging(),
-        move: (config) => archives.move({
-            path: path + config.path,
-            target: config.target,
-            targetPath: config.targetPath,
-        }),
-        copy: (config) => archives.copy({
-            path: path + config.path,
-            target: config.target,
-            targetPath: config.targetPath,
-        }),
-        assertPathValid: (path: string) => archives.assertPathValid(path),
-        getBaseArchives: () => {
-            let result = { parentPath: path, archives };
-            let base = archives.getBaseArchives?.();
-            if (base) {
-                result.parentPath = base.parentPath + path;
-                result.archives = base.archives;
-            }
-            return result;
-        },
-        getURL: archives.getURL ? (filePath: string) => archives.getURL!(path + stripFilePrefix(filePath)) : undefined,
-        getDownloadAuthorization: archives.getDownloadAuthorization && ((...args: Args<typeof archives.getDownloadAuthorization>) => archives.getDownloadAuthorization!(...args)),
-    };
-}
-
-let archivesLogging = false;
-export function enableArchivesLogging() {
-    archivesLogging = true;
-}
-
-let cached = new Map<string, Archives>();
-export function getArchives(rootPath = ""): Archives {
-    let value = cached.get(rootPath);
-    if (!value) {
-        value = getArchivesBase(rootPath);
-        cached.set(rootPath, value);
-    }
-    return value;
-}
-export function getArchivesBase(rootPath = ""): Archives {
-    let archives: Archives;
-    let domain = getDomain();
-    if (isNoNetwork()) {
-        archives = getArchivesLocal(domain);
-        archives = nestArchives(rootPath, archives);
-    } else {
-        archives = getArchivesBackblaze(domain);
-        archives = nestArchives(rootPath, archives);
-        // Why did we think we should always cache? We absolutely should not. Even if we were checking
-        //  file hashes on every read (which we do not), it would still potentially fill the cache up
-        //  with large temporary files we have no intention of using (which might evict entries we
-        //  intend to use).
-        //archives = wrapArchivesWithCache(archives);
-    }
-    if (archivesLogging) {
-        archives.enableLogging();
-    }
-    return archives;
-}
-
-export function hasArchivesPermissions() {
-    if (isNoNetwork()) {
-        return true;
-    }
-    return hasBackblazePermissions();
-}
-
-if (isNode()) {
-    if (isNoNetwork()) {
-        console.log("Primary data storage: " + magenta(getStorageDir()));
-    } else {
-        console.log("Config storage path:  " + magenta(getStorageDir()));
-        console.log("Primary data storage: " + magenta(`backblaze:${getDomain()}`));
-    }
+import { isNode } from "socket-function/src/misc";
+import { getDomain, isNoNetwork } from "../config";
+import { getStorageDir, getSubFolder } from "../fs";
+import { getArchivesBackblaze, hasBackblazePermissions } from "./archivesBackBlaze";
+import { getArchivesLocal } from "./archivesDisk";
+import { magenta } from "socket-function/src/formatting/logColors";
+import debugbreak from "debugbreak";
+import { isClient } from "../config2";
+import { wrapArchivesWithCache } from "./archiveCache";
+import { Args } from "socket-function/src/types";
+
+export interface Archives {
+    getDebugName(): string;
+
+    get(path: string, config?: {
+        range?: { start: number; end: number; };
+        retryCount?: number;
+        // Skips checking if the file exists on the remote. Only appropriate when the file name is fresh (recently obtained from find).
+        fastRead?: boolean;
+    }): Promise<Buffer | undefined>;
+    set(path: string, data: Buffer): Promise<void>;
+    del(path: string): Promise<void>;
+
+    append(path: string, data: Buffer): Promise<void>;
+
+    getInfo(path: string): Promise<{ writeTime: number; size: number; } | undefined>;
+
+    setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined>; }): Promise<void>;
+
+    // For example findFileNames("ips/")
+    /** Returns to full paths. Returns paths WILL NOT end with / */
+    find(
+        prefix: string,
+        config?: {
+            shallow?: boolean;
+            // Defaults to files
+            type: "files" | "folders";
+        }
+    ): Promise<string[]>;
+    findInfo(
+        prefix: string,
+        config?: {
+            shallow?: boolean;
+            // Defaults to files
+            type: "files" | "folders";
+        }
+    ): Promise<{ path: string; createTime: number; size: number; }[]>;
+
+    move(config: {
+        path: string;
+        target: Archives;
+        targetPath: string;
+    }): Promise<void>;
+    copy(config: {
+        path: string;
+        target: Archives;
+        targetPath: string;
+    }): Promise<void>;
+
+
+    enableLogging(): void;
+
+    /** Might throw due to too long of a length. Should never throw due to characters, as we can always encode special characters. */
+    assertPathValid(path: string): Promise<void>;
+
+    getBaseArchives?: () => { parentPath: string; archives: Archives; } | undefined;
+
+    /** Get a download URL for a file. Only implemented for some archive types (like Backblaze). */
+    getURL?: (path: string) => Promise<string>;
+
+    // NOTE: This isn't part of getURL, as it is much more efficient to call it once and reuse the authorization. This is only for backblaze as of right now.
+    // NOTE: This also gives authorization for ALL files, so... ONLY use it for internal download links. If we need external links, we should use a public folder with hash links, or... make a new endpoint.
+    getDownloadAuthorization?: (config: {
+        validDurationInSeconds: number;
+    }) => Promise<{
+        bucketId: string;
+        fileNamePrefix: string;
+        authorizationToken: string;
+    }>;
+}
+
+export function createArchivesOverride<T extends Partial<Archives>>(
+    baseArchives: Archives,
+    overrideArchives: T
+): T & Archives {
+    return {
+        getDebugName: () => baseArchives.getDebugName(),
+        get: (fileName: string, config) => baseArchives.get(fileName, config),
+        set: (fileName: string, data: Buffer) => baseArchives.set(fileName, data),
+        append: (fileName: string, data: Buffer) => baseArchives.append(fileName, data),
+        del: (fileName: string) => baseArchives.del(fileName),
+        getInfo: (fileName: string) => baseArchives.getInfo(fileName),
+        setLargeFile: (config) => baseArchives.setLargeFile(config),
+        find: async (prefix: string, config) => baseArchives.find(prefix, config),
+        findInfo: async (prefix: string, config) => baseArchives.findInfo(prefix, config),
+        enableLogging: () => baseArchives.enableLogging(),
+        move: (config) => baseArchives.move(config),
+        copy: (config) => baseArchives.copy(config),
+        assertPathValid: (path: string) => baseArchives.assertPathValid(path),
+        getBaseArchives: () => baseArchives.getBaseArchives?.() ?? ({ archives: baseArchives, parentPath: "" }),
+        getURL: baseArchives.getURL ? (filePath: string) => baseArchives.getURL!(filePath) : undefined,
+        getDownloadAuthorization: baseArchives.getDownloadAuthorization ? (config: { validDurationInSeconds: number }) => baseArchives.getDownloadAuthorization!(config) : undefined,
+        ...overrideArchives,
+    };
+}
+
+export function nestArchives(path: string, archives: Archives): Archives {
+    if (!path) return archives;
+    if (!path.endsWith("/")) {
+        path = path + "/";
+    }
+    function stripFilePrefix(file: string) {
+        if (file.startsWith("/")) {
+            return file.slice(1);
+        }
+        return file;
+    }
+    return {
+        getDebugName: () => archives.getDebugName() + "/" + path,
+        get: (fileName: string, config) => archives.get(path + stripFilePrefix(fileName), config),
+        setLargeFile: (config) => archives.setLargeFile({ ...config, path: path + stripFilePrefix(config.path) }),
+        set: (fileName: string, data: Buffer) => archives.set(path + stripFilePrefix(fileName), data),
+        append: (fileName: string, data: Buffer) => archives.append(path + stripFilePrefix(fileName), data),
+        del: (fileName: string) => archives.del(path + stripFilePrefix(fileName)),
+        getInfo: (fileName: string) => archives.getInfo(path + stripFilePrefix(fileName)),
+        find: async (prefix: string, config) => archives.find(path + prefix, config).then(x => x.map(x => x.slice(path.length))),
+        findInfo: async (prefix: string, config) => archives.findInfo(path + prefix, config).then(x => x.map(x => ({ path: x.path.slice(path.length), createTime: x.createTime, size: x.size }))),
+        enableLogging: () => archives.enableLogging(),
+        move: (config) => archives.move({
+            path: path + config.path,
+            target: config.target,
+            targetPath: config.targetPath,
+        }),
+        copy: (config) => archives.copy({
+            path: path + config.path,
+            target: config.target,
+            targetPath: config.targetPath,
+        }),
+        assertPathValid: (path: string) => archives.assertPathValid(path),
+        getBaseArchives: () => {
+            let result = { parentPath: path, archives };
+            let base = archives.getBaseArchives?.();
+            if (base) {
+                result.parentPath = base.parentPath + path;
+                result.archives = base.archives;
+            }
+            return result;
+        },
+        getURL: archives.getURL ? (filePath: string) => archives.getURL!(path + stripFilePrefix(filePath)) : undefined,
+        getDownloadAuthorization: archives.getDownloadAuthorization && ((...args: Args<typeof archives.getDownloadAuthorization>) => archives.getDownloadAuthorization!(...args)),
+    };
+}
+
+let archivesLogging = false;
+export function enableArchivesLogging() {
+    archivesLogging = true;
+}
+
+let cached = new Map<string, Archives>();
+export function getArchives(rootPath = ""): Archives {
+    let value = cached.get(rootPath);
+    if (!value) {
+        value = getArchivesBase(rootPath);
+        cached.set(rootPath, value);
+    }
+    return value;
+}
+export function getArchivesBase(rootPath = ""): Archives {
+    let archives: Archives;
+    let domain = getDomain();
+    if (isNoNetwork()) {
+        archives = getArchivesLocal(domain);
+        archives = nestArchives(rootPath, archives);
+    } else {
+        archives = getArchivesBackblaze(domain);
+        archives = nestArchives(rootPath, archives);
+        // Why did we think we should always cache? We absolutely should not. Even if we were checking
+        //  file hashes on every read (which we do not), it would still potentially fill the cache up
+        //  with large temporary files we have no intention of using (which might evict entries we
+        //  intend to use).
+        //archives = wrapArchivesWithCache(archives);
+    }
+    if (archivesLogging) {
+        archives.enableLogging();
+    }
+    return archives;
+}
+
+export function hasArchivesPermissions() {
+    if (isNoNetwork()) {
+        return true;
+    }
+    return hasBackblazePermissions();
+}
+
+if (isNode()) {
+    if (isNoNetwork()) {
+        console.log("Primary data storage: " + magenta(getStorageDir()));
+    } else {
+        console.log("Config storage path:  " + magenta(getStorageDir()));
+        console.log("Primary data storage: " + magenta(`backblaze:${getDomain()}`));
+    }
 }

package/src/-a-archives/archivesBackBlaze.ts

@@ -13,7 +13,6 @@ import debugbreak from "debugbreak";
 import { onTimeProfile } from "../-0-hooks/hooks";
 import dns from "dns";
 
-
 export function hasBackblazePermissions() {
     return isNode() && fs.existsSync(getBackblazePath());
 }