querysub 0.312.0 → 0.313.0

package/src/diagnostics/logs/FastArchiveController.ts

@@ -0,0 +1,524 @@
+module.hotreload = true;
+module.noserverhotreload = false;
+import { measureBlock, measureFnc, measureWrap } from "socket-function/src/profiling/measure";
+import { getMachineId, getOwnMachineId } from "../../-a-auth/certs";
+import { isDefined, parseFileNameKVP, parsePath, partialCopyObject, streamToIteratable, sum, toFileNameKVP } from "../../misc";
+import { registerShutdownHandler } from "../periodic";
+import { batchFunction, delay, runInSerial, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { PromiseObj, isNode, keyByArray, nextId, sort, timeInDay, timeInHour, timeInMinute } from "socket-function/src/misc";
+import os from "os";
+import { getOwnThreadId } from "../../-f-node-discovery/NodeDiscovery";
+import fs from "fs";
+import { MaybePromise, canHaveChildren } from "socket-function/src/types";
+import { formatNumber, formatTime } from "socket-function/src/formatting/format";
+import { cache, lazy } from "socket-function/src/caching";
+import { getArchives, nestArchives } from "../../-a-archives/archives";
+import { Zip } from "../../zip";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { assertIsManagementUser } from "../managementPages";
+import { getControllerNodeIdList } from "../../-g-core-values/NodeCapabilities";
+import { errorToUndefined, ignoreErrors, timeoutToUndefinedSilent } from "../../errors";
+import { getCallObj } from "socket-function/src/nodeProxy";
+import { getSyncedController } from "../../library-components/SyncedController";
+import { getBrowserUrlNode, getOwnNodeId } from "../../-f-node-discovery/NodeDiscovery";
+import { secureRandom } from "../../misc/random";
+import { getPathIndex, getPathStr2 } from "../../path";
+import { onNextPaint } from "../../functional/onNextPaint";
+import { getArchivesBackblazePrivateImmutable, getArchivesBackblazePublicImmutable } from "../../-a-archives/archivesBackBlaze";
+import { httpsRequest } from "socket-function/src/https";
+import { getDomain } from "../../config";
+import { getIPDomain } from "../../-e-certs/EdgeCertController";
+import { getArchivesPrivateFileSystem } from "../../-a-archives/archivesPrivateFileSystem";
+import { createArchivesLimitedCache } from "../../-a-archives/archivesLimitedCache";
+import { sha256 } from "js-sha256";
+import { assertIsNetworkTrusted } from "../../-d-trust/NetworkTrust2";
+import { blue, magenta } from "socket-function/src/formatting/logColors";
+import { FastArchiveAppendable, getFileTimeStamp } from "./FastArchiveAppendable";
+import { IdentityController_getMachineId, IdentityController_getReconnectNodeId } from "../../-c-identity/IdentityController";
+
+export type FileMetadata = {
+    nodeId?: string;
+    path: string;
+    url: string;
+    size: number;
+    startTime: number;
+    endTime: number;
+};
+
+type InternalFileMetadata = {
+    path: string;
+    size: number;
+};
+
+type SynchronizeInfo = {
+    id: string;
+    nodeId: string;
+    config: {
+        range: {
+            startTime: number;
+            endTime: number;
+        };
+        rootPath: string;
+    };
+    createTime: number;
+};
+
+// Excluding authorize tokens, then hashed
+export function getFileMetadataHash(file: FileMetadata): string {
+    let urlObj = new URL(file.url);
+    if (!file.nodeId) {
+        urlObj.search = "";
+    } else {
+        let args = JSON.parse(urlObj.searchParams.get("args") || "");
+        // The syncId shouldn't be part of the hash though
+        args[0] = "";
+        urlObj.searchParams.set("args", JSON.stringify(args));
+    }
+    return sha256(urlObj.toString());
+}
+
+export class FastArchiveAppendableControllerBase {
+    private static activeSynchronizes = new Map<string, SynchronizeInfo>();
+    public static progressCallbacks = new Map<string, (progress: { section: string; value: number; max: number; }) => void>();
+
+    /** Get all pending local files for this rootPath that haven't been uploaded to Backblaze yet */
+    public async getPendingFiles(rootPath: string, timeRange: { startTime: number; endTime: number; }): Promise<InternalFileMetadata[]> {
+
+        let rootCacheFolder = new FastArchiveAppendable(rootPath).getLocalPathRoot();
+
+        //console.log(`Searching for pending files in ${rootCacheFolder} for time range ${new Date(timeRange.startTime).toISOString()} to ${new Date(timeRange.endTime).toISOString()}`);
+
+        let result: InternalFileMetadata[] = [];
+
+        try {
+            let allFolders = await fs.promises.readdir(rootCacheFolder);
+            for (let threadId of allFolders) {
+                let threadDir = rootCacheFolder + threadId + "/";
+                let files = await fs.promises.readdir(threadDir);
+
+                for (let file of files) {
+                    if (file === "heartbeat") continue;
+
+                    let fileTimestamp = getFileTimeStamp(file);
+                    //console.log(`Found ${new Date(fileTimestamp).toISOString()} in ${threadDir + file}`);
+                    if (fileTimestamp < timeRange.startTime || fileTimestamp > timeRange.endTime) {
+                        continue;
+                    }
+
+                    let fullPath = threadDir + file;
+                    let stat = await fs.promises.stat(fullPath);
+
+                    result.push({
+                        path: fullPath,
+                        size: stat.size,
+                    });
+                }
+            }
+        } catch (e: any) {
+            // If directory doesn't exist, return empty array
+            if (e.code === "ENOENT") {
+                return [];
+            }
+            throw e;
+        }
+
+        return result;
+    }
+
+    /** Download a local file from another server (coordinator forwards request).
+     *      NOTE: This doesn't support streaming, which isn't great. However, eventually most logs will make their way to backblaze. So not supporting streaming here doesn't mean we don't support streaming in general. We also compress the file. So for the compressed file to be so big that you can't send it all at once would mean your uncompressed data would just be ridiculously large. 
+     */
+    public async downloadLocalFile(syncId: string, targetNodeId: string, path: string): Promise<Buffer> {
+        const caller = SocketFunction.getCaller();
+        let syncInfo = FastArchiveAppendableControllerBase.activeSynchronizes.get(syncId);
+        if (!syncInfo) {
+            throw new Error(`Invalid sync ID: ${syncId}`);
+        }
+
+        try {
+            // Forward the request to the actual server that has the file
+            let targetController = FastArchiveAppendableController.nodes[targetNodeId];
+            return await targetController.downloadLocalFileInternal(path);
+        } catch {
+            let machineId = getMachineId(targetNodeId);
+            let nodeIds = await getControllerNodeIdList(FastArchiveAppendableController);
+            let byMachineId = keyByArray(nodeIds, x => getMachineId(x.nodeId));
+            let nodeIdsOfMachine = byMachineId.get(machineId) || [];
+
+            let firstAliveNode = new PromiseObj<string>();
+            let allFinished = Promise.all(nodeIdsOfMachine.map(async ({ nodeId, entryPoint }) => {
+                if (await timeoutToUndefinedSilent(5000, FastArchiveAppendableController.nodes[nodeId].isNodeAlive(nodeId))) {
+                    firstAliveNode.resolve(nodeId);
+                }
+            }));
+            let aliveNodeId = await Promise.race([firstAliveNode.promise, allFinished]);
+            if (Array.isArray(aliveNodeId)) {
+                throw new Error(`No alive node found for machine ${machineId}`);
+            }
+            console.log(magenta(`Remapped request for ${targetNodeId} to ${aliveNodeId} for machine ${machineId}`));
+            let targetController = FastArchiveAppendableController.nodes[aliveNodeId];
+            return await targetController.downloadLocalFileInternal(path);
+        }
+    }
+
+    /** Internal method to download local file (called by coordinator) */
+    public async downloadLocalFileInternal(path: string): Promise<Buffer> {
+        return await Zip.gzip(await fs.promises.readFile(path));
+    }
+    public async isNodeAlive(nodeId: string): Promise<boolean> {
+        return true;
+    }
+
+    /** Create a new sync session and return the sync ID */
+    public async createSyncSession(): Promise<string> {
+        let caller = SocketFunction.getCaller();
+        let syncId = nextId();
+
+        let syncInfo: SynchronizeInfo = {
+            id: syncId,
+            nodeId: caller.nodeId,
+            config: {
+                range: { startTime: 0, endTime: 0 }, // Will be set by startSynchronize
+                rootPath: "",
+            },
+            createTime: Date.now(),
+        };
+
+        FastArchiveAppendableControllerBase.activeSynchronizes.set(syncId, syncInfo);
+
+        // Clean up on disconnect
+        SocketFunction.onNextDisconnect(caller.nodeId, () => {
+            FastArchiveAppendableControllerBase.activeSynchronizes.delete(syncId);
+        });
+
+        return syncId;
+    }
+
+    public async startSynchronize(config: {
+        syncId: string;
+        range: {
+            startTime: number;
+            endTime: number;
+        };
+        rootPath: string;
+        noLocalFiles?: boolean;
+    }): Promise<{
+        files: FileMetadata[];
+    }> {
+        let caller = SocketFunction.getCaller();
+        // If the user uses a bad syncId and it gets leak, screw it.
+        let syncId = config.syncId;
+
+        let syncInfo: SynchronizeInfo = {
+            id: syncId,
+            nodeId: caller.nodeId,
+            config: {
+                range: config.range,
+                rootPath: config.rootPath,
+            },
+            createTime: Date.now(),
+        };
+
+        FastArchiveAppendableControllerBase.activeSynchronizes.set(syncId, syncInfo);
+
+        // Clean up on disconnect
+        SocketFunction.onNextDisconnect(caller.nodeId, () => {
+            FastArchiveAppendableControllerBase.activeSynchronizes.delete(syncId);
+        });
+
+        return await this.startSynchronizeBase({
+            syncId,
+            range: config.range,
+            rootPath: config.rootPath,
+            noLocalFiles: config.noLocalFiles,
+        });
+    }
+
+    public async startSynchronizeInternal(config: {
+        range: {
+            startTime: number;
+            endTime: number;
+        };
+        rootPath: string;
+        noLocalFiles?: boolean;
+    }): Promise<{
+        files: FileMetadata[];
+    }> {
+        let syncId = "no-progress" + nextId();
+        FastArchiveAppendableControllerBase.activeSynchronizes.set(syncId, {
+            id: syncId,
+            nodeId: getOwnNodeId(),
+            config: {
+                range: config.range,
+                rootPath: config.rootPath,
+            },
+            createTime: Date.now(),
+        });
+        return await this.startSynchronizeBase({
+            syncId,
+            range: config.range,
+            rootPath: config.rootPath,
+            noLocalFiles: config.noLocalFiles,
+        });
+    }
+
+    private async startSynchronizeBase(config: {
+        syncId?: string;
+        range: {
+            startTime: number;
+            endTime: number;
+        };
+        rootPath: string;
+        noLocalFiles?: boolean;
+    }): Promise<{
+        files: FileMetadata[];
+    }> {
+        let syncId = config.syncId ?? "";
+
+        // Define inline functions for parallel execution
+        const searchBackblazeFiles = async (): Promise<FileMetadata[]> => {
+            let archives = new FastArchiveAppendable(config.rootPath).getArchives();
+            let backblazeFiles: FileMetadata[] = [];
+            if (!archives.getDownloadAuthorization) throw new Error(`archives.getDownloadAuthorization is missing?`);
+            let authorization = await archives.getDownloadAuthorization({
+                validDurationInSeconds: timeInDay * 6 / 1000,
+            });
+            let authToken = authorization.authorizationToken;
+
+            const folderProgress = this.updateProgress(syncId, "Backblaze folder search", 0);
+            let folderMax = 0;
+            let folderValue = 0;
+
+            async function searchBackblazeFilesRecursive(prefix: string, level: "year" | "month" | "day" | "hour") {
+                folderMax++;
+                let folders = await archives.find(prefix, { shallow: true, type: "folders" });
+                folderValue++;
+                folderProgress(folderValue, folderMax);
+                // It's actually really annoying because the time ranges aren't the same amount of time. It's probably fine though. This code only needs to exist in one place. 
+
+                await Promise.all(folders.map(async (folder) => {
+                    let folderName = folder.split("/").pop()!;
+                    let folderNum = parseInt(folderName, 10);
+
+                    if (level === "year") {
+                        let yearStart = Date.UTC(folderNum, 0, 1);
+                        let yearEnd = Date.UTC(folderNum + 1, 0, 1);
+
+                        if (yearEnd > config.range.startTime && yearStart <= config.range.endTime) {
+                            await searchBackblazeFilesRecursive(folder + "/", "month");
+                        }
+                    } else if (level === "month") {
+                        // folderName is 1-based month (01, 02, etc)
+                        let year = parseInt(folder.split("/")[0], 10);
+                        let monthStart = Date.UTC(year, folderNum - 1, 1);
+                        let monthEnd = Date.UTC(year, folderNum, 1);
+
+                        if (monthEnd > config.range.startTime && monthStart <= config.range.endTime) {
+                            await searchBackblazeFilesRecursive(folder + "/", "day");
+                        }
+                    } else if (level === "day") {
+                        let pathParts = folder.split("/");
+                        let year = parseInt(pathParts[0], 10);
+                        let month = parseInt(pathParts[1], 10) - 1; // Date constructor expects 0-based month
+                        let dayStart = Date.UTC(year, month, folderNum);
+                        let dayEnd = Date.UTC(year, month, folderNum + 1);
+
+                        if (dayEnd > config.range.startTime && dayStart <= config.range.endTime) {
+                            await searchBackblazeFilesRecursive(folder + "/", "hour");
+                        }
+                    } else if (level === "hour") {
+                        let pathParts = folder.split("/");
+                        let year = parseInt(pathParts[0], 10);
+                        let month = parseInt(pathParts[1], 10) - 1;
+                        let day = parseInt(pathParts[2], 10);
+                        let hourStart = Date.UTC(year, month, day, folderNum);
+                        let hourEnd = Date.UTC(year, month, day, folderNum + 1);
+
+                        let inRange = hourEnd > config.range.startTime && hourStart <= config.range.endTime;
+                        if (!inRange) return;
+                        // This hour folder is in range, get all files from it
+                        let filePaths = await archives.findInfo(folder + "/", { shallow: true, type: "files" });
+                        for (let info of filePaths) {
+                            if (!info.path.endsWith(".log")) continue;
+
+                            if (!archives.getURL) throw new Error(`archives.getURL is missing?`);
+                            let url = await archives.getURL(info.path);
+                            let urlObj = new URL(url);
+                            // IMPORTANT! This is CASE SENSITIVE! Ugh...
+                            urlObj.searchParams.set("Authorization", authToken);
+                            url = urlObj.toString();
+
+                            backblazeFiles.push({
+                                path: info.path,
+                                url: url,
+                                size: info.size,
+                                startTime: hourStart,
+                                endTime: hourEnd,
+                            });
+                        }
+                    }
+                }));
+            }
+
+            await searchBackblazeFilesRecursive("", "year");
+
+            return backblazeFiles;
+        };
+
+        const getRemoteFiles = async (): Promise<FileMetadata[]> => {
+            const getControllerProgress = this.updateProgress(syncId, "Discovering remote machines", 0);
+
+            let localFiles: FileMetadata[] = [];
+
+            let nodeIds = await getControllerNodeIdList(FastArchiveAppendableController);
+            let byMachineId = keyByArray(nodeIds, x => getMachineId(x.nodeId));
+            getControllerProgress(byMachineId.size, byMachineId.size);
+
+            let remoteProgress = this.updateProgress(syncId, "Discovering remote files", byMachineId.size);
+            let remoteValue = 0;
+
+            await Promise.all(Array.from(byMachineId).map(async ([machineId, nodeObjs]) => {
+                let firstAliveNode = new PromiseObj<string>();
+                let allFinished = Promise.all(nodeObjs.map(async ({ nodeId, entryPoint }) => {
+                    if (await timeoutToUndefinedSilent(5000, FastArchiveAppendableController.nodes[nodeId].isNodeAlive(nodeId))) {
+                        firstAliveNode.resolve(nodeId);
+                    }
+                }));
+                let aliveNodeId = await Promise.race([firstAliveNode.promise, allFinished]);
+                if (Array.isArray(aliveNodeId)) {
+                    console.log(blue(`No alive nodes found for machine ${machineId}`), nodeObjs);
+                    remoteValue++;
+                    remoteProgress(remoteValue);
+                    return;
+                }
+
+                let controller = FastArchiveAppendableController.nodes[aliveNodeId];
+
+                let pendingFiles = await errorToUndefined(
+                    controller.getPendingFiles(config.rootPath, config.range));
+                console.log(blue(`Found ${pendingFiles?.length} pending files on node ${aliveNodeId}`), nodeObjs);
+
+                remoteValue++;
+                remoteProgress(remoteValue);
+
+                if (!pendingFiles) return;
+                for (let file of pendingFiles) {
+                    // Create download URL that points to the coordinator (this server)
+                    // The coordinator will forward the request to the actual file server
+                    let coordinatorNodeId = getOwnNodeId();
+                    let coordinatorController = FastArchiveAppendableController.nodes[coordinatorNodeId];
+                    let downloadCall = coordinatorController.downloadLocalFile[getCallObj](
+                        syncId,
+                        aliveNodeId, // Target server that has the file
+                        file.path
+                    );
+                    let url = SocketFunction.getHTTPCallLink(downloadCall);
+                    // Have to use the IP domain, as it's externally available. That, plus the port, should uniquely identify us.
+                    let ipDomain = await getIPDomain();
+                    let urlObj = new URL(url);
+                    urlObj.hostname = ipDomain;
+                    url = urlObj.toString();
+                    let startTime = getFileTimeStamp(file.path);
+                    let endTime = startTime + timeInHour;
+
+                    localFiles.push({
+                        nodeId: aliveNodeId,
+                        path: file.path,
+                        url: url,
+                        size: file.size,
+                        startTime: startTime,
+                        endTime: endTime,
+                    });
+                }
+            }));
+
+            return localFiles;
+        };
+
+        // Execute both operations in parallel
+        let filePromises: Promise<FileMetadata[]>[] = [];
+        filePromises.push(searchBackblazeFiles());
+        if (!config.noLocalFiles) filePromises.push(getRemoteFiles());
+
+        let allFilesList = await Promise.all(filePromises);
+        let allFiles = allFilesList.flat();
+        sort(allFiles, x => -x.startTime);
+
+        return {
+            files: allFiles,
+        };
+    }
+
+    /** Update progress for a synchronization session - returns a function that takes only the progress value */
+    public updateProgress(syncId: string, section: string, max: number) {
+        let cancelled: Error | undefined;
+        let baseBatch = batchFunction({ delay: 150, },
+            async (config: { value: number, overrideMax?: number }[]) => {
+                if (cancelled) return;
+                let syncInfo = FastArchiveAppendableControllerBase.activeSynchronizes.get(syncId);
+                if (!syncInfo) {
+                    cancelled = new Error(`Cancelled (sync for id ${syncId} is missing)`);
+                    return;
+                }
+
+                let value = config.at(-1)!.value;
+                let usedMax = config.map(c => c.overrideMax).filter(isDefined).at(-1) ?? max;
+                value = Math.min(value, usedMax);
+
+                try {
+                    let result = await FastArchiveAppendableController.nodes[syncInfo.nodeId]
+                        .onSynchronizeProgress(syncId, { section, value, max: usedMax });
+                    if (result === "stop") {
+                        cancelled = new Error(`Sync for id ${syncId} was stopped by the client`);
+                        return;
+                    }
+                } catch (e: any) {
+                    cancelled = e;
+                }
+            }
+        );
+        let onProgress = (value: number, overrideMax?: number) => {
+            if (!syncId) return;
+            if (syncId.startsWith("no-progress")) return;
+            if (cancelled) throw cancelled;
+            void baseBatch({ value, overrideMax });
+        };
+        onProgress(0);
+        return onProgress;
+    }
+
+    /** Progress callback - called by server on client */
+    public async onSynchronizeProgress(syncId: string, progress: {
+        section: string;
+        value: number;
+        max: number;
+    }): Promise<"stop" | undefined> {
+        const callback = FastArchiveAppendableControllerBase.progressCallbacks.get(syncId);
+        if (!callback) {
+            return "stop";
+        }
+        callback(progress);
+    }
+}
+
+export const FastArchiveAppendableController = SocketFunction.register(
+    "FastArchiveAppendableController-b8c9e4d5-1f2a-4b6c-8d7e-9f0a1b2c3d4e",
+    new FastArchiveAppendableControllerBase(),
+    () => ({
+        getPendingFiles: { hooks: [assertIsManagementUser] },
+        // Secured via syncId (can't use assertIsManagementUser, because it is an HTTP call, so there is no negotiation step)
+        downloadLocalFile: {},
+        downloadLocalFileInternal: { hooks: [assertIsManagementUser] },
+        startSynchronize: { hooks: [assertIsManagementUser] },
+        updateProgress: { hooks: [assertIsManagementUser] },
+        onSynchronizeProgress: { hooks: [assertIsManagementUser] },
+        isNodeAlive: { hooks: [assertIsNetworkTrusted] },
+        createSyncSession: { hooks: [assertIsManagementUser] },
+    }),
+    () => ({
+
+    }),
+);
+
+export const fastArchiveAppendableController = getSyncedController(FastArchiveAppendableController);
+