querysub 0.374.0 → 0.375.0

package/src/diagnostics/logs/FastArchiveController.ts

@@ -1,573 +0,0 @@
-module.hotreload = true;
-module.noserverhotreload = false;
-import { measureBlock, measureFnc, measureWrap } from "socket-function/src/profiling/measure";
-import { getMachineId, getOwnMachineId } from "../../-a-auth/certs";
-import { isDefined, parseFileNameKVP, parsePath, partialCopyObject, streamToIteratable, sum, toFileNameKVP } from "../../misc";
-import { registerShutdownHandler } from "../periodic";
-import { batchFunction, delay, runInSerial, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
-import { PromiseObj, isNode, keyByArray, nextId, sort, timeInDay, timeInHour, timeInMinute } from "socket-function/src/misc";
-import os from "os";
-import { getOwnThreadId } from "../../-f-node-discovery/NodeDiscovery";
-import fs from "fs";
-import { MaybePromise, canHaveChildren } from "socket-function/src/types";
-import { formatNumber, formatTime } from "socket-function/src/formatting/format";
-import { cache, lazy } from "socket-function/src/caching";
-import { getArchives, nestArchives } from "../../-a-archives/archives";
-import { Zip } from "../../zip";
-import { SocketFunction } from "socket-function/SocketFunction";
-import { assertIsManagementUser } from "../managementPages";
-import { getControllerNodeIdList } from "../../-g-core-values/NodeCapabilities";
-import { errorToUndefined, ignoreErrors, timeoutToUndefinedSilent } from "../../errors";
-import { getCallObj } from "socket-function/src/nodeProxy";
-import { getSyncedController } from "../../library-components/SyncedController";
-import { getBrowserUrlNode, getOwnNodeId } from "../../-f-node-discovery/NodeDiscovery";
-import { secureRandom } from "../../misc/random";
-import { getPathIndex, getPathStr2 } from "../../path";
-import { onNextPaint } from "../../functional/onNextPaint";
-import { getArchivesBackblazePrivateImmutable, getArchivesBackblazePublicImmutable } from "../../-a-archives/archivesBackBlaze";
-import { httpsRequest } from "socket-function/src/https";
-import { getDomain, isPublic } from "../../config";
-import { getIPDomain } from "../../-e-certs/EdgeCertController";
-import { getArchivesPrivateFileSystem } from "../../-a-archives/archivesPrivateFileSystem";
-import { createArchivesLimitedCache } from "../../-a-archives/archivesLimitedCache";
-import { sha256 } from "js-sha256";
-import { assertIsNetworkTrusted } from "../../-d-trust/NetworkTrust2";
-import { blue, magenta } from "socket-function/src/formatting/logColors";
-import { FastArchiveAppendable, getFileTimeStamp } from "./FastArchiveAppendable";
-import { IdentityController_getMachineId, IdentityController_getReconnectNodeId } from "../../-c-identity/IdentityController";
-import { fsExistsAsync } from "../../fs";
-import { Querysub } from "../../4-querysub/QuerysubController";
-
-export type FileMetadata = {
-    nodeId?: string;
-    machineId?: string;
-    path: string;
-    url: string;
-    size: number;
-    startTime: number;
-    endTime: number;
-};
-
-type InternalFileMetadata = {
-    path: string;
-    size: number;
-};
-
-type SynchronizeInfo = {
-    id: string;
-    nodeId: string;
-    config: {
-        range: {
-            startTime: number;
-            endTime: number;
-        };
-        rootPath: string;
-    };
-    createTime: number;
-};
-
-// Excluding authorize tokens, then hashed
-export function getFileMetadataHash(file: FileMetadata): string {
-    // NOTE: This size will be at least equal or older to the size when we read it. And so for the file to have new data, it means the size will have to be larger. And because this size is always equal older to when we read it and never newer, it means it won't include the newer writes. And so even for local files which are constantly changing, this will properly invalidate when we write to them again. 
-    return sha256(JSON.stringify({ path: file.path, size: file.size, machineId: file.machineId }));
-}
-
-export class FastArchiveAppendableControllerBase {
-    private static activeSynchronizes = new Map<string, SynchronizeInfo>();
-    public static progressCallbacks = new Map<string, (progress: { section: string; value: number; max: number; }) => void>();
-
-    /** Get all pending local files for this rootPath that haven't been uploaded to Backblaze yet */
-    public async getPendingFiles(rootPath: string, timeRange: { startTime: number; endTime: number; }): Promise<InternalFileMetadata[]> {
-
-        let rootCacheFolder = new FastArchiveAppendable(rootPath).baseGetLocalPathRoot();
-        if (!await fsExistsAsync(rootCacheFolder)) return [];
-
-        //console.log(`Searching for pending files in ${rootCacheFolder} for time range ${new Date(timeRange.startTime).toISOString()} to ${new Date(timeRange.endTime).toISOString()}`);
-
-        let result: InternalFileMetadata[] = [];
-
-        try {
-            let allFolders = await fs.promises.readdir(rootCacheFolder);
-            for (let threadId of allFolders) {
-                let threadDir = rootCacheFolder + threadId + "/";
-                let files = await fs.promises.readdir(threadDir);
-
-                for (let file of files) {
-                    if (file === "heartbeat") continue;
-
-                    let fileTimestamp = getFileTimeStamp(file);
-                    //console.log(`Found ${new Date(fileTimestamp).toISOString()} in ${threadDir + file}`);
-                    if (fileTimestamp.endTime < timeRange.startTime || fileTimestamp.startTime > timeRange.endTime) {
-                        continue;
-                    }
-
-                    let fullPath = threadDir + file;
-                    let stat = await fs.promises.stat(fullPath);
-
-                    result.push({
-                        path: fullPath,
-                        size: stat.size,
-                    });
-                }
-            }
-        } catch (e: any) {
-            // If directory doesn't exist, return empty array
-            if (e.code === "ENOENT") {
-                return [];
-            }
-            throw e;
-        }
-
-        return result;
-    }
-
-    /** Download a local file from another server (coordinator forwards request).
-     *      NOTE: This doesn't support streaming, which isn't great. However, eventually most logs will make their way to backblaze. So not supporting streaming here doesn't mean we don't support streaming in general. We also compress the file. So for the compressed file to be so big that you can't send it all at once would mean your uncompressed data would just be ridiculously large. 
-     */
-    public async downloadLocalFile(syncId: string, targetNodeId: string, path: string, fromLocalArchives?: boolean): Promise<Buffer> {
-        const caller = SocketFunction.getCaller();
-        let syncInfo = FastArchiveAppendableControllerBase.activeSynchronizes.get(syncId);
-        if (!syncInfo) {
-            throw new Error(`Invalid sync ID: ${syncId}`);
-        }
-
-        if (fromLocalArchives) {
-            let archives = new FastArchiveAppendable(syncInfo.config.rootPath).getArchives(false);
-            let file = await archives.get(path);
-            if (!file) {
-                throw new Error(`File not found in local archives: ${path}`);
-            }
-            return file;
-        }
-        try {
-            // Forward the request to the actual server that has the file
-            let targetController = FastArchiveAppendableController.nodes[targetNodeId];
-            return await targetController.downloadLocalFileInternal(path);
-        } catch {
-            let machineId = getMachineId(targetNodeId);
-            let nodeIds = await getControllerNodeIdList(FastArchiveAppendableController);
-            let byMachineId = keyByArray(nodeIds, x => getMachineId(x.nodeId));
-            let nodeIdsOfMachine = byMachineId.get(machineId) || [];
-
-            let firstAliveNode = new PromiseObj<string>();
-            let allFinished = Promise.all(nodeIdsOfMachine.map(async ({ nodeId, entryPoint }) => {
-                if (await timeoutToUndefinedSilent(5000, FastArchiveAppendableController.nodes[nodeId].isNodeAlive(nodeId))) {
-                    firstAliveNode.resolve(nodeId);
-                }
-            }));
-            let aliveNodeId = await Promise.race([firstAliveNode.promise, allFinished]);
-            if (Array.isArray(aliveNodeId)) {
-                throw new Error(`No alive node found for machine ${machineId}`);
-            }
-            console.log(magenta(`Remapped request for ${targetNodeId} to ${aliveNodeId} for machine ${machineId}`));
-            let targetController = FastArchiveAppendableController.nodes[aliveNodeId];
-            return await targetController.downloadLocalFileInternal(path);
-        }
-    }
-
-    /** Internal method to download local file (called by coordinator) */
-    public async downloadLocalFileInternal(path: string): Promise<Buffer> {
-        return await Zip.gzip(await fs.promises.readFile(path));
-    }
-    public async isNodeAlive(nodeId: string): Promise<boolean> {
-        return true;
-    }
-
-    /** Create a new sync session and return the sync ID */
-    public async createSyncSession(): Promise<string> {
-        let caller = SocketFunction.getCaller();
-        let syncId = nextId();
-
-        let syncInfo: SynchronizeInfo = {
-            id: syncId,
-            nodeId: caller.nodeId,
-            config: {
-                range: { startTime: 0, endTime: 0 }, // Will be set by startSynchronize
-                rootPath: "",
-            },
-            createTime: Date.now(),
-        };
-
-        FastArchiveAppendableControllerBase.activeSynchronizes.set(syncId, syncInfo);
-
-        // Clean up on disconnect
-        SocketFunction.onNextDisconnect(caller.nodeId, () => {
-            FastArchiveAppendableControllerBase.activeSynchronizes.delete(syncId);
-        });
-
-        return syncId;
-    }
-
-    public async startSynchronize(config: {
-        syncId: string;
-        range: {
-            startTime: number;
-            endTime: number;
-        };
-        rootPath: string;
-        noLocalFiles?: boolean;
-        forceGetPublic?: boolean;
-    }): Promise<{
-        files: FileMetadata[];
-    }> {
-        let caller = SocketFunction.getCaller();
-        // If the user uses a bad syncId and it gets leak, screw it.
-        let syncId = config.syncId;
-
-        let syncInfo: SynchronizeInfo = {
-            id: syncId,
-            nodeId: caller.nodeId,
-            config: {
-                range: config.range,
-                rootPath: config.rootPath,
-            },
-            createTime: Date.now(),
-        };
-
-        FastArchiveAppendableControllerBase.activeSynchronizes.set(syncId, syncInfo);
-
-        // Clean up on disconnect
-        SocketFunction.onNextDisconnect(caller.nodeId, () => {
-            FastArchiveAppendableControllerBase.activeSynchronizes.delete(syncId);
-        });
-
-        return await this.startSynchronizeBase({
-            syncId,
-            range: config.range,
-            rootPath: config.rootPath,
-            noLocalFiles: config.noLocalFiles,
-            forceGetPublic: config.forceGetPublic,
-        });
-    }
-
-    public async startSynchronizeInternal(config: {
-        range: {
-            startTime: number;
-            endTime: number;
-        };
-        rootPath: string;
-        noLocalFiles?: boolean;
-        forceGetPublic?: boolean;
-    }): Promise<{
-        files: FileMetadata[];
-    }> {
-        let syncId = "no-progress" + nextId();
-        FastArchiveAppendableControllerBase.activeSynchronizes.set(syncId, {
-            id: syncId,
-            nodeId: getOwnNodeId(),
-            config: {
-                range: config.range,
-                rootPath: config.rootPath,
-            },
-            createTime: Date.now(),
-        });
-        return await this.startSynchronizeBase({
-            syncId,
-            range: config.range,
-            rootPath: config.rootPath,
-            noLocalFiles: config.noLocalFiles,
-            forceGetPublic: config.forceGetPublic,
-        });
-    }
-
-    private async startSynchronizeBase(config: {
-        syncId?: string;
-        range: {
-            startTime: number;
-            endTime: number;
-        };
-        rootPath: string;
-        noLocalFiles?: boolean;
-        forceGetPublic?: boolean;
-    }): Promise<{
-        files: FileMetadata[];
-    }> {
-        if (!SocketFunction.mountedNodeId) {
-            throw new Error(`Cannot use FastArchiveAppendableController before SocketFunction is mounted`);
-        }
-        let syncId = config.syncId ?? "";
-
-        let isPublicValue = isPublic() || config.forceGetPublic;
-
-        const getLocalFileMetadata = async (config: {
-            nodeId: string;
-            path: string;
-            size: number;
-            fromLocalArchives?: boolean;
-        }): Promise<FileMetadata> => {
-            let { nodeId, path, size, fromLocalArchives } = config;
-            // Create download URL that points to the coordinator (this server)
-            // The coordinator will forward the request to the actual file server
-            let coordinatorNodeId = getOwnNodeId();
-            let coordinatorController = FastArchiveAppendableController.nodes[coordinatorNodeId];
-            let downloadCall = coordinatorController.downloadLocalFile[getCallObj](
-                syncId,
-                nodeId, // Target server that has the file
-                path,
-                fromLocalArchives,
-            );
-            let url = SocketFunction.getHTTPCallLink(downloadCall);
-            // Have to use the IP domain, as it's externally available. That, plus the port, should uniquely identify us.
-            let ipDomain = await getIPDomain();
-            let urlObj = new URL(url);
-            urlObj.hostname = ipDomain;
-            url = urlObj.toString();
-
-            let timeStamp = getFileTimeStamp(path);
-            let startTime = timeStamp.startTime;
-            let endTime = timeStamp.endTime;
-
-            return {
-                nodeId: nodeId,
-                machineId: getMachineId(nodeId),
-                path: path,
-                url: url,
-                size: size,
-                startTime: startTime,
-                endTime: endTime,
-            };
-        };
-
-        // Define inline functions for parallel execution
-        const searchBackblazeFiles = async (): Promise<FileMetadata[]> => {
-            let archives = new FastArchiveAppendable(config.rootPath).getArchives(config.forceGetPublic ?? false);
-            let backblazeFiles: FileMetadata[] = [];
-            if (!archives.getDownloadAuthorization && isPublicValue) {
-                throw new Error(`archives.getDownloadAuthorization is `);
-            }
-
-            if (!archives.getURL && isPublicValue) throw new Error(`archives.getURL is missing?`);
-            let authorization = await archives.getDownloadAuthorization?.({
-                validDurationInSeconds: timeInDay * 6 / 1000,
-            });
-            let authToken = authorization?.authorizationToken;
-
-            const folderProgress = this.updateProgress(syncId, "Backblaze folder search", 0);
-            let folderMax = 0;
-            let folderValue = 0;
-
-            async function searchBackblazeFilesRecursive(prefix: string, level: "year" | "month" | "day" | "hour") {
-                folderMax++;
-                let folders = await archives.find(prefix, { shallow: true, type: "folders" });
-                folderValue++;
-                folderProgress(folderValue, folderMax);
-                // It's actually really annoying because the time ranges aren't the same amount of time. It's probably fine though. This code only needs to exist in one place. 
-
-                await Promise.all(folders.map(async (folder) => {
-                    let folderName = folder.split("/").pop()!;
-                    let folderNum = parseInt(folderName, 10);
-
-                    if (level === "year") {
-                        let yearStart = Date.UTC(folderNum, 0, 1);
-                        let yearEnd = Date.UTC(folderNum + 1, 0, 1);
-
-                        if (yearEnd > config.range.startTime && yearStart <= config.range.endTime) {
-                            await searchBackblazeFilesRecursive(folder + "/", "month");
-                        }
-                    } else if (level === "month") {
-                        // folderName is 1-based month (01, 02, etc)
-                        let year = parseInt(folder.split("/")[0], 10);
-                        let monthStart = Date.UTC(year, folderNum - 1, 1);
-                        let monthEnd = Date.UTC(year, folderNum, 1);
-
-                        if (monthEnd > config.range.startTime && monthStart <= config.range.endTime) {
-                            await searchBackblazeFilesRecursive(folder + "/", "day");
-                        }
-                    } else if (level === "day") {
-                        let pathParts = folder.split("/");
-                        let year = parseInt(pathParts[0], 10);
-                        let month = parseInt(pathParts[1], 10) - 1; // Date constructor expects 0-based month
-                        let dayStart = Date.UTC(year, month, folderNum);
-                        let dayEnd = Date.UTC(year, month, folderNum + 1);
-
-                        if (dayEnd > config.range.startTime && dayStart <= config.range.endTime) {
-                            await searchBackblazeFilesRecursive(folder + "/", "hour");
-                        }
-                    } else if (level === "hour") {
-                        let pathParts = folder.split("/");
-                        let year = parseInt(pathParts[0], 10);
-                        let month = parseInt(pathParts[1], 10) - 1;
-                        let day = parseInt(pathParts[2], 10);
-                        let hourStart = Date.UTC(year, month, day, folderNum);
-                        let hourEnd = Date.UTC(year, month, day, folderNum + 1);
-
-                        let inRange = hourEnd > config.range.startTime && hourStart <= config.range.endTime;
-                        if (!inRange) return;
-                        // This hour folder is in range, get all files from it
-                        let filePaths = await archives.findInfo(folder + "/", { shallow: true, type: "files" });
-                        for (let info of filePaths) {
-                            if (!info.path.endsWith(".log")) continue;
-                            if (archives.getURL && authToken !== undefined) {
-                                let url = await archives.getURL(info.path);
-                                let urlObj = new URL(url);
-                                // IMPORTANT! This is CASE SENSITIVE! Ugh...
-                                urlObj.searchParams.set("Authorization", authToken);
-                                url = urlObj.toString();
-
-                                backblazeFiles.push({
-                                    path: info.path,
-                                    url: url,
-                                    size: info.size,
-                                    startTime: hourStart,
-                                    endTime: hourEnd,
-                                });
-                            } else {
-                                backblazeFiles.push(await getLocalFileMetadata({
-                                    nodeId: getOwnNodeId(),
-                                    path: info.path,
-                                    size: info.size,
-                                    fromLocalArchives: true,
-                                }));
-                            }
-                        }
-                    }
-                }));
-            }
-
-            await searchBackblazeFilesRecursive("", "year");
-
-            return backblazeFiles;
-        };
-
-        // NOTE: Disk files are on our machine, and on every other server (as we want to search live logs as well).
-        const getDiskFiles = async (): Promise<FileMetadata[]> => {
-            const getControllerProgress = this.updateProgress(syncId, "Discovering remote machines", 0);
-
-            let localFiles: FileMetadata[] = [];
-
-            let nodeIds = await getControllerNodeIdList(FastArchiveAppendableController);
-            // This filtering is somewhat of a hack, as machines can have mixed public status, and change it easliy. However in practice... they don't... We COULD have two sets of pending logs, for public and non-public (like we have backblaze and local archives for non-pending logs). But... at the moment... I don't think it's needed.
-            let ownMachineId = getOwnMachineId();
-            if (!isPublicValue) {
-                nodeIds = nodeIds.filter(x => getMachineId(x.nodeId) === ownMachineId);
-            } else if (isPublicValue && !isPublic()) {
-                nodeIds = nodeIds.filter(x => getMachineId(x.nodeId) !== ownMachineId);
-            }
-            let byMachineId = keyByArray(nodeIds, x => getMachineId(x.nodeId));
-            getControllerProgress(byMachineId.size, byMachineId.size);
-
-            let remoteProgress = this.updateProgress(syncId, "Discovering remote files", byMachineId.size);
-            let remoteValue = 0;
-
-
-            await Promise.all(Array.from(byMachineId).map(async ([machineId, nodeObjs]) => {
-                let firstAliveNode = new PromiseObj<string>();
-                let allFinished = Promise.all(nodeObjs.map(async ({ nodeId, entryPoint }) => {
-                    if (await timeoutToUndefinedSilent(5000, FastArchiveAppendableController.nodes[nodeId].isNodeAlive(nodeId))) {
-                        firstAliveNode.resolve(nodeId);
-                    }
-                }));
-                let aliveNodeId = await Promise.race([firstAliveNode.promise, allFinished]);
-                if (Array.isArray(aliveNodeId)) {
-                    console.log(blue(`No alive nodes found for machine ${machineId}`), nodeObjs);
-                    remoteValue++;
-                    remoteProgress(remoteValue);
-                    return;
-                }
-
-                let controller = FastArchiveAppendableController.nodes[aliveNodeId];
-
-                let pendingFiles = await errorToUndefined(
-                    controller.getPendingFiles(config.rootPath, config.range)
-                );
-                console.log(blue(`Found ${pendingFiles?.length} pending files on node ${aliveNodeId}`));
-
-                remoteValue++;
-                remoteProgress(remoteValue);
-
-                if (!pendingFiles) return;
-                for (let file of pendingFiles) {
-                    localFiles.push(await getLocalFileMetadata({ nodeId: aliveNodeId, path: file.path, size: file.size }));
-                }
-            }));
-
-            return localFiles;
-        };
-        // Execute both operations in parallel
-        let filePromises: Promise<FileMetadata[]>[] = [];
-        filePromises.push(searchBackblazeFiles());
-        if (!config.noLocalFiles) filePromises.push(getDiskFiles());
-
-        let allFilesList = await Promise.all(filePromises);
-        let allFiles = allFilesList.flat();
-        // Newest first, so recent errors are found quickly
-        sort(allFiles, x => -x.startTime);
-
-        return {
-            files: allFiles,
-        };
-    }
-
-    /** Update progress for a synchronization session - returns a function that takes only the progress value */
-    public updateProgress(syncId: string, section: string, max: number) {
-        let cancelled: Error | undefined;
-        let baseBatch = batchFunction({ delay: 150, },
-            async (config: { value: number, overrideMax?: number }[]) => {
-                if (cancelled) return;
-                let syncInfo = FastArchiveAppendableControllerBase.activeSynchronizes.get(syncId);
-                if (!syncInfo) {
-                    cancelled = new Error(`Cancelled (sync for id ${syncId} is missing)`);
-                    return;
-                }
-
-                let value = config.at(-1)!.value;
-                let usedMax = config.map(c => c.overrideMax).filter(isDefined).at(-1) ?? max;
-                value = Math.min(value, usedMax);
-
-                try {
-                    let result = await FastArchiveAppendableController.nodes[syncInfo.nodeId]
-                        .onSynchronizeProgress(syncId, { section, value, max: usedMax });
-                    if (result === "stop") {
-                        cancelled = new Error(`Sync for id ${syncId} was stopped by the client`);
-                        return;
-                    }
-                } catch (e: any) {
-                    cancelled = e;
-                }
-            }
-        );
-        let onProgress = (value: number, overrideMax?: number) => {
-            if (!syncId) return;
-            if (syncId.startsWith("no-progress")) return;
-            if (cancelled) throw cancelled;
-            void baseBatch({ value, overrideMax });
-        };
-        onProgress(0);
-        return onProgress;
-    }
-
-    /** Progress callback - called by server on client */
-    public async onSynchronizeProgress(syncId: string, progress: {
-        section: string;
-        value: number;
-        max: number;
-    }): Promise<"stop" | undefined> {
-        const callback = FastArchiveAppendableControllerBase.progressCallbacks.get(syncId);
-        if (!callback) {
-            return "stop";
-        }
-        callback(progress);
-    }
-}
-
-export const FastArchiveAppendableController = SocketFunction.register(
-    "FastArchiveAppendableController-b8c9e4d5-1f2a-4b6c-8d7e-9f0a1b2c3d4e",
-    new FastArchiveAppendableControllerBase(),
-    () => ({
-        getPendingFiles: { hooks: [assertIsManagementUser] },
-        // Secured via syncId (can't use assertIsManagementUser, because it is an HTTP call, so there is no negotiation step)
-        downloadLocalFile: {},
-        downloadLocalFileInternal: { hooks: [assertIsManagementUser] },
-        startSynchronize: { hooks: [assertIsManagementUser] },
-        updateProgress: { hooks: [assertIsManagementUser] },
-        onSynchronizeProgress: { hooks: [assertIsManagementUser] },
-        isNodeAlive: { hooks: [assertIsNetworkTrusted] },
-        createSyncSession: { hooks: [assertIsManagementUser] },
-    }),
-    () => ({
-
-    }),
-);
-
-export const fastArchiveAppendableController = getSyncedController(FastArchiveAppendableController);
-