querysub 0.356.0 → 0.358.0

package/src/diagnostics/logs/FastArchiveController.ts

@@ -26,7 +26,7 @@ import { getPathIndex, getPathStr2 } from "../../path";
 import { onNextPaint } from "../../functional/onNextPaint";
 import { getArchivesBackblazePrivateImmutable, getArchivesBackblazePublicImmutable } from "../../-a-archives/archivesBackBlaze";
 import { httpsRequest } from "socket-function/src/https";
-import { getDomain } from "../../config";
+import { getDomain, isPublic } from "../../config";
 import { getIPDomain } from "../../-e-certs/EdgeCertController";
 import { getArchivesPrivateFileSystem } from "../../-a-archives/archivesPrivateFileSystem";
 import { createArchivesLimitedCache } from "../../-a-archives/archivesLimitedCache";
@@ -40,6 +40,7 @@ import { Querysub } from "../../4-querysub/QuerysubController";
 
 export type FileMetadata = {
     nodeId?: string;
+    machineId?: string;
     path: string;
     url: string;
     size: number;
@@ -67,16 +68,8 @@ type SynchronizeInfo = {
 
 // Excluding authorize tokens, then hashed
 export function getFileMetadataHash(file: FileMetadata): string {
-    let urlObj = new URL(file.url);
-    if (!file.nodeId) {
-        urlObj.search = "";
-    } else {
-        let args = JSON.parse(urlObj.searchParams.get("args") || "");
-        // The syncId shouldn't be part of the hash though
-        args[0] = "";
-        urlObj.searchParams.set("args", JSON.stringify(args));
-    }
-    return sha256(urlObj.toString());
+    // NOTE: This size will be at least equal or older to the size when we read it. And so for the file to have new data, it means the size will have to be larger. And because this size is always equal older to when we read it and never newer, it means it won't include the newer writes. And so even for local files which are constantly changing, this will properly invalidate when we write to them again. 
+    return sha256(JSON.stringify({ path: file.path, size: file.size, machineId: file.machineId }));
 }
 
 export class FastArchiveAppendableControllerBase {
@@ -131,13 +124,21 @@ export class FastArchiveAppendableControllerBase {
     /** Download a local file from another server (coordinator forwards request).
      *      NOTE: This doesn't support streaming, which isn't great. However, eventually most logs will make their way to backblaze. So not supporting streaming here doesn't mean we don't support streaming in general. We also compress the file. So for the compressed file to be so big that you can't send it all at once would mean your uncompressed data would just be ridiculously large. 
      */
-    public async downloadLocalFile(syncId: string, targetNodeId: string, path: string): Promise<Buffer> {
+    public async downloadLocalFile(syncId: string, targetNodeId: string, path: string, fromLocalArchives?: boolean): Promise<Buffer> {
         const caller = SocketFunction.getCaller();
         let syncInfo = FastArchiveAppendableControllerBase.activeSynchronizes.get(syncId);
         if (!syncInfo) {
             throw new Error(`Invalid sync ID: ${syncId}`);
         }
 
+        if (fromLocalArchives) {
+            let archives = new FastArchiveAppendable(syncInfo.config.rootPath).getArchives(false);
+            let file = await archives.get(path);
+            if (!file) {
+                throw new Error(`File not found in local archives: ${path}`);
+            }
+            return file;
+        }
         try {
             // Forward the request to the actual server that has the file
             let targetController = FastArchiveAppendableController.nodes[targetNodeId];
@@ -205,6 +206,7 @@ export class FastArchiveAppendableControllerBase {
         };
         rootPath: string;
         noLocalFiles?: boolean;
+        forceGetPublic?: boolean;
     }): Promise<{
         files: FileMetadata[];
     }> {
@@ -234,6 +236,7 @@ export class FastArchiveAppendableControllerBase {
             range: config.range,
             rootPath: config.rootPath,
             noLocalFiles: config.noLocalFiles,
+            forceGetPublic: config.forceGetPublic,
         });
     }
 
@@ -244,6 +247,7 @@ export class FastArchiveAppendableControllerBase {
         };
         rootPath: string;
         noLocalFiles?: boolean;
+        forceGetPublic?: boolean;
     }): Promise<{
         files: FileMetadata[];
     }> {
@@ -262,6 +266,7 @@ export class FastArchiveAppendableControllerBase {
             range: config.range,
             rootPath: config.rootPath,
             noLocalFiles: config.noLocalFiles,
+            forceGetPublic: config.forceGetPublic,
         });
     }
 
@@ -273,6 +278,7 @@ export class FastArchiveAppendableControllerBase {
         };
         rootPath: string;
         noLocalFiles?: boolean;
+        forceGetPublic?: boolean;
     }): Promise<{
         files: FileMetadata[];
     }> {
@@ -281,15 +287,60 @@ export class FastArchiveAppendableControllerBase {
         }
         let syncId = config.syncId ?? "";
 
+        let isPublicValue = isPublic() || config.forceGetPublic;
+
+        const getLocalFileMetadata = async (config: {
+            nodeId: string;
+            path: string;
+            size: number;
+            fromLocalArchives?: boolean;
+        }): Promise<FileMetadata> => {
+            let { nodeId, path, size, fromLocalArchives } = config;
+            // Create download URL that points to the coordinator (this server)
+            // The coordinator will forward the request to the actual file server
+            let coordinatorNodeId = getOwnNodeId();
+            let coordinatorController = FastArchiveAppendableController.nodes[coordinatorNodeId];
+            let downloadCall = coordinatorController.downloadLocalFile[getCallObj](
+                syncId,
+                nodeId, // Target server that has the file
+                path,
+                fromLocalArchives,
+            );
+            let url = SocketFunction.getHTTPCallLink(downloadCall);
+            // Have to use the IP domain, as it's externally available. That, plus the port, should uniquely identify us.
+            let ipDomain = await getIPDomain();
+            let urlObj = new URL(url);
+            urlObj.hostname = ipDomain;
+            url = urlObj.toString();
+
+            let timeStamp = getFileTimeStamp(path);
+            let startTime = timeStamp.startTime;
+            let endTime = timeStamp.endTime;
+
+            return {
+                nodeId: nodeId,
+                machineId: getMachineId(nodeId),
+                path: path,
+                url: url,
+                size: size,
+                startTime: startTime,
+                endTime: endTime,
+            };
+        };
+
         // Define inline functions for parallel execution
         const searchBackblazeFiles = async (): Promise<FileMetadata[]> => {
-            let archives = new FastArchiveAppendable(config.rootPath).getArchives();
+            let archives = new FastArchiveAppendable(config.rootPath).getArchives(config.forceGetPublic ?? false);
             let backblazeFiles: FileMetadata[] = [];
-            if (!archives.getDownloadAuthorization) throw new Error(`archives.getDownloadAuthorization is missing?`);
-            let authorization = await archives.getDownloadAuthorization({
+            if (!archives.getDownloadAuthorization && isPublicValue) {
+                throw new Error(`archives.getDownloadAuthorization is `);
+            }
+
+            if (!archives.getURL && isPublicValue) throw new Error(`archives.getURL is missing?`);
+            let authorization = await archives.getDownloadAuthorization?.({
                 validDurationInSeconds: timeInDay * 6 / 1000,
             });
-            let authToken = authorization.authorizationToken;
+            let authToken = authorization?.authorizationToken;
 
             const folderProgress = this.updateProgress(syncId, "Backblaze folder search", 0);
             let folderMax = 0;
@@ -346,21 +397,28 @@ export class FastArchiveAppendableControllerBase {
                         let filePaths = await archives.findInfo(folder + "/", { shallow: true, type: "files" });
                         for (let info of filePaths) {
                             if (!info.path.endsWith(".log")) continue;
-
-                            if (!archives.getURL) throw new Error(`archives.getURL is missing?`);
-                            let url = await archives.getURL(info.path);
-                            let urlObj = new URL(url);
-                            // IMPORTANT! This is CASE SENSITIVE! Ugh...
-                            urlObj.searchParams.set("Authorization", authToken);
-                            url = urlObj.toString();
-
-                            backblazeFiles.push({
-                                path: info.path,
-                                url: url,
-                                size: info.size,
-                                startTime: hourStart,
-                                endTime: hourEnd,
-                            });
+                            if (archives.getURL && authToken !== undefined) {
+                                let url = await archives.getURL(info.path);
+                                let urlObj = new URL(url);
+                                // IMPORTANT! This is CASE SENSITIVE! Ugh...
+                                urlObj.searchParams.set("Authorization", authToken);
+                                url = urlObj.toString();
+
+                                backblazeFiles.push({
+                                    path: info.path,
+                                    url: url,
+                                    size: info.size,
+                                    startTime: hourStart,
+                                    endTime: hourEnd,
+                                });
+                            } else {
+                                backblazeFiles.push(await getLocalFileMetadata({
+                                    nodeId: getOwnNodeId(),
+                                    path: info.path,
+                                    size: info.size,
+                                    fromLocalArchives: true,
+                                }));
+                            }
                         }
                     }
                 }));
@@ -371,12 +429,20 @@ export class FastArchiveAppendableControllerBase {
             return backblazeFiles;
         };
 
-        const getRemoteFiles = async (): Promise<FileMetadata[]> => {
+        // NOTE: Disk files are on our machine, and on every other server (as we want to search live logs as well).
+        const getDiskFiles = async (): Promise<FileMetadata[]> => {
             const getControllerProgress = this.updateProgress(syncId, "Discovering remote machines", 0);
 
             let localFiles: FileMetadata[] = [];
 
             let nodeIds = await getControllerNodeIdList(FastArchiveAppendableController);
+            // This filtering is somewhat of a hack, as machines can have mixed public status, and change it easliy. However in practice... they don't... We COULD have two sets of pending logs, for public and non-public (like we have backblaze and local archives for non-pending logs). But... at the moment... I don't think it's needed.
+            let ownMachineId = getOwnMachineId();
+            if (!isPublicValue) {
+                nodeIds = nodeIds.filter(x => getMachineId(x.nodeId) === ownMachineId);
+            } else if (isPublicValue && !isPublic()) {
+                nodeIds = nodeIds.filter(x => getMachineId(x.nodeId) !== ownMachineId);
+            }
             let byMachineId = keyByArray(nodeIds, x => getMachineId(x.nodeId));
             getControllerProgress(byMachineId.size, byMachineId.size);
 
@@ -402,7 +468,8 @@ export class FastArchiveAppendableControllerBase {
                 let controller = FastArchiveAppendableController.nodes[aliveNodeId];
 
                 let pendingFiles = await errorToUndefined(
-                    controller.getPendingFiles(config.rootPath, config.range));
+                    controller.getPendingFiles(config.rootPath, config.range)
+                );
                 console.log(blue(`Found ${pendingFiles?.length} pending files on node ${aliveNodeId}`));
 
                 remoteValue++;
@@ -410,44 +477,16 @@ export class FastArchiveAppendableControllerBase {
 
                 if (!pendingFiles) return;
                 for (let file of pendingFiles) {
-                    // Create download URL that points to the coordinator (this server)
-                    // The coordinator will forward the request to the actual file server
-                    let coordinatorNodeId = getOwnNodeId();
-                    let coordinatorController = FastArchiveAppendableController.nodes[coordinatorNodeId];
-                    let downloadCall = coordinatorController.downloadLocalFile[getCallObj](
-                        syncId,
-                        aliveNodeId, // Target server that has the file
-                        file.path
-                    );
-                    let url = SocketFunction.getHTTPCallLink(downloadCall);
-                    // Have to use the IP domain, as it's externally available. That, plus the port, should uniquely identify us.
-                    let ipDomain = await getIPDomain();
-                    let urlObj = new URL(url);
-                    urlObj.hostname = ipDomain;
-                    url = urlObj.toString();
-
-                    let timeStamp = getFileTimeStamp(file.path);
-                    let startTime = timeStamp.startTime;
-                    let endTime = timeStamp.endTime;
-
-                    localFiles.push({
-                        nodeId: aliveNodeId,
-                        path: file.path,
-                        url: url,
-                        size: file.size,
-                        startTime: startTime,
-                        endTime: endTime,
-                    });
+                    localFiles.push(await getLocalFileMetadata({ nodeId: aliveNodeId, path: file.path, size: file.size }));
                 }
             }));
 
             return localFiles;
         };
-
         // Execute both operations in parallel
         let filePromises: Promise<FileMetadata[]>[] = [];
         filePromises.push(searchBackblazeFiles());
-        if (!config.noLocalFiles) filePromises.push(getRemoteFiles());
+        if (!config.noLocalFiles) filePromises.push(getDiskFiles());
 
         let allFilesList = await Promise.all(filePromises);
         let allFiles = allFilesList.flat();

package/src/diagnostics/logs/FastArchiveViewer.tsx

@@ -32,11 +32,11 @@ export const filterParam2 = new URLParam("filter2", "");
 export const cacheBustParam = new URLParam("cacheBust", 0);
 const caseInsensitiveParam = new URLParam("caseInsensitive", false);
 const hideAllDataParam = new URLParam("hideAllData", false);
-
 export type ScanFnc = (posStart: number, posEnd: number, data: Buffer) => boolean;
 
 export class FastArchiveViewer<T> extends qreact.Component<{
-    fastArchives: FastArchiveAppendable<T>[];
+    fastArchives: () => FastArchiveAppendable<T>[];
+    forceGetPublic?: boolean;
     runOnLoad?: boolean;
     onStart: () => MaybePromise<void>;
     getScanFnc?: () => ScanFnc;
@@ -83,6 +83,8 @@ export class FastArchiveViewer<T> extends qreact.Component<{
     private limitedScanCount = 0;
     private limitedMatchCount = 0;
 
+    private fileErrors: Array<{ file: FileMetadata; error: Error }> = [];
+
     private progressBars: Record<string, {
         section: string,
         value: number,
@@ -105,7 +107,7 @@ export class FastArchiveViewer<T> extends qreact.Component<{
     }
 
     cancelAllSynchronizes() {
-        for (let fastArchive of this.props.fastArchives) {
+        for (let fastArchive of this.props.fastArchives()) {
             fastArchive.cancelAllSynchronizes();
         }
     }
@@ -161,7 +163,7 @@ export class FastArchiveViewer<T> extends qreact.Component<{
                 filterString,
                 startTime: timeRange.startTime,
                 endTime: timeRange.endTime,
-                fastArchivePaths: fastArchives.map(archive => archive.rootPath),
+                fastArchivePaths: fastArchives().map(archive => archive.rootPath),
             };
         });
 
@@ -174,6 +176,7 @@ export class FastArchiveViewer<T> extends qreact.Component<{
                 delete this.progressBars[key];
             }
             this.state.fileMetadata = [];
+            this.fileErrors = [];
             this.limitedScanCount = 0;
             this.limitedMatchCount = 0;
             this.allSize = 0;
@@ -331,7 +334,7 @@ export class FastArchiveViewer<T> extends qreact.Component<{
             const limitedBuffer = Buffer.from(LOG_LIMIT_FLAG);
 
 
-            await Promise.all(fastArchives.map(async (fastArchive, index) => {
+            await Promise.all(fastArchives().map(async (fastArchive, index) => {
                 const result = await fastArchive.synchronizeData({
                     range: timeRange,
                     cacheBust: Querysub.fastRead(() => cacheBustParam.value),
@@ -407,6 +410,10 @@ export class FastArchiveViewer<T> extends qreact.Component<{
                             }
                         }
                     }),
+                    onError: (error, file) => {
+                        if (!isLatestSync()) return;
+                        this.fileErrors.push({ file, error });
+                    },
                     onProgress,
                     onFinish: () => {
                         ifLatest(() => {
@@ -615,7 +622,7 @@ export class FastArchiveViewer<T> extends qreact.Component<{
     private getOutdatedInfo(): string[] {
         const currentTimeRange = getTimeRange();
         const currentFilterString = filterParam.value;
-        const currentArchivePaths = this.props.fastArchives.map(archive => archive.rootPath);
+        const currentArchivePaths = this.props.fastArchives().map(archive => archive.rootPath);
 
         // Get stored sync parameters
         const storedParams = this.state.currentSyncParams;
@@ -735,6 +742,19 @@ export class FastArchiveViewer<T> extends qreact.Component<{
                     <div className={css.vbox(10)}>
                         {this.state.runCount > 0 && (() => {
                             const outdatedWarnings = this.getOutdatedInfo();
+                            const errorCount = this.fileErrors.length;
+
+                            // Group errors by file path
+                            const errorsByFile = new Map<string, number>();
+                            for (let errorInfo of this.fileErrors) {
+                                const path = errorInfo.file.path;
+                                errorsByFile.set(path, (errorsByFile.get(path) || 0) + 1);
+                            }
+
+                            const errorTooltip = Array.from(errorsByFile.entries())
+                                .map(([path, count]) => `${path}: ${count} errors`)
+                                .join("\n");
+
                             return (
                                 <div className={css.hbox(10).wrap}>
                                     <div
@@ -743,8 +763,16 @@ export class FastArchiveViewer<T> extends qreact.Component<{
                                             `${x.path} (${formatNumber(x.size)})`
                                         ).join("\n")}
                                     >
-                                        File count: {formatNumber(totalFileCount)}, Backblaze size: {formatNumber(totalBackblazeByteCount)}B (compressed), Disk size: {formatNumber(totalLocalByteCount)}B (uncompressed)
+                                        File count: {formatNumber(totalFileCount)}{errorCount > 0 && ` (${errorCount} erred)`}, Backblaze size: {formatNumber(totalBackblazeByteCount)}B (compressed), Disk size: {formatNumber(totalLocalByteCount)}B (uncompressed)
                                     </div>
+                                    {errorCount > 0 && (
+                                        <div
+                                            className={infoDisplay(45)}
+                                            title={errorTooltip}
+                                        >
+                                            {errorCount} file{errorCount > 1 ? "s" : ""} failed to load
+                                        </div>
+                                    )}
                                     {outdatedWarnings.length > 0 && (
                                         <div
                                             className={infoDisplay(30).button}
@@ -772,9 +800,9 @@ export class FastArchiveViewer<T> extends qreact.Component<{
                             </div>
                         )}
                         {this.state.finished && (() => {
-                            if (readLocalTimes.length === 0) return null;
 
                             const timeRange = getTimeRange();
+                            if (timeRange.endTime && readLocalTimes.length === 0) return null;
                             const earliestTime = Math.min(...readLocalTimes);
 
                             if (earliestTime >= timeRange.endTime) return null;