querysub 0.373.0 → 0.375.0

package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx

@@ -26,11 +26,10 @@ import { IndexedLogResults, createEmptyIndexedLogResults, mergeIndexedLogResults
 import { TimeFilePath } from "./TimeFileTree";
 import { isPublic } from "../../../config";
 import { Zip } from "../../../zip";
+import { readProductionLogsURL, searchTextURL } from "./LogViewerParams";
 
-let searchText = new URLParam("searchText", "");
 let excludePendingResults = new URLParam("excludePendingResults", false);
 let limitURL = new URLParam("limit", 100);
-let readPublicLogs = new URLParam("readPublicLogs", false);
 
 let savedPathsURL = new URLParam("savedPaths", "");
 let selectedFieldsURL = new URLParam("selectedFields", {} as Record<string, boolean>);
@@ -105,6 +104,218 @@ export class LogViewer3 extends qreact.Component {
         await this.loadPaths();
     }
 
+    async loadPaths() {
+        if (savedPathsURL.value) {
+            return;
+        }
+
+        Querysub.commitLocal(() => {
+            this.state.loadingPaths = true;
+            this.state.results = [];
+            this.state.stats = undefined;
+        });
+
+        let loggers = await getLoggers2Async();
+        let selectedLoggers: typeof loggers.logLogs[] = [];
+        Querysub.localRead(() => {
+            if (this.state.enableLogs) selectedLoggers.push(loggers.logLogs);
+            if (this.state.enableInfos) selectedLoggers.push(loggers.infoLogs);
+            if (this.state.enableWarnings) selectedLoggers.push(loggers.warnLogs);
+            if (this.state.enableErrors) selectedLoggers.push(loggers.errorLogs);
+        });
+
+        if (selectedLoggers.length === 0) {
+            console.error("No log sources selected");
+            Querysub.commitLocal(() => {
+                this.state.loadingPaths = false;
+            });
+            return;
+        }
+
+        let allPaths: TimeFilePathWithSize[] = [];
+        let range = Querysub.localRead(() => getTimeRange());
+
+        await Promise.all(selectedLoggers.map(async (logger) => {
+            let paths = await logger.clientGetPaths({
+                startTime: range.startTime,
+                endTime: range.endTime,
+                only: excludePendingResults.value ? "public" : undefined,
+                forceReadProduction: readProductionLogsURL.value,
+            });
+            allPaths.push(...paths);
+        }));
+        sort(allPaths, x => -x.startTime);
+
+        Querysub.commitLocal(() => {
+            this.state.paths = allPaths;
+            this.state.loadingPaths = false;
+        });
+        return allPaths;
+    }
+
+    async updateToLatestResults() {
+        let prevPaths = this.getPaths();
+        let newPaths = await this.loadPaths();
+        Querysub.commitLocal(() => {
+            function getHash(path: TimeFilePath): string {
+                return path.threadId + "_" + path.machineId;
+            }
+            let keep = new Set(prevPaths.map(getHash));
+            this.state.paths = newPaths?.filter(x => keep.has(getHash(x))) || [];
+        });
+        await this.search();
+    }
+
+    cancel = async () => {
+        this.searchSequenceNumber++;
+        let loggers = await getLoggers2Async();
+        for (let logger of Object.values(loggers)) {
+            logger.clientCancelAllCallbacks();
+        }
+        Querysub.commitLocal(() => {
+            this.state.searching = false;
+            this.state.hasSearched = false;
+        });
+    };
+
+
+    search = async () => {
+        await this.cancel();
+        Querysub.commitLocal(() => {
+            this.state.searching = true;
+            this.state.results = [];
+            this.state.stats = undefined;
+            this.state.searchingLogs = false;
+            this.state.searchingInfos = false;
+            this.state.searchingWarnings = false;
+            this.state.searchingErrors = false;
+        });
+
+
+        let loggers = await getLoggers2Async();
+
+        let selectedLoggers: typeof loggers.logLogs[] = [];
+        Querysub.localRead(() => {
+            if (this.state.enableLogs) {
+                selectedLoggers.push(loggers.logLogs);
+                this.state.searchingLogs = true;
+            }
+            if (this.state.enableInfos) {
+                selectedLoggers.push(loggers.infoLogs);
+                this.state.searchingInfos = true;
+            }
+            if (this.state.enableWarnings) {
+                selectedLoggers.push(loggers.warnLogs);
+                this.state.searchingWarnings = true;
+            }
+            if (this.state.enableErrors) {
+                selectedLoggers.push(loggers.errorLogs);
+                this.state.searchingErrors = true;
+            }
+        });
+
+        this.searchSequenceNumber++;
+        let currentSequenceNumber = this.searchSequenceNumber;
+
+        let startTime = Date.now();
+
+        let hasPaths = Querysub.localRead(() => this.getPaths().length > 0);
+        let getFilesTime = 0;
+        if (!hasPaths) {
+            let startTime = Date.now();
+            await this.loadPaths();
+            getFilesTime = Date.now() - startTime;
+        }
+
+
+        if (selectedLoggers.length === 0) {
+            console.error("No log sources selected");
+            Querysub.commitLocal(() => {
+                this.state.searching = false;
+            });
+            return;
+        }
+
+        let searchBuffer = Querysub.localRead(() => Buffer.from(searchTextURL.value, "utf8"));
+        let results: LogDatum[] = [];
+        let stats: IndexedLogResults = createEmptyIndexedLogResults();
+        stats.fileFindTime += getFilesTime;
+
+        let paths = Querysub.localRead(() => this.getPaths());
+
+        let range = Querysub.localRead(() => getTimeRange());
+
+        let updateResults = throttleFunction(100, () => {
+            if (this.searchSequenceNumber !== currentSequenceNumber) return;
+            Querysub.commitLocal(() => {
+                this.state.results = results;
+            });
+        });
+
+        let loggerResults = new Map<typeof loggers.logLogs, IndexedLogResults>();
+
+        let updateStats = () => {
+            if (this.searchSequenceNumber !== currentSequenceNumber) return;
+
+            let mergedStats: IndexedLogResults = createEmptyIndexedLogResults();
+
+            for (let loggerResult of loggerResults.values()) {
+                mergedStats = mergeIndexedLogResults(mergedStats, loggerResult);
+            }
+
+            mergedStats.totalSearchTime = Date.now() - startTime;
+
+            Querysub.commitLocal(() => {
+                this.state.stats = mergedStats;
+            });
+        };
+
+        await Promise.all(selectedLoggers.map(async (logger) => {
+            let done = false;
+            let result = await errorToUndefined(logger.clientFind({
+                params: {
+                    startTime: range.startTime,
+                    endTime: range.endTime,
+                    limit: limitURL.value,
+                    findBuffer: searchBuffer,
+                    pathOverrides: paths,
+                    only: excludePendingResults.value ? "local" : undefined,
+                    forceReadProduction: readProductionLogsURL.value,
+                },
+                onResult: (match: LogDatum) => {
+                    console.log("onResult", match);
+                    results.push(match);
+                    void updateResults();
+                },
+                onResults: (loggerStats: IndexedLogResults) => {
+                    if (done) return;
+                    loggerResults.set(logger, loggerStats);
+                    updateStats();
+                },
+            }));
+            done = true;
+
+            if (result) {
+                loggerResults.set(logger, result);
+                updateStats();
+            }
+
+            Querysub.commitLocal(() => {
+                if (logger === loggers.logLogs) this.state.searchingLogs = false;
+                if (logger === loggers.infoLogs) this.state.searchingInfos = false;
+                if (logger === loggers.warnLogs) this.state.searchingWarnings = false;
+                if (logger === loggers.errorLogs) this.state.searchingErrors = false;
+            });
+        }));
+
+
+        Querysub.commitLocal(() => {
+            this.state.results = results;
+            this.state.searching = false;
+            this.state.hasSearched = true;
+        });
+    };
+
     renderSearchStats() {
         const stats = this.state.stats;
         if (!stats) return undefined;
@@ -288,7 +499,7 @@ export class LogViewer3 extends qreact.Component {
                                 <span>is not moving logs to remote storage. {formatNumber(warning.count)} files ({formatNumber(warning.totalSize)}B) are {formatTime(now - warning.oldestTime)} old</span>
                             </div>
                         ))}
-                        {!isPublic() && !readPublicLogs.value && <Button
+                        {!isPublic() && !readProductionLogsURL.value && <Button
                             onClick={async () => {
                                 Querysub.commitLocal(() => {
                                     this.state.forceMoveStartTime = Date.now();
@@ -336,217 +547,6 @@ export class LogViewer3 extends qreact.Component {
         );
     }
 
-    async loadPaths() {
-        if (savedPathsURL.value) {
-            return;
-        }
-
-        Querysub.commitLocal(() => {
-            this.state.loadingPaths = true;
-            this.state.results = [];
-            this.state.stats = undefined;
-        });
-
-        let loggers = await getLoggers2Async();
-        let selectedLoggers: typeof loggers.logLogs[] = [];
-        Querysub.localRead(() => {
-            if (this.state.enableLogs) selectedLoggers.push(loggers.logLogs);
-            if (this.state.enableInfos) selectedLoggers.push(loggers.infoLogs);
-            if (this.state.enableWarnings) selectedLoggers.push(loggers.warnLogs);
-            if (this.state.enableErrors) selectedLoggers.push(loggers.errorLogs);
-        });
-
-        if (selectedLoggers.length === 0) {
-            console.error("No log sources selected");
-            Querysub.commitLocal(() => {
-                this.state.loadingPaths = false;
-            });
-            return;
-        }
-
-        let allPaths: TimeFilePathWithSize[] = [];
-        let range = Querysub.localRead(() => getTimeRange());
-
-        await Promise.all(selectedLoggers.map(async (logger) => {
-            let paths = await logger.clientGetPaths({
-                startTime: range.startTime,
-                endTime: range.endTime,
-                only: excludePendingResults.value ? "public" : undefined,
-                forceReadPublic: readPublicLogs.value,
-            });
-            allPaths.push(...paths);
-        }));
-        sort(allPaths, x => -x.startTime);
-
-        Querysub.commitLocal(() => {
-            this.state.paths = allPaths;
-            this.state.loadingPaths = false;
-        });
-        return allPaths;
-    }
-
-    async updateToLatestResults() {
-        let prevPaths = this.getPaths();
-        let newPaths = await this.loadPaths();
-        Querysub.commitLocal(() => {
-            function getHash(path: TimeFilePath): string {
-                return path.threadId + "_" + path.machineId;
-            }
-            let keep = new Set(prevPaths.map(getHash));
-            this.state.paths = newPaths?.filter(x => keep.has(getHash(x))) || [];
-        });
-        await this.search();
-    }
-
-    cancel = async () => {
-        this.searchSequenceNumber++;
-        let loggers = await getLoggers2Async();
-        for (let logger of Object.values(loggers)) {
-            logger.clientCancelAllCallbacks();
-        }
-        Querysub.commitLocal(() => {
-            this.state.searching = false;
-            this.state.hasSearched = false;
-        });
-    };
-
-
-    search = async () => {
-        await this.cancel();
-        Querysub.commitLocal(() => {
-            this.state.searching = true;
-            this.state.results = [];
-            this.state.stats = undefined;
-            this.state.searchingLogs = false;
-            this.state.searchingInfos = false;
-            this.state.searchingWarnings = false;
-            this.state.searchingErrors = false;
-        });
-
-
-        let loggers = await getLoggers2Async();
-
-        let selectedLoggers: typeof loggers.logLogs[] = [];
-        Querysub.localRead(() => {
-            if (this.state.enableLogs) {
-                selectedLoggers.push(loggers.logLogs);
-                this.state.searchingLogs = true;
-            }
-            if (this.state.enableInfos) {
-                selectedLoggers.push(loggers.infoLogs);
-                this.state.searchingInfos = true;
-            }
-            if (this.state.enableWarnings) {
-                selectedLoggers.push(loggers.warnLogs);
-                this.state.searchingWarnings = true;
-            }
-            if (this.state.enableErrors) {
-                selectedLoggers.push(loggers.errorLogs);
-                this.state.searchingErrors = true;
-            }
-        });
-
-        this.searchSequenceNumber++;
-        let currentSequenceNumber = this.searchSequenceNumber;
-
-        let startTime = Date.now();
-
-        let hasPaths = Querysub.localRead(() => this.getPaths().length > 0);
-        let getFilesTime = 0;
-        if (!hasPaths) {
-            let startTime = Date.now();
-            await this.loadPaths();
-            getFilesTime = Date.now() - startTime;
-        }
-
-
-        if (selectedLoggers.length === 0) {
-            console.error("No log sources selected");
-            Querysub.commitLocal(() => {
-                this.state.searching = false;
-            });
-            return;
-        }
-
-        let searchBuffer = Querysub.localRead(() => Buffer.from(searchText.value, "utf8"));
-        let results: LogDatum[] = [];
-        let stats: IndexedLogResults = createEmptyIndexedLogResults();
-        stats.fileFindTime += getFilesTime;
-
-        let paths = Querysub.localRead(() => this.getPaths());
-
-        let range = Querysub.localRead(() => getTimeRange());
-
-        let updateResults = throttleFunction(100, () => {
-            if (this.searchSequenceNumber !== currentSequenceNumber) return;
-            Querysub.commitLocal(() => {
-                this.state.results = results;
-            });
-        });
-
-        let loggerResults = new Map<typeof loggers.logLogs, IndexedLogResults>();
-
-        let updateStats = () => {
-            if (this.searchSequenceNumber !== currentSequenceNumber) return;
-
-            let mergedStats: IndexedLogResults = createEmptyIndexedLogResults();
-
-            for (let loggerResult of loggerResults.values()) {
-                mergedStats = mergeIndexedLogResults(mergedStats, loggerResult);
-            }
-
-            mergedStats.totalSearchTime = Date.now() - startTime;
-
-            Querysub.commitLocal(() => {
-                this.state.stats = mergedStats;
-            });
-        };
-
-        await Promise.all(selectedLoggers.map(async (logger) => {
-            let done = false;
-            let result = await errorToUndefined(logger.clientFind({
-                params: {
-                    startTime: range.startTime,
-                    endTime: range.endTime,
-                    limit: limitURL.value,
-                    findBuffer: searchBuffer,
-                    pathOverrides: paths,
-                    only: excludePendingResults.value ? "local" : undefined,
-                    forceReadPublic: readPublicLogs.value,
-                },
-                onResult: (match: LogDatum) => {
-                    console.log("onResult", match);
-                    results.push(match);
-                    void updateResults();
-                },
-                onResults: (loggerStats: IndexedLogResults) => {
-                    if (done) return;
-                    loggerResults.set(logger, loggerStats);
-                    updateStats();
-                },
-            }));
-            done = true;
-
-            if (result) {
-                loggerResults.set(logger, result);
-                updateStats();
-            }
-
-            Querysub.commitLocal(() => {
-                if (logger === loggers.logLogs) this.state.searchingLogs = false;
-                if (logger === loggers.infoLogs) this.state.searchingInfos = false;
-                if (logger === loggers.warnLogs) this.state.searchingWarnings = false;
-                if (logger === loggers.errorLogs) this.state.searchingErrors = false;
-            });
-        }));
-
-
-        Querysub.commitLocal(() => {
-            this.state.results = results;
-            this.state.searching = false;
-            this.state.hasSearched = true;
-        });
-    };
 
     renderResults() {
         let fieldNames = new Set<string>();
@@ -695,8 +695,8 @@ export class LogViewer3 extends qreact.Component {
                     />
                     {!isPublic() && <InputLabelURL
                         checkbox
-                        label="Read Public Logs"
-                        url={readPublicLogs}
+                        label="Read Production Logs"
+                        url={readProductionLogsURL}
                         onChangeValue={(newValue) => {
                             if (newValue) {
                                 savedPathsURL.value = "";
@@ -754,7 +754,7 @@ export class LogViewer3 extends qreact.Component {
                     </Button>
                     <TimeRangeSelector />
                 </div>
-                {!isPublic() && readPublicLogs.value && <div className={css.fontSize(40).pad2(12, 6).hsl(280, 40, 50).colorhsl(0, 0, 100).boldStyle}>
+                {!isPublic() && readProductionLogsURL.value && <div className={css.fontSize(40).pad2(12, 6).hsl(280, 40, 50).colorhsl(0, 0, 100).boldStyle}>
                     Reading public logs
                 </div>}
 
@@ -763,10 +763,10 @@ export class LogViewer3 extends qreact.Component {
                         flavor="large"
                         fillWidth
                         focusOnMount
-                        url={searchText}
+                        url={searchTextURL}
                         onKeyDown={(e) => {
                             if (e.key === "Enter") {
-                                searchText.value = e.currentTarget.value;
+                                searchTextURL.value = e.currentTarget.value;
                                 void this.search();
                             }
                         }}
@@ -776,6 +776,7 @@ export class LogViewer3 extends qreact.Component {
                             flavor="large"
                             onClick={() => void this.loadPaths()}
                             hue={this.state.paths.length ? 200 : 120}
+                            className={css + (this.state.loadingPaths && css.opacity(0.5))}
                         >
                             {this.getPaths().length && "Reset Files" || "Preview Files"}
                         </Button>
@@ -835,7 +836,7 @@ export class LogViewer3 extends qreact.Component {
                     >
                         {savedPathsURL.value ? `Clear Frozen Files (${this.getPaths().length})` : `Freeze Files (${this.state.paths.length})`}
                     </Button>
-                    {!savedPathsURL.value && (
+                    {savedPathsURL.value && (
                         <div className={css.pad2(8, 4).hsl(40, 60, 50).colorhsl(40, 0, 100)}>
                             Files frozen in memory. Click Preview Files to refresh.
                         </div>

package/src/diagnostics/logs/IndexedLogs/LogViewerParams.ts

@@ -0,0 +1,21 @@
+import { URLParam } from "../../../library-components/URLParam";
+import { managementPageURL } from "../../managementPages";
+import { LogDatum } from "../diskLogger";
+
+export let searchTextURL = new URLParam("searchText", "");
+export let readProductionLogsURL = new URLParam("readProductionLogs", false);
+
+export function formatSearchString(obj: Record<string, unknown>): string {
+    return Object.entries(obj).map(([key, value]) => {
+        return `${JSON.stringify(key)}:${JSON.stringify(value)}`;
+    }).join("&");
+}
+
+export function getLogViewerParams(logObj: Partial<LogDatum>) {
+    return [
+        managementPageURL.getOverride("LogViewer3"),
+        searchTextURL.getOverride(formatSearchString(logObj)),
+        // If we're linking to the logs, it's almost certainly for production logs. 
+        readProductionLogsURL.getOverride(true),
+    ];
+}

package/src/diagnostics/logs/IndexedLogs/bufferMatcher.ts

@@ -26,7 +26,7 @@ export type MatchStructure = {
 function parseMatchStructure(pattern: Buffer): MatchStructure {
     let text = pattern.toString("utf-8");
 
-    let orParts = text.split(SEARCH_OR_CHAR);
+    let orParts = text.split(SEARCH_OR_CHAR).map(x => x.trim()).filter(x => x.length > 0);
     if (orParts.length > 1) {
         return {
             type: "or",
@@ -34,7 +34,7 @@ function parseMatchStructure(pattern: Buffer): MatchStructure {
         };
     }
 
-    let andParts = text.split(SEARCH_AND_CHAR);
+    let andParts = text.split(SEARCH_AND_CHAR).map(x => x.trim()).filter(x => x.length > 0);
     if (andParts.length > 1) {
         return {
             type: "and",
@@ -42,7 +42,7 @@ function parseMatchStructure(pattern: Buffer): MatchStructure {
         };
     }
 
-    let wildcardParts = text.split(WILD_CARD_CHAR).filter(p => p.length > 0);
+    let wildcardParts = text.split(WILD_CARD_CHAR).map(x => x.trim()).filter(x => x.length > 0);
     if (wildcardParts.length > 1) {
         return {
             type: "wildcard",

package/src/diagnostics/logs/diskLogger.ts

@@ -5,7 +5,6 @@ import { lazy } from "socket-function/src/caching";
 import { timeInMinute } from "socket-function/src/misc";
 import { formatTime } from "socket-function/src/formatting/format";
 import { addEpsilons } from "../../bits";
-import { FileMetadata } from "./FastArchiveController";
 import { getPathStr2 } from "../../path";
 import { isPublic } from "../../config";
 // IMPORTANT! We can't have any real imports here, because we are depended on so early in startup!
@@ -32,8 +31,6 @@ export type LogDatum = Record<string, unknown> & {
     __DIR__?: string;
     __NAME__?: string;
     __LINE__?: string;
-    /** Dynamically set at runtime in the frontend. */
-    __metadata?: FileMetadata;
     /** Dynamically set when matching recent errors only. */
     __matchedOutdatedSuppressionKey?: string;
 };
@@ -68,36 +65,6 @@ export const LOG_LINE_LIMIT_FLAG = String.fromCharCode(44534) + "LOGS_LINE_LIMIT
 /** If this key exists in the logged object, as in a key in one of the objects logged, then we will use the value of it as the limit ID. This is useful as it allows us to either override a limit or limit something independently from other logs in the file. */
 export const LOG_LINE_LIMIT_ID = "LIMIT_LINE_ID";
 
-export const getLoggers = lazy(function () {
-    const { FastArchiveAppendable } = require("./FastArchiveAppendable") as typeof import("./FastArchiveAppendable");
-    if (!FastArchiveAppendable) {
-        setImmediate(() => {
-            getLoggers.reset();
-        });
-        return undefined;
-    }
-    return {
-        logLogs: new FastArchiveAppendable<LogDatum>("logs-log/"),
-        warnLogs: new FastArchiveAppendable<LogDatum>("logs-warn/"),
-        infoLogs: new FastArchiveAppendable<LogDatum>("logs-info/"),
-        errorLogs: new FastArchiveAppendable<LogDatum>("logs-error/"),
-    };
-});
-setImmediate(() => {
-    // If we don't import it at all, then it doesn't work client-side. 
-    require("./FastArchiveAppendable") as typeof import("./FastArchiveAppendable");
-});
-const getNotifyErrors = lazy(function () {
-    const { notifyWatchersOfError: notifyErrors } = require("./errorNotifications/ErrorNotificationController") as typeof import("./errorNotifications/ErrorNotificationController");
-    if (typeof notifyErrors !== "function") {
-        setImmediate(() => {
-            getNotifyErrors.reset();
-        });
-        return undefined;
-    }
-    return notifyErrors;
-});
-
 export const getLoggers2 = lazy(function () {
     const { IndexedLogs } = require("./IndexedLogs/IndexedLogs") as typeof import("./IndexedLogs/IndexedLogs");
 
@@ -146,7 +113,7 @@ void Promise.resolve().then(() => {
 
 
 const logDiskDontShim = logDisk;
-/** NOTE: Calling this directly means we lose __FILE__ tracking. But... that's probably fine... */
+/** @deprecated, Don't call this directly, call console info instead, which our shim will prevent from logging to the console, but it will still call logDisk. */
 export function logDisk(type: "log" | "warn" | "info" | "error", ...args: unknown[]) {
     if (!isNode()) return;
     try {
@@ -189,11 +156,6 @@ export function logDisk(type: "log" | "warn" | "info" | "error", ...args: unknow
             }
         }
 
-        if (type === "warn" || type === "error") {
-            // Dropping notifies is fine, as long as they get added to the logs, we'll see them eventually...
-            void getNotifyErrors()?.(logObj);
-        }
-
     } catch (e: any) {
         process.stdout.write(`Error writing to disk logs: ${e.stack || e}\n\t${String(args[0])}\n`);
     }

package/src/diagnostics/logs/diskShimConsoleLogs.ts

@@ -55,6 +55,8 @@ export function shimConsoleLogs() {
                 // Some arguments might throw if accessed (as they might be proxies), so
                 //  catch and ignore errors
             }
+            // NOTE: Info really has absolutely no purpose. There's no reason to use info instead of log, so we're going to give it a purpose. Infos are not going to be shown in the console and are only going to be logged to disk. This helps fix our shimming issue, where when we call log disk directly, we lose out on the source file information.
+            if (fncName === "info") return;
             return originalFnc(...args);
         };
     }

package/src/diagnostics/logs/errorNotifications2/errorNotifications2.ts

@@ -0,0 +1,3 @@
+
+//todonext
+// FIRST, finish up our logs stuff, which is just about done...

package/src/diagnostics/logs/injectFileLocationToConsole.ts

@@ -41,4 +41,7 @@ if (isNode()) {
 
     let diskShimConsoleLogs = require("./diskShimConsoleLogs") as typeof import("./diskShimConsoleLogs");
     diskShimConsoleLogs.shimConsoleLogs();
+} else {
+    // NOTE: Disable console.info, as serverside we use it for disk only logs
+    console.info = () => { };
 }