querysub 0.394.0 → 0.395.0

package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycleMatching.ts

@@ -0,0 +1,49 @@
+import { LifeCycle, LifeCycleEntry } from "./lifeCycles";
+import { LogDatum } from "../diskLogger";
+import { createMatchesPatternCached } from "../IndexedLogs/bufferSearchFindMatcher";
+
+export type LifecycleMatch = {
+    lifecycle: LifeCycle;
+    entry: LifeCycleEntry;
+    datum: Record<string, unknown>;
+};
+
+export function getLifecycleMatchesForDatum(
+    lifecycles: LifeCycle[],
+    datum: LogDatum
+): LifecycleMatch[] {
+    let matches: LifecycleMatch[] = [];
+
+    let matchStr = String(datum.param0);
+    let matchBuffer = Buffer.from(matchStr, "utf8");
+
+    for (let lifecycle of lifecycles) {
+        for (let entry of lifecycle.entries) {
+            let matcher = createMatchesPatternCached(entry.matchPattern);
+            if (!matcher(matchBuffer)) continue;
+
+            let extractedDatum: Record<string, unknown> = {};
+            let hasAllKeys = true;
+
+            for (let keyConfig of entry.groupByKeys) {
+                let keyToLookup = keyConfig.startKey || keyConfig.ourKey;
+                let value = datum[keyToLookup];
+                if (value === undefined) {
+                    hasAllKeys = false;
+                    break;
+                }
+                extractedDatum[keyConfig.ourKey] = value;
+            }
+
+            if (!hasAllKeys || entry.groupByKeys.length === 0) continue;
+
+            matches.push({
+                lifecycle,
+                entry,
+                datum: extractedDatum,
+            });
+        }
+    }
+
+    return matches;
+}

package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycleSearch.tsx

@@ -0,0 +1,553 @@
+import { Querysub } from "../../../4-querysub/QuerysubController";
+import { getTimeRange } from "../TimeRangeSelector";
+import { URLParam } from "../../../library-components/URLParam";
+import { throttleFunction, sort } from "socket-function/src/misc";
+import { getLoggers2Async, LogDatum } from "../diskLogger";
+import { IndexedLogResults, createEmptyIndexedLogResults, mergeIndexedLogResults } from "../IndexedLogs/BufferIndexHelpers";
+import { errorToUndefined } from "../../../errors";
+import { unique, maybeUndefined } from "../../../misc";
+import { formatSearchString } from "../IndexedLogs/LogViewerParams";
+import { createMatchesPatternCached } from "../IndexedLogs/bufferSearchFindMatcher";
+import { getPathStr } from "../../../path";
+import { LifeCycle, LifeCycleEntry, LifeCyclesController } from "./lifeCycles";
+
+export let limitURL = new URLParam("lifecyclelimit", 100);
+export let additionalSearchURL = new URLParam("lifecyclesearch", "");
+
+type LifecycleInstance = {
+    key: string;
+    keys: { key: string; value: unknown }[];
+    entries: {
+        datum: LogDatum;
+        matchPattern: string;
+    }[];
+    isComplete?: boolean;
+    isTruncated?: boolean;
+    isWarning?: boolean;
+    warningMessage?: string;
+    startTime: number;
+    endTime?: number;
+};
+
+export function createLifeCycleSearch(
+    context: {
+        controller: ReturnType<typeof LifeCyclesController>;
+        state: {
+            searchingLifeCycleId: string | undefined;
+            phase1Results: LogDatum[];
+            phase1InvalidResults: Array<{ datum: LogDatum; missingKeys: string[] }>;
+            phase1Stats: IndexedLogResults | undefined;
+            phase1Searching: boolean;
+            phase2Results: LogDatum[];
+            phase2Stats: IndexedLogResults | undefined;
+            phase2Searching: boolean;
+            lifecycleInstances: LifecycleInstance[];
+        };
+    }
+) {
+    let searchSequenceNumber = 0;
+
+    let cancel = async () => {
+        searchSequenceNumber++;
+        let loggers = await getLoggers2Async();
+        for (let logger of Object.values(loggers)) {
+            logger.clientCancelAllCallbacks();
+        }
+        Querysub.commitLocal(() => {
+            context.state.phase1Searching = false;
+            context.state.phase2Searching = false;
+        });
+    };
+
+    // Entry point for lifecycle search. Currently only supports "find all" mode by passing empty seedDatums.
+    // Future: Will be parameterized to accept specific log datums from other pages.
+    let searchLifeCycle = async (lifeCycleId: string) => {
+        let lifeCycles = await context.controller.getLifeCycles.promise();
+        if (!lifeCycles) return;
+
+        let lifeCycle = lifeCycles.find(lc => lc.id === lifeCycleId);
+        if (!lifeCycle) return;
+
+        if (lifeCycle.entries.length === 0) {
+            return;
+        }
+
+        await cancel();
+
+        Querysub.commitLocal(() => {
+            context.state.searchingLifeCycleId = lifeCycleId;
+            context.state.phase1Results = [];
+            context.state.phase1InvalidResults = [];
+            context.state.phase1Stats = undefined;
+            context.state.phase2Results = [];
+            context.state.phase2Stats = undefined;
+            context.state.lifecycleInstances = [];
+        });
+
+        searchSequenceNumber++;
+        let currentSequenceNumber = searchSequenceNumber;
+
+        let phase1Results = await searchPhase1({
+            lifeCycle,
+            currentSequenceNumber,
+        });
+
+        if (searchSequenceNumber !== currentSequenceNumber) return;
+
+        if (phase1Results.length === 0) {
+            return;
+        }
+
+        let phase2Data = await searchPhase2({
+            lifeCycle,
+            phase1Results,
+            currentSequenceNumber,
+        });
+
+        if (searchSequenceNumber !== currentSequenceNumber) return;
+
+        let range = Querysub.localRead(() => getTimeRange());
+
+        let instances = groupLifecycleResults({
+            phase2Results: phase2Data.results,
+            lifeCycle,
+            searchEndTime: range.endTime,
+            hitLimit: phase2Data.hitLimit,
+        });
+
+        Querysub.commitLocal(() => {
+            context.state.lifecycleInstances = instances;
+        });
+    };
+
+    let searchPhase1 = async (params: {
+        lifeCycle: LifeCycle;
+        currentSequenceNumber: number;
+    }) => {
+        let { lifeCycle, currentSequenceNumber } = params;
+        let startEntries = lifeCycle.entries.filter(e => e.isStart);
+
+        if (startEntries.length === 0) {
+            return [];
+        }
+
+        Querysub.commitLocal(() => {
+            context.state.phase1Searching = true;
+            context.state.phase1Results = [];
+            context.state.phase1InvalidResults = [];
+            context.state.phase1Stats = undefined;
+        });
+
+        let searchQuery = startEntries.map(entry => entry.matchPattern).join("|");
+        let additionalSearch = additionalSearchURL.value.trim();
+        if (additionalSearch) {
+            searchQuery = searchQuery + "&" + additionalSearch;
+        }
+        let searchBuffer = Buffer.from(searchQuery, "utf8");
+
+        let sourceTypes = new Set<string>();
+        for (let entry of startEntries) {
+            sourceTypes.add(entry.sourceType);
+        }
+
+        let loggers = await getLoggers2Async();
+        let selectedLoggers: typeof loggers.logLogs[] = [];
+
+        if (sourceTypes.has("log")) selectedLoggers.push(loggers.logLogs);
+        if (sourceTypes.has("info")) selectedLoggers.push(loggers.infoLogs);
+        if (sourceTypes.has("warning")) selectedLoggers.push(loggers.warnLogs);
+        if (sourceTypes.has("error")) selectedLoggers.push(loggers.errorLogs);
+
+        if (selectedLoggers.length === 0) {
+            Querysub.commitLocal(() => {
+                context.state.phase1Searching = false;
+            });
+            return [];
+        }
+
+        let startTime = Date.now();
+        let results: LogDatum[] = [];
+
+        let range = Querysub.localRead(() => getTimeRange());
+
+        let updateResults = throttleFunction(100, () => {
+            if (searchSequenceNumber !== currentSequenceNumber) return;
+            Querysub.commitLocal(() => {
+                context.state.phase1Results = results;
+            });
+        });
+
+        let loggerResults = new Map<typeof loggers.logLogs, IndexedLogResults>();
+
+        let updateStats = () => {
+            if (searchSequenceNumber !== currentSequenceNumber) return;
+
+            let mergedStats: IndexedLogResults = createEmptyIndexedLogResults();
+
+            for (let loggerResult of loggerResults.values()) {
+                mergedStats = mergeIndexedLogResults(mergedStats, loggerResult);
+            }
+
+            mergedStats.totalSearchTime = Date.now() - startTime;
+
+            Querysub.commitLocal(() => {
+                context.state.phase1Stats = mergedStats;
+            });
+        };
+
+        await Promise.all(selectedLoggers.map(async (logger) => {
+            let done = false;
+            let result = await errorToUndefined(logger.clientFind({
+                params: {
+                    startTime: range.startTime,
+                    endTime: range.endTime,
+                    limit: limitURL.value,
+                    findBuffer: searchBuffer,
+                    searchFromStart: range.searchFromStart,
+                },
+                onResult: (match: LogDatum) => {
+                    results.push(match);
+                    void updateResults();
+                },
+                onResults: (loggerStats: IndexedLogResults) => {
+                    if (done) return;
+                    loggerResults.set(logger, loggerStats);
+                    updateStats();
+                },
+            }));
+            done = true;
+
+            if (result) {
+                loggerResults.set(logger, result);
+                updateStats();
+            }
+        }));
+
+        let validResults: LogDatum[] = [];
+        let invalidResults: Array<{ datum: LogDatum; missingKeys: string[] }> = [];
+
+        for (let datum of results) {
+            let matchStr = String(datum.param0);
+            let matchBuffer = Buffer.from(matchStr, "utf8");
+            let matchedEntry: LifeCycleEntry | undefined = undefined;
+
+            for (let startEntry of startEntries) {
+                let matcher = createMatchesPatternCached(startEntry.matchPattern);
+                if (matcher(matchBuffer)) {
+                    matchedEntry = startEntry;
+                    break;
+                }
+            }
+
+            if (!matchedEntry) {
+                validResults.push(datum);
+                continue;
+            }
+
+            let missingKeys: string[] = [];
+            for (let keyConfig of matchedEntry.groupByKeys) {
+                let value = datum[keyConfig.ourKey];
+                if (value === undefined) {
+                    missingKeys.push(keyConfig.ourKey);
+                }
+            }
+
+            if (missingKeys.length > 0) {
+                invalidResults.push({ datum, missingKeys });
+            } else {
+                validResults.push(datum);
+            }
+        }
+
+        Querysub.commitLocal(() => {
+            context.state.phase1Results = validResults;
+            context.state.phase1InvalidResults = invalidResults;
+            context.state.phase1Searching = false;
+        });
+
+        return validResults;
+    };
+
+    let searchPhase2 = async (params: {
+        lifeCycle: LifeCycle;
+        phase1Results: LogDatum[];
+        currentSequenceNumber: number;
+    }): Promise<{ results: LogDatum[]; hitLimit: boolean }> => {
+        let { lifeCycle, phase1Results, currentSequenceNumber } = params;
+
+        if (phase1Results.length === 0) {
+            return { results: [], hitLimit: false };
+        }
+
+        Querysub.commitLocal(() => {
+            context.state.phase2Searching = true;
+            context.state.phase2Results = [];
+            context.state.phase2Stats = undefined;
+        });
+
+        let allResults: LogDatum[] = [];
+        let startTime = Date.now();
+        let loggers = await getLoggers2Async();
+
+        let updateResults = throttleFunction(100, () => {
+            if (searchSequenceNumber !== currentSequenceNumber) return;
+            Querysub.commitLocal(() => {
+                context.state.phase2Results = allResults;
+            });
+        });
+
+        let loggerResults = new Map<typeof loggers.logLogs, IndexedLogResults>();
+
+        let updateStats = () => {
+            if (searchSequenceNumber !== currentSequenceNumber) return;
+
+            let mergedStats: IndexedLogResults = createEmptyIndexedLogResults();
+
+            for (let loggerResult of loggerResults.values()) {
+                mergedStats = mergeIndexedLogResults(mergedStats, loggerResult);
+            }
+
+            mergedStats.totalSearchTime = Date.now() - startTime;
+
+            Querysub.commitLocal(() => {
+                context.state.phase2Stats = mergedStats;
+            });
+        };
+
+        let range = Querysub.localRead(() => getTimeRange());
+
+        let startEntries = lifeCycle.entries.filter(e => e.isStart);
+        if (startEntries.length === 0) {
+            Querysub.commitLocal(() => {
+                context.state.phase2Searching = false;
+            });
+            return { results: [], hitLimit: false };
+        }
+
+        let searchesByLogger = new Map<string, Set<string>>();
+
+        for (let datum of phase1Results) {
+            let matchStr = String(datum.param0);
+            let matchBuffer = Buffer.from(matchStr, "utf8");
+
+            for (let startEntry of startEntries) {
+                let matcher = createMatchesPatternCached(startEntry.matchPattern);
+                if (!matcher(matchBuffer)) continue;
+
+                if (startEntry.groupByKeys.length === 0) continue;
+
+                // TODO: For most of these, only the first entry is unique, so we should really create a cache to be able to know all of the unique keys that we need to search for for each value. However, it probably won't matter, as we're not going to have that many results we're searching for. 
+                for (let entry of lifeCycle.entries) {
+                    if (entry.groupByKeys.length === 0) continue;
+
+                    let queries = searchesByLogger.get(entry.sourceType);
+                    if (!queries) {
+                        queries = new Set<string>();
+                        searchesByLogger.set(entry.sourceType, queries);
+                    }
+
+                    let queryPart: Record<string, unknown> = {};
+                    let matchedAll = true;
+
+                    for (let startKeyObj of startEntry.groupByKeys) {
+                        let remappedKey = entry.groupByKeys.find(k => (k.startKey || k.ourKey) === startKeyObj.ourKey)?.ourKey;
+                        if (!remappedKey) {
+                            matchedAll = false;
+                            break;
+                        }
+                        let value = datum[startKeyObj.ourKey];
+                        if (value === undefined) {
+                            matchedAll = false;
+                            break;
+                        }
+                        queryPart[remappedKey] = value;
+                    }
+
+                    if (matchedAll) {
+                        queries.add(formatSearchString(queryPart));
+                    }
+                }
+            }
+        }
+
+        if (searchesByLogger.size === 0) {
+            Querysub.commitLocal(() => {
+                context.state.phase2Searching = false;
+            });
+            return { results: [], hitLimit: false };
+        }
+
+        let anyHitLimit = false;
+
+        await Promise.all(Array.from(searchesByLogger.entries()).map(async ([sourceType, queries]) => {
+            let logger =
+                sourceType === "log" && loggers.logLogs ||
+                sourceType === "info" && loggers.infoLogs ||
+                sourceType === "warning" && loggers.warnLogs ||
+                loggers.errorLogs;
+
+            let searchQuery = Array.from(queries).join("|");
+            let searchBuffer = Buffer.from(searchQuery, "utf8");
+
+            let results: LogDatum[] = [];
+            let done = false;
+            let searchLimit = limitURL.value * lifeCycle.entries.length * queries.size * 3;
+            let result = await errorToUndefined(logger.clientFind({
+                params: {
+                    startTime: range.startTime,
+                    endTime: range.endTime,
+                    limit: searchLimit,
+                    findBuffer: searchBuffer,
+                    searchFromStart: range.searchFromStart,
+                },
+                onResult: (match: LogDatum) => {
+                    results.push(match);
+                    allResults.push(match);
+                    void updateResults();
+                },
+                onResults: (loggerStats: IndexedLogResults) => {
+                    if (done) return;
+                    loggerResults.set(logger, loggerStats);
+                    updateStats();
+                },
+            }));
+            done = true;
+
+            if (result) {
+                loggerResults.set(logger, result);
+                updateStats();
+            }
+
+            if (results.length >= searchLimit) {
+                anyHitLimit = true;
+            }
+        }));
+
+        Querysub.commitLocal(() => {
+            context.state.phase2Results = allResults;
+            context.state.phase2Searching = false;
+        });
+
+        return { results: allResults, hitLimit: anyHitLimit };
+    };
+
+    let groupLifecycleResults = (params: {
+        phase2Results: LogDatum[];
+        lifeCycle: LifeCycle;
+        searchEndTime: number;
+        hitLimit: boolean;
+    }): LifecycleInstance[] => {
+        let { phase2Results, lifeCycle, searchEndTime, hitLimit } = params;
+
+        let startEntries = lifeCycle.entries.filter(e => e.isStart);
+        if (startEntries.length === 0) return [];
+
+        let keyFields = startEntries[0].groupByKeys;
+        if (keyFields.length === 0) return [];
+
+        let hasExplicitEnd = lifeCycle.entries.some(e => e.isEnd);
+
+        sort(phase2Results, x => x.time);
+
+        let instancesByKey: Map<string, LifecycleInstance[]> = new Map();
+
+        for (let datum of phase2Results) {
+            let keyValuePairs: { key: string; value: unknown }[] = [];
+            let hasAllKeys = true;
+            for (let keyConfig of keyFields) {
+                let value = datum[keyConfig.ourKey];
+                if (value === undefined) {
+                    hasAllKeys = false;
+                    break;
+                }
+                keyValuePairs.push({ key: keyConfig.ourKey, value });
+            }
+            if (!hasAllKeys) continue;
+            let compositeKey = getPathStr(keyValuePairs.flatMap(kv => [kv.key, String(kv.value)]));
+
+            let matchStr = String(datum.param0);
+            let matchBuffer = Buffer.from(matchStr, "utf8");
+
+            for (let entry of lifeCycle.entries) {
+                let matcher = createMatchesPatternCached(entry.matchPattern);
+                if (!matcher(matchBuffer)) continue;
+
+                let instanceList = instancesByKey.get(compositeKey);
+                if (!instanceList) {
+                    instanceList = [];
+                    instancesByKey.set(compositeKey, instanceList);
+                }
+
+                let currentInstance = maybeUndefined(instanceList[instanceList.length - 1]);
+
+                if (entry.isStart) {
+                    if (currentInstance && !currentInstance.isComplete) {
+                        if (hasExplicitEnd) {
+                            currentInstance.isTruncated = true;
+                        } else {
+                            currentInstance.isComplete = true;
+                        }
+                        currentInstance.endTime = datum.time;
+                    }
+                    let newInstance: LifecycleInstance = {
+                        key: keyFields.map(k => k.ourKey).join(","),
+                        keys: keyValuePairs,
+                        entries: [{ datum, matchPattern: entry.matchPattern }],
+                        startTime: 0,
+                    };
+                    instanceList.push(newInstance);
+                } else if (entry.isEnd) {
+                    if (currentInstance && !currentInstance.isComplete) {
+                        currentInstance.entries.push({ datum, matchPattern: entry.matchPattern });
+                        currentInstance.isComplete = true;
+                        currentInstance.endTime = datum.time;
+                    } else {
+                        instanceList.push({
+                            key: keyFields.map(k => k.ourKey).join(","),
+                            keys: keyValuePairs,
+                            entries: [{ datum, matchPattern: entry.matchPattern }],
+                            isWarning: true,
+                            warningMessage: "End entry without matching start",
+                            startTime: 0,
+                        });
+                    }
+                } else {
+                    if (currentInstance && !currentInstance.isComplete) {
+                        currentInstance.entries.push({ datum, matchPattern: entry.matchPattern });
+                    } else {
+                        instanceList.push({
+                            key: keyFields.map(k => k.ourKey).join(","),
+                            keys: keyValuePairs,
+                            entries: [{ datum, matchPattern: entry.matchPattern }],
+                            isWarning: true,
+                            warningMessage: "Entry without matching start",
+                            startTime: 0,
+                        });
+                    }
+                }
+            }
+        }
+
+        let allInstances = Array.from(instancesByKey.values()).flat();
+
+        for (let instance of allInstances) {
+            if (instance.entries.length > 0) {
+                instance.startTime = instance.entries[0].datum.time;
+            }
+        }
+
+        sort(allInstances, x => x.startTime);
+
+        return allInstances;
+    };
+
+    return {
+        searchLifeCycle,
+        searchPhase1,
+        searchPhase2,
+        groupLifecycleResults,
+        cancel,
+    };
+}
+
+export type { LifecycleInstance };