querysub 0.394.0 → 0.396.0

package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycleSearch.tsx

@@ -0,0 +1,580 @@
+import { Querysub } from "../../../4-querysub/QuerysubController";
+import { getTimeRange } from "../TimeRangeSelector";
+import { throttleFunction, sort } from "socket-function/src/misc";
+import { getLoggers2Async, LogDatum } from "../diskLogger";
+import { IndexedLogResults, createEmptyIndexedLogResults, mergeIndexedLogResults } from "../IndexedLogs/BufferIndexHelpers";
+import { errorToUndefined } from "../../../errors";
+import { unique, maybeUndefined } from "../../../misc";
+import { formatSearchString } from "../IndexedLogs/LogViewerParams";
+import { createMatchesPatternCached } from "../IndexedLogs/bufferSearchFindMatcher";
+import { getPathStr } from "../../../path";
+import { LifeCycle, LifeCycleEntry, LifeCyclesController } from "./lifeCycles";
+
+const PHASE_2_FACTOR = 100;
+
+type LifecycleInstance = {
+    key: string;
+    keys: { key: string; value: unknown }[];
+    entries: {
+        datum: LogDatum;
+        matchPattern: string;
+    }[];
+    isComplete?: boolean;
+    isTruncated?: boolean;
+    isMissingStart?: boolean;
+    isMissingEnd?: boolean;
+    startTime: number;
+    endTime?: number;
+};
+
+export function createLifeCycleSearch(
+    context: {
+        controller: ReturnType<typeof LifeCyclesController>;
+        state: {
+            searchingLifeCycleId: string | undefined;
+            phase1Results: LogDatum[];
+            phase1InvalidResults: Array<{ datum: LogDatum; missingKeys: string[] }>;
+            phase1Stats: IndexedLogResults | undefined;
+            phase1Searching: boolean;
+            phase2Results: LogDatum[];
+            phase2Stats: IndexedLogResults | undefined;
+            phase2Searching: boolean;
+            phase2HitLimit: boolean;
+            lifecycleInstances: LifecycleInstance[];
+        };
+    }
+) {
+    let searchSequenceNumber = 0;
+
+    let clearSearchingFlags = () => {
+        context.state.phase1Searching = false;
+        context.state.phase2Searching = false;
+        context.state.phase2HitLimit = false;
+    };
+
+    let resetAllSearchState = (lifeCycleId?: string) => {
+        context.state.searchingLifeCycleId = lifeCycleId;
+        context.state.lifecycleInstances = [];
+        resetPhase1State();
+        resetPhase2State();
+    };
+
+    let resetPhase1State = () => {
+        context.state.phase1Searching = true;
+        context.state.phase1Results = [];
+        context.state.phase1InvalidResults = [];
+        context.state.phase1Stats = undefined;
+    };
+
+    let resetPhase2State = () => {
+        context.state.phase2Searching = true;
+        context.state.phase2Results = [];
+        context.state.phase2Stats = undefined;
+        context.state.phase2HitLimit = false;
+    };
+
+    let cancel = async () => {
+        searchSequenceNumber++;
+        let loggers = await getLoggers2Async();
+        for (let logger of Object.values(loggers)) {
+            logger.clientCancelAllCallbacks();
+        }
+        Querysub.commitLocal(() => {
+            clearSearchingFlags();
+        });
+    };
+
+    // Entry point for lifecycle search. Currently only supports "find all" mode by passing empty seedDatums.
+    // Future: Will be parameterized to accept specific log datums from other pages.
+    let searchLifeCycle = async (params: {
+        lifeCycleId: string;
+        additionalSearch: string;
+        limit: number;
+    }) => {
+        let { lifeCycleId, additionalSearch, limit } = params;
+        let lifeCycles = await context.controller.getLifeCycles.promise();
+        if (!lifeCycles) return;
+
+        let lifeCycle = lifeCycles.find(lc => lc.id === lifeCycleId);
+        if (!lifeCycle) return;
+
+        if (lifeCycle.entries.length === 0) {
+            return;
+        }
+
+        await cancel();
+
+        Querysub.commitLocal(() => {
+            resetAllSearchState(lifeCycleId);
+        });
+
+        searchSequenceNumber++;
+        let currentSequenceNumber = searchSequenceNumber;
+
+        let phase1Results = await searchPhase1({
+            lifeCycle,
+            currentSequenceNumber,
+            additionalSearch,
+            limit,
+        });
+
+        if (searchSequenceNumber !== currentSequenceNumber) return;
+
+        if (phase1Results.length === 0) {
+            return;
+        }
+
+        let phase2Data = await searchPhase2({
+            lifeCycle,
+            phase1Results,
+            currentSequenceNumber,
+            limit,
+        });
+
+        if (searchSequenceNumber !== currentSequenceNumber) return;
+
+        let range = Querysub.localRead(() => getTimeRange());
+
+        let instances = groupLifecycleResults({
+            phase2Results: phase2Data.results,
+            lifeCycle,
+            searchEndTime: range.endTime,
+            hitLimit: phase2Data.hitLimit,
+        });
+
+        Querysub.commitLocal(() => {
+            context.state.lifecycleInstances = instances;
+        });
+    };
+
+    let searchPhase1 = async (params: {
+        lifeCycle: LifeCycle;
+        currentSequenceNumber: number;
+        additionalSearch: string;
+        limit: number;
+    }) => {
+        let { lifeCycle, currentSequenceNumber, additionalSearch, limit } = params;
+        let startEntries = lifeCycle.entries.filter(e => e.isStart);
+
+        if (startEntries.length === 0) {
+            return [];
+        }
+
+        Querysub.commitLocal(() => {
+            resetPhase1State();
+        });
+
+        let searchQuery = startEntries.map(entry => entry.matchPattern).join("|");
+        if (additionalSearch) {
+            searchQuery = searchQuery + "&" + additionalSearch;
+        }
+        let searchBuffer = Buffer.from(searchQuery, "utf8");
+
+        let sourceTypes = new Set<string>();
+        for (let entry of startEntries) {
+            sourceTypes.add(entry.sourceType);
+        }
+
+        let loggers = await getLoggers2Async();
+        let selectedLoggers: typeof loggers.logLogs[] = [];
+
+        if (sourceTypes.has("log")) selectedLoggers.push(loggers.logLogs);
+        if (sourceTypes.has("info")) selectedLoggers.push(loggers.infoLogs);
+        if (sourceTypes.has("warning")) selectedLoggers.push(loggers.warnLogs);
+        if (sourceTypes.has("error")) selectedLoggers.push(loggers.errorLogs);
+
+        if (selectedLoggers.length === 0) {
+            Querysub.commitLocal(() => {
+                context.state.phase1Searching = false;
+            });
+            return [];
+        }
+
+        let startTime = Date.now();
+        let results: LogDatum[] = [];
+
+        let range = Querysub.localRead(() => getTimeRange());
+
+        let updateResults = throttleFunction(100, () => {
+            if (searchSequenceNumber !== currentSequenceNumber) return;
+            Querysub.commitLocal(() => {
+                context.state.phase1Results = results;
+            });
+        });
+
+        let loggerResults = new Map<typeof loggers.logLogs, IndexedLogResults>();
+
+        let updateStats = () => {
+            if (searchSequenceNumber !== currentSequenceNumber) return;
+
+            let mergedStats: IndexedLogResults = createEmptyIndexedLogResults();
+
+            for (let loggerResult of loggerResults.values()) {
+                mergedStats = mergeIndexedLogResults(mergedStats, loggerResult);
+            }
+
+            mergedStats.totalSearchTime = Date.now() - startTime;
+
+            Querysub.commitLocal(() => {
+                context.state.phase1Stats = mergedStats;
+            });
+        };
+
+        await Promise.all(selectedLoggers.map(async (logger) => {
+            let done = false;
+            let result = await errorToUndefined(logger.clientFind({
+                params: {
+                    startTime: range.startTime,
+                    endTime: range.endTime,
+                    limit: limit,
+                    findBuffer: searchBuffer,
+                    searchFromStart: range.searchFromStart,
+                },
+                onResult: (match: LogDatum) => {
+                    results.push(match);
+                    void updateResults();
+                },
+                onResults: (loggerStats: IndexedLogResults) => {
+                    if (done) return;
+                    loggerResults.set(logger, loggerStats);
+                    updateStats();
+                },
+            }));
+            done = true;
+
+            if (result) {
+                loggerResults.set(logger, result);
+                updateStats();
+            }
+        }));
+
+        let validResults: LogDatum[] = [];
+        let invalidResults: Array<{ datum: LogDatum; missingKeys: string[] }> = [];
+
+        for (let datum of results) {
+            let matchStr = String(datum.param0);
+            let matchBuffer = Buffer.from(matchStr, "utf8");
+            let matchedEntry: LifeCycleEntry | undefined = undefined;
+
+            for (let startEntry of startEntries) {
+                let matcher = createMatchesPatternCached(startEntry.matchPattern);
+                if (matcher(matchBuffer)) {
+                    matchedEntry = startEntry;
+                    break;
+                }
+            }
+
+            if (!matchedEntry) {
+                validResults.push(datum);
+                continue;
+            }
+
+            let missingKeys: string[] = [];
+            for (let keyConfig of matchedEntry.groupByKeys) {
+                let value = datum[keyConfig.ourKey];
+                if (value === undefined) {
+                    missingKeys.push(keyConfig.ourKey);
+                }
+            }
+
+            if (missingKeys.length > 0) {
+                invalidResults.push({ datum, missingKeys });
+            } else {
+                validResults.push(datum);
+            }
+        }
+
+        Querysub.commitLocal(() => {
+            context.state.phase1Results = validResults;
+            context.state.phase1InvalidResults = invalidResults;
+            context.state.phase1Searching = false;
+        });
+
+        return validResults;
+    };
+
+    let searchPhase2 = async (params: {
+        lifeCycle: LifeCycle;
+        phase1Results: LogDatum[];
+        currentSequenceNumber: number;
+        limit: number;
+    }): Promise<{ results: LogDatum[]; hitLimit: boolean }> => {
+        let { lifeCycle, phase1Results, currentSequenceNumber, limit } = params;
+
+        if (phase1Results.length === 0) {
+            return { results: [], hitLimit: false };
+        }
+
+        Querysub.commitLocal(() => {
+            resetPhase2State();
+        });
+
+        let allResults: LogDatum[] = [];
+        let startTime = Date.now();
+        let loggers = await getLoggers2Async();
+
+        let updateResults = throttleFunction(100, () => {
+            if (searchSequenceNumber !== currentSequenceNumber) return;
+            Querysub.commitLocal(() => {
+                context.state.phase2Results = allResults;
+            });
+        });
+
+        let loggerResults = new Map<typeof loggers.logLogs, IndexedLogResults>();
+
+        let updateStats = () => {
+            if (searchSequenceNumber !== currentSequenceNumber) return;
+
+            let mergedStats: IndexedLogResults = createEmptyIndexedLogResults();
+
+            for (let loggerResult of loggerResults.values()) {
+                mergedStats = mergeIndexedLogResults(mergedStats, loggerResult);
+            }
+
+            mergedStats.totalSearchTime = Date.now() - startTime;
+
+            Querysub.commitLocal(() => {
+                context.state.phase2Stats = mergedStats;
+            });
+        };
+
+        let range = Querysub.localRead(() => getTimeRange());
+
+        let startEntries = lifeCycle.entries.filter(e => e.isStart);
+        if (startEntries.length === 0) {
+            Querysub.commitLocal(() => {
+                context.state.phase2Searching = false;
+            });
+            return { results: [], hitLimit: false };
+        }
+
+        let searchesByLogger = new Map<string, Set<string>>();
+
+        for (let datum of phase1Results) {
+            let matchStr = String(datum.param0);
+            let matchBuffer = Buffer.from(matchStr, "utf8");
+
+            for (let startEntry of startEntries) {
+                let matcher = createMatchesPatternCached(startEntry.matchPattern);
+                if (!matcher(matchBuffer)) continue;
+
+                if (startEntry.groupByKeys.length === 0) continue;
+
+                // TODO: For most of these, only the first entry is unique, so we should really create a cache to be able to know all of the unique keys that we need to search for for each value. However, it probably won't matter, as we're not going to have that many results we're searching for. 
+                for (let entry of lifeCycle.entries) {
+                    if (entry.groupByKeys.length === 0) continue;
+
+                    let queries = searchesByLogger.get(entry.sourceType);
+                    if (!queries) {
+                        queries = new Set<string>();
+                        searchesByLogger.set(entry.sourceType, queries);
+                    }
+
+                    let queryPart: Record<string, unknown> = {};
+                    let matchedAll = true;
+
+                    for (let startKeyObj of startEntry.groupByKeys) {
+                        let remappedKey = entry.groupByKeys.find(k => (k.startKey || k.ourKey) === startKeyObj.ourKey)?.ourKey;
+                        if (!remappedKey) {
+                            matchedAll = false;
+                            break;
+                        }
+                        let value = datum[startKeyObj.ourKey];
+                        if (value === undefined) {
+                            matchedAll = false;
+                            break;
+                        }
+                        queryPart[remappedKey] = value;
+                    }
+
+                    if (matchedAll) {
+                        queries.add(formatSearchString(queryPart));
+                    }
+                }
+            }
+        }
+
+        if (searchesByLogger.size === 0) {
+            Querysub.commitLocal(() => {
+                context.state.phase2Searching = false;
+            });
+            return { results: [], hitLimit: false };
+        }
+
+        let anyHitLimit = false;
+
+        await Promise.all(Array.from(searchesByLogger.entries()).map(async ([sourceType, queries]) => {
+            let logger =
+                sourceType === "log" && loggers.logLogs ||
+                sourceType === "info" && loggers.infoLogs ||
+                sourceType === "warning" && loggers.warnLogs ||
+                loggers.errorLogs;
+
+            let searchQuery = Array.from(queries).join("|");
+            let searchBuffer = Buffer.from(searchQuery, "utf8");
+
+            let results: LogDatum[] = [];
+            let done = false;
+            let searchLimit = Math.ceil(limit * lifeCycle.entries.length * PHASE_2_FACTOR);
+            let result = await errorToUndefined(logger.clientFind({
+                params: {
+                    startTime: range.startTime,
+                    endTime: range.endTime,
+                    limit: searchLimit,
+                    findBuffer: searchBuffer,
+                    searchFromStart: range.searchFromStart,
+                },
+                onResult: (match: LogDatum) => {
+                    results.push(match);
+                    allResults.push(match);
+                    void updateResults();
+                },
+                onResults: (loggerStats: IndexedLogResults) => {
+                    if (done) return;
+                    loggerResults.set(logger, loggerStats);
+                    updateStats();
+                },
+            }));
+            done = true;
+
+            if (result) {
+                loggerResults.set(logger, result);
+                updateStats();
+            }
+
+            if (results.length >= searchLimit) {
+                anyHitLimit = true;
+            }
+        }));
+
+        Querysub.commitLocal(() => {
+            context.state.phase2Results = allResults;
+            context.state.phase2Searching = false;
+            context.state.phase2HitLimit = anyHitLimit;
+        });
+
+        return { results: allResults, hitLimit: anyHitLimit };
+    };
+
+    let groupLifecycleResults = (params: {
+        phase2Results: LogDatum[];
+        lifeCycle: LifeCycle;
+        searchEndTime: number;
+        hitLimit: boolean;
+    }): LifecycleInstance[] => {
+        let { phase2Results, lifeCycle, searchEndTime, hitLimit } = params;
+
+        let startEntries = lifeCycle.entries.filter(e => e.isStart);
+        if (startEntries.length === 0) return [];
+
+        let keyFields = startEntries[0].groupByKeys;
+        if (keyFields.length === 0) return [];
+
+        let hasExplicitEnd = lifeCycle.entries.some(e => e.isEnd);
+
+        sort(phase2Results, x => x.time);
+
+        let instancesByKey: Map<string, LifecycleInstance[]> = new Map();
+
+        for (let datum of phase2Results) {
+            let keyValuePairs: { key: string; value: unknown }[] = [];
+            let hasAllKeys = true;
+            for (let keyConfig of keyFields) {
+                let value = datum[keyConfig.ourKey];
+                if (value === undefined) {
+                    hasAllKeys = false;
+                    break;
+                }
+                keyValuePairs.push({ key: keyConfig.ourKey, value });
+            }
+            if (!hasAllKeys) continue;
+            let keyHash = getPathStr(keyValuePairs.flatMap(kv => [kv.key, String(kv.value)]));
+
+            let matchStr = String(datum.param0);
+            let matchBuffer = Buffer.from(matchStr, "utf8");
+
+            for (let entry of lifeCycle.entries) {
+                let matcher = createMatchesPatternCached(entry.matchPattern);
+                if (!matcher(matchBuffer)) continue;
+
+                let instanceList = instancesByKey.get(keyHash);
+                if (!instanceList) {
+                    instanceList = [];
+                    instancesByKey.set(keyHash, instanceList);
+                }
+
+                let currentInstance = maybeUndefined(instanceList[instanceList.length - 1]);
+
+                if (entry.isStart) {
+                    if (currentInstance && !currentInstance.isComplete) {
+                        if (hasExplicitEnd) {
+                            currentInstance.isTruncated = true;
+                        } else {
+                            currentInstance.isComplete = true;
+                        }
+                        currentInstance.endTime = datum.time;
+                    }
+                    let newInstance: LifecycleInstance = {
+                        key: keyFields.map(k => k.ourKey).join(","),
+                        keys: keyValuePairs,
+                        entries: [{ datum, matchPattern: entry.matchPattern }],
+                        startTime: 0,
+                    };
+                    instanceList.push(newInstance);
+                } else if (entry.isEnd) {
+                    if (currentInstance && !currentInstance.isComplete) {
+                        currentInstance.entries.push({ datum, matchPattern: entry.matchPattern });
+                        currentInstance.isComplete = true;
+                        currentInstance.endTime = datum.time;
+                    } else {
+                        instanceList.push({
+                            key: keyFields.map(k => k.ourKey).join(","),
+                            keys: keyValuePairs,
+                            entries: [{ datum, matchPattern: entry.matchPattern }],
+                            isMissingStart: true,
+                            startTime: 0,
+                        });
+                    }
+                } else {
+                    if (currentInstance && !currentInstance.isComplete) {
+                        currentInstance.entries.push({ datum, matchPattern: entry.matchPattern });
+                    } else {
+                        instanceList.push({
+                            key: keyFields.map(k => k.ourKey).join(","),
+                            keys: keyValuePairs,
+                            entries: [{ datum, matchPattern: entry.matchPattern }],
+                            isMissingStart: true,
+                            startTime: 0,
+                        });
+                    }
+                }
+            }
+        }
+
+        let allInstances = Array.from(instancesByKey.values()).flat();
+
+        for (let instance of allInstances) {
+            if (instance.entries.length > 0) {
+                instance.startTime = instance.entries[0].datum.time;
+            }
+            if (!instance.isComplete && !instance.isTruncated && !instance.isMissingStart) {
+                instance.isMissingEnd = true;
+            }
+        }
+
+        sort(allInstances, x => x.startTime);
+
+        return allInstances;
+    };
+
+    return {
+        searchLifeCycle,
+        searchPhase1,
+        searchPhase2,
+        groupLifecycleResults,
+        cancel,
+        resetAllSearchState,
+    };
+}
+
+export type { LifecycleInstance };