querysub 0.395.0 → 0.397.0

package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycleSearch.tsx

@@ -1,6 +1,5 @@
 import { Querysub } from "../../../4-querysub/QuerysubController";
 import { getTimeRange } from "../TimeRangeSelector";
-import { URLParam } from "../../../library-components/URLParam";
 import { throttleFunction, sort } from "socket-function/src/misc";
 import { getLoggers2Async, LogDatum } from "../diskLogger";
 import { IndexedLogResults, createEmptyIndexedLogResults, mergeIndexedLogResults } from "../IndexedLogs/BufferIndexHelpers";
@@ -11,8 +10,7 @@ import { createMatchesPatternCached } from "../IndexedLogs/bufferSearchFindMatch
 import { getPathStr } from "../../../path";
 import { LifeCycle, LifeCycleEntry, LifeCyclesController } from "./lifeCycles";
 
-export let limitURL = new URLParam("lifecyclelimit", 100);
-export let additionalSearchURL = new URLParam("lifecyclesearch", "");
+const PHASE_2_FACTOR = 100;
 
 type LifecycleInstance = {
     key: string;
@@ -23,8 +21,8 @@ type LifecycleInstance = {
     }[];
     isComplete?: boolean;
     isTruncated?: boolean;
-    isWarning?: boolean;
-    warningMessage?: string;
+    isMissingStart?: boolean;
+    isMissingEnd?: boolean;
     startTime: number;
     endTime?: number;
 };
@@ -41,12 +39,40 @@ export function createLifeCycleSearch(
             phase2Results: LogDatum[];
             phase2Stats: IndexedLogResults | undefined;
             phase2Searching: boolean;
+            phase2HitLimit: boolean;
             lifecycleInstances: LifecycleInstance[];
         };
     }
 ) {
     let searchSequenceNumber = 0;
 
+    let clearSearchingFlags = () => {
+        context.state.phase1Searching = false;
+        context.state.phase2Searching = false;
+        context.state.phase2HitLimit = false;
+    };
+
+    let resetAllSearchState = (lifeCycleId?: string) => {
+        context.state.searchingLifeCycleId = lifeCycleId;
+        context.state.lifecycleInstances = [];
+        resetPhase1State();
+        resetPhase2State();
+    };
+
+    let resetPhase1State = () => {
+        context.state.phase1Searching = true;
+        context.state.phase1Results = [];
+        context.state.phase1InvalidResults = [];
+        context.state.phase1Stats = undefined;
+    };
+
+    let resetPhase2State = () => {
+        context.state.phase2Searching = true;
+        context.state.phase2Results = [];
+        context.state.phase2Stats = undefined;
+        context.state.phase2HitLimit = false;
+    };
+
     let cancel = async () => {
         searchSequenceNumber++;
         let loggers = await getLoggers2Async();
@@ -54,14 +80,18 @@ export function createLifeCycleSearch(
             logger.clientCancelAllCallbacks();
         }
         Querysub.commitLocal(() => {
-            context.state.phase1Searching = false;
-            context.state.phase2Searching = false;
+            clearSearchingFlags();
         });
     };
 
     // Entry point for lifecycle search. Currently only supports "find all" mode by passing empty seedDatums.
     // Future: Will be parameterized to accept specific log datums from other pages.
-    let searchLifeCycle = async (lifeCycleId: string) => {
+    let searchLifeCycle = async (params: {
+        lifeCycleId: string;
+        additionalSearch: string;
+        limit: number;
+    }) => {
+        let { lifeCycleId, additionalSearch, limit } = params;
         let lifeCycles = await context.controller.getLifeCycles.promise();
         if (!lifeCycles) return;
 
@@ -75,13 +105,7 @@ export function createLifeCycleSearch(
         await cancel();
 
         Querysub.commitLocal(() => {
-            context.state.searchingLifeCycleId = lifeCycleId;
-            context.state.phase1Results = [];
-            context.state.phase1InvalidResults = [];
-            context.state.phase1Stats = undefined;
-            context.state.phase2Results = [];
-            context.state.phase2Stats = undefined;
-            context.state.lifecycleInstances = [];
+            resetAllSearchState(lifeCycleId);
         });
 
         searchSequenceNumber++;
@@ -90,6 +114,8 @@ export function createLifeCycleSearch(
         let phase1Results = await searchPhase1({
             lifeCycle,
             currentSequenceNumber,
+            additionalSearch,
+            limit,
         });
 
         if (searchSequenceNumber !== currentSequenceNumber) return;
@@ -102,6 +128,7 @@ export function createLifeCycleSearch(
             lifeCycle,
             phase1Results,
             currentSequenceNumber,
+            limit,
         });
 
         if (searchSequenceNumber !== currentSequenceNumber) return;
@@ -123,8 +150,10 @@ export function createLifeCycleSearch(
     let searchPhase1 = async (params: {
         lifeCycle: LifeCycle;
         currentSequenceNumber: number;
+        additionalSearch: string;
+        limit: number;
     }) => {
-        let { lifeCycle, currentSequenceNumber } = params;
+        let { lifeCycle, currentSequenceNumber, additionalSearch, limit } = params;
         let startEntries = lifeCycle.entries.filter(e => e.isStart);
 
         if (startEntries.length === 0) {
@@ -132,14 +161,10 @@ export function createLifeCycleSearch(
         }
 
         Querysub.commitLocal(() => {
-            context.state.phase1Searching = true;
-            context.state.phase1Results = [];
-            context.state.phase1InvalidResults = [];
-            context.state.phase1Stats = undefined;
+            resetPhase1State();
         });
 
         let searchQuery = startEntries.map(entry => entry.matchPattern).join("|");
-        let additionalSearch = additionalSearchURL.value.trim();
         if (additionalSearch) {
             searchQuery = searchQuery + "&" + additionalSearch;
         }
@@ -201,7 +226,7 @@ export function createLifeCycleSearch(
                 params: {
                     startTime: range.startTime,
                     endTime: range.endTime,
-                    limit: limitURL.value,
+                    limit: limit,
                     findBuffer: searchBuffer,
                     searchFromStart: range.searchFromStart,
                 },
@@ -272,17 +297,16 @@ export function createLifeCycleSearch(
         lifeCycle: LifeCycle;
         phase1Results: LogDatum[];
         currentSequenceNumber: number;
+        limit: number;
     }): Promise<{ results: LogDatum[]; hitLimit: boolean }> => {
-        let { lifeCycle, phase1Results, currentSequenceNumber } = params;
+        let { lifeCycle, phase1Results, currentSequenceNumber, limit } = params;
 
         if (phase1Results.length === 0) {
             return { results: [], hitLimit: false };
         }
 
         Querysub.commitLocal(() => {
-            context.state.phase2Searching = true;
-            context.state.phase2Results = [];
-            context.state.phase2Stats = undefined;
+            resetPhase2State();
         });
 
         let allResults: LogDatum[] = [];
@@ -391,7 +415,7 @@ export function createLifeCycleSearch(
 
             let results: LogDatum[] = [];
             let done = false;
-            let searchLimit = limitURL.value * lifeCycle.entries.length * queries.size * 3;
+            let searchLimit = Math.ceil(limit * lifeCycle.entries.length * PHASE_2_FACTOR);
             let result = await errorToUndefined(logger.clientFind({
                 params: {
                     startTime: range.startTime,
@@ -426,6 +450,7 @@ export function createLifeCycleSearch(
         Querysub.commitLocal(() => {
             context.state.phase2Results = allResults;
             context.state.phase2Searching = false;
+            context.state.phase2HitLimit = anyHitLimit;
         });
 
         return { results: allResults, hitLimit: anyHitLimit };
@@ -463,7 +488,7 @@ export function createLifeCycleSearch(
                 keyValuePairs.push({ key: keyConfig.ourKey, value });
             }
             if (!hasAllKeys) continue;
-            let compositeKey = getPathStr(keyValuePairs.flatMap(kv => [kv.key, String(kv.value)]));
+            let keyHash = getPathStr(keyValuePairs.flatMap(kv => [kv.key, String(kv.value)]));
 
             let matchStr = String(datum.param0);
             let matchBuffer = Buffer.from(matchStr, "utf8");
@@ -472,10 +497,10 @@ export function createLifeCycleSearch(
                 let matcher = createMatchesPatternCached(entry.matchPattern);
                 if (!matcher(matchBuffer)) continue;
 
-                let instanceList = instancesByKey.get(compositeKey);
+                let instanceList = instancesByKey.get(keyHash);
                 if (!instanceList) {
                     instanceList = [];
-                    instancesByKey.set(compositeKey, instanceList);
+                    instancesByKey.set(keyHash, instanceList);
                 }
 
                 let currentInstance = maybeUndefined(instanceList[instanceList.length - 1]);
@@ -506,8 +531,7 @@ export function createLifeCycleSearch(
                             key: keyFields.map(k => k.ourKey).join(","),
                             keys: keyValuePairs,
                             entries: [{ datum, matchPattern: entry.matchPattern }],
-                            isWarning: true,
-                            warningMessage: "End entry without matching start",
+                            isMissingStart: true,
                             startTime: 0,
                         });
                     }
@@ -519,8 +543,7 @@ export function createLifeCycleSearch(
                             key: keyFields.map(k => k.ourKey).join(","),
                             keys: keyValuePairs,
                             entries: [{ datum, matchPattern: entry.matchPattern }],
-                            isWarning: true,
-                            warningMessage: "Entry without matching start",
+                            isMissingStart: true,
                             startTime: 0,
                         });
                     }
@@ -534,6 +557,9 @@ export function createLifeCycleSearch(
             if (instance.entries.length > 0) {
                 instance.startTime = instance.entries[0].datum.time;
             }
+            if (!instance.isComplete && !instance.isTruncated && !instance.isMissingStart) {
+                instance.isMissingEnd = true;
+            }
         }
 
         sort(allInstances, x => x.startTime);
@@ -547,6 +573,7 @@ export function createLifeCycleSearch(
         searchPhase2,
         groupLifecycleResults,
         cancel,
+        resetAllSearchState,
     };
 }
 

package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx

@@ -30,12 +30,15 @@ export type LifeCycleEntry = {
         ourKey: string;
         // Defaults to the same key, which is usually the case
         startKey?: string;
+
+        aliases?: string[];
     }[];
 
     // All of the special variables, which are marked. The title defaults to the key, but can be overriden for the purposes of adding notes.
     variables: {
         [key: string]: {
             title?: string;
+            aliases?: string[];
         };
     };
 };

package/src/diagnostics/misc-pages/ArchiveViewer.tsx

@@ -190,7 +190,8 @@ export class ArchiveViewer extends qreact.Component {
             this.state.parsedValues = atomicObjectWrite({});
         });
 
-        const MAX_FILES = 1000;
+        // NOTE: I don't understand why we were grouping this. Doing a thousand calls is not that slow, there's not that much overhead. And we want to have small groups, that way, we get more progress. 
+        const MAX_FILES = 1;
         const MAX_SIZE = 500 * 1024 * 1024;
         let bufferObjs: { path: string; buffer: Buffer }[] = [];
         let newParsedValues: { [path: string]: PathValue[] } = {};
@@ -248,7 +249,7 @@ export class ArchiveViewer extends qreact.Component {
             this.state.progress.filesParsed.max = remainingFiles.size + bufferObjs.length;
         });
 
-        for (let group of fileGroups) {
+        await Promise.all(fileGroups.map(async group => {
             //let time = Date.now();
             let newBuffers = await controller.getRawValueFiles(group.file, {
                 // 1) Allows viewing old archives
@@ -276,7 +277,7 @@ export class ArchiveViewer extends qreact.Component {
                 this.state.progress.filesDownloaded.value += group.file.length;
                 this.state.progress.bytesDownloaded += newBuffers.reduce((sum, buffer) => sum + (buffer?.buffer.byteLength || 0), 0);
             });
-        }
+        }));
         Querysub.commitLocal(() => {
             this.state.progress.filesDownloaded.endTime = Date.now();
         });

package/src/diagnostics/misc-pages/ArchiveViewerTree.tsx

@@ -82,6 +82,9 @@ export class ArchiveViewerTree extends qreact.Component<{
                 if (scaleType === "size") {
                     total = node.totalSizeEstimate;
                 }
+                if (node.values.length === 1 && total > 1000 * 1000 * 1000) {
+                    debugger;
+                }
                 countInfo += `(${formatNumber(node.values.length)} /// ${sumFormatter(total)}) `;
             } else {
                 countInfo += `(${formatNumber(node.values.length)}) `;
@@ -395,7 +398,7 @@ export class ArchiveViewerTree extends qreact.Component<{
                 <div class={css.hbox(20)}>
                     <ButtonSelector<typeof archiveTreeScale.value>
                         options={[
-                            { title: this.props.valueLabel || "Sum", value: "size" },
+                            { title: this.props.valueLabel || "Size", value: "size" },
                             { title: "Datum Count", value: "count" },
                             { title: "Equal", value: "equal" },
                         ]}
@@ -516,11 +519,11 @@ function treeifyValues(values: PathValue[], config: {
         return node;
     }
     function estimateSize(value: PathValue) {
-        return Number(value.value) || 1;
+        //return 1;
         // if (config.sizeValueEstimateHack) {
         //     return Number(value.value) || 1;
         // }
-        // return getSingleSizeEstimate(value);
+        return getSingleSizeEstimate(value);
     }
 
     for (let value of values) {

package/src/library-components/StartEllipsis.tsx

@@ -6,7 +6,7 @@ export class StartEllipsis extends qreact.Component<{
 }> {
     render() {
         return <div className={css.relative.maxWidth(this.props.maxWidth).overflowHidden.whiteSpace("nowrap").direction("rtl")}>
-            <span className={css.relative.direction("ltr")}>{this.props.children}</span>
+            <span className={css.relative.direction("ltr").unicodeBidi("isolate")}>{this.props.children}</span>
             <span className={css.absolute.left(`calc((100% - ${this.props.maxWidth}px) * -10000)`).top(0).height("100%").width(30).background("linear-gradient(to right, white, transparent)").pointerEvents("none")}></span>
         </div>;
     }