querysub 0.349.0 → 0.350.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.349.0",
+  "version": "0.350.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/diagnostics/logs/FastArchiveAppendable.ts

@@ -35,6 +35,7 @@ import { assertIsNetworkTrusted } from "../../-d-trust/NetworkTrust2";
 import { blue, magenta } from "socket-function/src/formatting/logColors";
 import { FileMetadata, FastArchiveAppendableControllerBase, FastArchiveAppendableController, getFileMetadataHash } from "./FastArchiveController";
 import { fsExistsAsync } from "../../fs";
+import { ScanFnc } from "./FastArchiveViewer";
 
 // NOTE: In a single command line micro-test it looks like we can write about 40K writes of 500 per once, when using 10X parallel, on a fairly potato server. We should probably batch though, and only do 1X parallel.
 /*
@@ -360,6 +361,7 @@ export class FastArchiveAppendable<Datum> {
             endTime: number;
         };
         cacheBust: number;
+        scanFnc?: ScanFnc;
         getWantData?: (file: FileMetadata) => Promise<((posStart: number, posEnd: number, data: Buffer) => boolean) | undefined>;
         onData: (datum: Datum[], file: FileMetadata) => void;
         // Called after onData
@@ -519,7 +521,7 @@ export class FastArchiveAppendable<Datum> {
             let processProgress = await createProgress("Processing (datums)", 0);
             let corruptDatumsProgress = await createProgress("Corrupt Datums (datums)", 0);
 
-            const self = this;
+            let scanFnc = config.scanFnc;
 
 
             async function downloadAndParseFile(file: FileMetadata, runInner: (code: () => Promise<void>) => Promise<void>) {
@@ -546,6 +548,9 @@ export class FastArchiveAppendable<Datum> {
                         scanProgressCount++;
                     }
                     if (buffer !== "done") {
+                        if (scanFnc) {
+                            scanFnc(posStart, posEnd, buffer);
+                        }
                         if (wantData && !wantData(posStart, posEnd, buffer)) {
                             notMatchedSize += (posEnd - posStart);
                             notMatchedCount++;

package/src/diagnostics/logs/FastArchiveViewer.tsx

@@ -25,15 +25,22 @@ const RENDER_INTERVAL = 1000;
 const HISTOGRAM_RERENDER_INTERVAL = 10000;
 
 export const filterParam = new URLParam("filter", "");
+// If filter2 is set, we also need it. Not great (why not support arbitrary counts of these), but for now... this should be fine (and support arbitrary counts might slow down our incredibly optimized scan function, where most of our time is spent).
+// NOTE: This supports less than filterParam (no object parsing, it only does text contains)
+export const filterParam2 = new URLParam("filter2", "");
+
 export const cacheBustParam = new URLParam("cacheBust", 0);
 const caseInsensitiveParam = new URLParam("caseInsensitive", false);
 const hideAllDataParam = new URLParam("hideAllData", false);
 
+export type ScanFnc = (posStart: number, posEnd: number, data: Buffer) => boolean;
+
 export class FastArchiveViewer<T> extends qreact.Component<{
     fastArchives: FastArchiveAppendable<T>[];
     runOnLoad?: boolean;
     onStart: () => MaybePromise<void>;
-    getWantData?: (file: FileMetadata) => Promise<((posStart: number, posEnd: number, data: Buffer) => boolean) | undefined>;
+    getScanFnc?: () => ScanFnc;
+    getWantData?: (file: FileMetadata) => Promise<ScanFnc | undefined>;
     onDatums: (source: FastArchiveAppendable<T>, datums: T[], metadata: FileMetadata) => void;
     // Called after onData
     onStats?: (source: FastArchiveAppendable<T>, stats: DatumStats, metadata: FileMetadata) => void;
@@ -108,6 +115,7 @@ export class FastArchiveViewer<T> extends qreact.Component<{
 
     @measureFnc
     private async synchronizeData() {
+        console.log("synchronizeData");
         let onStart = Querysub.fastRead(() => this.props.onStart);
         let onDatums = Querysub.fastRead(() => this.props.onDatums);
         let onStats = Querysub.fastRead(() => this.props.onStats);
@@ -144,7 +152,7 @@ export class FastArchiveViewer<T> extends qreact.Component<{
 
         const timeRange = getTimeRange();
         let filterString = filterParam.value;
-
+        let filterString2 = filterParam2.value;
         let scannedValueCount = 0;
 
         // Store current sync parameters BEFORE calling synchronizeData
@@ -310,6 +318,15 @@ export class FastArchiveViewer<T> extends qreact.Component<{
             });
             let { scanMatch, datumMatch } = joinMatches(andMatches, "||");
 
+            let andMatches2 = filterString2.split("|").map(orParts => {
+                let andParts = orParts.split("&");
+                let andMatches = andParts.map(part => parsePart(part.trim()));
+                return joinMatches(andMatches, "&&");
+            });
+            let scanObj2 = joinMatches(andMatches2, "||");
+            let scanMatch2 = scanObj2.scanMatch;
+            let datumMatch2 = scanObj2.datumMatch;
+            let useScan2 = !!filterString2.trim();
 
             const limitedBuffer = Buffer.from(LOG_LIMIT_FLAG);
 
@@ -318,6 +335,7 @@ export class FastArchiveViewer<T> extends qreact.Component<{
                 const result = await fastArchive.synchronizeData({
                     range: timeRange,
                     cacheBust: Querysub.fastRead(() => cacheBustParam.value),
+                    scanFnc: Querysub.fastRead(() => this.props.getScanFnc?.()),
                     getWantData: async (file) => {
                         let wantData = await getWantData?.(file);
                         return (posStart: number, posEnd: number, data: Buffer) => {
@@ -341,6 +359,9 @@ export class FastArchiveViewer<T> extends qreact.Component<{
 
                             // scanMatch is faster than wantData (generally), so run it first
                             let matched = scanMatch(posStart, posEnd, data);
+                            if (matched && useScan2) {
+                                matched = scanMatch2(posStart, posEnd, data);
+                            }
                             if (matched && wantData) {
                                 matched = wantData(posStart, posEnd, data);
                             }
@@ -635,8 +656,7 @@ export class FastArchiveViewer<T> extends qreact.Component<{
         return warnings;
     }
 
-    public handleDownload = throttleFunction(500, () => {
-        console.log("handleDownload");
+    public synchronizeDataThrottled = throttleFunction(500, () => {
         Querysub.onCommitFinished(() => {
             logErrors(this.synchronizeData());
         });
@@ -683,7 +703,7 @@ export class FastArchiveViewer<T> extends qreact.Component<{
                         label={
                             <div className={css.hbox(10)}>
                                 <Button onClick={() => {
-                                    void this.handleDownload();
+                                    void this.synchronizeDataThrottled();
                                 }}>
                                     Run
                                 </Button>
@@ -694,10 +714,10 @@ export class FastArchiveViewer<T> extends qreact.Component<{
                         hot
                         flavor="large"
                         fillWidth
-                        onKeyUp={this.handleDownload}
+                        onKeyUp={this.synchronizeDataThrottled}
                         ref2={() => {
                             if (this.props.runOnLoad) {
-                                void this.handleDownload();
+                                void this.synchronizeDataThrottled();
                             }
                         }}
                         noEnterKeyBlur
@@ -708,7 +728,7 @@ export class FastArchiveViewer<T> extends qreact.Component<{
                         checkbox
                         url={caseInsensitiveParam}
                         onChange={() => {
-                            void this.handleDownload();
+                            void this.synchronizeDataThrottled();
                         }}
                         flavor="small"
                     />
@@ -729,7 +749,7 @@ export class FastArchiveViewer<T> extends qreact.Component<{
                                         <div
                                             className={infoDisplay(30).button}
                                             onClick={() => {
-                                                void this.handleDownload();
+                                                void this.synchronizeDataThrottled();
                                             }}
                                             title={outdatedWarnings.join(", ")}
                                         >
@@ -746,7 +766,7 @@ export class FastArchiveViewer<T> extends qreact.Component<{
                         })()}
                         {this.state.runCount === 0 && (
                             <div className={infoDisplay(200).button} onClick={() => {
-                                void this.handleDownload();
+                                void this.synchronizeDataThrottled();
                             }}>
                                 No data downloaded yet. Click here to download data.
                             </div>
@@ -766,7 +786,7 @@ export class FastArchiveViewer<T> extends qreact.Component<{
                                         Querysub.commit(() => {
                                             cacheBustParam.value = Date.now();
                                         });
-                                        void this.handleDownload();
+                                        void this.synchronizeDataThrottled();
                                     }}
                                 >
                                     <div className={css.hbox(8).alignItems("center").wrap}>
@@ -783,7 +803,7 @@ export class FastArchiveViewer<T> extends qreact.Component<{
                             <div className={infoDisplay(60).boldStyle.button}
                                 onClick={() => {
                                     filterParam.value = LOG_LIMIT_FLAG;
-                                    void this.handleDownload();
+                                    void this.synchronizeDataThrottled();
                                 }}
                             >
                                 Click here to see {formatNumber(this.limitedScanCount)} scanned logs were rate limited at a file level. This means lines might be missing. If this happens a lot, use pass {`{ [LOG_LINE_LIMIT_ID]: "INSERT RANDOM GUID HERE" }`} in the error/warn message to limit only the spamming line.
@@ -793,7 +813,7 @@ export class FastArchiveViewer<T> extends qreact.Component<{
                             <div className={infoDisplay(0).boldStyle.button}
                                 onClick={() => {
                                     filterParam.value += " & " + LOG_LIMIT_FLAG;
-                                    void this.handleDownload();
+                                    void this.synchronizeDataThrottled();
                                 }}
                             >
                                 Click here to see {formatNumber(this.limitedMatchCount)} matched logs were rate limited at a file level. This means lines might be missing. If this happens a lot, use pass {`{ [LOG_LINE_LIMIT_ID]: "INSERT RANDOM GUID HERE" }`} in the error/warn message to limit only the spamming line.

package/src/diagnostics/logs/LogViewer2.tsx

@@ -13,7 +13,7 @@ import { logErrors } from "../../errors";
 import { batchFunction, runInSerial } from "socket-function/src/batching";
 import { Querysub } from "../../4-querysub/QuerysubController";
 import { sort, timeInDay, timeInHour } from "socket-function/src/misc";
-import { FastArchiveViewer, cacheBustParam, filterParam } from "./FastArchiveViewer";
+import { FastArchiveViewer, cacheBustParam, filterParam, filterParam2 } from "./FastArchiveViewer";
 import { LogDatum, getLoggers, LOG_LIMIT_FLAG, LOG_LINE_LIMIT_FLAG } from "./diskLogger";
 import { ColumnType, Table, TableType } from "../../5-diagnostics/Table";
 import { formatDateJSX } from "../../misc/formatJSX";
@@ -28,6 +28,7 @@ import { RecentErrorsController, SuppressionListController, getSuppressEntryChec
 import { SocketFunction } from "socket-function/SocketFunction";
 import { throttleRender } from "../../functional/throttleRender";
 import { isNode } from "typesafecss";
+import { createLogViewerExtractField } from "./logViewerExtractField";
 
 const RENDER_INTERVAL = 1000;
 
@@ -56,6 +57,7 @@ const defaultSelectedFields = {
 export class LogViewer2 extends qreact.Component {
     state = t.state({
         datumsSeqNum: t.atomic<number>(0),
+        showCountByFile: t.boolean,
     });
 
     private example: string | undefined = undefined;
@@ -66,6 +68,7 @@ export class LogViewer2 extends qreact.Component {
     private errors = 0;
     private notMatchedCount = 0;
     private datums: LogDatum[] = [];
+    private countsPerName = new Map<string, number>();
 
     private lastRenderTime = 0;
     private suppressionCounts = new Map<string, number>();
@@ -74,7 +77,7 @@ export class LogViewer2 extends qreact.Component {
 
     rerun() {
         cacheBustParam.value = Date.now();
-        void this.fastArchiveViewer?.handleDownload();
+        void this.fastArchiveViewer?.synchronizeDataThrottled();
     }
 
     render() {
@@ -166,6 +169,7 @@ export class LogViewer2 extends qreact.Component {
                         this.datums = [];
                         this.suppressionCounts = new Map();
                         this.expiredSuppressionCounts = new Map();
+                        this.countsPerName = new Map();
                         this.operationSequenceNum++;
                         void (async () => {
                             const currentSequenceNum = this.operationSequenceNum;
@@ -183,10 +187,21 @@ export class LogViewer2 extends qreact.Component {
                         suppressionList = await suppressionController.getSuppressionList.promise();
 
                     }}
+                    getScanFnc={() => {
+                        if (!this.state.showCountByFile) {
+                            return () => false;
+                        }
+                        return createLogViewerExtractField("name", (value) => {
+                            let prevCount = this.countsPerName.get(value) ?? 0;
+                            prevCount++;
+                            this.countsPerName.set(value, prevCount);
+                        });
+                    }}
                     getWantData={async (file) => {
                         if (!hasErrorNotifyToggle) return undefined;
                         // By defaulting to the synchronous one, the list will be updated if there's any changes. However, we will also just get it asynchronously if the list hasn't been updated by the time we call get1data. And because we assign it back to the variable, it'll be cached. 
                         suppressionList = suppressionList || await suppressionController.getSuppressionList.promise();
+
                         let suppressionFull = getSuppressionFull({
                             entries: suppressionList,
                             blockTimeRange: {
@@ -329,6 +344,7 @@ export class LogViewer2 extends qreact.Component {
                                 </div>;
                             }
                         };
+                        let includedFiles = filterParam2.value.split("|").map(x => x.trim());
                         return <>
                             <div className={css.hbox(10)}>
                                 <InputPicker
@@ -373,6 +389,79 @@ export class LogViewer2 extends qreact.Component {
                             {this.datums.length === 0 && <div className={css.hsl(40, 50, 50).colorhsl(60, 50, 100).boldStyle.pad2(10).ellipsis}>
                                 No logs matched, either increase the time range or decrease the filter specificity.
                             </div>}
+                            {includedFiles.length > 0 && includedFiles[0] && <div className={css.hbox(20)}>
+                                <div className={css.hbox(10)}>
+                                    <div className={css.fontSize(18).hsl(0, 50, 50).colorhsl(0, 50, 95).pad2(8, 4)}>
+                                        Only showing {includedFiles.length} files:
+                                    </div>
+                                    <Button onClick={() => {
+                                        filterParam2.value = "";
+                                        this.rerun();
+                                    }}>
+                                        Clear All Filters
+                                    </Button>
+                                </div>
+                                <div className={css.hbox(10, 10).wrap}>
+                                    {includedFiles.filter(x => x).map((fileName) => (
+                                        <div
+                                            key={fileName}
+                                            className={css.pad2(8, 4).hsl(120, 40, 80).bord2(120, 40, 60, 1).button}
+                                            onClick={() => {
+                                                let newFiles = includedFiles.filter(x => x !== fileName);
+                                                filterParam2.value = newFiles.join("|");
+                                            }}
+                                            title={`Click to remove ${fileName} from filter`}
+                                        >
+                                            {fileName} ✕
+                                        </div>
+                                    ))}
+                                </div>
+                            </div>}
+
+                            {this.state.showCountByFile && (() => {
+                                let counts = Array.from(this.countsPerName.entries());
+                                sort(counts, x => -x[1]);
+                                let topCounts = counts.slice(0, 20);
+                                let totalCount = counts.reduce((sum, [, count]) => sum + count, 0);
+
+                                return <div className={css.vbox(10)}>
+                                    <div className={css.hbox(10, 10).wrap}>
+                                        {topCounts.map(([name, count]) => {
+                                            let percentage = totalCount > 0 ? (count / totalCount) * 100 : 0;
+                                            let isFiltered = includedFiles.includes(name);
+
+                                            return <div
+                                                key={name}
+                                                className={css.relative.pad2(8, 4).bord2(210, 30, 60, 1).ellipsis.button}
+                                                title={isFiltered ? `Click to remove ${name} from filter` : `Click to add ${name} to filter`}
+                                                onClick={() => {
+                                                    if (isFiltered) {
+                                                        // Remove from filter
+                                                        let newFiles = includedFiles.filter(x => x !== name);
+                                                        filterParam2.value = newFiles.join("|");
+                                                    } else {
+                                                        // Add to filter
+                                                        let newFiles = [...includedFiles.filter(x => x), name];
+                                                        filterParam2.value = newFiles.join("|");
+                                                    }
+                                                }}
+                                            >
+                                                <div
+                                                    className={css.absolute.pos(0, 0).height("100%").width(`${percentage}%`).hsl(isFiltered ? 120 : 210, 40, 80)}
+                                                />
+                                                <div className={css.relative.maxWidth(200).ellipsis}>
+                                                    {formatNumber(count)} | {name || "(no name present)"} {isFiltered && "✓" || ""}
+                                                </div>
+                                            </div>;
+                                        })}
+                                    </div>
+                                </div>;
+                            })() || <Button onClick={() => {
+                                this.state.showCountByFile = true;
+                                this.rerun();
+                            }}>
+                                    Show Count by File
+                                </Button>}
                             <Table
                                 rows={this.datums}
                                 columns={columns}

package/src/diagnostics/logs/errorNotifications/errorWatchEntry.tsx

@@ -121,7 +121,12 @@ const sendIMs = batchFunction(({ delay: BATCH_TIME }), async (logsAll: LogDatum[
             return;
         }
         let url = createLink(getErrorLogsLink());
-        let message = Object.values(info.perFile).flat().map(
+        let flat = Object.values(info.perFile).flat();
+        if (flat.length === 0) {
+            console.log(`All messages have hit their IM limit, so not sending any message at all.`);
+            return;
+        }
+        let message = flat.map(
             x => `[${formatDateTime(x.time)}](${url}) | ${ellipsisMiddle(String(x.param0), 100)} (${x.__NAME__})`
         ).join("\n");
         message = ellipsisMiddle(message, 900);

package/src/diagnostics/logs/logViewerExtractField.ts

@@ -0,0 +1,37 @@
+import { ScanFnc } from "./FastArchiveViewer";
+
+export function createLogViewerExtractField(
+    fieldName: string,
+    onFieldValue: (value: string) => void,
+): ScanFnc {
+    let fieldNameBuffer = Buffer.from(`${JSON.stringify(fieldName)}:"`);
+    let quoteChar = Buffer.from(`"`)[0];
+    let escapeChar = Buffer.from(`\\`)[0];
+    return (posStart: number, posEnd: number, data: Buffer) => {
+        for (let i = posStart; i < posEnd; i++) {
+            if (data[i] === fieldNameBuffer[0]) {
+                for (let j = 1; j < fieldNameBuffer.length; j++) {
+                    if (data[i + j] !== fieldNameBuffer[j]) {
+                        break;
+                    }
+                    if (j === fieldNameBuffer.length - 1) {
+                        // Parse until the string ends, ignoring escaped characters
+                        let start = i + j + 1;
+                        for (let k = start; k < posEnd; k++) {
+                            if (data[k] === quoteChar) {
+                                onFieldValue(data.slice(start, k).toString());
+                                break;
+                            }
+                            if (data[k] === escapeChar) {
+                                k++;
+                            }
+                        }
+                        return true;
+                    }
+                }
+            }
+        }
+        onFieldValue("");
+        return false;
+    };
+}

package/src/library-components/InputPicker.tsx

@@ -35,7 +35,9 @@ export class InputPicker<T> extends qreact.Component<{
     state = {
         pendingText: "",
         focused: false,
+        showLimit: this.props.matchLimit ?? 10,
     };
+    inputElem: HTMLInputElement | null = null;
     render() {
         const { singleOption, fillWidth } = this.props;
         // Input, and beside it the picked values
@@ -63,7 +65,7 @@ export class InputPicker<T> extends qreact.Component<{
             pendingMatches = resolvedOptions;
         }
         let extra = pendingMatches.length;
-        pendingMatches = pendingMatches.slice(0, this.props.matchLimit ?? 10);
+        pendingMatches = pendingMatches.slice(0, this.state.showLimit);
         extra -= pendingMatches.length;
         return (
             <div class={
@@ -78,6 +80,7 @@ export class InputPicker<T> extends qreact.Component<{
                         {this.props.label}
                     </div>
                     <Input
+                        inputRef={x => this.inputElem = x}
                         value={this.state.pendingText}
                         hot
                         alwaysUseLatestValueWhenFocused
@@ -119,7 +122,17 @@ export class InputPicker<T> extends qreact.Component<{
                             </Button>
                         ))}
                         {extra > 0 && (
-                            <Button class={css.hbox(5).button} disabled>
+                            <Button class={css.hbox(5).button + " keepModalsOpen"}
+                                onMouseDown={e => {
+                                    this.state.showLimit *= 2;
+                                    // HACK: Trying to prevent the event from causing a blur doesn't seem to work, so we'll just refocus it. And also waiting for 0 milliseconds doesn't work, so we'll wait for 100 milliseconds. 
+                                    //  - If the user clicks too fast this doesn't work, but the on-mouse out handler also won't work, Because we'll blur before on mouse up and then will be removed. 
+                                    // TODO: Instead of keeping track of the focus state, we should make it a palette and when the palette goes away we should hide the open state. 
+                                    setTimeout(() => {
+                                        this.inputElem?.focus();
+                                    }, 1000);
+                                }}
+                            >
                                 + {extra} more...
                             </Button>
                         )}