querysub 0.326.0 → 0.327.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.326.0",
+  "version": "0.327.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/diagnostics/logs/FastArchiveAppendable.ts

@@ -516,6 +516,7 @@ export class FastArchiveAppendable<Datum> {
                 });
 
                 const onDecompressedData = createLogScanner({
+                    debugName: file.path,
                     onParsedData,
                 });
                 let batchedData: Buffer[] = [];
@@ -697,14 +698,18 @@ export class FastArchiveAppendable<Datum> {
 
 
 export function createLogScanner(config: {
+    debugName: string;
     onParsedData: (posStart: number, posEnd: number, buffer: Buffer | "done") => MaybePromise<void>;
 }): (data: Buffer | "done") => Promise<void> {
     const { onParsedData } = config;
     let pendingData: Buffer[] = [];
 
+    let finished = false;
+
     let delimitterMatchIndex = 0;
-    return runInSerial(async (data: Buffer | "done") => {
+    return (async (data: Buffer | "done") => {
         if (data === "done") {
+            finished = true;
             // Flush any pending data, even though we have no delimitter. It will probably fail to parse, but... maybe it will work?
             if (pendingData.length > 0) {
                 let combinedBuffer = Buffer.concat(pendingData);
@@ -714,6 +719,9 @@ export function createLogScanner(config: {
             await onParsedData(0, 0, "done");
             return;
         }
+        if (finished) {
+            throw new Error(`Finished scan, but we received more data: ${data.length}, sample is: ${data.slice(0, 100).toString("hex")}, ${config.debugName}`);
+        }
 
         let lastStart = 0;
         await measureBlock(async () => {
@@ -734,6 +742,7 @@ export function createLogScanner(config: {
                             ...pendingData,
                             data.slice(lastStart, i + 1),
                         ]).slice(0, -objectDelimitterBuffer.length);
+                        pendingData = [];
                         posStart = 0;
                         posEnd = buffer.length;
                     } else {
@@ -741,19 +750,12 @@ export function createLogScanner(config: {
                         posStart = lastStart;
                         posEnd = i + 1 - objectDelimitterBuffer.length;
                     }
-                    // Delimitter was the start of the chunk, and it's the first chunk. Just skip it.
-                    if (posStart === posEnd && i === 0) {
-                        lastStart = i + 1;
-                        continue;
-                    }
-
                     // Only sometimes awaiting here makes scanning almost 2X faster, in the normal case, somehow?
                     let maybePromise = onParsedData(posStart, posEnd, buffer);
                     if (maybePromise) {
                         await maybePromise;
                     }
 
-                    pendingData = [];
                     lastStart = i + 1;
                 }
             }

package/src/diagnostics/logs/FastArchiveViewer.tsx

@@ -15,7 +15,7 @@ import { ButtonSelector } from "../../library-components/ButtonSelector";
 import { Button } from "../../library-components/Button";
 import { lazy } from "socket-function/src/caching";
 import { LOG_LIMIT_FLAG } from "./diskLogger";
-import { canHaveChildren } from "socket-function/src/types";
+import { MaybePromise, canHaveChildren } from "socket-function/src/types";
 import { niceParse } from "../../niceStringify";
 import { FileMetadata } from "./FastArchiveController";
 
@@ -30,7 +30,8 @@ const caseInsensitiveParam = new URLParam("caseInsensitive", false);
 
 export class FastArchiveViewer<T> extends qreact.Component<{
     fastArchives: FastArchiveAppendable<T>[];
-    onStart: () => void;
+    runOnLoad?: boolean;
+    onStart: () => MaybePromise<void>;
     getWantData?: (file: FileMetadata) => Promise<((posStart: number, posEnd: number, data: Buffer) => boolean) | undefined>;
     onDatums: (source: FastArchiveAppendable<T>, datums: T[], metadata: FileMetadata) => void;
     // Called after onData
@@ -38,12 +39,13 @@ export class FastArchiveViewer<T> extends qreact.Component<{
     onFinish?: () => void;
 }> {
     state = t.state({
+        runCount: t.atomic<number>(0),
         // rootPath =>
         fileMetadata: t.atomic<({
             files: FileMetadata[];
             createTime?: number;
         } | undefined)[]>([]),
-        finished: t.atomic(false),
+        finished: t.atomic(true),
         error: t.atomic<string | undefined>(undefined),
         pendingSyncInitializations: t.atomic<number>(0),
 
@@ -108,6 +110,11 @@ export class FastArchiveViewer<T> extends qreact.Component<{
         this.latestSequenceNumber = this.currentSequenceNumber;
         const mySequenceNumber = this.currentSequenceNumber;
 
+        // Increment run count for each new run
+        Querysub.commit(() => {
+            this.state.runCount++;
+        });
+
         // Helper function to check if this sequence number is still the latest
         const isLatestSync = () => mySequenceNumber === this.latestSequenceNumber;
 
@@ -150,7 +157,7 @@ export class FastArchiveViewer<T> extends qreact.Component<{
 
             this.histogramStartTime = timeRange.startTime - this.histogramBucketTime * 2;
             this.histogramEndTime = timeRange.endTime + this.histogramBucketTime * 2;
-            const bucketCount = Math.ceil((this.histogramEndTime - this.histogramStartTime) / this.histogramBucketTime);
+            const bucketCount = clamp(Math.ceil((this.histogramEndTime - this.histogramStartTime) / this.histogramBucketTime), 1, 10000);
 
             this.histogramAllDataCounts = new Float64Array(bucketCount);
             this.histogramSelectedDataCounts = new Float64Array(bucketCount);
@@ -181,9 +188,7 @@ export class FastArchiveViewer<T> extends qreact.Component<{
             }
         };
         try {
-            ifLatest(() => {
-                onStart();
-            });
+            await onStart();
 
             const caseInsensitive = Querysub.fastRead(() => caseInsensitiveParam.value);
             let caseInsensitiveMapping = new Uint8Array(256);
@@ -571,6 +576,11 @@ export class FastArchiveViewer<T> extends qreact.Component<{
                         flavor="large"
                         fillWidth
                         onKeyUp={this.handleDownload}
+                        ref2={() => {
+                            if (this.props.runOnLoad) {
+                                void this.handleDownload();
+                            }
+                        }}
                         noEnterKeyBlur
                         placeholder="Filter terms, ex x | y & z"
                     />
@@ -584,17 +594,21 @@ export class FastArchiveViewer<T> extends qreact.Component<{
                         flavor="small"
                     />
                     <div className={css.vbox(10)}>
-                        {this.state.fileMetadata
-                            && (
-                                <div
-                                    className={infoDisplay(120)}
-                                    title={this.state.fileMetadata.map(x => x?.files || []).flat().map(x =>
-                                        `${x.path} (${formatNumber(x.size)})`
-                                    ).join("\n")}
-                                >
-                                    File count: {formatNumber(totalFileCount)}, Backblaze size: {formatNumber(totalBackblazeByteCount)}B (compressed), Disk size: {formatNumber(totalLocalByteCount)}B (uncompressed)
-                                </div>
-                            )}
+                        {this.state.runCount > 0 && (
+                            <div
+                                className={infoDisplay(120)}
+                                title={this.state.fileMetadata.map(x => x?.files || []).flat().map(x =>
+                                    `${x.path} (${formatNumber(x.size)})`
+                                ).join("\n")}
+                            >
+                                File count: {formatNumber(totalFileCount)}, Backblaze size: {formatNumber(totalBackblazeByteCount)}B (compressed), Disk size: {formatNumber(totalLocalByteCount)}B (uncompressed)
+                            </div>
+                        )}
+                        {this.state.runCount === 0 && (
+                            <div className={infoDisplay(200)}>
+                                No data downloaded yet. Click Run to download data.
+                            </div>
+                        )}
                         {this.state.finished && <div
                             className={infoDisplay(60).button}
                             onClick={() => {

package/src/diagnostics/logs/LogViewer2.tsx

@@ -144,7 +144,8 @@ export class LogViewer2 extends qreact.Component {
                 <FastArchiveViewer
                     ref2={x => this.fastArchiveViewer = x}
                     fastArchives={logs}
-                    onStart={() => {
+                    runOnLoad={errorNotifyToggleURL.value}
+                    onStart={async () => {
                         this.datumCount = 0;
                         this.notMatchedCount = 0;
                         this.errors = 0;
@@ -167,6 +168,9 @@ export class LogViewer2 extends qreact.Component {
                                 });
                             }
                         })();
+                        // ALWAYS update it, as the synchronous one might be out of date, and if we use an outdated one extra errors show up.
+                        suppressionList = await suppressionController.getSuppressionList.promise();
+
                     }}
                     getWantData={async (file) => {
                         if (!hasErrorNotifyToggle) return undefined;

package/src/diagnostics/logs/errorNotifications/ErrorNotificationController.ts

@@ -224,22 +224,26 @@ class SuppressionList {
         suppressionUpdatedChannel.watch(() => {
             void this.updateEntriesNow();
         });
+        await runInfinitePollCallAtStart(SUPPRESSION_POLL_INTERVAL, async () => {
+            await this.updateEntriesNow();
+        });
     });
+    private cacheEntries: SuppressionListBase | undefined = undefined;
     private updateEntriesNow = async () => {
         let entries = await suppressionListArchive.get(suppressionListKey);
         if (!entries) {
             entries = { entries: {} };
         }
-        this.getEntries.set(Promise.resolve(entries));
+        this.cacheEntries = entries;
     };
-    private getEntries = lazy(async (): Promise<SuppressionListBase> => {
+    private async getEntries(): Promise<SuppressionListBase> {
         await this.init();
-        await runInfinitePollCallAtStart(SUPPRESSION_POLL_INTERVAL, async () => {
-            await this.updateEntriesNow();
-        });
+        if (!this.cacheEntries) {
+            throw new Error("Cache entries not set? Should be impossible.");
+        }
         // Infinite poll will have set this, so we don't infinitely loop
-        return await this.getEntries();
-    });
+        return this.cacheEntries;
+    }
 
     public async filterObjsToNonSuppressed(objs: LogDatum[]): Promise<LogDatum[]> {
         // NOTE: Streamed data should be rare enough, that handling this inefficiently is okay.
@@ -259,6 +263,7 @@ class SuppressionList {
         }
         let buffer = Buffer.concat(parts);
         let scanner = await this.scanForRecentErrors({
+            debugName: "filterObjsToNonSuppressed",
             startTime,
             endTime,
         });
@@ -266,6 +271,7 @@ class SuppressionList {
         return await scanner.finish();
     }
     public async scanForRecentErrors(config: {
+        debugName: string;
         startTime: number;
         endTime: number;
     }): Promise<{
@@ -285,6 +291,7 @@ class SuppressionList {
         // for the suppression key. 
         let obj: { outdatedSuppressionKey?: string } = {};
         let callback = createLogScanner({
+            debugName: config.debugName,
             onParsedData: (posStart, posEnd, buffer) => {
                 if (buffer === "done") {
                     return;
@@ -297,7 +304,7 @@ class SuppressionList {
                 try {
                     datum = JSON.parse(buffer.slice(posStart, posEnd).toString()) as LogDatum;
                 } catch (e: any) {
-                    process.stderr.write(`Failed to parse log datum in around ${buffer.slice(posStart, posEnd).slice(0, 100).toString("hex")}, error is:\n${e.stack}`);
+                    process.stderr.write(`Failed to parse log datum in around ${buffer.slice(posStart, posEnd).slice(0, 100).toString("hex")}, in source ${config.debugName}, error is:\n${e.stack}`);
                     return;
                 }
                 if (obj.outdatedSuppressionKey) {
@@ -307,18 +314,19 @@ class SuppressionList {
             },
         });
         let lastWaitTime = Date.now();
+        const stream = runInSerial(async (buffer: Buffer | "done") => {
+            // TODO: Maybe we should add this pattern to batching.ts? Basically, if we get called fast, we allow the calls through. BUT, if we called slowly OR we are doing a lot of processing (and so we are working for all of SELF_THROTTLE_INTERVAL), then we wait. This prevents this from taking over the machine. The back off is steep though, and if the machine is lagging we might reduce to a trickle, just getting 1 call in per SELF_THROTTLE_DELAY + synchronous lag from work in other parts of the program.
+            let now = Date.now();
+            if (now - lastWaitTime > SELF_THROTTLE_INTERVAL) {
+                await delay(SELF_THROTTLE_DELAY);
+                lastWaitTime = now;
+            }
+            await callback(buffer);
+        });
         return {
-            onData: runInSerial(async (buffer) => {
-                // TODO: Maybe we should add this pattern to batching.ts? Basically, if we get called fast, we allow the calls through. BUT, if we called slowly OR we are doing a lot of processing (and so we are working for all of SELF_THROTTLE_INTERVAL), then we wait. This prevents this from taking over the machine. The back off is steep though, and if the machine is lagging we might reduce to a trickle, just getting 1 call in per SELF_THROTTLE_DELAY + synchronous lag from work in other parts of the program.
-                let now = Date.now();
-                if (now - lastWaitTime > SELF_THROTTLE_INTERVAL) {
-                    await delay(SELF_THROTTLE_DELAY);
-                    lastWaitTime = now;
-                }
-                await callback(buffer);
-            }),
+            onData: stream,
             finish: async () => {
-                await callback("done");
+                await stream("done");
                 // NOTE: We COULD limit as we run, however... how many errors are we really going to encounter that AREN'T suppressed? Suppression is supposed to prevent overload anyways. I guess worst case scenario, yes, we could get overloaded, but... if we are logging more NEW errors than we can store in memory, we have bigger problems...
                 return limitRecentErrors(datums);
             },
@@ -341,7 +349,8 @@ class SuppressionList {
     }
 
     public async getSuppressionList(): Promise<SuppressionEntry[]> {
-        return Object.values((await this.getEntries()).entries);
+        let entries = Object.values((await this.getEntries()).entries);
+        return entries;
     }
 }
 const suppressionList = new SuppressionList();
@@ -597,6 +606,7 @@ class RecentErrors {
                     path = await urlCache.getURLLocalPath(file.url, hash);
                     if (!path) continue;
                     let scanner = await suppressionList.scanForRecentErrors({
+                        debugName: file.url,
                         startTime: file.startTime,
                         endTime: file.endTime,
                     });

package/src/diagnostics/logs/errorNotifications/ErrorWarning.tsx

@@ -10,7 +10,7 @@ import { ATag } from "../../../library-components/ATag";
 import { managementPageURL, showingManagementURL } from "../../managementPages";
 import { errorNotifyToggleURL } from "../LogViewer2";
 import { Querysub } from "../../../4-querysub/QuerysubController";
-import { nextId, timeInDay, timeInHour } from "socket-function/src/misc";
+import { deepCloneJSON, nextId, timeInDay, timeInHour } from "socket-function/src/misc";
 import { formatNumber } from "socket-function/src/formatting/format";
 import { Icon } from "../../../library-components/icons";
 import { filterParam } from "../FastArchiveViewer";
@@ -133,14 +133,15 @@ export class ErrorWarning extends qreact.Component {
                 {topExpired &&
                     <div className={css.hbox(8)}>
                         <Button onClick={() => {
+                            let newObj = deepCloneJSON({
+                                ...topExpired!,
+                                expiresAt: Date.now() + timeInDay,
+                            });
                             void Querysub.onCommitFinished(async () => {
-                                await suppressionController.setSuppressionEntry.promise({
-                                    ...topExpired!,
-                                    expiresAt: Date.now() + timeInDay,
-                                });
+                                await suppressionController.setSuppressionEntry.promise(newObj);
                             });
                         }}>
-                            Ignore Again
+                            Ignore Again ({formatDateJSX(topExpired.expiresAt)})
                         </Button>
                         <div>
                             Match Pattern =

package/src/diagnostics/logs/errorNotifications/errorWatchEntry.tsx

@@ -4,24 +4,28 @@ import { RecentErrorsController, recentErrorsChannel, watchRecentErrors } from "
 import { timeInSecond } from "socket-function/src/misc";
 import { formatDateTime } from "socket-function/src/formatting/format";
 
+//todonext
 
+// 1) Fix whatever is causing our memory usage to highly fluctuate. We seem to just go up and up and up from 100 megabytes all the way up to 300 megabytes and then we GC going back to 100. I turned on the socket function logging and I don't think we're sending network messages. And I ran a profile and it doesn't seem to be doing anything slow, but what are we even doing? We need to fix this. 
+//      - It could be the FPS display. It's pretty easy to check. 
+// YEP! I think it's the FPS display. I disabled it. Now we could see if the air parsing lag goes away and if it does and then comes back when we turn back on the FPS display then you know we're gonna have to fix that. I could understand lagging but why would it be allocating so much? Is it the animation rendering? It definitely could be that because the animations are slow to render and can lag Chrome and can lag the GPU and can lag other windows. 
 
-// 3) Dismissing of certain errors is not working. They keep coming back.
-//      - I think our expiry date comparison code might be wrong. It seems like once they leave the maybe expired range they come back immediately. If we can reproduce this locally, it'll be trivial to debug because the suppression stuff is global, so we can just see if there are any errors, and if there are, we break in on them. 
-// UGH... To debug this, we need to ignore the changes and then we need to work on the other stuff and then we need to come back later and see if those changes have shown up again. We basically need to debug it when it happens. We can't debug it now. It's too late, Now the errors should be showing up because they are expired. 
+// 2) The error parsing page is clearly lagging. Even when loading just the error data for the last few days, which is less than a megabytes of total logs, the error loading animation is locking up. Stuttering that is. We need to fix this. 
 
 
+// Deploy everything and then update the server runner itself. 
+
 // The constant error notifications might be fixed now. We'll see tomorrow after all the rolling updates finish. 
 // 4) fix whatever's causing constant error notifications. Something is broadcasting on the Recent Errors Change channel constantly. 
 //      - I guess usually there's no server that's going to be listening on it. So it's... Not that big of a deal, but it's still annoying. 
 
 
-//  4.1) update channel watching so you can specify that you want to watch only on a specific node ID and then update our code so we only watch it on the controller node ID that we're interfacing with. 
-
 
-// 5) Verify our suppression updates broadcast across the channel correctly, causing us to be able to suppress a notification and our watching script to then stop seeing the new updates. Realistically, it's the calling script that stops setting them, but same thing. 
 
+// Back to getting a Node.js watcher that works smoothly without receiving a whole bunch of extra errors etc.
+//  4.1) update channel watching so you can specify that you want to watch only on a specific node ID and then update our code so we only watch it on the controller node ID that we're interfacing with. 
 
+// 5) Verify our suppression updates broadcast across the channel correctly, causing us to be able to suppress a notification and our watching script to then stop seeing the new updates. Realistically, it's the calling script that stops setting them, but same thing. 
 
 // 5) Set up the Instant Messaging Sending API. 
 //  - Discord. With beeper it won't really matter what we're messaging. We could also do WhatsApp. It's really all the same.  

package/src/library-components/SyncedController.ts

@@ -318,9 +318,11 @@ export function getSyncedController<T extends SocketRegistered>(
                         }
                         // Don't cache promise calls
                         void promise.finally(() => {
-                            if (obj.promise === promise) {
-                                obj.promise = undefined;
-                            }
+                            Querysub.fastRead(() => {
+                                if (obj.promise === promise) {
+                                    obj.promise = undefined;
+                                }
+                            });
                         });
                         return promise;
                     });