querysub 0.321.0 → 0.323.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.321.0",
+  "version": "0.323.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/2-proxy/PathValueProxyWatcher.ts

@@ -2255,15 +2255,8 @@ registerPeriodic(function checkForZombieWatches() {
     let zombies = Array.from(proxyWatcher.getAllWatchers()).filter(isZombieWatch);
     lastZombieCount = zombies.length;
     if (zombies.length > 0) {
-        console.warn(red(`Found ${zombies.length} zombie watchers`));
         let namesSet = new Set(zombies.map(x => x.debugName));
-        let topNames = Array.from(namesSet).slice(0, 5);
-        for (let name of topNames) {
-            console.warn(red(`    ${name}`));
-        }
-        if (namesSet.size > topNames.length) {
-            console.warn(red(`    ... and ${namesSet.size - topNames.length} more`));
-        }
+        console.warn(`Found ${zombies.length} zombie watchers`, { names: Array.from(namesSet) });
     }
 });
 

package/src/4-querysub/Querysub.ts

@@ -517,9 +517,9 @@ export class Querysub {
     public static getOwnMachineId = getOwnMachineId;
     public static getSelfMachineId = getOwnMachineId;
 
-    public static getOwnNodeId = getOwnNodeId;
-    public static getSelfNodeId = getOwnNodeId;
-    public static getOwnThreadId = getOwnThreadId;
+    public static getOwnNodeId = () => getOwnNodeId();
+    public static getSelfNodeId = () => getOwnNodeId();
+    public static getOwnThreadId = () => getOwnThreadId();
 
     /** Set ClientWatcher.DEBUG_SOURCES to true for to be populated */
     public static getTriggerReason() {

package/src/diagnostics/logs/FastArchiveAppendable.ts

@@ -99,7 +99,10 @@ export type DatumStats = {
 };
 
 
-export function getFileTimeStamp(path: string): number {
+export function getFileTimeStamp(path: string): {
+    startTime: number;
+    endTime: number;
+} {
     let file = path.replaceAll("\\", "/").split("/").at(-1)!;
     // Remove .log extension and parse as ISO date
     let dateStr = file.replace(/\.log$/, "");
@@ -107,7 +110,11 @@ export function getFileTimeStamp(path: string): number {
     if (dateStr.length === 13) { // YYYY-MM-DDTHH format
         dateStr += ":00:00.000Z";
     }
-    return new Date(dateStr).getTime();
+    let startTime = new Date(dateStr).getTime();
+    return {
+        startTime,
+        endTime: startTime + timeInHour,
+    };
 }
 
 
@@ -284,7 +291,7 @@ export class FastArchiveAppendable<Datum> {
                     try {
                         // We could use modified time here? Although, this is nice if we move files around, and then manually have them moved, although even then... this could cause problem be tripping while we are copying the file, so... maybe this is just wrong?
                         let timeStamp = getFileTimeStamp(fullPath);
-                        if (timeStamp > Date.now() - UPLOAD_THRESHOLD) continue;
+                        if (timeStamp.endTime > Date.now() - UPLOAD_THRESHOLD) continue;
 
                         // NOTE: Because we use the same target path, if multiple services do this at the same time it's fine. Not great, but... fine.
                         let backblazePath = FastArchiveAppendable.getBackblazePath({ fileName: file, threadId });

package/src/diagnostics/logs/FastArchiveController.ts

@@ -101,7 +101,7 @@ export class FastArchiveAppendableControllerBase {
 
                     let fileTimestamp = getFileTimeStamp(file);
                     //console.log(`Found ${new Date(fileTimestamp).toISOString()} in ${threadDir + file}`);
-                    if (fileTimestamp < timeRange.startTime || fileTimestamp > timeRange.endTime) {
+                    if (fileTimestamp.endTime < timeRange.startTime || fileTimestamp.startTime > timeRange.endTime) {
                         continue;
                     }
 
@@ -418,8 +418,9 @@ export class FastArchiveAppendableControllerBase {
                     let urlObj = new URL(url);
                     urlObj.hostname = ipDomain;
                     url = urlObj.toString();
-                    let startTime = getFileTimeStamp(file.path);
-                    let endTime = startTime + timeInHour;
+                    let timeStamp = getFileTimeStamp(file.path);
+                    let startTime = timeStamp.startTime;
+                    let endTime = timeStamp.endTime;
 
                     localFiles.push({
                         nodeId: aliveNodeId,

package/src/diagnostics/logs/TimeRangeSelector.tsx

@@ -7,7 +7,7 @@ import { formatTime } from "socket-function/src/formatting/format";
 import { InputLabel } from "../../library-components/InputLabel";
 import { Button } from "../../library-components/Button";
 import { Querysub } from "../../4-querysub/QuerysubController";
-import { timeInHour, timeInMinute } from "socket-function/src/misc";
+import { timeInDay, timeInHour, timeInMinute } from "socket-function/src/misc";
 
 // URL parameters for time range
 export const startTimeParam = new URLParam("startTime", undefined as number | undefined);
@@ -73,6 +73,15 @@ export class TimeRangeSelector extends qreact.Component {
                         }}
                         outerClass={!endTimeParam.value && css.opacity(0.5) || ""}
                     />
+                    <Button
+                        hue={110} onClick={() => {
+                            let now = Date.now();
+                            startTimeParam.value = now;
+                            endTimeParam.value = undefined;
+                        }}
+                    >
+                        Set to future data
+                    </Button>
                     {(endTimeParam.value || startTimeParam.value) && <Button
                         hue={110} onClick={resetToLastDay}
                     >

package/src/diagnostics/logs/errorNotifications/ErrorNotificationController.ts

@@ -386,7 +386,8 @@ class RecentErrors {
     });
 
     private _recentErrors: LogDatum[] = [];
-    private updateRecentErrors = (objs: LogDatum[]) => {
+    private updateRecentErrors = async (objs: LogDatum[]) => {
+        objs = await suppressionList.filterObjsToNonSuppressed(objs);
         let newRecentErrors = limitRecentErrors(objs);
         // If any changed
         let prev = new Set(this._recentErrors);
@@ -413,15 +414,15 @@ class RecentErrors {
         recentErrorsChannel.broadcast(true);
     });
 
-    private addErrors(objs: LogDatum[]) {
+    private async addErrors(objs: LogDatum[]) {
         for (let obj of objs) {
             this._recentErrors.push(obj);
         }
-        this.updateRecentErrors(this._recentErrors);
+        await this.updateRecentErrors(this._recentErrors);
     }
 
     public async onSuppressionChanged() {
-        this.updateRecentErrors(await suppressionList.filterObjsToNonSuppressed(this._recentErrors));
+        await this.updateRecentErrors(this._recentErrors);
     }
 
     private scannedHashes = new Set<string>();
@@ -440,7 +441,7 @@ class RecentErrors {
                 noLocalFiles: config.noLocalFiles,
             });
             // Filter again, as new suppressions change the errors
-            this.updateRecentErrors(await suppressionList.filterObjsToNonSuppressed(this._recentErrors));
+            await this.updateRecentErrors(this._recentErrors);
             let recentLimit = 0;
             const applyRecentLimit = () => {
                 if (this._recentErrors.length < MAX_RECENT_ERRORS) return;
@@ -512,7 +513,7 @@ class RecentErrors {
                         await fd.close();
                     }
                     let newErrors = await scanner.finish();
-                    this.addErrors(newErrors);
+                    await this.addErrors(newErrors);
                     applyRecentLimit();
                 } catch (e: any) {
                     console.error(`Failed to scan file ${file.url}, size is ${size}, error is:\n${e.stack}`);

package/src/diagnostics/logs/lifeCycleAnalysis/spec.md

@@ -4,7 +4,8 @@ Very small amount of data
     https://127-0-0-1.querysubtest.com:7007/?hot&enableLogs&page=login&filter=%22431%22&showingmanagement&endTime=1755140880000&startTime=1754950020000&managementpage=LogViewer2
 
 
-4) WTF... our http server is trying to run the queue. But we told it not to... Ugh...
+3) Our time range is not being correctly applied. We have data from 4-5, and are filtering from 4:25 to 5:32, and not matching that data, even though it overlaps a lot.
+4) API watchers are repeatedly dying. I feel like this is fixable, especially with our logs...
 
 5) Update all services, and move them to that machine
 5) Verify the hezner server can run the site well
@@ -18,6 +19,8 @@ Very small amount of data
     - https://127-0-0-1.querysubtest.com:7007/?enableLogs&page=login&showingmanagement&endTime=1757835685102.667&managementpage=LogViewer2&machineview=service-detail&startTime=1757745685102.667&serviceId=service-1756340309836&filter=__machineId%20%3D%20a794fbcf7b104c68%20%26%20Edge
     - Or... maybe logs are lost SOMETIMES, and ALWAYS when we kill the server? Although... that would mean we have multiple issues. Ugh...
 
+11) API ranges overlapped? What? I think our code to pick an empty range is wrong?
+
 6) Update URLParam to allow linking it to other parameters, resetting when they change.
     - With a function, and have standard one beside URLParam (that uses page and tab)
         - ALSO managementPageURL