querysub 0.608.0 → 0.610.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.608.0",
+  "version": "0.610.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -75,7 +75,7 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
-    "sliftutils": "^1.7.83",
+    "sliftutils": "^1.7.84",
     "socket-function": "^1.2.30",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -1,5 +1,5 @@
 import { SocketFunction } from "socket-function/SocketFunction";
-import { getArchives2 } from "../-a-archives/archives2";
+import { getArchives2, getArchives2Buffered } from "../-a-archives/archives2";
 import { getDomain, isDevDebugbreak, isNoNetwork, isPublic } from "../config";
 import { measureBlock } from "socket-function/src/profiling/measure";
 import { isNode, keyByArray, sha256Hash, throttleFunction, timeInMinute, timeInSecond } from "socket-function/src/misc";
@@ -56,7 +56,7 @@ let DEAD_NODE_POLL_COOLDOWN = timeInMinute * 5;
 
 let shutdown = false;
 
-const archives = lazy(() => getArchives2("nodes/"));
+const archives = lazy(() => getArchives2Buffered("nodes/"));
 
 let logging = true;
 export function enableNodeDiscoveryLogging() {

package/src/deployManager/components/storage/operationsView.tsx

@@ -128,6 +128,7 @@ class OperationSummaryTable extends qreact.Component<{ serverUrl: string; operat
         }
         if (!summaries) return <div>Loading...</div>;
         if (!summaries.length) return <div className={css.colorhsl(OP_METRIC_COLOR.h, OP_METRIC_COLOR.s, OP_METRIC_COLOR.l)}>No accesses recorded.</div>;
+        sort(summaries, entry => -entry.weight);
         let rows = summaries.map(entry => ({
             path: entry.path,
             value: formatAccessMetric(entry.weight, asBytes),

package/src/diagnostics/logs/errorNotifications2/errorNotifications.ts

@@ -54,6 +54,48 @@ export async function getSuppressionEntries(): Promise<SuppressionEntry[]> {
 
 let getMatcher = cacheLimited(10000, (text: string) => createMatchesPattern(Buffer.from(text), false));
 
+// The watch loop checks every incoming error against the suppression list. Reading the suppression archive once per error produced thousands of storage listings per minute, so the hot path is served from a cached copy: the first pass tolerates 15 minute staleness, and only errors that pass unsuppressed force a fresher (30 second) re-read.
+const SUPPRESSION_CACHE_TTL = timeInMinute * 15;
+const SUPPRESSION_FRESH_CACHE_TTL = 1000 * 30;
+let cachedSuppressionEntries: SuppressionEntry[] = [];
+let suppressionCacheTime = 0;
+let suppressionFreshCacheTime = 0;
+
+async function refreshSuppressionEntriesCache() {
+    cachedSuppressionEntries = await getSuppressionEntries();
+    suppressionCacheTime = Date.now();
+    suppressionFreshCacheTime = Date.now();
+    return cachedSuppressionEntries;
+}
+async function getSuppressionEntriesCached() {
+    if (Date.now() - suppressionCacheTime > SUPPRESSION_CACHE_TTL) {
+        return await refreshSuppressionEntriesCache();
+    }
+    return cachedSuppressionEntries;
+}
+async function getSuppressionEntriesFresh() {
+    if (Date.now() - suppressionFreshCacheTime > SUPPRESSION_FRESH_CACHE_TTL) {
+        return await refreshSuppressionEntriesCache();
+    }
+    return cachedSuppressionEntries;
+}
+function invalidateSuppressionEntriesCaches() {
+    suppressionCacheTime = 0;
+    suppressionFreshCacheTime = 0;
+}
+
+// Side-effect-free version of applySuppression's "suppressed" verdict, for deciding which entry set the watch loop should apply against.
+function isSuppressedByAny(error: LogDatum, suppressionEntries: SuppressionEntry[]): boolean {
+    let errorBuffer = Buffer.from(JSON.stringify(error));
+    for (let suppressionEntry of suppressionEntries) {
+        let isMatch = getMatcher(suppressionEntry.pattern);
+        if (isMatch(errorBuffer) && error.time <= suppressionEntry.timeout) {
+            return true;
+        }
+    }
+    return false;
+}
+
 const MAX_UNMATCHED = 10000;
 const MAX_EXAMPLES = 100;
 let unmatchedIndex = 0;
@@ -299,7 +341,11 @@ export async function exposeErrorWatchService() {
     let errorLogs = await getErrorLogs();
     for await (let error of watchAllValues(errorLogs)) {
         try {
-            let suppressionEntries = await getSuppressionEntries();
+            // First pass: a long-lived cached list is enough to confirm an already-known suppression (the common case). Only if it doesn't already suppress this error do we re-read from a fresher cache, before treating it as unmatched and potentially spending an AI pattern generation on it.
+            let suppressionEntries = await getSuppressionEntriesCached();
+            if (!isSuppressedByAny(error, suppressionEntries)) {
+                suppressionEntries = await getSuppressionEntriesFresh();
+            }
             let suppressed = false;
             let matchedSuppressionId: string | undefined = undefined;
 
@@ -336,6 +382,7 @@ export async function exposeErrorWatchService() {
                     };
 
                     await suppression.set(newEntry.id, newEntry);
+                    invalidateSuppressionEntriesCaches();
                     applySuppression(error, newEntry);
                     queueDiscordNotification(newEntry.id, 1, true);
 
@@ -413,11 +460,13 @@ class ErrorNotificationService {
 
     public async setSuppressionEntry(entry: SuppressionEntry) {
         await suppression.set(entry.id, entry);
+        invalidateSuppressionEntriesCaches();
         await reapplySuppressions();
     }
 
     public async deleteSuppressionEntry(id: string) {
         await suppression.delete(id);
+        invalidateSuppressionEntriesCaches();
         await reapplySuppressions();
     }
 
@@ -428,6 +477,7 @@ class ErrorNotificationService {
             throw new Error(`Suppression entry ${id} not found`);
         }
         await suppression.set(id, { ...entry, notes, lastUpdatedTime: Date.now() });
+        invalidateSuppressionEntriesCaches();
     }
 }