querysub 0.522.0 → 0.523.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.522.0",
+  "version": "0.523.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-f-node-discovery/TrafficTracking.ts

@@ -2,9 +2,10 @@
     Per-server, in-memory traffic counters (never written to the database). Everything is just an integer increment
     into a 5-minute bucket; we keep the last 60 minutes (12 buckets) and drop older ones, so memory is bounded.
 
-    Reads return PER-SECOND RATES, not totals. Each bucket also records the timestamp it first received data, so the
-    rate window runs from the oldest retained bucket's first data up to now — which stays accurate right after startup
-    (when only a partial window of data exists) instead of dividing a small burst by the full 60-minute window.
+    Reads return PER-SECOND RATES: the window's total divided by how long data has ACTUALLY been collecting in that
+    window — from the later of the window's start and when we first saw any data, up to now (so it's dynamic, and on a
+    server that just started it divides by the real short duration, not the full window). Under a second of data falls
+    back to the window length so we never divide by ~0.
 
     Byte traffic comes from SocketFunction.trackMessageSizes (upload/download per connection). The connection nodeId is
     sometimes a raw client-connection id, so we map it to the nice node id (debugNodeId) lazily at read time — the hot
@@ -19,10 +20,8 @@ import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
 import { getCachedNodeIds } from "./NodeDiscovery";
 import { debugNodeId } from "../-c-identity/IdentityController";
 
-const BUCKET_MS = timeInMinute * 5;
-const BUCKET_COUNT = 12;
-// Floor on the rate window so a burst in the first fraction of a second after startup can't divide out to an absurd rate.
-const MIN_WINDOW_SECONDS = 1;
+const BUCKET_SIZE = timeInMinute * 5;
+const BUCKET_HISTORY = BUCKET_SIZE * 20;
 // Sentinel key for traffic that isn't a network node (client connections, etc.). Aggregated so memory stays O(nodes),
 // not O(client connections) — clients churn through many raw ids per hour and we never want a bucket per client.
 const OUTSIDE_KEY = "";
@@ -39,16 +38,8 @@ export type TrafficStats = {
     outside: NodeDataTraffic;
 };
 
-// count = amount accumulated in the bucket; firstMs = when the bucket first received data (for the rate window).
-type Bucket = { count: number; firstMs: number; };
-type BucketMap = Map<number, Bucket>;
-
-function currentBucket() {
-    return Math.floor(Date.now() / BUCKET_MS);
-}
-function oldestBucket() {
-    return currentBucket() - BUCKET_COUNT + 1;
-}
+// bucket index => amount accumulated in that bucket
+type BucketMap = Map<number, number>;
 
 let functionsExecuted: BucketMap = new Map();
 let pathValuesSent: BucketMap = new Map();
@@ -59,13 +50,8 @@ let bytesSent = new Map<string, BucketMap>();
 let bytesReceived = new Map<string, BucketMap>();
 
 function addSimple(buckets: BucketMap, amount: number) {
-    let bucket = currentBucket();
-    let entry = buckets.get(bucket);
-    if (!entry) {
-        entry = { count: 0, firstMs: Date.now() };
-        buckets.set(bucket, entry);
-    }
-    entry.count += amount;
+    let bucket = Math.floor(Date.now() / BUCKET_SIZE) * BUCKET_SIZE;
+    buckets.set(bucket, (buckets.get(bucket) || 0) + amount);
 }
 function addKeyed(map: Map<string, BucketMap>, key: string, amount: number) {
     let buckets = map.get(key);
@@ -99,26 +85,30 @@ function trafficKey(nodeId: string): string {
 SocketFunction.trackMessageSizes.upload.push((size, nodeId) => addKeyed(bytesSent, trafficKey(nodeId), size));
 SocketFunction.trackMessageSizes.download.push((size, nodeId) => addKeyed(bytesReceived, trafficKey(nodeId), size));
 
-// Per-second rate over the retained window: total accumulated / (now - oldest retained bucket's first-data time).
-// Also prunes expired buckets as it goes.
-function bucketRate(buckets: BucketMap): number {
-    let oldest = oldestBucket();
+// Per-second rate: total within the window divided by the actual span the matched buckets cover (oldest bucket's start
+// time → now). Also prunes expired buckets as it goes.
+function bucketRate(buckets: BucketMap, windowSize: number): number {
+    let now = Date.now();
+    let retentionOldest = now - BUCKET_HISTORY;
+    let targetTime = now - windowSize;
     let total = 0;
-    let firstMs = Infinity;
-    for (let [bucket, entry] of Array.from(buckets)) {
-        if (bucket < oldest) {
+    let oldestTime = now;
+    for (let [bucket, count] of Array.from(buckets)) {
+        if (bucket < retentionOldest) {
             buckets.delete(bucket);
             continue;
         }
-        total += entry.count;
-        if (entry.firstMs < firstMs) firstMs = entry.firstMs;
+        if (bucket < targetTime) continue;
+        oldestTime = Math.min(oldestTime, bucket);
+        total += count;
+    }
+    if (oldestTime === now) {
+        oldestTime = targetTime;
     }
-    if (total <= 0 || !Number.isFinite(firstMs)) return 0;
-    let windowSeconds = Math.max(MIN_WINDOW_SECONDS, (Date.now() - firstMs) / 1000);
-    return total / windowSeconds;
+    return total / ((now - oldestTime) / 1000);
 }
 
-function aggregateData(): { perNode: { [nodeId: string]: NodeDataTraffic }; outside: NodeDataTraffic; } {
+function aggregateData(windowSize: number): { perNode: { [nodeId: string]: NodeDataTraffic }; outside: NodeDataTraffic; } {
     let known = new Set(getCachedNodeIds());
     let perNode: { [nodeId: string]: NodeDataTraffic } = {};
     let outside: NodeDataTraffic = { sent: 0, received: 0 };
@@ -143,31 +133,31 @@ function aggregateData(): { perNode: { [nodeId: string]: NodeDataTraffic }; outs
         }
     };
     for (let [rawNodeId, buckets] of Array.from(bytesSent)) {
-        attribute(rawNodeId, "sent", bucketRate(buckets));
+        attribute(rawNodeId, "sent", bucketRate(buckets, windowSize));
         if (buckets.size === 0) bytesSent.delete(rawNodeId);
     }
     for (let [rawNodeId, buckets] of Array.from(bytesReceived)) {
-        attribute(rawNodeId, "received", bucketRate(buckets));
+        attribute(rawNodeId, "received", bucketRate(buckets, windowSize));
         if (buckets.size === 0) bytesReceived.delete(rawNodeId);
     }
     return { perNode, outside };
 }
 
-export function getTrafficStats(): TrafficStats {
-    let data = aggregateData();
+export function getTrafficStats(windowSize: number): TrafficStats {
+    let data = aggregateData(windowSize);
     return {
-        functionsExecuted: bucketRate(functionsExecuted),
-        pathValuesSent: bucketRate(pathValuesSent),
-        pathValuesReceived: bucketRate(pathValuesReceived),
-        querysubCalls: bucketRate(querysubCalls),
+        functionsExecuted: bucketRate(functionsExecuted, windowSize),
+        pathValuesSent: bucketRate(pathValuesSent, windowSize),
+        pathValuesReceived: bucketRate(pathValuesReceived, windowSize),
+        querysubCalls: bucketRate(querysubCalls, windowSize),
         perNode: data.perNode,
         outside: data.outside,
     };
 }
 
 class TrafficControllerBase {
-    public async getTrafficStats(): Promise<TrafficStats> {
-        return getTrafficStats();
+    public async getTrafficStats(windowSize: number): Promise<TrafficStats> {
+        return getTrafficStats(windowSize);
     }
 }
 export const TrafficController = SocketFunction.register(

package/src/0-path-value-core/startupAuthority.ts

@@ -34,7 +34,7 @@ export async function startupAuthority(spec: AuthoritySpec) {
             pathValues: values,
             parentSyncs: [],
             initialTriggers: { values: new Set(), parentPaths: new Set() },
-            doNotArchive: new Set(values),
+            doNotArchive: true,
         });
         console.log(blue(`Finished ingesting values from source ${source.nodeId} (values: ${formatNumber(values.length)}`), { nodeId: source.nodeId, spec: debugSpec(source), values: values.length });
     }
@@ -62,7 +62,7 @@ export async function startupAuthority(spec: AuthoritySpec) {
         let flat = Object.values(snapshot.values).flat();
         console.log(blue(`Loaded snapshot in memory, found ${formatNumber(flat.length)} values`), { spec: debugSpec(spec), values: flat.length });
         // NOTE: If it's on disk, it can't be rejected, so there's no point in passing it through our validStateComputer (and passing it through is VERY slow).
-        authorityStorage.ingestValues(flat, { doNotArchive: new Set(flat) });
+        authorityStorage.ingestValues(flat, { doNotArchive: true });
         console.log(blue(`Finished ingesting values from snapshot, found ${formatNumber(flat.length)} values`), { spec: debugSpec(spec), values: flat.length });
     })();
 

package/src/4-querysub/querysubPrediction.ts

@@ -308,7 +308,7 @@ function predictCallBase(config: {
             pathValues: predictions.writes,
             parentSyncs: [],
             initialTriggers: { values: new Set(), parentPaths: new Set() },
-            doNotArchive: new Set(predictions.writes),
+            doNotArchive: true,
         });
 
         if (Querysub.DEBUG_PREDICTIONS) {
@@ -337,7 +337,7 @@ function predictCallBase(config: {
             pathValues: rejectedWrites,
             parentSyncs: [],
             initialTriggers: { values: new Set(), parentPaths: new Set() },
-            doNotArchive: new Set(rejectedWrites),
+            doNotArchive: true,
         });
     }
 

package/src/diagnostics/misc-pages/RoutingTablePage.tsx

@@ -14,7 +14,7 @@ import { getSyncedController } from "../../library-components/SyncedController";
 import { assertIsManagementUser } from "../managementPages";
 import { NodeCapabilitiesController } from "../../-g-core-values/NodeCapabilities";
 import { timeoutToUndefinedSilent } from "../../errors";
-import { sort } from "socket-function/src/misc";
+import { sort, timeInMinute } from "socket-function/src/misc";
 import type { AuthoritySpec } from "../../0-path-value-core/PathRouter";
 import { getFunctionRunnerIndex, FunctionRunnerNodeInfo } from "../../4-querysub/FunctionRunnerTracking";
 import { formatTime, formatNumber } from "socket-function/src/formatting/format";
@@ -34,6 +34,7 @@ const LATENCY_GRAPH_HEIGHT_PX = 720;
 
 // Clicking a machine node in the graph (or a row) sorts that machine's nodes to the top of the table.
 const selectedMachineParam = new URLParam("rtSelectedMachine", "");
+const trafficWindowParam = new URLParam<number>("rtTrafficWindow", timeInMinute * 60);
 
 // The per-node table columns, in order. `color` tints the function/path-value/querysub columns to match the graph;
 // `perMachine` columns (the IP and machine id) only print on the first row of each machine group.
@@ -106,9 +107,10 @@ class RoutingTablePageControllerBase {
     public async getNodeLatencies(nodeId: string): Promise<{ [nodeId: string]: number } | undefined> {
         return timeoutToUndefinedSilent(PROBE_TIMEOUT_MS, LatencyController.nodes[nodeId].getLatencies());
     }
-    // Renamed (was getNodeTraffic) because the returned shape changed — a new name busts any stale synced-call cache.
-    public async getNodeTrafficStats(nodeId: string): Promise<TrafficStats | undefined> {
-        return timeoutToUndefinedSilent(PROBE_TIMEOUT_MS, TrafficController.nodes[nodeId].getTrafficStats());
+    // Values are now per-second rates over the window (were near-totals). Renamed so a stale synced-call cache from the
+    // old meaning is dropped instead of being served forever.
+    public async getNodeTrafficRates(nodeId: string, windowSize: number): Promise<TrafficStats | undefined> {
+        return timeoutToUndefinedSilent(PROBE_TIMEOUT_MS, TrafficController.nodes[nodeId].getTrafficStats(windowSize));
     }
 }
 
@@ -118,7 +120,7 @@ export const RoutingTablePageController = SocketFunction.register(
     () => ({
         getAllNodeAuthoritySpecs: {},
         getNodeLatencies: {},
-        getNodeTrafficStats: {},
+        getNodeTrafficRates: {},
         getNodeIps: {},
     }),
     () => ({
@@ -387,12 +389,13 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
         let infoByNode = new Map(this.props.infos.map(info => [info.nodeId, info]));
 
         // One synced call per node for each data source; each resolves independently so the graph fills in progressively.
+        let trafficWindow = trafficWindowParam.value;
         let latencyMaps = new Map<string, { [nodeId: string]: number }>();
         let trafficMaps = new Map<string, TrafficStats>();
         for (let nodeId of nodeIds) {
             let latencies = synced.getNodeLatencies(nodeId);
             if (latencies) latencyMaps.set(nodeId, latencies);
-            let traffic = synced.getNodeTrafficStats(nodeId);
+            let traffic = synced.getNodeTrafficRates(nodeId, trafficWindow);
             if (traffic) trafficMaps.set(nodeId, traffic);
         }
         // Only graph nodes that reported their latencies — ones that never respond probably don't exist.
@@ -512,7 +515,19 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
         sort(rows, row => `${row.machineId === selected ? 0 : 1}${sep}${row.machineId}${sep}${row.threadPort}`);
 
         return <div className={css.vbox(8).fillWidth}>
-            <h2 className={css.margin(0)}>Latency Graph ({machineRespondedThreads.size} machines · {respondedSet.size}/{nodeIds.length} nodes reported)</h2>
+            <div className={css.hbox(14)}>
+                <h2 className={css.margin(0)}>Latency Graph ({machineRespondedThreads.size} machines · {respondedSet.size}/{nodeIds.length} nodes reported)</h2>
+                <div className={css.hbox(0).bord2(0, 0, 70)}>
+                    {([timeInMinute * 5, timeInMinute * 60]).map(windowSize => {
+                        let selected = trafficWindow === windowSize;
+                        return <div
+                            className={css.pad2(12, 6).cursor("pointer")
+                                .hsl(210, selected ? 70 : 0, selected ? 45 : 96).colorhsl(0, 0, selected ? 100 : 30)}
+                            onMouseDown={() => trafficWindowParam.value = windowSize}
+                        >{formatTime(windowSize)}</div>;
+                    })}
+                </div>
+            </div>
             <div className={css.relative.fillWidth.height(LATENCY_GRAPH_HEIGHT_PX).bord2(0, 0, 85)}>
                 <LatencyGraph
                     nodes={nodes}

package/src/library-components/LatencyGraph.tsx

@@ -821,15 +821,20 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
             this.builtSig = sig;
             this.builtWeightSig = weightSig;
             this.build();
-        } else if (weightSig !== this.builtWeightSig) {
-            // Traffic weights changed but the node/link set didn't — just refresh sizing, no re-layout.
-            this.builtWeightSig = weightSig;
-            this.computeWeights();
-        } else if (latencyExponentParam.value !== this.builtLatencyExponent) {
-            this.computeSolveEdges();
-            this.applyLayout();
-        } else if (Math.max(1, Math.floor(connectionsParam.value)) !== this.builtConnections) {
-            this.computeRenderEdges();
+        } else {
+            // Same node/link counts (same order): refresh the node objects so label text that changed as data streamed
+            // in — or when the traffic window toggled — shows up, without disturbing the settled layout/positions.
+            this.nodes = this.props.nodes.slice();
+            if (weightSig !== this.builtWeightSig) {
+                this.builtWeightSig = weightSig;
+                this.computeWeights();
+            }
+            if (latencyExponentParam.value !== this.builtLatencyExponent) {
+                this.computeSolveEdges();
+                this.applyLayout();
+            } else if (Math.max(1, Math.floor(connectionsParam.value)) !== this.builtConnections) {
+                this.computeRenderEdges();
+            }
         }
         this.scheduleFrame();