querysub 0.520.0 → 0.522.0

package/bin/function-public.js

@@ -2,8 +2,6 @@
 
 process.argv.push("--nobreak");
 process.argv.push("--public");
-process.argv.push("--network");
-process.argv.push("public");
 
 require("typenode");
 require("../src/3-path-functions/PathFunctionRunnerMain");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.520.0",
+  "version": "0.522.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -70,7 +70,7 @@
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
     "sliftutils": "^1.7.5",
-    "socket-function": "^1.2.11",
+    "socket-function": "^1.2.12",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",
     "typesafecss": "^0.32.0",

package/src/-f-node-discovery/TrafficTracking.ts

@@ -2,9 +2,13 @@
     Per-server, in-memory traffic counters (never written to the database). Everything is just an integer increment
     into a 5-minute bucket; we keep the last 60 minutes (12 buckets) and drop older ones, so memory is bounded.
 
+    Reads return PER-SECOND RATES, not totals. Each bucket also records the timestamp it first received data, so the
+    rate window runs from the oldest retained bucket's first data up to now — which stays accurate right after startup
+    (when only a partial window of data exists) instead of dividing a small burst by the full 60-minute window.
+
     Byte traffic comes from SocketFunction.trackMessageSizes (upload/download per connection). The connection nodeId is
     sometimes a raw client-connection id, so we map it to the nice node id (debugNodeId) lazily at read time — the hot
-    path is just a Map increment keyed by the raw id. At read we split traffic into per-known-node buckets and a single
+    path is just a Map increment keyed by the raw id. At read we split traffic into per-known-node rates and a single
     "outside the network" aggregate for anything not in our cached node list (clients, etc.).
 */
 
@@ -17,21 +21,28 @@ import { debugNodeId } from "../-c-identity/IdentityController";
 
 const BUCKET_MS = timeInMinute * 5;
 const BUCKET_COUNT = 12;
+// Floor on the rate window so a burst in the first fraction of a second after startup can't divide out to an absurd rate.
+const MIN_WINDOW_SECONDS = 1;
 // Sentinel key for traffic that isn't a network node (client connections, etc.). Aggregated so memory stays O(nodes),
 // not O(client connections) — clients churn through many raw ids per hour and we never want a bucket per client.
 const OUTSIDE_KEY = "";
 
 export type NodeDataTraffic = { sent: number; received: number; };
+// All numbers are PER-SECOND rates (see file header).
 export type TrafficStats = {
     functionsExecuted: number;
     pathValuesSent: number;
     pathValuesReceived: number;
     querysubCalls: number;
-    // Bytes to/from each known network node, plus one aggregate for everything outside the node list.
+    // Byte rates to/from each known network node, plus one aggregate for everything outside the node list.
     perNode: { [nodeId: string]: NodeDataTraffic };
     outside: NodeDataTraffic;
 };
 
+// count = amount accumulated in the bucket; firstMs = when the bucket first received data (for the rate window).
+type Bucket = { count: number; firstMs: number; };
+type BucketMap = Map<number, Bucket>;
+
 function currentBucket() {
     return Math.floor(Date.now() / BUCKET_MS);
 }
@@ -39,20 +50,24 @@ function oldestBucket() {
     return currentBucket() - BUCKET_COUNT + 1;
 }
 
-// bucket => count
-let functionsExecuted = new Map<number, number>();
-let pathValuesSent = new Map<number, number>();
-let pathValuesReceived = new Map<number, number>();
-let querysubCalls = new Map<number, number>();
-// raw connection nodeId => (bucket => bytes)
-let bytesSent = new Map<string, Map<number, number>>();
-let bytesReceived = new Map<string, Map<number, number>>();
+let functionsExecuted: BucketMap = new Map();
+let pathValuesSent: BucketMap = new Map();
+let pathValuesReceived: BucketMap = new Map();
+let querysubCalls: BucketMap = new Map();
+// raw connection nodeId => buckets
+let bytesSent = new Map<string, BucketMap>();
+let bytesReceived = new Map<string, BucketMap>();
 
-function addSimple(buckets: Map<number, number>, amount: number) {
+function addSimple(buckets: BucketMap, amount: number) {
     let bucket = currentBucket();
-    buckets.set(bucket, (buckets.get(bucket) || 0) + amount);
+    let entry = buckets.get(bucket);
+    if (!entry) {
+        entry = { count: 0, firstMs: Date.now() };
+        buckets.set(bucket, entry);
+    }
+    entry.count += amount;
 }
-function addKeyed(map: Map<string, Map<number, number>>, key: string, amount: number) {
+function addKeyed(map: Map<string, BucketMap>, key: string, amount: number) {
     let buckets = map.get(key);
     if (!buckets) {
         buckets = new Map();
@@ -84,17 +99,23 @@ function trafficKey(nodeId: string): string {
 SocketFunction.trackMessageSizes.upload.push((size, nodeId) => addKeyed(bytesSent, trafficKey(nodeId), size));
 SocketFunction.trackMessageSizes.download.push((size, nodeId) => addKeyed(bytesReceived, trafficKey(nodeId), size));
 
-function sumRecent(buckets: Map<number, number>): number {
+// Per-second rate over the retained window: total accumulated / (now - oldest retained bucket's first-data time).
+// Also prunes expired buckets as it goes.
+function bucketRate(buckets: BucketMap): number {
     let oldest = oldestBucket();
     let total = 0;
-    for (let [bucket, count] of Array.from(buckets)) {
+    let firstMs = Infinity;
+    for (let [bucket, entry] of Array.from(buckets)) {
         if (bucket < oldest) {
             buckets.delete(bucket);
             continue;
         }
-        total += count;
+        total += entry.count;
+        if (entry.firstMs < firstMs) firstMs = entry.firstMs;
     }
-    return total;
+    if (total <= 0 || !Number.isFinite(firstMs)) return 0;
+    let windowSeconds = Math.max(MIN_WINDOW_SECONDS, (Date.now() - firstMs) / 1000);
+    return total / windowSeconds;
 }
 
 function aggregateData(): { perNode: { [nodeId: string]: NodeDataTraffic }; outside: NodeDataTraffic; } {
@@ -103,10 +124,10 @@ function aggregateData(): { perNode: { [nodeId: string]: NodeDataTraffic }; outs
     let outside: NodeDataTraffic = { sent: 0, received: 0 };
     // At most (node count + 1) keys, so this whole pass is O(nodes). debugNodeId (a linear lookup) is only used for
     // the rare server-style id that isn't already a known node.
-    let attribute = (rawNodeId: string, field: "sent" | "received", amount: number) => {
-        if (amount <= 0) return;
+    let attribute = (rawNodeId: string, field: "sent" | "received", rate: number) => {
+        if (rate <= 0) return;
         if (rawNodeId === OUTSIDE_KEY) {
-            outside[field] += amount;
+            outside[field] += rate;
             return;
         }
         let nice = known.has(rawNodeId) ? rawNodeId : debugNodeId(rawNodeId);
@@ -116,17 +137,17 @@ function aggregateData(): { perNode: { [nodeId: string]: NodeDataTraffic }; outs
                 entry = { sent: 0, received: 0 };
                 perNode[nice] = entry;
             }
-            entry[field] += amount;
+            entry[field] += rate;
         } else {
-            outside[field] += amount;
+            outside[field] += rate;
         }
     };
     for (let [rawNodeId, buckets] of Array.from(bytesSent)) {
-        attribute(rawNodeId, "sent", sumRecent(buckets));
+        attribute(rawNodeId, "sent", bucketRate(buckets));
         if (buckets.size === 0) bytesSent.delete(rawNodeId);
     }
     for (let [rawNodeId, buckets] of Array.from(bytesReceived)) {
-        attribute(rawNodeId, "received", sumRecent(buckets));
+        attribute(rawNodeId, "received", bucketRate(buckets));
         if (buckets.size === 0) bytesReceived.delete(rawNodeId);
     }
     return { perNode, outside };
@@ -135,10 +156,10 @@ function aggregateData(): { perNode: { [nodeId: string]: NodeDataTraffic }; outs
 export function getTrafficStats(): TrafficStats {
     let data = aggregateData();
     return {
-        functionsExecuted: sumRecent(functionsExecuted),
-        pathValuesSent: sumRecent(pathValuesSent),
-        pathValuesReceived: sumRecent(pathValuesReceived),
-        querysubCalls: sumRecent(querysubCalls),
+        functionsExecuted: bucketRate(functionsExecuted),
+        pathValuesSent: bucketRate(pathValuesSent),
+        pathValuesReceived: bucketRate(pathValuesReceived),
+        querysubCalls: bucketRate(querysubCalls),
         perNode: data.perNode,
         outside: data.outside,
     };

package/src/3-path-functions/PathFunctionRunner.ts

@@ -21,7 +21,7 @@ import { getOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
 import { getScheduledShutdownTime } from "../-g-core-values/scheduledShutdown";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
-import { getDomain, isPublic, DEFAULT_NETWORK } from "../config";
+import { getDomain, isPublic } from "../config";
 import { getGitRefSync, getGitURLSync } from "../4-deploy/git";
 import type { DeployProgress } from "../4-deploy/deployFunctions";
 import { getRoutingOverride, getRoutingOverridePart } from "../0-path-value-core/PathRouterRouteOverride";
@@ -363,14 +363,15 @@ export class PathFunctionRunner {
         secondaryShardRange?: { startFraction: number, endFraction: number };
         PermissionsChecker: PermissionsCheckType | undefined;
         // The networks we listen to calls on. Unset is equivalent to ["default"].
-        networks?: string[];
+        networks: string[];
     }) {
+        if (!config.networks.length) throw new Error(`networks is required`);
         PathFunctionRunner.hostControllers();
         debugFunctionRunnerShards.push({
             domainName: config.domainName,
             shardRange: config.shardRange,
             secondaryShardRange: config.secondaryShardRange,
-            networks: config.networks && config.networks.length > 0 && config.networks || [DEFAULT_NETWORK],
+            networks: config.networks,
             isPublic: isPublic(),
         });
         logErrors(this.startWatching());
@@ -401,7 +402,7 @@ export class PathFunctionRunner {
 
         let outstandingCalls = 0;
 
-        let networks = this.config.networks && this.config.networks.length > 0 && this.config.networks || [DEFAULT_NETWORK];
+        let networks = this.config.networks;
 
         let watchModuleCalls = cache((moduleId: string) => {
             for (let network of networks) {
@@ -691,7 +692,7 @@ export class PathFunctionRunner {
         }
 
         let callNetwork = callSpec.network;
-        let ourNetworks = this.config.networks && this.config.networks.length > 0 && this.config.networks || [DEFAULT_NETWORK];
+        let ourNetworks = this.config.networks;
         if (!ourNetworks.includes(callNetwork)) {
             return;
         }

package/src/3-path-functions/PathFunctionRunnerMain.ts

@@ -15,7 +15,7 @@ import { SocketFunction } from "socket-function/SocketFunction";
 import { getThreadKeyCert } from "sliftutils/misc/https/certs";
 import { ClientWatcher } from "../1-path-client/pathValueClientWatcher";
 import { timeInMinute } from "socket-function/src/misc";
-import { getDomain, getNetworks, isLocal, isPublic, DEFAULT_NETWORK } from "../config";
+import { getDomain, getNetworks, isLocal, isPublic } from "../config";
 import { green, magenta } from "socket-function/src/formatting/logColors";
 import path from "path";
 import { IndexedLogs } from "../diagnostics/logs/IndexedLogs/IndexedLogs";
@@ -32,6 +32,9 @@ async function main() {
     // ClientWatcher.DEBUG_TRIGGERS = "heavy";
     // authorityStorage.DEBUG_UNWATCH = true;
 
+    let networks = getNetworks();
+    if (!networks.length) throw new Error(`No networks found. Use --network to specify the networks this function runner should listen on.`);
+
     PathFunctionRunner.DEBUG_CALLS = true;
     // debugCoreMode();
 
@@ -60,8 +63,6 @@ async function main() {
         console.log(green(`Sharding from ${shardStart} to ${shardEnd}`));
     }
 
-    let networks = getNetworks();
-
     new PathFunctionRunner({
         domainName: getDomain(),
         shardRange: { startFraction: shardStart, endFraction: shardEnd },
@@ -76,9 +77,5 @@ async function main() {
         let deployPath = path.resolve("./deploy.ts");
         await import(deployPath);
     }
-
-    if (networks.length !== 1 || networks[0] !== DEFAULT_NETWORK) {
-        console.log(magenta(`Only running functions on the network(s): ${networks.join(", ")}. Use --network ${networks[0]} in your http server to route calls here.`));
-    }
 }
 logErrors(main());

package/src/config.ts

@@ -44,8 +44,6 @@ if (isNode()) {
     }
 }
 
-export const DEFAULT_NETWORK = "default";
-
 export function expandHomePath(path: string): string {
     if (path === "~" || path.startsWith("~/") || path.startsWith("~\\")) {
         return os.homedir() + path.slice(1);
@@ -76,7 +74,7 @@ export function getNetworks(): string[] {
         result = [...result, ...networkFileNetworks()];
     }
     result = Array.from(new Set(result));
-    if (result.length === 0) return [DEFAULT_NETWORK];
+    if (result.length === 0) return [];
     return result;
 }
 

package/src/deployManager/components/ServiceDetailPage.tsx

@@ -169,20 +169,24 @@ export class ServiceDetailPage extends qreact.Component {
     private renderTemplateVariables(config: ServiceConfig) {
         let templateVariables = getCommandTemplateVariables(config.parameters.command);
         let targets = getMachineTargets(config.parameters);
-        let anyVariablesSet = targets.some(target => Object.keys(target.variables).length > 0);
-        if (templateVariables.length === 0 && !anyVariablesSet) return undefined;
+        if (targets.length === 0) return undefined;
 
-        const setVariable = (entryIndex: number, name: string, value: string) => {
+        const updateTargets = (change: (targets: ReturnType<typeof getMachineTargets>) => void) => {
             let updated = deepCloneJSON(config);
             let updatedTargets = getMachineTargets(updated.parameters);
-            if (value) {
-                updatedTargets[entryIndex].variables[name] = value;
-            } else {
-                delete updatedTargets[entryIndex].variables[name];
-            }
+            change(updatedTargets);
             setMachineTargets(updated.parameters, updatedTargets);
             this.updateEditorState(updated);
         };
+        const setVariable = (entryIndex: number, name: string, value: string) => {
+            updateTargets(targets => {
+                if (value) {
+                    targets[entryIndex].variables[name] = value;
+                } else {
+                    delete targets[entryIndex].variables[name];
+                }
+            });
+        };
 
         let dupIndexes = new Map<string, number>();
         return <div className={css.vbox(10).fillWidth.pad2(12).bord2(0, 0, 20)}>
@@ -201,6 +205,26 @@ export class ServiceDetailPage extends qreact.Component {
                             onChangeValue={value => setVariable(entryIndex, name, value)}
                         />;
                     })}
+                    <button
+                        className={css.pad2(10, 4).button.bord2(0, 0, 20).hsl(120, 70, 90)}
+                        title="Duplicate this entry (same machine and variables)"
+                        onClick={() => {
+                            updateTargets(targets => {
+                                targets.splice(entryIndex + 1, 0, deepCloneJSON(targets[entryIndex]));
+                            });
+                        }}>
+                        +
+                    </button>
+                    <button
+                        className={css.pad2(10, 4).button.bord2(0, 0, 20).hsl(0, 70, 90)}
+                        title="Remove this entry"
+                        onClick={() => {
+                            updateTargets(targets => {
+                                targets.splice(entryIndex, 1);
+                            });
+                        }}>
+                        −
+                    </button>
                 </div>;
             })}
         </div>;