querysub 0.616.0 → 0.619.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.616.0",
+  "version": "0.619.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-a-archives/archives2.ts

@@ -73,7 +73,7 @@ function archiveBuilder(bucket: string, overrides: Partial<RemoteConfigBase>) {
     ]);
 
     return createArchives({
-        version: 21,
+        version: 22,
         sources,
     });
 }
@@ -131,28 +131,28 @@ function getSafeDomain() {
 }
 
 export function getArchives2(folder: string) {
-    return nestBucket(folder, archivesBuilderCache(getSafeDomain(), { noFullSync: true, }));
+    return nestBucket(folder, archivesBuilderCache(getSafeDomain(), {}));
 }
 
 /** The rights are buffered server-side, so if you do many writes at once, it won't slow down the disk and it won't result in many writes to back backblades. Of course, this does mean if the server crashes, you do lose some data.
  */
 export function getArchives2Buffered(folder: string, bufferDelay = timeInMinute * 5) {
-    return nestBucket(folder, archivesBuilderCache(getSafeDomain() + "-fast", { noFullSync: true, fast: true, writeDelay: bufferDelay }));
+    return nestBucket(folder, archivesBuilderCache(getSafeDomain() + "-fast", { fast: true, writeDelay: bufferDelay }));
 }
 
 export function getArchives2BufferedPublic(folder: string, bufferDelay = timeInMinute * 5) {
-    return nestBucket(folder, archivesBuilderCache(getSafeDomain() + "-public-fast", { noFullSync: true, fast: true, writeDelay: bufferDelay, public: true }));
+    return nestBucket(folder, archivesBuilderCache(getSafeDomain() + "-public-fast", { fast: true, writeDelay: bufferDelay, public: true }));
 }
 
 export function getArchives2PrivateImmutable(folder: string) {
-    return nestBucket(folder, archivesBuilderCache(getSafeDomain() + "-private-immutable", { noFullSync: true, immutable: true }));
+    return nestBucket(folder, archivesBuilderCache(getSafeDomain() + "-private-immutable", { immutable: true }));
 }
 function getAllowedOrigins(domain: string) {
     return [`https://${domain}`, `https://127-0-0-1.${domain}:7007`];
 }
 export function getArchives2PublicImmutable(folder: string) {
     let domain = getSafeDomain();
-    return nestBucket(folder, archivesBuilderCache(domain + "-public-immutable", { noFullSync: true, immutable: true, public: true, allowedOrigins: getAllowedOrigins(getDomain()) }));
+    return nestBucket(folder, archivesBuilderCache(domain + "-public-immutable", { immutable: true, public: true, allowedOrigins: getAllowedOrigins(getDomain()) }));
 }
 export function getArchives2Public(folder: string) {
     let domain = getSafeDomain();

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -541,9 +541,9 @@ async function runServerSyncLoops() {
             let suicideThreshold = Date.now() - SUICIDE_HEARTBEAT_THRESHOLD;
             if (!lastTime || lastTime < suicideThreshold) {
                 if (!lastTime) {
-                    console.error(red(`Self node was removed due to not heartbeating. Terminating self process, as it likely has very stale data.`));
+                    console.error(red(`Self node was removed due to heartbeat file not existing (${selfNodeId}). Terminating self process, as it likely has very stale data.`));
                 } else {
-                    console.error(red(`Self node was has very old heartbeat. Terminating self process, as it likely has very stale data.`));
+                    console.error(red(`Self node was has very old heartbeat. Terminating self process, as it likely has very stale data. ${formatDateTime(lastTime)} < ${formatDateTime(suicideThreshold)}, now is ${formatDateTime(Date.now())}`));
                 }
                 process.exit();
             }

package/src/0-path-value-core/PathValueController.ts

@@ -13,6 +13,7 @@ import { getSlowdown, isDiskAudit, getDomain } from "../config";
 import { decodeNodeId } from "sliftutils/misc/https/certs";
 import { areNodeIdsEqual, isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
 import { recordPathValuesSent, recordPathValuesReceived } from "../-f-node-discovery/TrafficTracking";
+import { recordPathValuesSentStats, recordPathValuesReceivedStats } from "./PathValueStats";
 import { getNodeIdIP } from "socket-function/src/nodeCache";
 import { authorityLookup } from "./AuthorityLookup";
 import { timeoutToError } from "../errors";
@@ -60,6 +61,7 @@ export class PathValueControllerBase {
         let { pathValues, nodeId } = config;
 
         pathValueSendCount += pathValues.length;
+        recordPathValuesSentStats(pathValues);
         let serializedValues = await measureBlock(() => pathValueSerializer.serialize(pathValues, { compress: Querysub.COMPRESS_NETWORK }), "createValues|serialize");
         if (isDebugLogEnabled()) {
             for (let value of pathValues) {
@@ -119,6 +121,7 @@ export class PathValueControllerBase {
         let { nodeId, initialTriggers } = config;
         pathValueSendCount += changes.length;
         recordPathValuesSent({ count: changes.length, nodeId });
+        recordPathValuesSentStats(changes);
         let buffers = await measureBlock(() => pathValueSerializer.serialize(changes, {
             noLocks: !config.keepLocks,
             compress: getCompressNetwork(),
@@ -156,6 +159,7 @@ export class PathValueControllerBase {
             values = await measureBlock(() => pathValueSerializer.deserialize(valueBuffers), "sendData|deserialize");
             ActionsHistory.OnRead(values);
             recordPathValuesReceived({ count: values.length, nodeId: callerId });
+            recordPathValuesReceivedStats(values);
         }
         if (debugOnSendData) {
             debugOnSendData(values, callerId);

package/src/0-path-value-core/PathValueStats.ts

@@ -0,0 +1,83 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
+import { TreeSummary } from "sliftutils/treeSummary";
+import type { SummaryEntry } from "sliftutils/treeSummary";
+
+/*
+    Per-server, in-memory stats for path values this node has sent and received (never persisted). Two things are
+    tracked: a running total count for each direction, and a TreeSummary of the paths in each direction — a bounded
+    prefix tree that stays O(1000s of nodes) no matter how many distinct paths flow through, so we can show which
+    paths dominate without keeping every path. Cleared wholesale by the clear endpoint.
+*/
+
+const PATH_SUMMARY_EXPECTED_OUTPUT_COUNT = 100;
+
+export type PathValueSummaryState = { count: number };
+export type PathValueTotals = { sent: number; received: number };
+export type PathValueDirection = "sent" | "received";
+
+// Only the path matters for the tree; callers pass whole PathValues but we never look at anything else.
+type PathValueLike = { path: string };
+
+function makeTree() {
+    return new TreeSummary<PathValueLike, PathValueSummaryState>({
+        getPath: value => value.path,
+        createSummary: () => ({ count: 0 }),
+        addToSummary: (_value, summary) => { summary.count++; },
+        mergeSummaries: (target, source) => { target.count += source.count; },
+        getWeight: summary => summary.count,
+        expectedOutputCount: PATH_SUMMARY_EXPECTED_OUTPUT_COUNT,
+    });
+}
+
+let totalSent = 0;
+let totalReceived = 0;
+let sentTree = makeTree();
+let receivedTree = makeTree();
+
+export function recordPathValuesSentStats(pathValues: PathValueLike[]) {
+    totalSent += pathValues.length;
+    for (let value of pathValues) {
+        sentTree.add(value);
+    }
+}
+export function recordPathValuesReceivedStats(pathValues: PathValueLike[]) {
+    totalReceived += pathValues.length;
+    for (let value of pathValues) {
+        receivedTree.add(value);
+    }
+}
+export function clearPathValueStats() {
+    totalSent = 0;
+    totalReceived = 0;
+    sentTree = makeTree();
+    receivedTree = makeTree();
+}
+
+class PathValueStatsControllerBase {
+    // Cheap: just the two running totals, so the server list can render every server without pulling any tree data.
+    public async getTotals(): Promise<PathValueTotals> {
+        return { sent: totalSent, received: totalReceived };
+    }
+    // Heavy: the top-N path breakdown for one direction. Only fetched when a server row is expanded.
+    public async getSummary(direction: PathValueDirection, maxCount: number): Promise<SummaryEntry<PathValueSummaryState>[]> {
+        let tree = direction === "sent" ? sentTree : receivedTree;
+        return tree.getSummaries(maxCount);
+    }
+    public async clear(): Promise<void> {
+        clearPathValueStats();
+    }
+}
+
+export const PathValueStatsController = SocketFunction.register(
+    "PathValueStatsController-3f9a1c72-2b64-4e8d-9c1a-7d5e0b3f6a21",
+    new PathValueStatsControllerBase(),
+    () => ({
+        getTotals: {},
+        getSummary: {},
+        clear: {},
+    }),
+    () => ({
+        hooks: [requiresNetworkTrustHook],
+    })
+);

package/src/0-path-value-core/archiveLocks/archiveSnapshots.ts

@@ -215,9 +215,4 @@ export async function loadSnapshot(config: {
     for (let i = 0; i < 10; i++) {
         console.log(green(`!!!Finished loading snapshot!!!`));
     }
-
-    if (!config.noExit && !isPublic()) {
-        // Exit so we know when this is done. Really, all the servers need to be restarted after we load the snapshot, anyway. 
-        process.exit();
-    }
 }

package/src/1-path-client/RemoteWatcher.ts

@@ -35,7 +35,9 @@ const SHUTDOWN_REHOME_WINDOW = timeInMinute * 3;
 const SHUTDOWN_REHOME_POLL_INTERVAL = timeInSecond * 30;
 
 // Rehome the watches on an authority when an equivalent authority has this factor lower latency. At 2 the router's full-confidence cutoff factor (also 2) then reliably routes everything to the better node on the rewatch.
-const LATENCY_REHOME_FACTOR = 2;
+const LATENCY_REHOME_FACTOR = 1.5;
+// Jitter floor added to the candidate's latency before the factor comparison. Below roughly this many milliseconds the difference between two nodes is noise, not speed (1ms vs 2ms isn't "twice as fast"), and without the floor a candidate that momentarily reads near-zero always looks infinitely faster, so we rehome on pure jitter. Added only to the candidate, keeping us biased against needless moves.
+const LATENCY_REHOME_JITTER_MS = 50;
 // Both latencies must be medians over this many samples — with fewer we don't trust either number enough to move watches over it.
 const LATENCY_REHOME_HISTORY_COUNT = 10;
 const LATENCY_REHOME_POLL_INTERVAL = timeInMinute;
@@ -186,7 +188,7 @@ export class RemoteWatcher {
                 if (!areAuthoritySpecsEquivalent(entry.authoritySpec, candidate.authoritySpec)) continue;
                 let candidateLatency = getNodeLatencyMedian({ nodeId: candidate.nodeId, historyCount: LATENCY_REHOME_HISTORY_COUNT });
                 if (!candidateLatency || candidateLatency.historyUsed < LATENCY_REHOME_HISTORY_COUNT) continue;
-                if (candidateLatency.latency * LATENCY_REHOME_FACTOR > current.latency) continue;
+                if ((candidateLatency.latency + LATENCY_REHOME_JITTER_MS) * LATENCY_REHOME_FACTOR > current.latency) continue;
                 console.log(yellow(`Authority ${authorityId} has high latency (${formatTime(current.latency)}), and the equivalent authority ${candidate.nodeId} is much faster (${formatTime(candidateLatency.latency)}), so we are rehoming all paths watched on it`));
                 logErrors(this.refreshAllWatches(authorityId));
                 break;

package/src/deployManager/MachinesPage.tsx

@@ -10,6 +10,7 @@ import { MachineDetailPage } from "./components/MachineDetailPage";
 import { Anchor } from "../library-components/ATag";
 import { DeployPage } from "./components/DeployPage";
 import { StoragePage } from "./components/storage/StoragePage";
+import { RoutingTablePage } from "../diagnostics/misc-pages/RoutingTablePage";
 
 export class MachinesPage extends qreact.Component {
     private renderTabs() {
@@ -22,6 +23,7 @@ export class MachinesPage extends qreact.Component {
                 { key: "services", label: "Services", otherKeys: ["service-detail"] },
                 { key: "deploy", label: "Deploy", otherKeys: [] },
                 { key: "storage", label: "Storage", otherKeys: [] },
+                { key: "routing", label: "PathValues / Functions Routing", otherKeys: [] },
             ].map(tab => {
                 let isActive = currentViewParam.value === tab.key || tab.otherKeys.includes(currentViewParam.value);
                 return <Anchor noStyles key={tab.key}
@@ -50,6 +52,7 @@ export class MachinesPage extends qreact.Component {
                 {currentViewParam.value === "machine-detail" && <MachineDetailPage />}
                 {currentViewParam.value === "deploy" && <DeployPage />}
                 {currentViewParam.value === "storage" && <StoragePage />}
+                {currentViewParam.value === "routing" && <RoutingTablePage />}
             </div>
         </div>;
     }

package/src/deployManager/urlParams.ts

@@ -1,6 +1,6 @@
 import { URLParam } from "../library-components/URLParam";
 
-export type ViewType = "services" | "machines" | "service-detail" | "machine-detail" | "deploy" | "storage";
+export type ViewType = "services" | "machines" | "service-detail" | "machine-detail" | "deploy" | "storage" | "routing";
 
 export const currentViewParam = new URLParam<ViewType>("machineview", "machines");
 export const selectedServiceIdParam = new URLParam("serviceId", "");

package/src/diagnostics/managementPages.tsx

@@ -80,12 +80,6 @@ export async function registerManagementPages2(config: {
     let inputPages: typeof config.pages = [];
 
 
-    inputPages.push({
-        title: "Routing Table",
-        componentName: "RoutingTablePage",
-        controllerName: "RoutingTablePageController",
-        getModule: () => import("./misc-pages/RoutingTablePage"),
-    });
     inputPages.push({
         title: "LOG VIEWER",
         componentName: "LogViewer3",
@@ -404,6 +398,10 @@ class ManagementRoot extends qreact.Component {
                             managementPageURL.getOverride("MachinesPage"),
                             currentViewParam.getOverride("storage"),
                         ]}>Storage</ATag>
+                        <ATag values={[
+                            managementPageURL.getOverride("MachinesPage"),
+                            currentViewParam.getOverride("routing"),
+                        ]}>PathValues / Functions Routing</ATag>
                         {pages.map(page =>
                             <ATag values={[{ param: managementPageURL, value: page.componentName }]}>{page.title}</ATag>
                         )}

package/src/diagnostics/misc-pages/FunctionRunnersSection.tsx

@@ -0,0 +1,38 @@
+module.allowclient = true;
+
+import { qreact } from "../../4-dom/qreact";
+import { css } from "typesafecss";
+import { formatTime } from "socket-function/src/formatting/format";
+import { getFunctionRunnerIndex } from "../../4-querysub/FunctionRunnerTracking";
+import { FUNCTION_RUNNER_COLOR } from "../../misc/nodeCategoryColors";
+import { AuthorityRangeBar } from "./RoutingTablePage";
+
+export class FunctionRunnersSection extends qreact.Component {
+    render() {
+        let index = getFunctionRunnerIndex();
+        let nodes = index?.nodes || [];
+        return <div className={css.vbox(8).fillWidth}>
+            <h2>Function Runners ({nodes.length})</h2>
+            {nodes.length === 0 && <div className={css.colorhsl(0, 0, 50)}>(no function runners found yet)</div>}
+            {nodes.map(node =>
+                <div className={css.vbox(4).pad2(10).fillWidth.bord2(0, 0, 85).hsl(0, 0, 99)}>
+                    <div className={css.hbox(10).fillWidth}>
+                        <span className={css.boldStyle}>{node.nodeId}</span>
+                        <span className={css.color(FUNCTION_RUNNER_COLOR)}>networks: {node.networks.join(", ")}</span>
+                        {!node.isPublic && <span className={css.colorhsl(0, 70, 35)}>(non-public)</span>}
+                        <span>latency {formatTime(node.averageLatency)}</span>
+                        <span>up for {formatTime(Date.now() - node.startupTime)}</span>
+                        <span className={css.colorhsl(0, 0, 40).ellipsis}>{node.entryPoint}</span>
+                    </div>
+                    {node.shards.map(shard =>
+                        <div className={css.hbox(10).fillWidth}>
+                            <span>{shard.shardRange.startFraction.toFixed(4)} - {shard.shardRange.endFraction.toFixed(4)}</span>
+                            <AuthorityRangeBar start={shard.shardRange.startFraction} end={shard.shardRange.endFraction} />
+                            {shard.secondaryShardRange && <span className={css.colorhsl(0, 0, 50)}>secondary {shard.secondaryShardRange.startFraction.toFixed(4)} - {shard.secondaryShardRange.endFraction.toFixed(4)}</span>}
+                        </div>
+                    )}
+                </div>
+            )}
+        </div>;
+    }
+}