querysub 0.616.0 → 0.617.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.616.0",
+  "version": "0.617.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-a-archives/archives2.ts

@@ -73,7 +73,7 @@ function archiveBuilder(bucket: string, overrides: Partial<RemoteConfigBase>) {
     ]);
 
     return createArchives({
-        version: 21,
+        version: 22,
         sources,
     });
 }
@@ -131,28 +131,28 @@ function getSafeDomain() {
 }
 
 export function getArchives2(folder: string) {
-    return nestBucket(folder, archivesBuilderCache(getSafeDomain(), { noFullSync: true, }));
+    return nestBucket(folder, archivesBuilderCache(getSafeDomain(), {}));
 }
 
 /** The rights are buffered server-side, so if you do many writes at once, it won't slow down the disk and it won't result in many writes to back backblades. Of course, this does mean if the server crashes, you do lose some data.
  */
 export function getArchives2Buffered(folder: string, bufferDelay = timeInMinute * 5) {
-    return nestBucket(folder, archivesBuilderCache(getSafeDomain() + "-fast", { noFullSync: true, fast: true, writeDelay: bufferDelay }));
+    return nestBucket(folder, archivesBuilderCache(getSafeDomain() + "-fast", { fast: true, writeDelay: bufferDelay }));
 }
 
 export function getArchives2BufferedPublic(folder: string, bufferDelay = timeInMinute * 5) {
-    return nestBucket(folder, archivesBuilderCache(getSafeDomain() + "-public-fast", { noFullSync: true, fast: true, writeDelay: bufferDelay, public: true }));
+    return nestBucket(folder, archivesBuilderCache(getSafeDomain() + "-public-fast", { fast: true, writeDelay: bufferDelay, public: true }));
 }
 
 export function getArchives2PrivateImmutable(folder: string) {
-    return nestBucket(folder, archivesBuilderCache(getSafeDomain() + "-private-immutable", { noFullSync: true, immutable: true }));
+    return nestBucket(folder, archivesBuilderCache(getSafeDomain() + "-private-immutable", { immutable: true }));
 }
 function getAllowedOrigins(domain: string) {
     return [`https://${domain}`, `https://127-0-0-1.${domain}:7007`];
 }
 export function getArchives2PublicImmutable(folder: string) {
     let domain = getSafeDomain();
-    return nestBucket(folder, archivesBuilderCache(domain + "-public-immutable", { noFullSync: true, immutable: true, public: true, allowedOrigins: getAllowedOrigins(getDomain()) }));
+    return nestBucket(folder, archivesBuilderCache(domain + "-public-immutable", { immutable: true, public: true, allowedOrigins: getAllowedOrigins(getDomain()) }));
 }
 export function getArchives2Public(folder: string) {
     let domain = getSafeDomain();

package/src/0-path-value-core/PathValueController.ts

@@ -13,6 +13,7 @@ import { getSlowdown, isDiskAudit, getDomain } from "../config";
 import { decodeNodeId } from "sliftutils/misc/https/certs";
 import { areNodeIdsEqual, isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
 import { recordPathValuesSent, recordPathValuesReceived } from "../-f-node-discovery/TrafficTracking";
+import { recordPathValuesSentStats, recordPathValuesReceivedStats } from "./PathValueStats";
 import { getNodeIdIP } from "socket-function/src/nodeCache";
 import { authorityLookup } from "./AuthorityLookup";
 import { timeoutToError } from "../errors";
@@ -60,6 +61,7 @@ export class PathValueControllerBase {
         let { pathValues, nodeId } = config;
 
         pathValueSendCount += pathValues.length;
+        recordPathValuesSentStats(pathValues);
         let serializedValues = await measureBlock(() => pathValueSerializer.serialize(pathValues, { compress: Querysub.COMPRESS_NETWORK }), "createValues|serialize");
         if (isDebugLogEnabled()) {
             for (let value of pathValues) {
@@ -119,6 +121,7 @@ export class PathValueControllerBase {
         let { nodeId, initialTriggers } = config;
         pathValueSendCount += changes.length;
         recordPathValuesSent({ count: changes.length, nodeId });
+        recordPathValuesSentStats(changes);
         let buffers = await measureBlock(() => pathValueSerializer.serialize(changes, {
             noLocks: !config.keepLocks,
             compress: getCompressNetwork(),
@@ -156,6 +159,7 @@ export class PathValueControllerBase {
             values = await measureBlock(() => pathValueSerializer.deserialize(valueBuffers), "sendData|deserialize");
             ActionsHistory.OnRead(values);
             recordPathValuesReceived({ count: values.length, nodeId: callerId });
+            recordPathValuesReceivedStats(values);
         }
         if (debugOnSendData) {
             debugOnSendData(values, callerId);

package/src/0-path-value-core/PathValueStats.ts

@@ -0,0 +1,83 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
+import { TreeSummary } from "sliftutils/treeSummary";
+import type { SummaryEntry } from "sliftutils/treeSummary";
+
+/*
+    Per-server, in-memory stats for path values this node has sent and received (never persisted). Two things are
+    tracked: a running total count for each direction, and a TreeSummary of the paths in each direction — a bounded
+    prefix tree that stays O(1000s of nodes) no matter how many distinct paths flow through, so we can show which
+    paths dominate without keeping every path. Cleared wholesale by the clear endpoint.
+*/
+
+const PATH_SUMMARY_EXPECTED_OUTPUT_COUNT = 100;
+
+export type PathValueSummaryState = { count: number };
+export type PathValueTotals = { sent: number; received: number };
+export type PathValueDirection = "sent" | "received";
+
+// Only the path matters for the tree; callers pass whole PathValues but we never look at anything else.
+type PathValueLike = { path: string };
+
+function makeTree() {
+    return new TreeSummary<PathValueLike, PathValueSummaryState>({
+        getPath: value => value.path,
+        createSummary: () => ({ count: 0 }),
+        addToSummary: (_value, summary) => { summary.count++; },
+        mergeSummaries: (target, source) => { target.count += source.count; },
+        getWeight: summary => summary.count,
+        expectedOutputCount: PATH_SUMMARY_EXPECTED_OUTPUT_COUNT,
+    });
+}
+
+let totalSent = 0;
+let totalReceived = 0;
+let sentTree = makeTree();
+let receivedTree = makeTree();
+
+export function recordPathValuesSentStats(pathValues: PathValueLike[]) {
+    totalSent += pathValues.length;
+    for (let value of pathValues) {
+        sentTree.add(value);
+    }
+}
+export function recordPathValuesReceivedStats(pathValues: PathValueLike[]) {
+    totalReceived += pathValues.length;
+    for (let value of pathValues) {
+        receivedTree.add(value);
+    }
+}
+export function clearPathValueStats() {
+    totalSent = 0;
+    totalReceived = 0;
+    sentTree = makeTree();
+    receivedTree = makeTree();
+}
+
+class PathValueStatsControllerBase {
+    // Cheap: just the two running totals, so the server list can render every server without pulling any tree data.
+    public async getTotals(): Promise<PathValueTotals> {
+        return { sent: totalSent, received: totalReceived };
+    }
+    // Heavy: the top-N path breakdown for one direction. Only fetched when a server row is expanded.
+    public async getSummary(direction: PathValueDirection, maxCount: number): Promise<SummaryEntry<PathValueSummaryState>[]> {
+        let tree = direction === "sent" ? sentTree : receivedTree;
+        return tree.getSummaries(maxCount);
+    }
+    public async clear(): Promise<void> {
+        clearPathValueStats();
+    }
+}
+
+export const PathValueStatsController = SocketFunction.register(
+    "PathValueStatsController-3f9a1c72-2b64-4e8d-9c1a-7d5e0b3f6a21",
+    new PathValueStatsControllerBase(),
+    () => ({
+        getTotals: {},
+        getSummary: {},
+        clear: {},
+    }),
+    () => ({
+        hooks: [requiresNetworkTrustHook],
+    })
+);

package/src/diagnostics/misc-pages/FunctionRunnersSection.tsx

@@ -0,0 +1,38 @@
+module.allowclient = true;
+
+import { qreact } from "../../4-dom/qreact";
+import { css } from "typesafecss";
+import { formatTime } from "socket-function/src/formatting/format";
+import { getFunctionRunnerIndex } from "../../4-querysub/FunctionRunnerTracking";
+import { FUNCTION_RUNNER_COLOR } from "../../misc/nodeCategoryColors";
+import { AuthorityRangeBar } from "./RoutingTablePage";
+
+export class FunctionRunnersSection extends qreact.Component {
+    render() {
+        let index = getFunctionRunnerIndex();
+        let nodes = index?.nodes || [];
+        return <div className={css.vbox(8).fillWidth}>
+            <h2>Function Runners ({nodes.length})</h2>
+            {nodes.length === 0 && <div className={css.colorhsl(0, 0, 50)}>(no function runners found yet)</div>}
+            {nodes.map(node =>
+                <div className={css.vbox(4).pad2(10).fillWidth.bord2(0, 0, 85).hsl(0, 0, 99)}>
+                    <div className={css.hbox(10).fillWidth}>
+                        <span className={css.boldStyle}>{node.nodeId}</span>
+                        <span className={css.color(FUNCTION_RUNNER_COLOR)}>networks: {node.networks.join(", ")}</span>
+                        {!node.isPublic && <span className={css.colorhsl(0, 70, 35)}>(non-public)</span>}
+                        <span>latency {formatTime(node.averageLatency)}</span>
+                        <span>up for {formatTime(Date.now() - node.startupTime)}</span>
+                        <span className={css.colorhsl(0, 0, 40).ellipsis}>{node.entryPoint}</span>
+                    </div>
+                    {node.shards.map(shard =>
+                        <div className={css.hbox(10).fillWidth}>
+                            <span>{shard.shardRange.startFraction.toFixed(4)} - {shard.shardRange.endFraction.toFixed(4)}</span>
+                            <AuthorityRangeBar start={shard.shardRange.startFraction} end={shard.shardRange.endFraction} />
+                            {shard.secondaryShardRange && <span className={css.colorhsl(0, 0, 50)}>secondary {shard.secondaryShardRange.startFraction.toFixed(4)} - {shard.secondaryShardRange.endFraction.toFixed(4)}</span>}
+                        </div>
+                    )}
+                </div>
+            )}
+        </div>;
+    }
+}