querysub 0.508.0 → 0.509.0

package/src/deployManager/setupMachineMain.ts

@@ -92,7 +92,8 @@ async function getGitHubApiKey(repoUrl: string, sshRemote: string, forceRefresh
     await open(tokenUrl);
 
     const rl = readline.createInterface({
-        input: process.stdin,
+        // Cast, as different @types/node versions disagree on the stream types
+        input: process.stdin as unknown as NodeJS.ReadableStream,
         output: process.stdout
     });
 

package/src/diagnostics/grossStats/GrossStatsController.ts

@@ -12,6 +12,7 @@ import {
 import { getOwnNodeId, isOwnNodeId } from "../../-f-node-discovery/NodeDiscovery";
 import { authorityLookup } from "../../0-path-value-core/AuthorityLookup";
 import { timeoutToUndefinedSilent } from "../../errors";
+import { spreadCallsOverTime } from "../../misc";
 
 const POLL_INTERVAL = timeInMinute;
 const PER_NODE_TIMEOUT = 30 * timeInSecond;
@@ -31,7 +32,7 @@ async function listClusterNodeIds(): Promise<string[]> {
 async function pollOnce() {
     let nodeIds = await listClusterNodeIds();
     let cutoff = Date.now() - RETENTION_MS;
-    await Promise.all(nodeIds.map(async nodeId => {
+    await spreadCallsOverTime(nodeIds, POLL_INTERVAL, async nodeId => {
         if (isOwnNodeId(nodeId)) {
             polledBuckets.set(nodeId, getGrossStatsBuckets().filter(b => b.time >= cutoff));
             return;
@@ -46,7 +47,7 @@ async function pollOnce() {
         let merged = prev ? (result ? prev.concat(result.buckets) : prev) : (result?.buckets ?? []);
         merged = merged.filter(b => b.time >= cutoff);
         polledBuckets.set(nodeId, merged);
-    }));
+    });
 }
 
 let ensurePolling = lazy(() => {

package/src/diagnostics/logs/errorTickets/autoFixer.ts

@@ -793,8 +793,9 @@ async function runClaude(prompt: string, takeNewUserComments: () => Promise<stri
             }
         });
     }
-    forwardLines(child.stdout!, "[claude]", line => handleClaudeStreamLine(line));
-    forwardLines(child.stderr!, "[claude:err]");
+    // Casts, as different @types/node versions disagree on the stream types
+    forwardLines(child.stdout as unknown as NodeJS.ReadableStream, "[claude]", line => handleClaudeStreamLine(line));
+    forwardLines(child.stderr as unknown as NodeJS.ReadableStream, "[claude:err]");
 
     let timedOut = false;
     let timeout = setTimeout(() => {

package/src/diagnostics/logs/logGitHashes.ts

@@ -23,7 +23,7 @@ async function execPromise(command: string, options: child_process.ExecOptions)
             if (err) {
                 reject(err);
             } else {
-                resolve(stdout.trim());
+                resolve(String(stdout).trim());
             }
         });
     });

package/src/diagnostics/managementPages.tsx

@@ -299,6 +299,8 @@ export async function isManagementUser() {
         ModuleId: schema.moduleId,
         FunctionId: functionId,
         runAtTime: getNextTime(),
+        // This call is only evaluated locally to check permissions, and is never committed, so it never reaches a FunctionRunner network
+        network: "",
     };
 
     let writes = await getCallWrites({

package/src/diagnostics/misc-pages/AuthoritySpecPage.tsx

@@ -10,6 +10,8 @@ import { NodeCapabilitiesController } from "../../-g-core-values/NodeCapabilitie
 import { timeoutToUndefinedSilent } from "../../errors";
 import { sort } from "socket-function/src/misc";
 import type { AuthoritySpec } from "../../0-path-value-core/PathRouter";
+import { getFunctionRunnerIndex } from "../../4-querysub/FunctionRunnerTracking";
+import { formatTime } from "socket-function/src/formatting/format";
 
 const PROBE_TIMEOUT_MS = 5000;
 const RANGE_BAR_WIDTH_PX = 360;
@@ -79,7 +81,6 @@ class AuthorityNodeRow extends qreact.Component<{ info: NodeAuthorityInfo }> {
                 <AuthorityRangeBar start={spec.routeStart} end={spec.routeEnd} />
                 <span className={css.colorhsl(0, 0, 50)}>width {(spec.routeEnd - spec.routeStart).toFixed(4)}</span>
                 {spec.excludeDefault && <span className={css.colorhsl(0, 70, 35)}>(excludes default)</span>}
-                {spec.networks && spec.networks.length > 0 && <span className={css.colorhsl(210, 60, 40)}>networks: {spec.networks === "all" && "all" || Array.isArray(spec.networks) && spec.networks.join(", ") || ""}</span>}
                 <span className={css.colorhsl(0, 0, 40).ellipsis.flexFillWidth}>{info.entryPoint || "(no entry point)"}</span>
             </div>
             {expanded &&
@@ -95,6 +96,36 @@ class AuthorityNodeRow extends qreact.Component<{ info: NodeAuthorityInfo }> {
     }
 }
 
+class FunctionRunnersSection extends qreact.Component {
+    render() {
+        let index = getFunctionRunnerIndex();
+        let nodes = index?.nodes || [];
+        return <div className={css.vbox(8).fillWidth}>
+            <h2>Function Runners ({nodes.length})</h2>
+            {nodes.length === 0 && <div className={css.colorhsl(0, 0, 50)}>(no function runners found yet)</div>}
+            {nodes.map(node =>
+                <div className={css.vbox(4).pad2(10).fillWidth.bord2(0, 0, 85).hsl(0, 0, 99)}>
+                    <div className={css.hbox(10).fillWidth}>
+                        <span className={css.boldStyle}>{node.nodeId}</span>
+                        <span className={css.colorhsl(210, 60, 40)}>networks: {node.networks.join(", ")}</span>
+                        {!node.isPublic && <span className={css.colorhsl(0, 70, 35)}>(non-public)</span>}
+                        <span>latency {formatTime(node.averageLatency)}</span>
+                        <span>up for {formatTime(Date.now() - node.startupTime)}</span>
+                        <span className={css.colorhsl(0, 0, 40).ellipsis}>{node.entryPoint}</span>
+                    </div>
+                    {node.shards.map(shard =>
+                        <div className={css.hbox(10).fillWidth}>
+                            <span>{shard.shardRange.startFraction.toFixed(4)} - {shard.shardRange.endFraction.toFixed(4)}</span>
+                            <AuthorityRangeBar start={shard.shardRange.startFraction} end={shard.shardRange.endFraction} />
+                            {shard.secondaryShardRange && <span className={css.colorhsl(0, 0, 50)}>secondary {shard.secondaryShardRange.startFraction.toFixed(4)} - {shard.secondaryShardRange.endFraction.toFixed(4)}</span>}
+                        </div>
+                    )}
+                </div>
+            )}
+        </div>;
+    }
+}
+
 export class AuthoritySpecPage extends qreact.Component {
     render() {
         let infos = AuthoritySpecSynced(getBrowserUrlNode()).getAllNodeAuthoritySpecs();
@@ -104,35 +135,12 @@ export class AuthoritySpecPage extends qreact.Component {
         infos = infos.filter(x => x.spec && x.spec.routeStart >= 0 && x.spec.routeEnd >= 0);
         sort(infos, x => x.spec!.routeStart);
 
-        let infosPerNetwork = new Map<string, NodeAuthorityInfo[]>();
-        for (let info of infos) {
-            let networks = info.spec!.networks;
-            if (networks === "all") {
-                networks = ["all"];
-            }
-            if (!networks || networks.length === 0) {
-                networks = ["default"];
-            }
-            for (let network of networks) {
-                let list = infosPerNetwork.get(network);
-                if (!list) {
-                    list = [];
-                    infosPerNetwork.set(network, list);
-                }
-                list.push(info);
-            }
-        }
-        let networkEntries = Array.from(infosPerNetwork.entries());
-        sort(networkEntries, entry => entry[0] === "default" && " " || entry[0]);
-
         return <div className={css.vbox(12).pad2(16).fillWidth}>
             <h2>Routing Table ({infos.length})</h2>
-            {networkEntries.map(([network, networkInfos]) =>
-                <div className={css.vbox(8).fillWidth}>
-                    <h3>{network} ({networkInfos.length})</h3>
-                    {networkInfos.map(info => <AuthorityNodeRow key={info.nodeId} info={info} />)}
-                </div>
-            )}
+            <div className={css.vbox(8).fillWidth}>
+                {infos.map(info => <AuthorityNodeRow key={info.nodeId} info={info} />)}
+            </div>
+            <FunctionRunnersSection />
         </div>;
     }
 }

package/src/diagnostics/misc-pages/FunctionCaptureAnalyzePage.tsx

@@ -158,7 +158,7 @@ export class FunctionCaptureAnalyzePage extends qreact.Component {
         return <div className={css.vbox(2).fillWidth}>
             <div className={css.fontWeight("bold")}>{title} ({formatNumber(rows.length)})</div>
             <div className={css.vbox(1).fillWidth.fontFamily("monospace").fontSize(12)}>
-                {shown.map(row => <div className={css.whiteSpace("pre-wrap").wordBreak("break-all")}>{row}</div>)}
+                {shown.map(row => <div className={css.whiteSpace("pre-wrap").overflowWrap("break-word")}>{row}</div>)}
             </div>
             {remaining > 0 &&
                 <Button onClick={() => Querysub.localCommit(() => { this.state.rowLimits[sectionId] = limit + ROW_LIMIT; })}>

package/src/library-components/NetworkSelector.tsx

@@ -0,0 +1,77 @@
+import { css } from "typesafecss";
+import { qreact } from "../4-dom/qreact";
+import { formatTime } from "socket-function/src/formatting/format";
+import { isCurrentUserSuperUser } from "../user-implementation/userData";
+import { forcedNetworkURL, getAutoSelectedNetwork, getNetworkSelectionInfos, getSelectedNetwork } from "../4-querysub/FunctionRunnerTracking";
+
+/** A dropdown showing all the function runner networks, which one our function calls are being put on, and allowing forcing a specific network (and unforcing it, going back to the automatic selection). */
+export class NetworkSelector extends qreact.Component<{}> {
+    render() {
+        let infos = getNetworkSelectionInfos();
+        let forced = forcedNetworkURL.value;
+        let selected = getSelectedNetwork({ noThrow: true });
+        let auto = getAutoSelectedNetwork();
+        let isSuperUser = isCurrentUserSuperUser();
+
+        // Only networks that actually exist (have function runners) can be picked
+        let networks = infos.map(x => x.network);
+        if (forced && !networks.includes(forced)) {
+            networks.push(forced);
+        }
+
+        if (!selected) {
+            return <div className={css.hbox(6)}>
+                <span className={css.opacity(0.7)}>Network</span>
+                <span className={css.pad2(6, 2).bord2(0, 85, 55).colorhsl(0, 85, 55).boldStyle}>⛔ no function runner networks</span>
+            </div>;
+        }
+
+        return <div className={css.hbox(6)}>
+            <span className={css.opacity(0.7)}>Network</span>
+            <select
+                value={selected}
+                className={css.pad2(4, 2).hsl(0, 0, 16).colorhsl(0, 0, 90).bord2(0, 0, 30)}
+                onChange={e => {
+                    forcedNetworkURL.value = e.currentTarget.value;
+                }}
+            >
+                {networks.map(network => {
+                    let info = infos.find(x => x.network === network);
+                    let label = network;
+                    if (info) {
+                        label += ` (${info.nodeCount} runner${info.nodeCount !== 1 && "s" || ""}, ${formatTime(info.averageLatency)} latency`;
+                        if (info.averageCallTime) {
+                            label += `, ${formatTime(info.averageCallTime)} avg call`;
+                        }
+                        if (!info.isPublic) {
+                            label += `, non-public`;
+                        }
+                        if (info.dying) {
+                            label += `, shutting down`;
+                        }
+                        label += `)`;
+                    }
+                    let disabled = !!info && !info.isPublic && !isSuperUser;
+                    return <option value={network} disabled={disabled}>{label}</option>;
+                })}
+            </select>
+            {forced && <span
+                className={css.button.pad2(6, 2).hsl(35, 60, 22).colorhsl(0, 0, 90).bord2(35, 60, 40)}
+                title="The network is forced. Click to reset back to the automatically picked network."
+                onMouseDown={() => forcedNetworkURL.value = ""}
+            >
+                auto: {auto}
+            </span>}
+            {(() => {
+                let selectedInfo = infos.find(x => x.network === selected);
+                if (!selectedInfo || selectedInfo.nodeCount === 0) {
+                    return <span className={css.pad2(6, 2).bord2(0, 85, 55).colorhsl(0, 85, 55).boldStyle}>⛔ no function runners on this network</span>;
+                }
+                if (selectedInfo.dying) {
+                    return <span className={css.colorhsl(35, 85, 55)}>⚠️ this network is shutting down</span>;
+                }
+                return undefined;
+            })()}
+        </div>;
+    }
+}

package/src/misc.ts

@@ -1,4 +1,5 @@
 import { canHaveChildren } from "socket-function/src/types";
+import { delay } from "socket-function/src/batching";
 
 // TIMING: About 20MB/s
 export function createRandomText(count: number): string {
@@ -194,4 +195,22 @@ export function isAsyncFunction(func: unknown): boolean {
 
 export function maybeUndefined<T>(value: T): T | undefined {
     return value;
+}
+
+/** Runs `callback` for every item, but staggers the calls evenly across `totalTime` instead of
+ *      firing them all at once, and awaits all of them before returning. Used inside a poll loop
+ *      (with totalTime = the poll interval) so fanning out to many nodes doesn't spike — otherwise
+ *      every dead node's timeout lands at the same instant each cycle. */
+export async function spreadCallsOverTime<T>(
+    items: T[],
+    totalTime: number,
+    callback: (item: T, index: number) => Promise<void>
+): Promise<void> {
+    await Promise.all(items.map(async (item, index) => {
+        let startOffset = index / items.length * totalTime;
+        if (startOffset > 0) {
+            await delay(startOffset);
+        }
+        await callback(item, index);
+    }));
 }

package/src/user-implementation/userData.ts

@@ -892,7 +892,7 @@ export const LOCAL_STORAGE_USER_TYPE_KEY = "userType";
 export const LOCAL_STORAGE_COMMIT_DELAY_KEY = "LOCAL_STORAGE_COMMIT_DELAY_KEY";
 if (!isNode()) {
     logErrors((async () => {
-        await Querysub.optionalStartupWait();
+        await Querysub.startupWait();
         let isLoggedIn = await Querysub.commitSynced(() => isCurrentUserType("user"));
         if (isLoggedIn) {
             Querysub.commit(() => functions.registerPageLoadTime());

package/src/zipThreaded.ts

@@ -64,8 +64,9 @@ async function spawnWorker() {
         let { promise, onFree } = messageQueue.shift()!;
         onFree();
         workerObj.queueDepth = messageQueue.length;
-        console.log(`Worker error: ${err.message}`);
-        promise.reject(err);
+        let error = err as Error;
+        console.log(`Worker error: ${error.stack || error}`);
+        promise.reject(error);
     });
     async function unzip(buffer: Buffer): Promise<Buffer> {
         let promise = new PromiseObj<Buffer>();

package/bin/server-dev.js

@@ -1,11 +0,0 @@
-#!/usr/bin/env node
-
-process.argv.push("--local");
-// Serves the dev network AND the default network, so one server can satisfy paths with no network as well as paths on our dev network.
-process.argv.push("--networkfile");
-process.argv.push("~/devnetwork.txt");
-process.argv.push("--network");
-process.argv.push("default");
-
-require("typenode");
-require("../src/server.ts");