querysub 0.526.0 → 0.528.0

package/.claude/settings.local.json

@@ -29,7 +29,8 @@
       "WebSearch",
       "WebFetch(domain:blog.cloudflare.com)",
       "WebFetch(domain:developers.cloudflare.com)",
-      "PowerShell(yarn type *)"
+      "PowerShell(yarn type *)",
+      "PowerShell(Remove-Item -Confirm:$false \"C:\\\\Users\\\\quent\\\\.claude\\\\projects\\\\D--repos-querysub\\\\memory\\\\no-naming-new-bins.md\")"
     ]
   }
 }

package/appSecrets.ts

@@ -0,0 +1,23 @@
+import fs from "fs";
+import { getBackblazePath } from "./src/-a-archives/archivesBackBlaze";
+import { getCloudflareCreds } from "./src/-b-authorities/cloudflareHelpers";
+import { Querysub } from "./src/4-querysub/Querysub";
+
+/** Serves the secret keys sliftutils' getSecret expects, reading them the way querysub already does: backblaze creds off disk (getBackblazePath), cloudflare creds from the keys archives bucket (with its local file fallback). */
+export async function getAppSecret(key: string): Promise<string | undefined> {
+    // Use this so some of our default imports get imported
+    Querysub;
+    if (key.startsWith("backblaze.json.")) {
+        let creds = JSON.parse(fs.readFileSync(getBackblazePath(), "utf8")) as { applicationKeyId?: string; applicationKey?: string };
+        if (key === "backblaze.json.applicationKeyId") return creds.applicationKeyId;
+        if (key === "backblaze.json.applicationKey") return creds.applicationKey;
+        return undefined;
+    }
+    if (key.startsWith("cloudflare.json.")) {
+        let creds = await getCloudflareCreds();
+        if (key === "cloudflare.json.key") return creds.key;
+        if (key === "cloudflare.json.email") return creds.email;
+        return undefined;
+    }
+    return undefined;
+}

package/bin/storageserve.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+
+require("typenode");
+require("../src/storageSetup.ts");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.526.0",
+  "version": "0.528.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -49,7 +49,8 @@
     "audit-imports": "./bin/audit-imports.js",
     "audit-disk-values": "./bin/audit-disk-values.js",
     "mcp-indexed-logs": "./bin/mcp-indexed-logs.js",
-    "autofix": "./bin/autofix.js"
+    "autofix": "./bin/autofix.js",
+    "storageserve": "./bin/storageserve.js"
   },
   "dependencies": {
     "@types/fs-ext": "^2.0.3",
@@ -70,7 +71,7 @@
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
     "sliftutils": "^1.7.5",
-    "socket-function": "^1.2.14",
+    "socket-function": "^1.2.16",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",
     "typesafecss": "^0.32.0",

package/src/-b-authorities/cdnAuthority.ts

@@ -1,6 +1,6 @@
 import { Archives } from "../-a-archives/archives";
 import { ArchivesBackblaze } from "../-a-archives/archivesBackBlaze";
-import { cloudflareGETCall, cloudflarePOSTCall } from "./cloudflareHelpers";
+import { cloudflareGETCall, cloudflarePOSTCall } from "sliftutils/misc/https/cloudflareHelpers";
 import { setRecord, getZoneId } from "./dnsAuthority";
 import debugbreak from "debugbreak";
 

package/src/-b-authorities/cloudflareHelpers.ts

@@ -1,11 +1,9 @@
-import { httpsRequest } from "socket-function/src/https";
 import { getArchives } from "../-a-archives/archives";
 import { lazy } from "socket-function/src/caching";
-import { getStorageDir, getSubFolder } from "../fs";
+import { getStorageDir } from "../fs";
 import fs from "fs";
-import { isNodeTrue } from "socket-function/src/misc";
 
-export const keys = lazy(() => getArchives("keys"));
+const keys = lazy(() => getArchives("keys"));
 
 export const getCloudflareCreds = lazy(async (): Promise<{ key: string; email: string }> => {
     let credsJSON = await keys().get("cloudflare.json");
@@ -20,44 +18,3 @@ export const getCloudflareCreds = lazy(async (): Promise<{ key: string; email: s
     }
     return JSON.parse(credsJSON.toString());
 });
-
-export async function cloudflareGETCall<T>(path: string, params?: { [key: string]: string }): Promise<T> {
-    let url = new URL(`https://api.cloudflare.com/client/v4` + path);
-    for (let key in params) {
-        url.searchParams.set(key, params[key]);
-    }
-    let creds = await getCloudflareCreds();
-    let result = await httpsRequest(url.toString(), [], "GET", undefined, {
-        headers: {
-            "Content-Type": "application/json",
-            "X-Auth-Email": creds.email,
-            "X-Auth-User-Service-Key": creds.key,
-            "X-Auth-Key": creds.key,
-        }
-    });
-    let result2 = JSON.parse(result.toString()) as { result: unknown; success: boolean; errors: { code: number; message: string }[] };
-    if (!result2.success) {
-        throw new Error(`Cloudflare call failed: ${result2.errors.map(x => x.message).join(", ")}`);
-    }
-    return result2.result as T;
-}
-export async function cloudflarePOSTCall<T>(path: string, params: { [key: string]: unknown }): Promise<T> {
-    return await cloudflareCall(path, Buffer.from(JSON.stringify(params)), "POST");
-}
-export async function cloudflareCall<T>(path: string, payload: Buffer, method: string): Promise<T> {
-    let url = new URL(`https://api.cloudflare.com/client/v4` + path);
-    let creds = await getCloudflareCreds();
-    let result = await httpsRequest(url.toString(), payload, method, undefined, {
-        headers: {
-            "Content-Type": "application/json",
-            "X-Auth-Email": creds.email,
-            "X-Auth-User-Service-Key": creds.key,
-            "X-Auth-Key": creds.key,
-        }
-    });
-    let result2 = JSON.parse(result.toString()) as { result: unknown; success: boolean; errors: { code: number; message: string }[] };
-    if (!result2.success) {
-        throw new Error(`Cloudflare call failed: ${result2.errors.map(x => x.message).join(", ")}`);
-    }
-    return result2.result as T;
-}

package/src/-b-authorities/dnsAuthority.ts

@@ -1,185 +1,44 @@
-import os from "os";
-import * as fs from "fs";
-import { cache, lazy } from "socket-function/src/caching";
-import { isNode, isNodeTrue, timeInDay } from "socket-function/src/misc";
-import { httpsRequest } from "../https";
-import { getStorageDir } from "../fs";
-import { SocketFunction } from "socket-function/SocketFunction";
-import { delay } from "socket-function/src/batching";
-import debugbreak from "debugbreak";
-import { isClient } from "../config2";
-import { getArchives } from "../-a-archives/archives";
-import { cloudflareCall, cloudflareGETCall, cloudflarePOSTCall, getCloudflareCreds } from "./cloudflareHelpers";
-import { magenta } from "socket-function/src/formatting/logColors";
-
-const DNS_TTLSeconds = {
-    "TXT": 60,
-    "A": 60,
-};
-
-const DNS_REFRESH_STALE_AFTER = timeInDay;
-
-// We stamp our own "last asserted" time into the record's comment, because Cloudflare's
-//  modified_on can only be moved by an actual content change - and re-asserting an already
-//  correct record has no content change to make (a create errors with 81058, a no-op edit
-//  doesn't bump the timestamp). Reading freshness from text we control sidesteps that entirely,
-//  and lets us refresh a stale-but-correct record in place instead of deleting and recreating it.
-const FRESHNESS_REGEX = /<set on:[^>]*>/;
-
-/** Strips any prior freshness tag and appends a new one, preserving other comment text. */
-function stampFreshness(comment?: string): string {
-    let base = (comment ?? "").replace(FRESHNESS_REGEX, "").trim();
-    let stamp = `<set on: ${new Date().toString()}>`;
-    return base ? `${base} ${stamp}` : stamp;
-}
-/** Parses our tag back out; 0 (i.e. always stale) when it's absent or unparseable. */
-export function freshnessTime(comment?: string): number {
-    let match = FRESHNESS_REGEX.exec(comment ?? "");
-    if (!match) return 0;
-    return new Date(match[0].replace("<set on:", "").replace(">", "").trim()).getTime() || 0;
-}
-
-export const hasDNSWritePermissions = lazy(async () => {
-    if (!isNode()) return false;
-    if (isClient()) return false;
-    try {
-        await getCloudflareCreds();
-        return true;
-    } catch {
-        return false;
-    }
-});
-
-export const getZoneId = cache(async (rootDomain: string): Promise<string> => {
-    let zones = await cloudflareGETCall<{ id: string; name: string }[]>("/zones", {});
-    let selected = zones.find(x => x.name === rootDomain);
-    if (!selected) {
-        throw new Error(`Could not find zone for ${rootDomain}. Found ${zones.map(x => x.name).join(", ")}`);
-    }
-    return selected.id;
-});
-
-function getRootDomain(key: string) {
-    if (key.endsWith(".")) {
-        key = key.slice(0, -1);
-    }
-    return key.split(".").slice(-2).join(".");
-}
-
-export async function getRecordsRaw(type: string, key: string) {
-    if (key.endsWith(".")) key = key.slice(0, -1);
-    let zoneId = await getZoneId(getRootDomain(key));
-    let results = await cloudflareGETCall<{
-        id: string;
-        type: string;
-        name: string;
-        content: string;
-        proxied: boolean;
-        modified_on: string;
-        // Omitted by Cloudflare when the record has no comment.
-        comment?: string;
-    }[]>(`/zones/${zoneId}/dns_records`);
-    // DNS names are case-insensitive and Cloudflare returns them lowercased, so a mixed-case key
-    //  (e.g. a machine id subdomain) would never match an exact-case compare - match lowercased.
-    let keyLower = key.toLowerCase();
-    return results.filter(x => x.type === type && x.name.toLowerCase() === keyLower);
-}
-
-/** Cloudflare's batch endpoint applies deletes, then patches, then posts in a single database
- *   transaction. We route edits (patches) through here because the standalone PATCH/PUT verbs
- *   aren't usable in our setup, and because it lets "remove others + assert target" happen
- *   without a window where the name resolves to nothing. */
-export async function batchRecords(zoneId: string, batch: {
-    deletes?: { id: string }[];
-    patches?: { id: string; comment?: string }[];
-    posts?: { type: string; name: string; content: string; ttl: number; proxied: boolean; comment?: string }[];
-}) {
-    let payload: { [key: string]: unknown } = {};
-    if (batch.deletes && batch.deletes.length > 0) payload.deletes = batch.deletes;
-    if (batch.patches && batch.patches.length > 0) payload.patches = batch.patches;
-    if (batch.posts && batch.posts.length > 0) payload.posts = batch.posts;
-    try {
-        await cloudflarePOSTCall(`/zones/${zoneId}/dns_records/batch`, payload);
-    } catch (error) {
-        console.error(`Error updating DNS records:`, { error: error, batch });
-        throw new Error(`Error updating DNS records. ${JSON.stringify(batch)}. Error: ${error}`);
-    }
-}
-export async function getRecords(type: string, key: string) {
-    if (key.endsWith(".")) key = key.slice(0, -1);
-    let raw = await getRecordsRaw(type, key);
-    return raw.map(x => x.content);
-}
-export async function deleteRecord(type: string, key: string, value: string) {
-    if (key.endsWith(".")) key = key.slice(0, -1);
-    let zoneId = await getZoneId(getRootDomain(key));
-    let prevValues = await getRecordsRaw(type, key);
-    prevValues = prevValues.filter(x => x.content === value);
-    if (prevValues.length === 0) {
-        if (!SocketFunction.silent) {
-            console.log(`No need to delete record, it was not found. ${JSON.stringify(value)} value was not in ${type} for ${key}, values ${JSON.stringify(prevValues.map(x => x.content))}`);
-        }
-        return;
-    }
-
-    console.log(`Removing records of ${type} for ${key}, values ${JSON.stringify(prevValues.map(x => x.content))}`);
-    for (let value of prevValues) {
-        await cloudflareCall(`/zones/${zoneId}/dns_records/${value.id}`, Buffer.from([]), "DELETE");
-    }
-}
-/** Removes all existing records (unless the record is already present and fresh) */
-export async function setRecord(type: string, key: string, value: string, proxied?: "proxied", staleAfter = DNS_REFRESH_STALE_AFTER) {
-    if (key.endsWith(".")) key = key.slice(0, -1);
-    let zoneId = await getZoneId(getRootDomain(key));
-    let prevValues = await getRecordsRaw(type, key);
-    let existing = prevValues.find(x => x.content === value);
-    let others = prevValues.filter(x => x.content !== value);
-
-    // Already correct and recently asserted - a prior run also cleaned up the other records,
-    //  so there is nothing left to do.
-    if (existing && Date.now() - freshnessTime(existing.comment) < staleAfter) return;
-
-    // A single atomic batch: drop the wrong records, and either refresh the existing record's
-    //  comment in place or create it - so the name is never left resolving to nothing.
-    let ttl = DNS_TTLSeconds[type as "A"] || 60;
-    let comment = stampFreshness(existing?.comment);
-    console.log(magenta(`Setting ${type} record for ${key} to ${value} (previously had ${JSON.stringify(prevValues.map(x => x.content))})`));
-    await batchRecords(zoneId, {
-        deletes: others.map(x => ({ id: x.id })),
-        patches: existing ? [{ id: existing.id, comment }] : [],
-        posts: existing ? [] : [{ type, name: key, content: value, ttl, proxied: proxied === "proxied", comment }],
-    });
-    // Only a brand new record needs to propagate; an in-place comment refresh doesn't change the answer.
-    if (!existing) {
-        console.log(`Waiting ${ttl} seconds for DNS to propagate...`);
-        for (let ttlLeft = ttl; ttlLeft > 0; ttlLeft--) {
-            await delay(1000);
-            console.log(`${ttlLeft} seconds left...`);
-        }
-        console.log(`Done waiting for DNS to update.`);
-    }
-}
-/** Keeps existing records */
-export async function addRecord(type: string, key: string, value: string, proxied?: "proxied", staleAfter = DNS_REFRESH_STALE_AFTER) {
-    if (key.endsWith(".")) key = key.slice(0, -1);
-    let zoneId = await getZoneId(getRootDomain(key));
-    let prevValues = await getRecordsRaw(type, key);
-    let existing = prevValues.find(x => x.content === value);
-    if (existing && Date.now() - freshnessTime(existing.comment) < staleAfter) return;
-
-    // Same single-batch flow as setRecord, minus the deletes (we keep sibling records here).
-    let ttl = DNS_TTLSeconds[type as "A"] || 60;
-    let comment = stampFreshness(existing?.comment);
-    console.log(`Adding ${type} record for ${key} to ${value} (previously had ${JSON.stringify(prevValues.map(x => x.content))})`);
-    await batchRecords(zoneId, {
-        patches: existing ? [{ id: existing.id, comment }] : [],
-        posts: existing ? [] : [{ type, name: key, content: value, ttl, proxied: proxied === "proxied", comment }],
-    });
-    if (existing) return;
-    console.log(`Waiting ${ttl} seconds for DNS to propagate...`);
-    for (let ttlLeft = ttl; ttlLeft > 0; ttlLeft--) {
-        await delay(1000);
-        console.log(`${ttlLeft} seconds left...`);
-    }
-    console.log(`Done waiting for DNS to update.`);
-}
+import { lazy } from "socket-function/src/caching";
+import { isNode } from "socket-function/src/misc";
+import { isClient } from "../config2";
+import {
+    addRecord as addRecordBase,
+    deleteRecord as deleteRecordBase,
+    freshnessTime as freshnessTimeBase,
+    getRecords as getRecordsBase,
+    getZoneId as getZoneIdBase,
+    hasDNSWritePermissions as hasDNSWritePermissionsBase,
+    setRecord as setRecordBase,
+} from "sliftutils/misc/https/dns";
+
+export function freshnessTime(comment?: string): number {
+    return freshnessTimeBase(comment);
+}
+
+export const hasDNSWritePermissions = lazy(async () => {
+    if (!isNode()) return false;
+    if (isClient()) return false;
+    return await hasDNSWritePermissionsBase();
+});
+
+export async function getZoneId(rootDomain: string): Promise<string> {
+    return await getZoneIdBase(rootDomain);
+}
+
+export async function getRecords(type: string, key: string): Promise<string[]> {
+    return await getRecordsBase(type, key);
+}
+
+export async function deleteRecord(type: string, key: string, value: string): Promise<void> {
+    return await deleteRecordBase(type, key, value);
+}
+
+/** Removes all existing records (unless the record is already present and fresh) */
+export async function setRecord(type: string, key: string, value: string, proxied?: "proxied", staleAfter?: number): Promise<void> {
+    return await setRecordBase(type, key, value, proxied, staleAfter);
+}
+
+/** Keeps existing records */
+export async function addRecord(type: string, key: string, value: string, proxied?: "proxied", staleAfter?: number): Promise<void> {
+    return await addRecordBase(type, key, value, proxied, staleAfter);
+}

package/src/-c-identity/IdentityController.ts

@@ -197,7 +197,7 @@ class IdentityControllerBase {
         });
 
         let duration = Date.now() - time;
-        console.log(`Authenticated identity for ${caller.nodeId} in ${formatTime(duration)}, at ${Date.now()}`, {
+        console.info(`Authenticated identity for ${caller.nodeId} in ${formatTime(duration)}, at ${Date.now()}`, {
             clientId: caller.nodeId,
             reconnectNodeId,
             duration,
@@ -208,7 +208,7 @@ class IdentityControllerBase {
         SocketFunction.onNextDisconnect(caller.nodeId, () => {
             // NOTE: I don't really see any purpose of deleting from caller info. I don't think we're going to run out of memory because of too many callers authenticating. 
             // However, logging here is useful as it allows us to complete the life cycle so we know how long a client was connected for.
-            console.log(`Disconnected client`, {
+            console.info(`Disconnected client`, {
                 clientId: caller.nodeId,
             });
         });
@@ -251,6 +251,7 @@ const changeIdentityOnce = cacheWeak(async function changeIdentityOnce(connectio
         () => new Error(`Timeout calling changeIdentity for ${nodeId}`)
     );
 });
+let startupTime = Date.now();
 SocketFunction.addGlobalClientHook(async function identityHook(context) {
     if (context.call.classGuid === IdentityController._classGuid) return;
     // This is for US to tell them our identity. And if they established the connection the identity will come from their original connection url (that they used to connect to us), and they validated it either being a real certificate, or they added the cert from the trusted backblaze bucket. If it just from a real certificate it means we identified them, but they might not have network trust. But that's fine, as IdentityController is JUST for identification, and if it's a real certificate we know who they are! (Which doesn't mean we trust them).
@@ -259,8 +260,9 @@ SocketFunction.addGlobalClientHook(async function identityHook(context) {
     }
     let time = Date.now();
     await changeIdentityOnce(context.connectionId);
-    let duration = Date.now() - time;
-    if (duration > 200) {
+    let now = Date.now();
+    let duration = now - time;
+    if (duration > 200 && now - startupTime > timeInMinute) {
         console.log(red(`IdentityHook took ${formatTime(duration)} for ${context.connectionId.nodeId} ${context.call.classGuid}.${context.call.functionName}`));
     }
 });

package/src/-d-trust/NetworkTrust2.ts

@@ -104,6 +104,11 @@ let populateTrustedCache = lazy(async () => {
     }, TRUSTED_CACHE_RESET_INTERVAL);
 });
 
+export async function getTrustedMachineIds() {
+    await populateTrustedCache();
+    return Array.from(trustedCache);
+}
+
 export async function isNodeTrusted(nodeId: string) {
     let domainName = getNodeIdDomainMaybeUndefined(nodeId);
     if (!domainName) return false;
@@ -119,7 +124,7 @@ export const loadServerCert = cache(async (machineId: string) => {
         console.warn(`Could not find certificate in archives for ${machineId}`);
         return;
     }
-    console.log(magenta(`Loading certificate for ${machineId}`));
+    console.info(magenta(`Loading certificate for ${machineId}`));
     trustCertificate(certFile);
 });
 

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -216,7 +216,7 @@ async function setNodeIds(nodeIds: string[]) {
     if (newNodeIds.length === 0 && removedNodeIds.length === 0) return;
     if (logging) {
         for (let nodeId of newNodeIds) {
-            console.log(blue(`Discovered node ${nodeId}`));
+            console.info(blue(`Discovered node ${nodeId}`));
         }
         for (let nodeId of removedNodeIds) {
             console.log(red(`Removed node from setNodeIds ${nodeId}`));

package/src/config.ts

@@ -35,6 +35,7 @@ let yargObj = parseArgsFactory()
     .option("slowdown", { type: "number", desc: "Delay all input data values by this amount of time, pretending like we didn't even receive it until this time is up." })
     .option("network", { type: "array", desc: `The networks this node is on (pass multiple arguments to be on multiple, ex: --network test --network default). Authorities only satisfy paths on their networks ("default" if unset). Function calls are put on the first network in the list. If no FunctionRunner is on a call's network, the call will fail to run.` })
     .option("networkfile", { type: "string", desc: `The same as --network, except the networks are read from the given file (one per line). Supports "~/" for the home directory. If the file doesn't exist, the process exits with an error.` })
+    .option("port", { type: "number", desc: "The storage port for `yarn storageserve`" })
     .argv
     ;
 
@@ -51,6 +52,10 @@ export function expandHomePath(path: string): string {
     return path;
 }
 
+export function getPort(): number | undefined {
+    return yargObj.port;
+}
+
 let networkFileNetworks = lazy((): string[] => {
     if (!yargObj.networkfile) return [];
     let path = expandHomePath(String(yargObj.networkfile));

package/src/diagnostics/managementPages.tsx

@@ -128,12 +128,6 @@ export async function registerManagementPages2(config: {
         controllerName: "SnapshotViewerController",
         getModule: () => import("./misc-pages/SnapshotViewer"),
     });
-    inputPages.push({
-        title: "DNS",
-        componentName: "DNSPage",
-        controllerName: "DNSPageController",
-        getModule: () => import("./misc-pages/DNSPage"),
-    });
     inputPages.push({
         title: "Fnc Capture",
         componentName: "FunctionCapturePage",

package/src/diagnostics/misc-pages/RoutingTablePage.tsx

@@ -36,7 +36,7 @@ const MACHINE_LATENCY_WIDTH_PX = 235;
 
 // Clicking a machine node in the graph (or a row) sorts that machine's nodes to the top of the table.
 const selectedMachineParam = new URLParam("rtSelectedMachine", "", { reset: [mainResets] });
-const trafficWindowParam = new URLParam<number>("rtTrafficWindow", timeInMinute * 60, { reset: [mainResets] });
+const trafficWindowParam = new URLParam<number>("rtTrafficWindow", timeInMinute * 5, { reset: [mainResets] });
 
 // The per-node table columns, in order. `color` tints the function/path-value/querysub columns to match the graph;
 // `perMachine` columns (the IP and machine id) only print on the first row of each machine group.
@@ -638,7 +638,7 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
             return {
                 id: machineId,
                 labelLines: machineLabelLines({ machineId, threads: allThreads, trafficMaps, ip: machineIp.get(machineId), networks: machineNetworks.get(machineId) || [] }),
-                weight: totals.dataSent + totals.dataReceived,
+                weight: totals.valuesSent + totals.valuesReceived,
             };
         });
         let links: LatencyGraphLink[] = [];
@@ -649,7 +649,7 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
             if (pv && (pv.fwd || pv.back)) {
                 extraLabel = { text: `↑${formatNumber(pv.fwd)}/s ↓${formatNumber(pv.back)}/s values`, color: PATHVALUE_COLOR };
             }
-            links.push({ source, destination, latencyMs, weight: machinePairTraffic.get(key) || 0, extraLabel });
+            links.push({ source, destination, latencyMs, weight: machinePairTraffic.get(key) || 0, valueWeight: pv && pv.fwd + pv.back || 0, extraLabel });
         }
 
         // One row per thread node, grouped so a machine's threads sit together; the selected machine sorts to the top.

package/src/library-components/EdgeNodeSelector.tsx

@@ -1,7 +1,7 @@
 import { css } from "typesafecss";
 import { qreact } from "../4-dom/qreact";
 import { Querysub } from "../4-querysub/Querysub";
-import { timeInSecond } from "socket-function/src/misc";
+import { sort, timeInSecond } from "socket-function/src/misc";
 import { formatTime } from "socket-function/src/formatting/format";
 import { isCurrentUserSuperUser } from "../user-implementation/userData";
 import type { EdgeNodeConfig, EdgeNodeStat } from "../4-deploy/edgeNodes";
@@ -43,6 +43,7 @@ export class EdgeNodeSelector extends qreact.Component<{}> {
             nodes = [...nodes, { host: booted.host, nodeId: booted.nodeId, public: booted.public, live: true, finished: false }];
         }
         let autoStat = nodes.find(x => x.host === stats?.autoPickedHost);
+        sort(nodes, x => x.latency);
 
         return <div className={css.hbox(6)}>
             <span className={css.opacity(0.7)}>Edge</span>

package/src/library-components/LatencyGraph.tsx

@@ -9,7 +9,7 @@ import { formatTime } from "socket-function/src/formatting/format";
 
 export type LatencyGraphLabelLine = { text: string; color?: string; };
 export type LatencyGraphNode = { id: string; label?: string; latitude?: number; longitude?: number; labelLines?: LatencyGraphLabelLine[]; weight?: number; };
-export type LatencyGraphLink = { source: string; destination: string; latencyMs: number; weight?: number; extraLabel?: LatencyGraphLabelLine; };
+export type LatencyGraphLink = { source: string; destination: string; latencyMs: number; weight?: number; valueWeight?: number; extraLabel?: LatencyGraphLabelLine; };
 export type LatencyGraphProps = {
     nodes: LatencyGraphNode[];
     links: LatencyGraphLink[];
@@ -59,7 +59,7 @@ const HIGHLIGHT_COLOR = "hsl(40, 90%, 60%)";
 const NODE_WEIGHT_MULT = 2.5;
 const LINE_BASE_WIDTH = 1.5;
 const LINE_MIN_WIDTH = 1;
-const LINE_MAX_WIDTH = 7;
+const LINE_MAX_WIDTH = 35;
 const LABEL_LINE_HEIGHT = 13;
 const ZOOM_STEP = 1.1;
 const MIN_SCALE = 0.05;
@@ -84,6 +84,8 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
     maxNodeWeight = 0;
     pairWeight = new Map<number, number>();
     maxPairWeight = 0;
+    pairValueWeight = new Map<number, number>();
+    maxPairValueWeight = 0;
     pairExtraLabel = new Map<number, LatencyGraphLabelLine>();
     // Cached from props in render() so draw() (a rAF callback, not synced) never reads this.props.
     formatWeight: ((weight: number) => string) | undefined = undefined;
@@ -156,15 +158,23 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
         }
         this.pairWeight = new Map();
         this.maxPairWeight = 0;
+        this.pairValueWeight = new Map();
+        this.maxPairValueWeight = 0;
         for (let link of this.props.links) {
-            if (!link.weight) continue;
             let a = index.get(link.source);
             let b = index.get(link.destination);
             if (a === undefined || b === undefined || a === b) continue;
             let pk = Math.min(a, b) * n + Math.max(a, b);
-            let combined = (this.pairWeight.get(pk) || 0) + link.weight;
-            this.pairWeight.set(pk, combined);
-            this.maxPairWeight = Math.max(this.maxPairWeight, combined);
+            if (link.weight) {
+                let combined = (this.pairWeight.get(pk) || 0) + link.weight;
+                this.pairWeight.set(pk, combined);
+                this.maxPairWeight = Math.max(this.maxPairWeight, combined);
+            }
+            if (link.valueWeight) {
+                let combined = (this.pairValueWeight.get(pk) || 0) + link.valueWeight;
+                this.pairValueWeight.set(pk, combined);
+                this.maxPairValueWeight = Math.max(this.maxPairValueWeight, combined);
+            }
         }
     }
 
@@ -554,10 +564,19 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
         this.drawHover(ctx, toScreenX, toScreenY);
     }
 
+    // Opacity encodes the values sent back and forth on the connection (normalized to the busiest pair). Latency-based opacity is only the fallback when no link has value data.
+    edgeOpacity(a: number, b: number, latency: number) {
+        if (this.maxPairValueWeight <= 0) {
+            return this.opacityFor(latency, this.renderMin, this.renderMax);
+        }
+        let valueWeight = this.pairValueWeight.get(Math.min(a, b) * this.nodes.length + Math.max(a, b)) || 0;
+        return Math.max(MIN_OPACITY, valueWeight / this.maxPairValueWeight);
+    }
+
     drawEdges(ctx: CanvasRenderingContext2D, toScreenX: (wx: number) => number, toScreenY: (wy: number) => number) {
         for (let edge of this.renderEdges) {
             ctx.lineWidth = this.lineWidthFor(edge.a, edge.b);
-            ctx.strokeStyle = `hsla(${NODE_HUE}, 70%, 62%, ${this.opacityFor(edge.latency, this.renderMin, this.renderMax)})`;
+            ctx.strokeStyle = `hsla(${NODE_HUE}, 70%, 62%, ${this.edgeOpacity(edge.a, edge.b, edge.latency)})`;
             ctx.beginPath();
             ctx.moveTo(toScreenX(this.positionsX[edge.a]), toScreenY(this.positionsY[edge.a]));
             ctx.lineTo(toScreenX(this.positionsX[edge.b]), toScreenY(this.positionsY[edge.b]));
@@ -836,7 +855,7 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
             weightSig += node.weight || 0;
         }
         for (let link of this.props.links) {
-            weightSig += link.weight || 0;
+            weightSig += (link.weight || 0) + (link.valueWeight || 0);
         }
         if (sig !== this.builtSig || geoParam.value !== this.builtGeo) {
             // Switching layout mode (geographic vs solved) changes the whole coordinate space, so re-fit the view.

package/src/storageSetup.ts

@@ -0,0 +1,27 @@
+import "./inject";
+import { getDomain, getPort } from "./config";
+import { getTrustedMachineIds } from "./-d-trust/NetworkTrust2";
+import { getExternalIP } from "socket-function/src/networking";
+
+// The storage server entry point (see bin/storageserve.js): sets up the CLI args the sliftutils storage server expects, synchronizes our network trust list into its trust store, and runs its CLI.
+
+async function main() {
+    if (!process.argv.includes("--url")) {
+        let ip = await getExternalIP();
+        let ipDomain = ip.replaceAll(".", "-");
+        process.argv.push("--url", `https://${ipDomain}.${getDomain()}:${getPort()}`);
+    }
+
+    let trustedMachineIds = await getTrustedMachineIds();
+
+    let { setTrustedMachines } = await import("sliftutils/storage/remoteStorage/storageServerState");
+    await setTrustedMachines({ account: "root", machineIds: trustedMachineIds });
+    console.log(`Synchronized ${trustedMachineIds.length} trusted machines`);
+
+    await import("sliftutils/storage/remoteStorage/storageServerCli");
+}
+
+main().catch(e => {
+    console.error(e.stack || e);
+    process.exit(1);
+});

package/test.ts

@@ -1,11 +1,11 @@
 import { chdir } from "process";
-import { getRecordsRaw, setRecord } from "./src/-b-authorities/dnsAuthority";
+import { getRecords, setRecord } from "./src/-b-authorities/dnsAuthority";
 import { timeInMinute } from "socket-function/src/misc";
 chdir("D:/repos/qs-cyoa/");
 
 
 async function main() {
-    let rawRecords = await getRecordsRaw("A", "test.querysubtest.com");
+    let rawRecords = await getRecords("A", "test.querysubtest.com");
     for (let record of rawRecords) {
         console.log(record);
     }

package/src/diagnostics/misc-pages/DNSPage.tsx

@@ -1,354 +0,0 @@
-module.allowclient = true;
-
-import { qreact } from "../../4-dom/qreact";
-import { css } from "typesafecss";
-import { SocketFunction } from "socket-function/SocketFunction";
-import { getBrowserUrlNode } from "../../-f-node-discovery/NodeDiscovery";
-import { getSyncedController } from "../../library-components/SyncedController";
-import { assertIsManagementUser } from "../managementPages";
-import { t } from "../../2-proxy/schema2";
-import { Querysub } from "../../4-querysub/Querysub";
-import { sort, timeInDay } from "socket-function/src/misc";
-import { isNode } from "typesafecss";
-import { formatDateJSX } from "../../misc/formatJSX";
-
-const RECORDS_PER_PAGE = 1000;
-const HEX_SUBDOMAIN_LENGTH = 17;
-const STALE_HEX_SUBDOMAIN_AGE = timeInDay * 7;
-const HEX_SUBDOMAIN_REGEX = new RegExp(`^(\\*\\.)?[0-9a-f]{${HEX_SUBDOMAIN_LENGTH}}\\..+$`);
-
-export type DNSRecord = {
-    id: string;
-    zoneId: string;
-    zoneName: string;
-    type: string;
-    name: string;
-    content: string;
-    proxied: boolean;
-    ttl: number;
-    createdOn: number;
-    modifiedOn: number;
-    // Time we last asserted this record, parsed from our freshness comment tag, falling back to
-    //  Cloudflare's modified_on when the tag is absent. This is the time used for staleness.
-    setOn: number;
-};
-
-class DNSPageControllerBase {
-    public async getAllRecords(): Promise<DNSRecord[]> {
-        if (!isNode()) throw new Error(`getAllRecords must be called serverside`);
-        const { cloudflareGETCall } = await import("../../-b-authorities/cloudflareHelpers");
-        const { freshnessTime } = await import("../../-b-authorities/dnsAuthority");
-        const { getDomain } = await import("../../config");
-
-        let zones = await cloudflareGETCall<{ id: string; name: string }[]>("/zones", {
-            per_page: String(RECORDS_PER_PAGE),
-        });
-        let domain = getDomain();
-        zones = zones.filter(z => z.name === domain);
-
-        let allRecords: DNSRecord[] = [];
-        for (let zone of zones) {
-            let page = 1;
-            while (true) {
-                let records = await cloudflareGETCall<{
-                    id: string;
-                    type: string;
-                    name: string;
-                    content: string;
-                    proxied: boolean;
-                    ttl: number;
-                    created_on: string;
-                    modified_on: string;
-                    comment?: string;
-                }[]>(`/zones/${zone.id}/dns_records`, {
-                    per_page: String(RECORDS_PER_PAGE),
-                    page: String(page),
-                });
-                for (let record of records) {
-                    let modifiedOn = new Date(record.modified_on).getTime();
-                    allRecords.push({
-                        id: record.id,
-                        zoneId: zone.id,
-                        zoneName: zone.name,
-                        type: record.type,
-                        name: record.name,
-                        content: record.content,
-                        proxied: record.proxied,
-                        ttl: record.ttl,
-                        createdOn: new Date(record.created_on).getTime(),
-                        modifiedOn,
-                        setOn: freshnessTime(record.comment) || modifiedOn,
-                    });
-                }
-                if (records.length < RECORDS_PER_PAGE) break;
-                page++;
-            }
-        }
-        return allRecords;
-    }
-
-    public async deleteRecord(config: { zoneId: string; recordId: string }): Promise<void> {
-        if (!isNode()) throw new Error(`deleteRecord must be called serverside`);
-        const { cloudflareCall } = await import("../../-b-authorities/cloudflareHelpers");
-        await cloudflareCall(`/zones/${config.zoneId}/dns_records/${config.recordId}`, Buffer.from([]), "DELETE");
-    }
-
-    public async deleteRecordsByContent(config: { content: string }): Promise<number> {
-        if (!isNode()) throw new Error(`deleteRecordsByContent must be called serverside`);
-        const { cloudflareCall } = await import("../../-b-authorities/cloudflareHelpers");
-        let all = await this.getAllRecords();
-        let matching = all.filter(x => x.content === config.content);
-        for (let record of matching) {
-            await cloudflareCall(`/zones/${record.zoneId}/dns_records/${record.id}`, Buffer.from([]), "DELETE");
-        }
-        return matching.length;
-    }
-}
-
-export const DNSPageController = SocketFunction.register(
-    "DNSPageController-3e7b1d92-44a8-4f7e-9bba-d12c4f87a6e0",
-    new DNSPageControllerBase(),
-    () => ({
-        getAllRecords: {},
-        deleteRecord: {},
-        deleteRecordsByContent: {},
-    }),
-    () => ({
-        hooks: [assertIsManagementUser],
-    }),
-    {
-        noAutoExpose: true,
-    }
-);
-
-const DNSPageSynced = getSyncedController(DNSPageController, {
-    reads: {
-        getAllRecords: ["dnsRecords"],
-    },
-    writes: {
-        deleteRecord: ["dnsRecords"],
-        deleteRecordsByContent: ["dnsRecords"],
-    },
-});
-
-export class DNSPage extends qreact.Component {
-    state = t.state({
-        expandedGroups: t.lookup(t.boolean),
-        busyKeys: t.lookup(t.boolean),
-        errorMessage: t.string,
-    });
-
-    private async deleteOne(record: DNSRecord) {
-        let typed = prompt(`Type the content "${record.content}" to confirm deleting this record:\n\n${record.type} ${record.name} → ${record.content}`);
-        if (typed === null) return;
-        if (typed !== record.content) {
-            alert(`Confirmation does not match. Aborted.`);
-            return;
-        }
-        let busyKey = `record:${record.id}`;
-        Querysub.commit(() => {
-            this.state.busyKeys[busyKey] = true;
-            this.state.errorMessage = "";
-        });
-        try {
-            await DNSPageSynced(getBrowserUrlNode()).deleteRecord.promise({
-                zoneId: record.zoneId,
-                recordId: record.id,
-            });
-        } catch (err) {
-            console.error(`DNS deleteRecord failed:`, (err as Error).stack ?? err);
-            Querysub.commit(() => {
-                this.state.errorMessage = (err as Error).stack ?? String(err);
-            });
-        } finally {
-            Querysub.commit(() => {
-                delete this.state.busyKeys[busyKey];
-            });
-        }
-    }
-
-    private async deleteStaleHexRecords(content: string, matching: DNSRecord[]) {
-        let listText = matching.map(r => `  ${r.type} ${r.name}  (set ${new Date(r.setOn).toISOString()})`).join("\n");
-        let typed = prompt(`Type "${content}" to confirm deleting ${matching.length} stale hex-subdomain record(s):\n\n${listText}`);
-        if (typed === null) return;
-        if (typed !== content) {
-            alert(`Confirmation does not match. Aborted.`);
-            return;
-        }
-        let busyKey = `staleHex:${content}`;
-        Querysub.commit(() => {
-            this.state.busyKeys[busyKey] = true;
-            this.state.errorMessage = "";
-        });
-        try {
-            for (let record of matching) {
-                await DNSPageSynced(getBrowserUrlNode()).deleteRecord.promise({
-                    zoneId: record.zoneId,
-                    recordId: record.id,
-                });
-            }
-        } catch (err) {
-            console.error(`DNS deleteStaleHexRecords failed:`, (err as Error).stack ?? err);
-            Querysub.commit(() => {
-                this.state.errorMessage = (err as Error).stack ?? String(err);
-            });
-        } finally {
-            Querysub.commit(() => {
-                delete this.state.busyKeys[busyKey];
-            });
-        }
-    }
-
-    private async deleteGroup(content: string, count: number) {
-        let typed = prompt(`Type "${content}" to confirm deleting ALL ${count} record(s) pointing to it:`);
-        if (typed === null) return;
-        if (typed !== content) {
-            alert(`Confirmation does not match. Aborted.`);
-            return;
-        }
-        let busyKey = `group:${content}`;
-        Querysub.commit(() => {
-            this.state.busyKeys[busyKey] = true;
-            this.state.errorMessage = "";
-        });
-        try {
-            await DNSPageSynced(getBrowserUrlNode()).deleteRecordsByContent.promise({ content });
-        } catch (err) {
-            console.error(`DNS deleteRecordsByContent failed:`, (err as Error).stack ?? err);
-            Querysub.commit(() => {
-                this.state.errorMessage = (err as Error).stack ?? String(err);
-            });
-        } finally {
-            Querysub.commit(() => {
-                delete this.state.busyKeys[busyKey];
-            });
-        }
-    }
-
-    render() {
-        let records = DNSPageSynced(getBrowserUrlNode()).getAllRecords();
-        if (!records) {
-            return <div className={css.pad2(16)}>Loading DNS records...</div>;
-        }
-
-        let groups = new Map<string, DNSRecord[]>();
-        for (let record of records) {
-            let list = groups.get(record.content);
-            if (!list) {
-                list = [];
-                groups.set(record.content, list);
-            }
-            list.push(record);
-        }
-
-        let groupEntries = Array.from(groups.entries()).map(([content, list]) => ({ content, records: list }));
-        sort(groupEntries, x => -x.records.length);
-        for (let group of groupEntries) {
-            sort(group.records, r => -r.setOn);
-        }
-
-        return <div className={css.vbox(12).pad2(16).fillWidth}>
-            <div className={css.hbox(12).alignItems("center")}>
-                <h2 className={css.flexGrow(1)}>DNS Records ({records.length} total, {groupEntries.length} unique contents)</h2>
-            </div>
-            {this.state.errorMessage && <pre className={css.colorhsl(0, 60, 40).whiteSpace("pre-wrap").pad2(8).bord2(0, 60, 60).hsl(0, 50, 95)}>{this.state.errorMessage}</pre>}
-            <div className={css.vbox(8).fillWidth}>
-                {groupEntries.map(({ content, records: list }) => {
-                    let expanded = !!this.state.expandedGroups[content];
-                    let groupBusy = !!this.state.busyKeys[`group:${content}`];
-                    return <div key={content} className={css.vbox(0).fillWidth.bord2(0, 0, 80).hsl(0, 0, 99)}>
-                        <div className={css.hbox(10).alignItems("center").pad2(10).button}
-                            onClick={() => {
-                                if (this.state.expandedGroups[content]) {
-                                    delete this.state.expandedGroups[content];
-                                } else {
-                                    this.state.expandedGroups[content] = true;
-                                }
-                            }}
-                        >
-                            <span>{expanded ? "▼" : "▶"}</span>
-                            <span
-                                className={css.boldStyle.fontFamily("monospace").button.pad2(4, 2).bord2(0, 0, 80).hsl(0, 0, 100).hbox(6).alignItems("center")}
-                                title="Click to copy"
-                                onClick={(e) => {
-                                    e.stopPropagation();
-                                    void navigator.clipboard.writeText(content);
-                                }}
-                            >
-                                <span>{content}</span>
-                                <span className={css.colorhsl(0, 0, 50)}>📋</span>
-                            </span>
-                            <span className={css.colorhsl(0, 0, 40)}>{list.length} record(s)</span>
-                            <div className={css.flexGrow(1)} />
-                            <button
-                                className={css.pad2(10, 6).button.bord2(0, 80, 50)
-                                    + (groupBusy ? css.hsl(0, 0, 90).colorhsl(0, 0, 50) : css.hsl(0, 80, 92).colorhsl(0, 80, 30))}
-                                disabled={groupBusy}
-                                onClick={(e) => {
-                                    e.stopPropagation();
-                                    void this.deleteGroup(content, list.length);
-                                }}
-                            >
-                                {groupBusy ? "Deleting..." : `🗑️ Delete all ${list.length}`}
-                            </button>
-                        </div>
-                        {expanded && <div className={css.vbox(4).pad2(10).hsl(0, 0, 100)}>
-                            {(() => {
-                                let staleHex = list.filter(r =>
-                                    HEX_SUBDOMAIN_REGEX.test(r.name)
-                                    && Date.now() - r.setOn > STALE_HEX_SUBDOMAIN_AGE
-                                );
-                                if (staleHex.length === 0) return undefined;
-                                let staleHexBusy = !!this.state.busyKeys[`staleHex:${content}`];
-                                return <div className={css.vbox(4).pad2(8).bord2(30, 60, 60).hsl(30, 70, 96)}>
-                                    <div className={css.hbox(10).alignItems("center")}>
-                                        <span className={css.boldStyle}>Stale hex-subdomain records (set &gt;7d ago): {staleHex.length}</span>
-                                        <div className={css.flexGrow(1)} />
-                                        <button
-                                            className={css.pad2(10, 6).button.bord2(0, 80, 50)
-                                                + (staleHexBusy ? css.hsl(0, 0, 90).colorhsl(0, 0, 50) : css.hsl(0, 80, 92).colorhsl(0, 80, 30))}
-                                            disabled={staleHexBusy}
-                                            onClick={() => void this.deleteStaleHexRecords(content, staleHex)}
-                                        >
-                                            {staleHexBusy ? "Deleting..." : `🗑️ Delete ${staleHex.length} stale hex record(s)`}
-                                        </button>
-                                    </div>
-                                    <div className={css.vbox(2).fontFamily("monospace").colorhsl(0, 0, 30)}>
-                                        {staleHex.map(r => <div key={r.id}>{r.type} {r.name} — set {formatDateJSX(r.setOn)}</div>)}
-                                    </div>
-                                </div>;
-                            })()}
-                            {list.map(record => {
-                                let recordBusy = !!this.state.busyKeys[`record:${record.id}`];
-                                return <div key={record.id} className={css.hbox(10).alignItems("center").pad2(6).bord2(0, 0, 92)}>
-                                    <span className={css.boldStyle.minWidth(50)}>{record.type}</span>
-                                    <span className={css.fontFamily("monospace").flexGrow(1)}>{record.name}</span>
-                                    {record.proxied && <span className={css.colorhsl(30, 80, 40).pad2(4, 2).bord2(30, 80, 70).hsl(30, 80, 95)}>proxied</span>}
-                                    <span className={css.colorhsl(0, 0, 50)}>ttl {record.ttl}</span>
-                                    <span className={css.colorhsl(0, 0, 50)}>
-                                        created {formatDateJSX(record.createdOn)}
-                                    </span>
-                                    <span className={css.colorhsl(0, 0, 50)}>
-                                        modified {formatDateJSX(record.modifiedOn)}
-                                    </span>
-                                    <span className={css.colorhsl(0, 0, 50)}>
-                                        set {formatDateJSX(record.setOn)}
-                                    </span>
-                                    <span className={css.colorhsl(0, 0, 50)}>{record.zoneName}</span>
-                                    <button
-                                        className={css.pad2(8, 4).button.bord2(0, 0, 60)
-                                            + (recordBusy ? css.hsl(0, 0, 90).colorhsl(0, 0, 50) : css.hsl(0, 0, 100))}
-                                        disabled={recordBusy}
-                                        onClick={() => void this.deleteOne(record)}
-                                    >
-                                        {recordBusy ? "Deleting..." : "🗑️ Delete"}
-                                    </button>
-                                </div>;
-                            })}
-                        </div>}
-                    </div>;
-                })}
-            </div>
-        </div>;
-    }
-}