querysub 0.588.0 → 0.590.0

package/src/0-path-value-core/archiveLocks/archiveSnapshots.ts

@@ -9,11 +9,12 @@ import { sort } from "socket-function/src/misc";
 import { lazy } from "socket-function/src/caching";
 import { decodeCborx, encodeCborx } from "../../misc/cloneHelpers";
 import { runInParallel } from "socket-function/src/batching";
-import { formatNumber } from "socket-function/src/formatting/format";
+import { formatNumber, formatTime } from "socket-function/src/formatting/format";
+import { getArchives2 } from "../../-a-archives/archives2";
 
 
 // Snapshot is just a newline delimitted list of files. The name contains the metadata and time of it.
-const snapshots = lazy(() => getArchives("snapshots"));
+const snapshots = lazy(() => getArchives2("snapshots"));
 
 export type ArchiveSnapshotOverview = {
     // If live, it isn't a real file that can be loaded
@@ -89,23 +90,26 @@ function fileNameToOverview(fileName: string): ArchiveSnapshotOverview {
     };
 }
 
-// pathValueArchives.decodeDataPath
-export async function getSnapshotList(): Promise<ArchiveSnapshotOverview[]> {
+export type ArchiveSnapshotList = {
+    loadTime: number;
+    snapshots: ArchiveSnapshotOverview[];
+};
+
+export async function getSavedSnapshotList(): Promise<ArchiveSnapshotList> {
+    let startTime = Date.now();
     let snapshotFiles = await snapshots().find("");
+    let overview = snapshotFiles.map(file => fileNameToOverview(file));
+    sort(overview, x => -x.time);
+    return { loadTime: Date.now() - startTime, snapshots: overview };
+}
 
-    let overview: ArchiveSnapshotOverview[] = [];
+export async function getLiveSnapshotList(): Promise<ArchiveSnapshotList> {
+    let startTime = Date.now();
     let locker = await pathValueArchives.getArchiveLocker();
     let allFiles = await locker.getAllValidFiles();
-
     let liveOverview = getSnapshotOverview(allFiles.map(x => x.file));
     liveOverview.file = "live";
-    overview.push(liveOverview);
-
-    overview.push(...snapshotFiles.map(file => fileNameToOverview(file)));
-
-    sort(overview, x => -x.time);
-
-    return overview;
+    return { loadTime: Date.now() - startTime, snapshots: [liveOverview] };
 }
 
 export async function getSnapshot(snapshotFile: string | "live"): Promise<ArchiveSnapshotRead> {

package/src/0-path-value-core/pathValueArchives.ts

@@ -1,7 +1,7 @@
 import { blue, green, magenta, red, yellow } from "socket-function/src/formatting/logColors";
 import { measureFnc } from "socket-function/src/profiling/measure";
 import { ARCHIVE_FLUSH_LIMIT, ARCHIVE_FLUSH_LIMIT_START, compareTime, debugPathValuePath, getCompressNetwork, isCoreQuiet, PathValue, PathValueSnapshot, registerGetCompressNetwork, Time } from "./pathValueCore";
-import { getArchives, nestArchives } from "../-a-archives/archives";
+import { getArchives2 } from "../-a-archives/archives2";
 import { cache, cacheLimited, lazy } from "socket-function/src/caching";
 import { getOwnNodeId, getOwnNodeIdAssert } from "../-f-node-discovery/NodeDiscovery";
 import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
@@ -11,7 +11,7 @@ import { list } from "socket-function/src/misc";
 import { NodeIdParts, decodeNodeId, decodeNodeIdAssert, encodeNodeId } from "sliftutils/misc/https/certs";
 import { createArchiveLocker2 } from "./archiveLocks/ArchiveLocks2";
 import { devDebugbreak, isNoNetwork, getDomain } from "../config";
-import { wrapArchivesWithCache } from "../-a-archives/archiveCache";
+import { wrapArchivesWithCache2 } from "../-a-archives/archiveCache2";
 import { AuthoritySpec, PathRouter, debugSpec } from "./PathRouter";
 import { authorityLookup } from "./AuthorityLookup";
 import { delay, retryFunctional } from "socket-function/src/batching";
@@ -21,10 +21,10 @@ import { shutdown } from "../diagnostics/periodic";
 
 // Kept separate from the cache-wrapped `archives` so we can probe the underlying archives
 //  directly (without the cache layer in the way) when diagnosing missing-file failures.
-const archivesBase = lazy(() => getArchives("path-values/"));
-export const archives = lazy(() => wrapArchivesWithCache(archivesBase()));
-export const archivesLocks = lazy(() => getArchives("path-values-locks/"));
-export const archivesRecycleBin = lazy(() => wrapArchivesWithCache(getArchives("path-values-recycle-bin/")));
+const archivesBase = lazy(() => getArchives2("path-values/"));
+export const archives = lazy(() => wrapArchivesWithCache2(archivesBase()));
+export const archivesLocks = lazy(() => getArchives2("path-values-locks/"));
+export const archivesRecycleBin = lazy(() => wrapArchivesWithCache2(getArchives2("path-values-recycle-bin/")));
 
 // If getInfo reports a just-written file as missing, recheck a few times before treating it
 //  as a fatal "written too slowly" condition.
@@ -314,7 +314,7 @@ export class PathValueArchives {
                 while (pendingDataPaths.length > 0) {
                     let dataPath = pendingDataPaths.pop()!;
                     if (readCache.has(dataPath)) continue;
-                    let data = await archives().get(dataPath, { fastRead: true });
+                    let data = await archives().get(dataPath);
                     if (!data) continue;
                     readCache.set(dataPath, data);
                 }
@@ -469,7 +469,7 @@ export class PathValueArchives {
         const BATCH_SIZE = 16;
         for (let i = 0; i < dataPath.length; i += BATCH_SIZE) {
             let cur = dataPath.slice(i, i + BATCH_SIZE);
-            let curResults = await Promise.all(cur.map(x => archives().get(x, { fastRead: true })));
+            let curResults = await Promise.all(cur.map(x => archives().get(x)));
             results.push(...curResults);
         }
         if (config?.includeRecycleBin) {
@@ -483,7 +483,7 @@ export class PathValueArchives {
             for (let i = 0; i < notFoundList.length; i += BATCH_SIZE) {
                 let curIndexes = notFoundList.slice(i, i + BATCH_SIZE);
                 let cur = curIndexes.map(x => dataPath[x]);
-                let curResults = await Promise.all(cur.map(x => archivesRecycleBin().get(x, { fastRead: true })));
+                let curResults = await Promise.all(cur.map(x => archivesRecycleBin().get(x)));
                 for (let j = 0; j < curResults.length; j++) {
                     results[curIndexes[j]] = curResults[j];
                 }

package/src/2-proxy/PathValueProxyWatcher.ts

@@ -664,7 +664,7 @@ export class PathValueProxyWatcher {
         return pathValue;
     };
     public getCallback = (pathStr: string, syncParentKeys?: "parentKeys", readTransparent?: "readTransparent"): { value: unknown } | undefined => {
-        if (PathValueProxyWatcher.BREAK_ON_READS.size > 0 && (proxyWatcher.isAllSynced() || this)) {
+        if (PathValueProxyWatcher.BREAK_ON_READS.size > 0 && (proxyWatcher.isAllSynced({ checkCommitAllRunsFlag: true }) || this)) {
             // NOTE: We can't do a recursive match, as the parent paths include the 
             //  root, which is constantly read, but not relevant.
             if (PathValueProxyWatcher.BREAK_ON_READS.has(pathStr)) {
@@ -750,7 +750,7 @@ export class PathValueProxyWatcher {
     };
 
     public setCallback = (pathStr: string, value: unknown, inRecursion = false, allowSpecial = false): void => {
-        if (PathValueProxyWatcher.BREAK_ON_WRITES.size > 0 && proxyWatcher.isAllSynced()) {
+        if (PathValueProxyWatcher.BREAK_ON_WRITES.size > 0 && proxyWatcher.isAllSynced({ checkCommitAllRunsFlag: true })) {
             if (isRecursiveMatch(PathValueProxyWatcher.BREAK_ON_WRITES, pathStr)) {
                 let unwatch = () => removeMatches(PathValueProxyWatcher.BREAK_ON_WRITES, pathStr);
                 debugger;
@@ -758,7 +758,7 @@ export class PathValueProxyWatcher {
             }
         }
 
-        if (PathValueProxyWatcher.SET_FUNCTION_WATCH_ON_WRITES.size > 0 && proxyWatcher.isAllSynced()) {
+        if (PathValueProxyWatcher.SET_FUNCTION_WATCH_ON_WRITES.size > 0 && proxyWatcher.isAllSynced({ checkCommitAllRunsFlag: true })) {
             if (isRecursiveMatch(PathValueProxyWatcher.SET_FUNCTION_WATCH_ON_WRITES, pathStr)) {
                 let unwatch = () => removeMatches(PathValueProxyWatcher.SET_FUNCTION_WATCH_ON_WRITES, pathStr);
                 PathValueProxyWatcher.BREAK_ON_CALL.add(
@@ -767,7 +767,7 @@ export class PathValueProxyWatcher {
             }
         }
 
-        if (PathValueProxyWatcher.LOG_WRITES_INCLUDES.size > 0 && proxyWatcher.isAllSynced()) {
+        if (PathValueProxyWatcher.LOG_WRITES_INCLUDES.size > 0 && proxyWatcher.isAllSynced({ checkCommitAllRunsFlag: true })) {
             if (isRecursiveMatch(PathValueProxyWatcher.LOG_WRITES_INCLUDES, pathStr)) {
                 let unwatch = () => removeMatches(PathValueProxyWatcher.LOG_WRITES_INCLUDES, pathStr);
                 console.log(`Write path "${pathStr}" = ${value}`);
@@ -2138,9 +2138,9 @@ export class PathValueProxyWatcher {
     }
 
     public isAllSynced(config?: {
-        ignoreAlwaysCommitAllRunsFlag?: boolean;
+        checkCommitAllRunsFlag?: boolean;
     }) {
-        if (!config?.ignoreAlwaysCommitAllRunsFlag && this.runningWatcher?.options.commitAllRuns) {
+        if (config?.checkCommitAllRunsFlag && this.runningWatcher?.options.commitAllRuns) {
             return true;
         }
         return !this.getTriggeredWatcherMaybeUndefined()?.hasAnyUnsyncedAccesses();

package/src/4-dom/qreact.tsx

@@ -744,7 +744,7 @@ class QRenderClass {
                     } else {
                         // We check here, and not in the caller, so the props are always up to date,
                         //  as the objects might be mutated or special (or have callback functions).
-                        if (statics.jsonComparePropUpdates && Querysub.isAllSynced()) {
+                        if (statics.jsonComparePropUpdates && Querysub.isAllSynced({ checkCommitAllRunsFlag: true })) {
 
                             let props = self.data().props;
 
@@ -809,7 +809,7 @@ class QRenderClass {
                 //  ALSO, this stops infinite loops caused by self triggering, which is really useful.
                 if (!QRenderClass.areVNodesEqual(comparePrevVNode, vNode)) {
                     self.data().vNodeForRender = frozen;
-                    if (Querysub.isAllSynced()) {
+                    if (Querysub.isAllSynced({ checkCommitAllRunsFlag: true })) {
                         comparePrevVNode = frozen.value;
                     }
                 } else {
@@ -1886,7 +1886,7 @@ class QRenderClass {
 export const __INTERNAL__QRenderClass = QRenderClass;
 
 let defaultErrorHandler: ErrorHandler = ({ error, debugName }) => {
-    if (Querysub.isAllSynced()) {
+    if (Querysub.isAllSynced({ checkCommitAllRunsFlag: true })) {
         console.error(`Render error in ${debugName}`, error);
         // Throw, so we get a good callstack
         setImmediate(() => { throw error; });
@@ -2673,7 +2673,7 @@ function watchUnsyncedComponents(): Set<ExternalRenderClass> {
     return unsyncedComponents;
 }
 function componentRendered(component: QRenderClass) {
-    if (component.disposed || Querysub.isAllSynced({ ignoreAlwaysCommitAllRunsFlag: true })) {
+    if (component.disposed || Querysub.isAllSynced()) {
         if (unsyncedComponents.has(component)) {
             unsyncedComponents.delete(component);
             triggerUnsyncedNow();

package/src/4-querysub/Querysub.ts

@@ -422,7 +422,7 @@ export class Querysub {
     public static anyUnsynced() {
         return !Querysub.allSynced();
     }
-    public static allSynced(config?: { ignoreAlwaysCommitAllRunsFlag?: boolean }) {
+    public static allSynced(config?: { checkCommitAllRunsFlag?: boolean }) {
         return proxyWatcher.isAllSynced(config);
     }
     public static fullySynced = Querysub.allSynced;
@@ -528,6 +528,13 @@ export class Querysub {
         });
     }
 
+    public static triggerOnPromiseFinish(promise: MaybePromise<unknown>, config: {
+        waitReason: string;
+        noWait?: boolean;
+    }) {
+        proxyWatcher.triggerOnPromiseFinish(promise, config);
+    }
+
     /** Returns true if any predictions are running. In which case, we will correctly rerun the function and consider it non-synced until the predictions finish. This allows you to check for predictions of the in your function and return if you have any. This is useful for functions that need a stable state before they run. However, if many functions use this and you trigger them at once, it might result in an n-squared situation, so this should be used with caution.
      *  - The best use case is if something is triggering maybe on focus and on blur, and you want to make sure the on blur changes happen before your on focus changes happen.
      */

package/src/deployManager/components/RouteConfigView.tsx

@@ -183,9 +183,18 @@ export function getRouteConfigGroups(servers: StorageServerBuckets[]): RouteConf
     let byBucket = new Map<string, Map<string, ConfigVariant>>();
     for (let server of servers) {
         for (let bucket of server.buckets || []) {
+            // An inactive bucket is not serving anything, so whatever routing it happens to hold says nothing about where values actually come from - and letting it vote would put a stale config up against the live ones as a conflict
+            if (!bucket.active) continue;
             let remoteConfig: RemoteConfig | undefined = bucket.config?.remoteConfig;
             if (!remoteConfig) continue;
-            let sources = remoteConfig.sources.map(normalizeSource);
+            let sources: Source[];
+            try {
+                sources = remoteConfig.sources.map(normalizeSource);
+            } catch (e: any) {
+                // One server's unreadable config must not cost us the grouping for every other bucket - it simply does not get a say in the consensus
+                console.error(`Ignoring the routing config ${server.url} reports for ${bucket.bucketName}, which could not be read:`, e.stack ?? e);
+                continue;
+            }
             let variants = byBucket.get(bucket.bucketName);
             if (!variants) {
                 variants = new Map();

package/src/deployManager/components/ServiceMeasureBars.tsx

@@ -0,0 +1,169 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { qreact } from "../../4-dom/qreact";
+import { css } from "typesafecss";
+import { formatNumber, formatTime, formatVeryNiceDateTime } from "socket-function/src/formatting/format";
+import { timeoutToUndefinedSilent, timeInSecond } from "socket-function/src/misc";
+import { StatsValue, getStatsTop } from "socket-function/src/profiling/stats";
+import type { MeasureChunk, MeasureSummary } from "../../diagnostics/watchdog";
+import { NodeCapabilitiesController } from "../../-g-core-values/NodeCapabilities";
+import { getSyncedController } from "../../library-components/SyncedController";
+import { assertIsManagementUser } from "../../diagnostics/managementPages";
+
+const NODE_CALL_TIMEOUT = timeInSecond * 10;
+const BAR_HEIGHT_PX = 10;
+const MIN_SEGMENT_WIDTH_PX = 30;
+
+class ServiceMeasureControllerBase {
+    public async getMeasureSummaries(nodeIds: string[]): Promise<Record<string, MeasureSummary | undefined>> {
+        let result: Record<string, MeasureSummary | undefined> = {};
+        await Promise.all(nodeIds.map(async nodeId => {
+            result[nodeId] = await timeoutToUndefinedSilent(NODE_CALL_TIMEOUT, NodeCapabilitiesController.nodes[nodeId].getMeasureSummary());
+        }));
+        return result;
+    }
+    public async getMeasureBreakdown(nodeId: string, range: { startTime: number; endTime: number }): Promise<{ chunk?: MeasureChunk }> {
+        let chunk = await timeoutToUndefinedSilent(NODE_CALL_TIMEOUT, NodeCapabilitiesController.nodes[nodeId].getMeasureBreakdown(range));
+        return { chunk };
+    }
+}
+export const ServiceMeasureController = getSyncedController(SocketFunction.register(
+    "ServiceMeasureController-3f8c1de2-5b74-4e0f-a6c9-8d21f47b90a5",
+    new ServiceMeasureControllerBase(),
+    () => ({
+        getMeasureSummaries: { hooks: [assertIsManagementUser], compress: true },
+        getMeasureBreakdown: { hooks: [assertIsManagementUser], compress: true },
+    }),
+    () => ({
+    })
+));
+
+function percent(value: number) {
+    return `${(value * 100).toFixed(2)}%`;
+}
+
+function nameHue(name: string) {
+    let hash = 0;
+    for (let i = 0; i < name.length; i++) {
+        hash = (hash * 31 + name.charCodeAt(i)) | 0;
+    }
+    return Math.abs(hash) % 360;
+}
+
+export class ServiceMeasureBars extends qreact.Component<{
+    serviceNodeIds: string[];
+    /** Every node id on the page, so all rows share one cached call. */
+    allNodeIds: string[];
+    expandedNodeId: string;
+    onToggleNode: (nodeId: string) => void;
+}> {
+    render() {
+        let controller = ServiceMeasureController(SocketFunction.browserNodeId());
+        let summaries = controller.getMeasureSummaries(this.props.allNodeIds);
+        if (!summaries) return undefined;
+
+        return <div className={css.vbox(2)}>
+            {this.props.serviceNodeIds.map(nodeId => {
+                let summary = summaries[nodeId];
+                if (!summary) return undefined;
+                let timeRunFor = summary.endTime - summary.startTime;
+                if (timeRunFor <= 0) return undefined;
+                let isExpanded = this.props.expandedNodeId === nodeId;
+                return <div key={nodeId}
+                    className={
+                        css.hbox(1).wrap.button
+                        + (isExpanded && css.outline("1px solid hsl(210, 60%, 50%)") || "")
+                    }
+                    onClick={() => this.props.onToggleNode(nodeId)}
+                >
+                    {summary.entries.map(entry => {
+                        let fraction = entry.ownTime / timeRunFor;
+                        return <div key={entry.name}
+                            className={css
+                                .height(BAR_HEIGHT_PX)
+                                .width(`${fraction * 100}%`)
+                                .minWidth(MIN_SEGMENT_WIDTH_PX)
+                                .background(`hsl(${nameHue(entry.name)}, 60%, 60%)`)
+                            }
+                            title={`${entry.name}\n${formatTime(entry.ownTime)} (${percent(fraction)} of ${formatTime(timeRunFor)} profiled)\n${nodeId}`}
+                        />;
+                    })}
+                </div>;
+            })}
+        </div>;
+    }
+}
+
+export class ServiceMeasureBreakdown extends qreact.Component<{
+    nodeId: string;
+    allNodeIds: string[];
+}> {
+    render() {
+        let controller = ServiceMeasureController(SocketFunction.browserNodeId());
+        let summaries = controller.getMeasureSummaries(this.props.allNodeIds);
+        let summary = summaries?.[this.props.nodeId];
+        if (!summary) return undefined;
+
+        let result = controller.getMeasureBreakdown(this.props.nodeId, { startTime: summary.startTime, endTime: summary.endTime });
+        if (!result) {
+            return <div className={css.pad2(10).hsl(0, 0, 98).bord2(0, 0, 20)}>Loading breakdown...</div>;
+        }
+        let chunk = result.chunk;
+        if (!chunk) {
+            return <div className={css.pad2(10).hsl(0, 0, 98).bord2(0, 0, 20)}>Breakdown is no longer available on the node (it only keeps the last hour, and restarts clear it).</div>;
+        }
+
+        let timeRunFor = chunk.endTime - chunk.startTime;
+        let cpuFraction = chunk.profiledTime / timeRunFor;
+        return <div className={css.vbox(6).pad2(10).hsl(0, 0, 98).bord2(0, 0, 20).fontFamily("monospace").fontSize(13)}>
+            <div className={css.boldStyle}>
+                Profiled {formatTime(chunk.profiledTime)} ({percent(cpuFraction)} CPU) (profile for {formatTime(timeRunFor)}, ending {formatVeryNiceDateTime(chunk.endTime)})
+            </div>
+            <div className={css.colorhsl(0, 0, 45).fontSize(12)}>{this.props.nodeId}</div>
+            <table className={css.borderCollapse("collapse")}>
+                {chunk.entries.map(entry => this.renderEntryRow(entry, chunk!))}
+            </table>
+        </div>;
+    }
+
+    renderEntryRow(entry: MeasureChunk["entries"][0], chunk: MeasureChunk) {
+        let fraction = entry.ownTime.sum / chunk.profiledTime;
+        let cellPad = css.pad2(6, 1).whiteSpace("pre");
+        return <tr key={entry.name}>
+            <td className={cellPad
+                .colorhsl(210, 70, 35)
+                .background(`linear-gradient(to right, hsla(210, 60%, 55%, 0.25) ${fraction * 100}%, transparent ${fraction * 100}%)`)
+            }>
+                {entry.name}
+            </td>
+            <td className={cellPad.textAlign("right")}>{percent(fraction)}</td>
+            <td className={cellPad.textAlign("right")}>{formatTime(entry.ownTime.sum)}</td>
+            <td className={cellPad}>=</td>
+            {this.renderEquationCell(entry.ownTime)}
+            <td className={cellPad}>
+                {entry.stillOpenCount > 0 && <span className={css.colorhsl(0, 70, 45)}>({entry.stillOpenCount} open)</span>}
+            </td>
+        </tr>;
+    }
+
+    renderEquationCell(stats: StatsValue) {
+        let cellPad = css.pad2(6, 1).whiteSpace("pre");
+        let top = getStatsTop(stats);
+        if (!top.topHeavy) {
+            return <td className={cellPad}>
+                {formatNumber(stats.count)} × {formatTime(stats.sum / stats.count)}
+            </td>;
+        }
+        let bottomCount = stats.count - top.count;
+        let bottomValue = stats.sum - top.value;
+        let splitAt = top.valueFraction * 100;
+        return <td className={cellPad
+            .background(`linear-gradient(to right, hsla(0, 70%, 55%, 0.2) ${splitAt}%, hsla(210, 60%, 55%, 0.15) ${splitAt}%)`)
+        }
+            title={`Top heavy: ${percent(top.valueFraction)} of the time is in ${percent(top.countFraction)} of the calls`}
+        >
+            <span className={css.colorhsl(0, 70, 40)}>{formatNumber(top.count)} × {formatTime(top.value / top.count)}</span>
+            {" + "}
+            <span>{formatNumber(bottomCount)} × {formatTime(bottomValue / bottomCount || 0)}</span>
+        </td>;
+    }
+}

package/src/deployManager/components/ServicesListPage.tsx

@@ -17,6 +17,7 @@ import { PendingDeployInfo, UpdateButtons, UpdateServiceButtons } from "./deploy
 import { isDefined } from "../../misc";
 import { formatDateJSX } from "../../misc/formatJSX";
 import { Tools } from "./Tools";
+import { ServiceMeasureBars, ServiceMeasureBreakdown } from "./ServiceMeasureBars";
 
 module.hotreload = true;
 
@@ -41,6 +42,10 @@ class ServiceCategoryBadge extends qreact.Component<{ config: ServiceConfig }> {
 }
 
 export class ServicesListPage extends qreact.Component {
+    state = t.state({
+        measurementsEnabled: t.atomic<boolean>(true),
+        expandedMeasureNodes: t.lookup(t.string),
+    });
 
     render() {
         let controller = MachineServiceController(SocketFunction.browserNodeId());
@@ -63,6 +68,21 @@ export class ServicesListPage extends qreact.Component {
             keyCounts.set(key, (keyCounts.get(key) || 0) + 1);
         }
 
+        let serviceNodeIds = new Map<string, string[]>();
+        if (this.state.measurementsEnabled) {
+            for (let [serviceId, config] of services) {
+                if (!config) continue;
+                let nodeIds: string[] = [];
+                for (let machineId of getMachineIdList(config.parameters)) {
+                    if (getMachineConfig(machineId)?.disabled) continue;
+                    let serviceInfo = getMachineInfo(machineId)?.services[serviceId];
+                    nodeIds.push(...Object.values(serviceInfo?.nodeIds || {}));
+                }
+                serviceNodeIds.set(serviceId, nodeIds);
+            }
+        }
+        let allNodeIds = sort(Array.from(new Set(Array.from(serviceNodeIds.values()).flat())), x => x);
+
         return <div className={css.vbox(16)}>
             <div className={css.hbox(12).wrap}>
                 <h2 className={css.flexGrow(1)}>Services</h2>
@@ -104,6 +124,14 @@ export class ServicesListPage extends qreact.Component {
                 </button>
                 <UpdateButtons services={services.map(x => x[1]).filter(isDefined)} />
             </div>
+            <div
+                className={css.pad2(12, 8).button.bord2(0, 0, 20).hsl(0, 0, 95).alignSelf("flex-start")}
+                onClick={() => {
+                    this.state.measurementsEnabled = !this.state.measurementsEnabled;
+                }}
+            >
+                <span className={css.boldStyle}>{this.state.measurementsEnabled ? "☑" : "☐"} Measurements</span>
+            </div>
             <Tools />
             <div className={css.vbox(8)}>
                 {services.map(([serviceId, config]) => {
@@ -138,9 +166,13 @@ export class ServicesListPage extends qreact.Component {
                     }, 0);
                     let unknown = enabledMachineIds.length - runningMachines.length - failingMachines.length - missingMachines.length;
                     let duplicateKey = (keyCounts.get(config.parameters.key || "") || 0) > 1;
-                    return <div className={css.hbox(10)}>
+                    let measureNodeIds = serviceNodeIds.get(serviceId) || [];
+                    let expandedMeasureNodeId = this.state.expandedMeasureNodes[serviceId] || "";
+                    return <div className={css.vbox(4)} key={serviceId}>
+                        <div className={css.hbox(10)}>
                         <ServiceCategoryBadge config={config} />
-                        <Anchor noStyles key={serviceId}
+                        <div className={css.vbox(4)}>
+                        <Anchor noStyles
                             values={[currentViewParam.getOverride("service-detail"), selectedServiceIdParam.getOverride(serviceId)]}
                             className={
                                 css.pad2(12).button.bord2(0, 0, 20)
@@ -206,8 +238,26 @@ export class ServicesListPage extends qreact.Component {
                                 </div>
                             </div>
                         </Anchor>
+                        {this.state.measurementsEnabled && measureNodeIds.length > 0 && <ServiceMeasureBars
+                            serviceNodeIds={measureNodeIds}
+                            allNodeIds={allNodeIds}
+                            expandedNodeId={expandedMeasureNodeId}
+                            onToggleNode={nodeId => {
+                                if (this.state.expandedMeasureNodes[serviceId] === nodeId) {
+                                    delete this.state.expandedMeasureNodes[serviceId];
+                                } else {
+                                    this.state.expandedMeasureNodes[serviceId] = nodeId;
+                                }
+                            }}
+                        />}
+                        </div>
                         <UpdateServiceButtons service={config} />
                         <PendingDeployInfo service={config} />
+                        </div>
+                        {this.state.measurementsEnabled && expandedMeasureNodeId && <ServiceMeasureBreakdown
+                            nodeId={expandedMeasureNodeId}
+                            allNodeIds={allNodeIds}
+                        />}
                     </div>
                         ;
                 })}

package/src/deployManager/components/StoragePage.tsx

@@ -41,6 +41,8 @@ export type StorageServerBuckets = StorageServer & {
     /** Still waiting on this server's own endpoint - each server loads independently */
     loading?: boolean;
     buckets?: ServerBucketInfo[];
+    /** This server could not be read at all. Kept per server rather than thrown, so one bad server costs its own row and nothing else. */
+    error?: string;
 };
 
 function getStorageUrls(parameters: ServiceParameters): string[] {
@@ -59,7 +61,7 @@ function getStorageUrls(parameters: ServiceParameters): string[] {
 export function getStorageServers(configs: ServiceConfig[]): StorageServer[] {
     let serviceByUrl = new Map<string, StorageServer>();
     for (let config of configs) {
-        let allParameters = [getLiveServiceParameters(config), config.parameters, config.oldParameters];
+        let allParameters = [getLiveServiceParameters(config)];
         for (let parameters of allParameters) {
             if (!parameters) continue;
             for (let url of getStorageUrls(parameters)) {
@@ -214,6 +216,10 @@ function getBucketRows(servers: StorageServerBuckets[]): BucketRow[] {
             syncing: undefined,
             error: "",
         };
+        if (server.error) {
+            rows.push({ ...baseRow, error: server.error });
+            continue;
+        }
         if (server.loading) {
             rows.push({ ...baseRow, bucket: "Loading..." });
             continue;
@@ -226,24 +232,34 @@ function getBucketRows(servers: StorageServerBuckets[]): BucketRow[] {
         }
         for (let [index, bucket] of buckets.entries()) {
             let config = bucket.config;
-            rows.push({
-                ...baseRow,
-                server: index === 0 && server.url || "",
-                bucket: bucket.bucketName,
-                state: bucket.active && ACTIVE_STATE || INACTIVE_STATE,
-                files: config?.index && formatNumber(config.index.fileCount) || "",
-                bytes: config?.index && formatNumber(config.index.byteCount) + "B" || "",
-                flags: getBucketFlags(server.url, bucket.bucketName, config?.remoteConfig),
-                indexSources: config?.indexSources,
-                readerDiskLimit: config?.readerDiskLimit && formatNumber(config.readerDiskLimit) + "B" || "",
-                writes: bucket.writeStats && formatNumber(bucket.writeStats.originalWrites) || "",
-                written: bucket.writeStats && formatNumber(bucket.writeStats.originalBytes) + "B" || "",
-                writeGain: getWriteGain(bucket.writeStats),
-                disk: bucket.disk,
-                diskError: bucket.diskError || "",
-                syncing: config?.syncing,
-                error: bucket.error || "",
-            });
+            // Per bucket for the same reason it is per server: a single unparseable config costs its own row, not the table
+            try {
+                rows.push({
+                    ...baseRow,
+                    server: index === 0 && server.url || "",
+                    bucket: bucket.bucketName,
+                    state: bucket.active && ACTIVE_STATE || INACTIVE_STATE,
+                    files: config?.index && formatNumber(config.index.fileCount) || "",
+                    bytes: config?.index && formatNumber(config.index.byteCount) + "B" || "",
+                    flags: getBucketFlags(server.url, bucket.bucketName, config?.remoteConfig),
+                    indexSources: config?.indexSources,
+                    readerDiskLimit: config?.readerDiskLimit && formatNumber(config.readerDiskLimit) + "B" || "",
+                    writes: bucket.writeStats && formatNumber(bucket.writeStats.originalWrites) || "",
+                    written: bucket.writeStats && formatNumber(bucket.writeStats.originalBytes) + "B" || "",
+                    writeGain: getWriteGain(bucket.writeStats),
+                    disk: bucket.disk,
+                    diskError: bucket.diskError || "",
+                    syncing: config?.syncing,
+                    error: bucket.error || "",
+                });
+            } catch (e: any) {
+                rows.push({
+                    ...baseRow,
+                    server: index === 0 && server.url || "",
+                    bucket: bucket.bucketName,
+                    error: e.stack ?? String(e),
+                });
+            }
         }
     }
     return rows;
@@ -537,9 +553,14 @@ export class StoragePage extends qreact.Component {
         const configs = (serviceList || []).map(serviceId => machineController.getServiceConfig(serviceId)).filter(isDefined);
         const storageServers = getStorageServers(configs);
         const servers: StorageServerBuckets[] = storageServers.map(server => {
-            let buckets = controller.getServerBuckets(server.url);
-            if (!buckets) return { ...server, loading: true };
-            return { ...server, buckets };
+            // Every other server still has something worth showing, so one that cannot be read becomes a row saying so instead of an empty page
+            try {
+                let buckets = controller.getServerBuckets(server.url);
+                if (!buckets) return { ...server, loading: true };
+                return { ...server, buckets };
+            } catch (e: any) {
+                return { ...server, error: e.stack ?? String(e) };
+            }
         });
         const header = <>
             <div className={css.hbox(12)}>