querysub 0.589.0 → 0.591.0

package/src/0-path-value-core/pathValueArchives.ts

@@ -1,7 +1,7 @@
 import { blue, green, magenta, red, yellow } from "socket-function/src/formatting/logColors";
 import { measureFnc } from "socket-function/src/profiling/measure";
 import { ARCHIVE_FLUSH_LIMIT, ARCHIVE_FLUSH_LIMIT_START, compareTime, debugPathValuePath, getCompressNetwork, isCoreQuiet, PathValue, PathValueSnapshot, registerGetCompressNetwork, Time } from "./pathValueCore";
-import { getArchives, nestArchives } from "../-a-archives/archives";
+import { getArchives2 } from "../-a-archives/archives2";
 import { cache, cacheLimited, lazy } from "socket-function/src/caching";
 import { getOwnNodeId, getOwnNodeIdAssert } from "../-f-node-discovery/NodeDiscovery";
 import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
@@ -11,7 +11,7 @@ import { list } from "socket-function/src/misc";
 import { NodeIdParts, decodeNodeId, decodeNodeIdAssert, encodeNodeId } from "sliftutils/misc/https/certs";
 import { createArchiveLocker2 } from "./archiveLocks/ArchiveLocks2";
 import { devDebugbreak, isNoNetwork, getDomain } from "../config";
-import { wrapArchivesWithCache } from "../-a-archives/archiveCache";
+import { wrapArchivesWithCache2 } from "../-a-archives/archiveCache2";
 import { AuthoritySpec, PathRouter, debugSpec } from "./PathRouter";
 import { authorityLookup } from "./AuthorityLookup";
 import { delay, retryFunctional } from "socket-function/src/batching";
@@ -21,10 +21,10 @@ import { shutdown } from "../diagnostics/periodic";
 
 // Kept separate from the cache-wrapped `archives` so we can probe the underlying archives
 //  directly (without the cache layer in the way) when diagnosing missing-file failures.
-const archivesBase = lazy(() => getArchives("path-values/"));
-export const archives = lazy(() => wrapArchivesWithCache(archivesBase()));
-export const archivesLocks = lazy(() => getArchives("path-values-locks/"));
-export const archivesRecycleBin = lazy(() => wrapArchivesWithCache(getArchives("path-values-recycle-bin/")));
+const archivesBase = lazy(() => getArchives2("path-values/"));
+export const archives = lazy(() => wrapArchivesWithCache2(archivesBase()));
+export const archivesLocks = lazy(() => getArchives2("path-values-locks/"));
+export const archivesRecycleBin = lazy(() => wrapArchivesWithCache2(getArchives2("path-values-recycle-bin/")));
 
 // If getInfo reports a just-written file as missing, recheck a few times before treating it
 //  as a fatal "written too slowly" condition.
@@ -314,7 +314,7 @@ export class PathValueArchives {
                 while (pendingDataPaths.length > 0) {
                     let dataPath = pendingDataPaths.pop()!;
                     if (readCache.has(dataPath)) continue;
-                    let data = await archives().get(dataPath, { fastRead: true });
+                    let data = await archives().get(dataPath);
                     if (!data) continue;
                     readCache.set(dataPath, data);
                 }
@@ -469,7 +469,7 @@ export class PathValueArchives {
         const BATCH_SIZE = 16;
         for (let i = 0; i < dataPath.length; i += BATCH_SIZE) {
             let cur = dataPath.slice(i, i + BATCH_SIZE);
-            let curResults = await Promise.all(cur.map(x => archives().get(x, { fastRead: true })));
+            let curResults = await Promise.all(cur.map(x => archives().get(x)));
             results.push(...curResults);
         }
         if (config?.includeRecycleBin) {
@@ -483,7 +483,7 @@ export class PathValueArchives {
             for (let i = 0; i < notFoundList.length; i += BATCH_SIZE) {
                 let curIndexes = notFoundList.slice(i, i + BATCH_SIZE);
                 let cur = curIndexes.map(x => dataPath[x]);
-                let curResults = await Promise.all(cur.map(x => archivesRecycleBin().get(x, { fastRead: true })));
+                let curResults = await Promise.all(cur.map(x => archivesRecycleBin().get(x)));
                 for (let j = 0; j < curResults.length; j++) {
                     results[curIndexes[j]] = curResults[j];
                 }

package/src/2-proxy/PathValueProxyWatcher.ts

@@ -664,7 +664,7 @@ export class PathValueProxyWatcher {
         return pathValue;
     };
     public getCallback = (pathStr: string, syncParentKeys?: "parentKeys", readTransparent?: "readTransparent"): { value: unknown } | undefined => {
-        if (PathValueProxyWatcher.BREAK_ON_READS.size > 0 && (proxyWatcher.isAllSynced() || this)) {
+        if (PathValueProxyWatcher.BREAK_ON_READS.size > 0 && (proxyWatcher.isAllSynced({ checkCommitAllRunsFlag: true }) || this)) {
             // NOTE: We can't do a recursive match, as the parent paths include the 
             //  root, which is constantly read, but not relevant.
             if (PathValueProxyWatcher.BREAK_ON_READS.has(pathStr)) {
@@ -750,7 +750,7 @@ export class PathValueProxyWatcher {
     };
 
     public setCallback = (pathStr: string, value: unknown, inRecursion = false, allowSpecial = false): void => {
-        if (PathValueProxyWatcher.BREAK_ON_WRITES.size > 0 && proxyWatcher.isAllSynced()) {
+        if (PathValueProxyWatcher.BREAK_ON_WRITES.size > 0 && proxyWatcher.isAllSynced({ checkCommitAllRunsFlag: true })) {
             if (isRecursiveMatch(PathValueProxyWatcher.BREAK_ON_WRITES, pathStr)) {
                 let unwatch = () => removeMatches(PathValueProxyWatcher.BREAK_ON_WRITES, pathStr);
                 debugger;
@@ -758,7 +758,7 @@ export class PathValueProxyWatcher {
             }
         }
 
-        if (PathValueProxyWatcher.SET_FUNCTION_WATCH_ON_WRITES.size > 0 && proxyWatcher.isAllSynced()) {
+        if (PathValueProxyWatcher.SET_FUNCTION_WATCH_ON_WRITES.size > 0 && proxyWatcher.isAllSynced({ checkCommitAllRunsFlag: true })) {
             if (isRecursiveMatch(PathValueProxyWatcher.SET_FUNCTION_WATCH_ON_WRITES, pathStr)) {
                 let unwatch = () => removeMatches(PathValueProxyWatcher.SET_FUNCTION_WATCH_ON_WRITES, pathStr);
                 PathValueProxyWatcher.BREAK_ON_CALL.add(
@@ -767,7 +767,7 @@ export class PathValueProxyWatcher {
             }
         }
 
-        if (PathValueProxyWatcher.LOG_WRITES_INCLUDES.size > 0 && proxyWatcher.isAllSynced()) {
+        if (PathValueProxyWatcher.LOG_WRITES_INCLUDES.size > 0 && proxyWatcher.isAllSynced({ checkCommitAllRunsFlag: true })) {
             if (isRecursiveMatch(PathValueProxyWatcher.LOG_WRITES_INCLUDES, pathStr)) {
                 let unwatch = () => removeMatches(PathValueProxyWatcher.LOG_WRITES_INCLUDES, pathStr);
                 console.log(`Write path "${pathStr}" = ${value}`);
@@ -2138,9 +2138,9 @@ export class PathValueProxyWatcher {
     }
 
     public isAllSynced(config?: {
-        ignoreAlwaysCommitAllRunsFlag?: boolean;
+        checkCommitAllRunsFlag?: boolean;
     }) {
-        if (!config?.ignoreAlwaysCommitAllRunsFlag && this.runningWatcher?.options.commitAllRuns) {
+        if (config?.checkCommitAllRunsFlag && this.runningWatcher?.options.commitAllRuns) {
             return true;
         }
         return !this.getTriggeredWatcherMaybeUndefined()?.hasAnyUnsyncedAccesses();

package/src/4-dom/qreact.tsx

@@ -744,7 +744,7 @@ class QRenderClass {
                     } else {
                         // We check here, and not in the caller, so the props are always up to date,
                         //  as the objects might be mutated or special (or have callback functions).
-                        if (statics.jsonComparePropUpdates && Querysub.isAllSynced()) {
+                        if (statics.jsonComparePropUpdates && Querysub.isAllSynced({ checkCommitAllRunsFlag: true })) {
 
                             let props = self.data().props;
 
@@ -809,7 +809,7 @@ class QRenderClass {
                 //  ALSO, this stops infinite loops caused by self triggering, which is really useful.
                 if (!QRenderClass.areVNodesEqual(comparePrevVNode, vNode)) {
                     self.data().vNodeForRender = frozen;
-                    if (Querysub.isAllSynced()) {
+                    if (Querysub.isAllSynced({ checkCommitAllRunsFlag: true })) {
                         comparePrevVNode = frozen.value;
                     }
                 } else {
@@ -1886,7 +1886,7 @@ class QRenderClass {
 export const __INTERNAL__QRenderClass = QRenderClass;
 
 let defaultErrorHandler: ErrorHandler = ({ error, debugName }) => {
-    if (Querysub.isAllSynced()) {
+    if (Querysub.isAllSynced({ checkCommitAllRunsFlag: true })) {
         console.error(`Render error in ${debugName}`, error);
         // Throw, so we get a good callstack
         setImmediate(() => { throw error; });
@@ -2673,7 +2673,7 @@ function watchUnsyncedComponents(): Set<ExternalRenderClass> {
     return unsyncedComponents;
 }
 function componentRendered(component: QRenderClass) {
-    if (component.disposed || Querysub.isAllSynced({ ignoreAlwaysCommitAllRunsFlag: true })) {
+    if (component.disposed || Querysub.isAllSynced()) {
         if (unsyncedComponents.has(component)) {
             unsyncedComponents.delete(component);
             triggerUnsyncedNow();

package/src/4-querysub/Querysub.ts

@@ -422,7 +422,7 @@ export class Querysub {
     public static anyUnsynced() {
         return !Querysub.allSynced();
     }
-    public static allSynced(config?: { ignoreAlwaysCommitAllRunsFlag?: boolean }) {
+    public static allSynced(config?: { checkCommitAllRunsFlag?: boolean }) {
         return proxyWatcher.isAllSynced(config);
     }
     public static fullySynced = Querysub.allSynced;
@@ -528,6 +528,13 @@ export class Querysub {
         });
     }
 
+    public static triggerOnPromiseFinish(promise: MaybePromise<unknown>, config: {
+        waitReason: string;
+        noWait?: boolean;
+    }) {
+        proxyWatcher.triggerOnPromiseFinish(promise, config);
+    }
+
     /** Returns true if any predictions are running. In which case, we will correctly rerun the function and consider it non-synced until the predictions finish. This allows you to check for predictions of the in your function and return if you have any. This is useful for functions that need a stable state before they run. However, if many functions use this and you trigger them at once, it might result in an n-squared situation, so this should be used with caution.
      *  - The best use case is if something is triggering maybe on focus and on blur, and you want to make sure the on blur changes happen before your on focus changes happen.
      */

package/src/deployManager/components/RouteConfigView.tsx

@@ -183,9 +183,18 @@ export function getRouteConfigGroups(servers: StorageServerBuckets[]): RouteConf
     let byBucket = new Map<string, Map<string, ConfigVariant>>();
     for (let server of servers) {
         for (let bucket of server.buckets || []) {
+            // An inactive bucket is not serving anything, so whatever routing it happens to hold says nothing about where values actually come from - and letting it vote would put a stale config up against the live ones as a conflict
+            if (!bucket.active) continue;
             let remoteConfig: RemoteConfig | undefined = bucket.config?.remoteConfig;
             if (!remoteConfig) continue;
-            let sources = remoteConfig.sources.map(normalizeSource);
+            let sources: Source[];
+            try {
+                sources = remoteConfig.sources.map(normalizeSource);
+            } catch (e: any) {
+                // One server's unreadable config must not cost us the grouping for every other bucket - it simply does not get a say in the consensus
+                console.error(`Ignoring the routing config ${server.url} reports for ${bucket.bucketName}, which could not be read:`, e.stack ?? e);
+                continue;
+            }
             let variants = byBucket.get(bucket.bucketName);
             if (!variants) {
                 variants = new Map();

package/src/deployManager/components/ServiceMeasureBars.tsx

@@ -0,0 +1,179 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { qreact } from "../../4-dom/qreact";
+import { css } from "typesafecss";
+import { formatNumber, formatTime, formatVeryNiceDateTime } from "socket-function/src/formatting/format";
+import { timeInSecond } from "socket-function/src/misc";
+import { timeoutToError } from "../../errors";
+import { StatsValue, getStatsTop } from "socket-function/src/profiling/stats";
+import type { MeasureChunk, MeasureSummary } from "../../diagnostics/watchdog";
+import { NodeCapabilitiesController } from "../../-g-core-values/NodeCapabilities";
+import { getSyncedController } from "../../library-components/SyncedController";
+import { assertIsManagementUser } from "../../diagnostics/managementPages";
+
+const NODE_CALL_TIMEOUT = timeInSecond * 10;
+const BAR_HEIGHT_PX = 6;
+const MIN_BAR_WIDTH_PX = 30;
+// Green at idle, red at fully busy.
+const BAR_IDLE_HUE = 120;
+const BAR_FLASH_THRESHOLD = 0.6;
+const FLASH_CLASS_NAME = "ServiceMeasureBars-flash";
+
+class ServiceMeasureControllerBase {
+    public async getMeasureSummary(nodeId: string): Promise<MeasureSummary | undefined> {
+        return await timeoutToError(NODE_CALL_TIMEOUT, NodeCapabilitiesController.nodes[nodeId].getMeasureSummary(), () => new Error(`Timed out asking ${nodeId} for its measure summary`));
+    }
+    public async getMeasureBreakdown(nodeId: string, range: { startTime: number; endTime: number }): Promise<{ chunk?: MeasureChunk }> {
+        let chunk = await timeoutToError(NODE_CALL_TIMEOUT, NodeCapabilitiesController.nodes[nodeId].getMeasureBreakdown(range), () => new Error(`Timed out asking ${nodeId} for its measure breakdown`));
+        return { chunk };
+    }
+}
+export const ServiceMeasureController = getSyncedController(SocketFunction.register(
+    "ServiceMeasureController-3f8c1de2-5b74-4e0f-a6c9-8d21f47b90a5",
+    new ServiceMeasureControllerBase(),
+    () => ({
+        getMeasureSummary: { hooks: [assertIsManagementUser], compress: true },
+        getMeasureBreakdown: { hooks: [assertIsManagementUser], compress: true },
+    }),
+    () => ({
+    })
+));
+
+function percent(value: number) {
+    return `${(value * 100).toFixed(2)}%`;
+}
+
+export class ServiceMeasureBars extends qreact.Component<{
+    serviceNodeIds: string[];
+    expandedNodeId: string;
+    onToggleNode: (nodeId: string) => void;
+}> {
+    render() {
+        let controller = ServiceMeasureController(SocketFunction.browserNodeId());
+
+        return <div className={css.vbox(1).alignItems("stretch")}>
+            {this.props.serviceNodeIds.map(nodeId => {
+                let isExpanded = this.props.expandedNodeId === nodeId;
+                let summary: MeasureSummary | undefined;
+                try {
+                    summary = controller.getMeasureSummary(nodeId);
+                } catch (err) {
+                    return <div key={nodeId}
+                        className={css.height(BAR_HEIGHT_PX).hsl(0, 70, 55)}
+                        title={`Node did not respond (it likely doesn't have the measure API yet)\n${nodeId}\n${(err as Error).stack ?? err}`}
+                    />;
+                }
+                if (!summary) return undefined;
+                let timeRunFor = summary.endTime - summary.startTime;
+                if (timeRunFor <= 0) return undefined;
+                let fraction = summary.profiledTime / timeRunFor;
+                let clampedFraction = Math.min(1, Math.max(0, fraction));
+                let hue = BAR_IDLE_HUE * (1 - clampedFraction);
+                let isFlashing = fraction > BAR_FLASH_THRESHOLD;
+                return <div key={nodeId} className={css.hbox(0)}>
+                    <div
+                        className={
+                            css.height(BAR_HEIGHT_PX)
+                                .width(`${fraction * 100}%`)
+                                .minWidth(MIN_BAR_WIDTH_PX)
+                                .button
+                                .hsl(hue, 70, isExpanded ? 40 : 55)
+                            + (isFlashing && (" " + FLASH_CLASS_NAME) || "")
+                        }
+                        title={`Profiled ${formatTime(summary.profiledTime)} of ${formatTime(timeRunFor)} (${percent(fraction)} CPU)\n${nodeId}`}
+                        onClick={() => this.props.onToggleNode(nodeId)}
+                    />
+                    {isFlashing && <style>{`
+                        @keyframes ${FLASH_CLASS_NAME}-anim {
+                            0%, 100% { opacity: 1; }
+                            50% { opacity: 0.3; }
+                        }
+                        .${FLASH_CLASS_NAME} {
+                            animation: ${FLASH_CLASS_NAME}-anim 0.8s infinite;
+                        }
+                    `}</style>}
+                </div>;
+            })}
+        </div>;
+    }
+}
+
+export class ServiceMeasureBreakdown extends qreact.Component<{
+    nodeId: string;
+}> {
+    render() {
+        let controller = ServiceMeasureController(SocketFunction.browserNodeId());
+        let summary: MeasureSummary | undefined;
+        let result: { chunk?: MeasureChunk } | undefined;
+        try {
+            summary = controller.getMeasureSummary(this.props.nodeId);
+            if (!summary) return undefined;
+            result = controller.getMeasureBreakdown(this.props.nodeId, { startTime: summary.startTime, endTime: summary.endTime });
+        } catch (err) {
+            return <div className={css.pad2(10).hsl(0, 70, 92).bord2(0, 0, 20)}>
+                <div className={css.boldStyle.colorhsl(0, 70, 35)}>Node did not respond (it likely doesn't have the measure API yet)</div>
+                <pre className={css.whiteSpace("pre-wrap").fontSize(12)}>{(err as Error).stack ?? String(err)}</pre>
+            </div>;
+        }
+        if (!result) {
+            return <div className={css.pad2(10).hsl(0, 0, 98).bord2(0, 0, 20)}>Loading breakdown...</div>;
+        }
+        let chunk = result.chunk;
+        if (!chunk) {
+            return <div className={css.pad2(10).hsl(0, 0, 98).bord2(0, 0, 20)}>Breakdown is no longer available on the node (it only keeps the last hour, and restarts clear it).</div>;
+        }
+
+        let timeRunFor = chunk.endTime - chunk.startTime;
+        let cpuFraction = chunk.profiledTime / timeRunFor;
+        return <div className={css.vbox(6).pad2(10).hsl(0, 0, 98).bord2(0, 0, 20).fontFamily("monospace").fontSize(13)}>
+            <div className={css.boldStyle}>
+                Profiled {formatTime(chunk.profiledTime)} ({percent(cpuFraction)} CPU) (profile for {formatTime(timeRunFor)}, ending {formatVeryNiceDateTime(chunk.endTime)})
+            </div>
+            <div className={css.colorhsl(0, 0, 45).fontSize(12)}>{this.props.nodeId}</div>
+            <table className={css.borderCollapse("collapse")}>
+                {chunk.entries.map(entry => this.renderEntryRow(entry, chunk!))}
+            </table>
+        </div>;
+    }
+
+    renderEntryRow(entry: MeasureChunk["entries"][0], chunk: MeasureChunk) {
+        let fraction = entry.ownTime.sum / chunk.profiledTime;
+        let cellPad = css.pad2(6, 1).whiteSpace("pre");
+        return <tr key={entry.name}>
+            <td className={cellPad
+                .colorhsl(210, 70, 35)
+                .background(`linear-gradient(to right, hsla(210, 60%, 55%, 0.25) ${fraction * 100}%, transparent ${fraction * 100}%)`)
+            }>
+                {entry.name}
+            </td>
+            <td className={cellPad.textAlign("right")}>{percent(fraction)}</td>
+            <td className={cellPad.textAlign("right")}>{formatTime(entry.ownTime.sum)}</td>
+            <td className={cellPad}>=</td>
+            {this.renderEquationCell(entry.ownTime)}
+            <td className={cellPad}>
+                {entry.stillOpenCount > 0 && <span className={css.colorhsl(0, 70, 45)}>({entry.stillOpenCount} open)</span>}
+            </td>
+        </tr>;
+    }
+
+    renderEquationCell(stats: StatsValue) {
+        let cellPad = css.pad2(6, 1).whiteSpace("pre");
+        let top = getStatsTop(stats);
+        if (!top.topHeavy) {
+            return <td className={cellPad}>
+                {formatNumber(stats.count)} × {formatTime(stats.sum / stats.count)}
+            </td>;
+        }
+        let bottomCount = stats.count - top.count;
+        let bottomValue = stats.sum - top.value;
+        let splitAt = top.valueFraction * 100;
+        return <td className={cellPad
+            .background(`linear-gradient(to right, hsla(0, 70%, 55%, 0.2) ${splitAt}%, hsla(210, 60%, 55%, 0.15) ${splitAt}%)`)
+        }
+            title={`Top heavy: ${percent(top.valueFraction)} of the time is in ${percent(top.countFraction)} of the calls`}
+        >
+            <span className={css.colorhsl(0, 70, 40)}>{formatNumber(top.count)} × {formatTime(top.value / top.count)}</span>
+            {" + "}
+            <span>{formatNumber(bottomCount)} × {formatTime(bottomValue / bottomCount || 0)}</span>
+        </td>;
+    }
+}

package/src/deployManager/components/ServicesListPage.tsx

@@ -17,6 +17,7 @@ import { PendingDeployInfo, UpdateButtons, UpdateServiceButtons } from "./deploy
 import { isDefined } from "../../misc";
 import { formatDateJSX } from "../../misc/formatJSX";
 import { Tools } from "./Tools";
+import { ServiceMeasureBars, ServiceMeasureBreakdown } from "./ServiceMeasureBars";
 
 module.hotreload = true;
 
@@ -41,6 +42,10 @@ class ServiceCategoryBadge extends qreact.Component<{ config: ServiceConfig }> {
 }
 
 export class ServicesListPage extends qreact.Component {
+    state = t.state({
+        measurementsEnabled: t.atomic<boolean>(true),
+        expandedMeasureNodes: t.lookup(t.string),
+    });
 
     render() {
         let controller = MachineServiceController(SocketFunction.browserNodeId());
@@ -63,6 +68,20 @@ export class ServicesListPage extends qreact.Component {
             keyCounts.set(key, (keyCounts.get(key) || 0) + 1);
         }
 
+        let serviceNodeIds = new Map<string, string[]>();
+        if (this.state.measurementsEnabled) {
+            for (let [serviceId, config] of services) {
+                if (!config) continue;
+                let nodeIds: string[] = [];
+                for (let machineId of getMachineIdList(config.parameters)) {
+                    if (getMachineConfig(machineId)?.disabled) continue;
+                    let serviceInfo = getMachineInfo(machineId)?.services[serviceId];
+                    nodeIds.push(...Object.values(serviceInfo?.nodeIds || {}));
+                }
+                serviceNodeIds.set(serviceId, nodeIds);
+            }
+        }
+
         return <div className={css.vbox(16)}>
             <div className={css.hbox(12).wrap}>
                 <h2 className={css.flexGrow(1)}>Services</h2>
@@ -104,6 +123,14 @@ export class ServicesListPage extends qreact.Component {
                 </button>
                 <UpdateButtons services={services.map(x => x[1]).filter(isDefined)} />
             </div>
+            <div
+                className={css.pad2(12, 8).button.bord2(0, 0, 20).hsl(0, 0, 95).alignSelf("flex-start")}
+                onClick={() => {
+                    this.state.measurementsEnabled = !this.state.measurementsEnabled;
+                }}
+            >
+                <span className={css.boldStyle}>{this.state.measurementsEnabled ? "☑" : "☐"} Measurements</span>
+            </div>
             <Tools />
             <div className={css.vbox(8)}>
                 {services.map(([serviceId, config]) => {
@@ -138,9 +165,13 @@ export class ServicesListPage extends qreact.Component {
                     }, 0);
                     let unknown = enabledMachineIds.length - runningMachines.length - failingMachines.length - missingMachines.length;
                     let duplicateKey = (keyCounts.get(config.parameters.key || "") || 0) > 1;
-                    return <div className={css.hbox(10)}>
+                    let measureNodeIds = serviceNodeIds.get(serviceId) || [];
+                    let expandedMeasureNodeId = this.state.expandedMeasureNodes[serviceId] || "";
+                    return <div className={css.vbox(4).alignItems("stretch")} key={serviceId}>
+                        <div className={css.hbox(10)}>
                         <ServiceCategoryBadge config={config} />
-                        <Anchor noStyles key={serviceId}
+                        <div className={css.vbox(4).alignItems("stretch")}>
+                        <Anchor noStyles
                             values={[currentViewParam.getOverride("service-detail"), selectedServiceIdParam.getOverride(serviceId)]}
                             className={
                                 css.pad2(12).button.bord2(0, 0, 20)
@@ -206,8 +237,24 @@ export class ServicesListPage extends qreact.Component {
                                 </div>
                             </div>
                         </Anchor>
+                        {this.state.measurementsEnabled && measureNodeIds.length > 0 && <ServiceMeasureBars
+                            serviceNodeIds={measureNodeIds}
+                            expandedNodeId={expandedMeasureNodeId}
+                            onToggleNode={nodeId => {
+                                if (this.state.expandedMeasureNodes[serviceId] === nodeId) {
+                                    delete this.state.expandedMeasureNodes[serviceId];
+                                } else {
+                                    this.state.expandedMeasureNodes[serviceId] = nodeId;
+                                }
+                            }}
+                        />}
+                        </div>
                         <UpdateServiceButtons service={config} />
                         <PendingDeployInfo service={config} />
+                        </div>
+                        {this.state.measurementsEnabled && expandedMeasureNodeId && <ServiceMeasureBreakdown
+                            nodeId={expandedMeasureNodeId}
+                        />}
                     </div>
                         ;
                 })}

package/src/deployManager/components/StoragePage.tsx

@@ -41,6 +41,8 @@ export type StorageServerBuckets = StorageServer & {
     /** Still waiting on this server's own endpoint - each server loads independently */
     loading?: boolean;
     buckets?: ServerBucketInfo[];
+    /** This server could not be read at all. Kept per server rather than thrown, so one bad server costs its own row and nothing else. */
+    error?: string;
 };
 
 function getStorageUrls(parameters: ServiceParameters): string[] {
@@ -59,7 +61,7 @@ function getStorageUrls(parameters: ServiceParameters): string[] {
 export function getStorageServers(configs: ServiceConfig[]): StorageServer[] {
     let serviceByUrl = new Map<string, StorageServer>();
     for (let config of configs) {
-        let allParameters = [getLiveServiceParameters(config), config.parameters, config.oldParameters];
+        let allParameters = [getLiveServiceParameters(config)];
         for (let parameters of allParameters) {
             if (!parameters) continue;
             for (let url of getStorageUrls(parameters)) {
@@ -214,6 +216,10 @@ function getBucketRows(servers: StorageServerBuckets[]): BucketRow[] {
             syncing: undefined,
             error: "",
         };
+        if (server.error) {
+            rows.push({ ...baseRow, error: server.error });
+            continue;
+        }
         if (server.loading) {
             rows.push({ ...baseRow, bucket: "Loading..." });
             continue;
@@ -226,24 +232,34 @@ function getBucketRows(servers: StorageServerBuckets[]): BucketRow[] {
         }
         for (let [index, bucket] of buckets.entries()) {
             let config = bucket.config;
-            rows.push({
-                ...baseRow,
-                server: index === 0 && server.url || "",
-                bucket: bucket.bucketName,
-                state: bucket.active && ACTIVE_STATE || INACTIVE_STATE,
-                files: config?.index && formatNumber(config.index.fileCount) || "",
-                bytes: config?.index && formatNumber(config.index.byteCount) + "B" || "",
-                flags: getBucketFlags(server.url, bucket.bucketName, config?.remoteConfig),
-                indexSources: config?.indexSources,
-                readerDiskLimit: config?.readerDiskLimit && formatNumber(config.readerDiskLimit) + "B" || "",
-                writes: bucket.writeStats && formatNumber(bucket.writeStats.originalWrites) || "",
-                written: bucket.writeStats && formatNumber(bucket.writeStats.originalBytes) + "B" || "",
-                writeGain: getWriteGain(bucket.writeStats),
-                disk: bucket.disk,
-                diskError: bucket.diskError || "",
-                syncing: config?.syncing,
-                error: bucket.error || "",
-            });
+            // Per bucket for the same reason it is per server: a single unparseable config costs its own row, not the table
+            try {
+                rows.push({
+                    ...baseRow,
+                    server: index === 0 && server.url || "",
+                    bucket: bucket.bucketName,
+                    state: bucket.active && ACTIVE_STATE || INACTIVE_STATE,
+                    files: config?.index && formatNumber(config.index.fileCount) || "",
+                    bytes: config?.index && formatNumber(config.index.byteCount) + "B" || "",
+                    flags: getBucketFlags(server.url, bucket.bucketName, config?.remoteConfig),
+                    indexSources: config?.indexSources,
+                    readerDiskLimit: config?.readerDiskLimit && formatNumber(config.readerDiskLimit) + "B" || "",
+                    writes: bucket.writeStats && formatNumber(bucket.writeStats.originalWrites) || "",
+                    written: bucket.writeStats && formatNumber(bucket.writeStats.originalBytes) + "B" || "",
+                    writeGain: getWriteGain(bucket.writeStats),
+                    disk: bucket.disk,
+                    diskError: bucket.diskError || "",
+                    syncing: config?.syncing,
+                    error: bucket.error || "",
+                });
+            } catch (e: any) {
+                rows.push({
+                    ...baseRow,
+                    server: index === 0 && server.url || "",
+                    bucket: bucket.bucketName,
+                    error: e.stack ?? String(e),
+                });
+            }
         }
     }
     return rows;
@@ -537,9 +553,14 @@ export class StoragePage extends qreact.Component {
         const configs = (serviceList || []).map(serviceId => machineController.getServiceConfig(serviceId)).filter(isDefined);
         const storageServers = getStorageServers(configs);
         const servers: StorageServerBuckets[] = storageServers.map(server => {
-            let buckets = controller.getServerBuckets(server.url);
-            if (!buckets) return { ...server, loading: true };
-            return { ...server, buckets };
+            // Every other server still has something worth showing, so one that cannot be read becomes a row saying so instead of an empty page
+            try {
+                let buckets = controller.getServerBuckets(server.url);
+                if (!buckets) return { ...server, loading: true };
+                return { ...server, buckets };
+            } catch (e: any) {
+                return { ...server, error: e.stack ?? String(e) };
+            }
         });
         const header = <>
             <div className={css.hbox(12)}>