querysub 0.545.0 → 0.547.0

package/appSecrets.ts

@@ -74,9 +74,8 @@ export const getCloudflareCreds = lazy(async (): Promise<{ key: string; email: s
 
 /** Serves the secret keys sliftutils' getSecret expects, reading them the way querysub already does: backblaze creds off disk (getBackblazePath), cloudflare creds from the keys archives bucket (with its local file fallback). */
 export async function getAppSecret(key: string): Promise<string | undefined> {
-    // todonext;
-    //  Add this back after we finish testing. We need this in order to get logging, but for testing, It just adds extra logging, which is annoying and slows down runs. 
-    // Querysub;
+    // Import query sub so our logs go to our actual log server. Otherwise we won't know if we get any errors. 
+    Querysub;
     if (key.startsWith("backblaze.json.")) {
         let creds = JSON.parse(fs.readFileSync(getBackblazePath(), "utf8")) as { applicationKeyId?: string; applicationKey?: string };
         if (key === "backblaze.json.applicationKeyId") return creds.applicationKeyId;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.545.0",
+  "version": "0.547.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -71,7 +71,7 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
-    "sliftutils": "^1.7.32",
+    "sliftutils": "^1.7.33",
     "socket-function": "^1.2.26",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",

package/src/-a-archives/archives2.ts

@@ -3,6 +3,9 @@ import { createArchives } from "sliftutils/storage/remoteStorage/createArchives"
 import { cache, cacheJSONArgsEqual } from "socket-function/src/caching";
 import { formatDateTime } from "socket-function/src/formatting/format";
 import { getDomain } from "../config";
+import { timeInMinute } from "socket-function/src/misc";
+
+export const STORAGE_ACCOUNT = "root";
 
 function createSourceWindows(
     overrides: Partial<RemoteConfigBase>,
@@ -32,36 +35,34 @@ function createSourceWindows(
 }
 
 function archiveBuilder(bucket: string, overrides: Partial<RemoteConfigBase>) {
-    const HETZNER = `https://65-109-93-113.querysubtest.com:5233/file/root/${bucket}/storage/storagerouting.json`;
-    const ONTARIO = `https://99-250-124-91.querysubtest.com:5234/file/root/${bucket}/storage/storagerouting.json`;
+    const HETZNER = `https://65-109-93-113.querysubtest.com:5233/file/${STORAGE_ACCOUNT}/${bucket}/storage/storagerouting.json`;
+    const ONTARIO = `https://99-250-124-91.querysubtest.com:5234/file/${STORAGE_ACCOUNT}/${bucket}/storage/storagerouting.json`;
     const BACKBLAZE = `https://f002.backblazeb2.com/file/${bucket}/storage/storagerouting.json`;
     let sources = createSourceWindows(overrides, [
         {
             startTime: 0,
             sources: [
-                { type: "remote", url: ONTARIO, route: [0.5, 1], },
-                { type: "remote", url: HETZNER, route: [0, 0.5], },
+                { type: "remote", url: ONTARIO, route: [0, 0.5], },
+                { type: "remote", url: HETZNER, route: [0.5, 1], },
                 { type: "remote", url: ONTARIO, },
                 { type: "remote", url: HETZNER, },
                 { type: "backblaze", url: BACKBLAZE, },
             ]
         },
-        /*
         {
-            startTime: +new Date("2026-07-20 02:30:00-04:00"),
+            startTime: +new Date("2026-07-20 18:00:00-04:00"),
             sources: [
-                { type: "remote", url: ONTARIO, route: [0, 0.5], },
-                { type: "remote", url: HETZNER, route: [0.5, 1], },
+                { type: "remote", url: ONTARIO, route: [0.5, 1], },
+                { type: "remote", url: HETZNER, route: [0, 0.5], },
                 { type: "remote", url: ONTARIO, },
                 { type: "remote", url: HETZNER, },
                 { type: "backblaze", url: BACKBLAZE, },
             ]
         }
-        */
     ]);
 
     return createArchives({
-        version: 12,
+        version: 14,
         sources,
     });
 }
@@ -100,6 +101,13 @@ function getSafeDomain() {
 export function getArchives2(folder: string) {
     return nestBucket(folder, archivesBuilderCache(getSafeDomain(), { noFullSync: true, }));
 }
+
+/** The rights are buffered server-side, so if you do many writes at once, it won't slow down the disk and it won't result in many writes to back backblades. Of course, this does mean if the server crashes, you do lose some data.
+ */
+export function getArchives2Buffered(folder: string, bufferDelay = timeInMinute * 5) {
+    return nestBucket(folder, archivesBuilderCache(getSafeDomain() + "-fast", { noFullSync: true, fast: true, writeDelay: bufferDelay }));
+}
+
 export function getArchives2PrivateImmutable(folder: string) {
     return nestBucket(folder, archivesBuilderCache(getSafeDomain() + "-private-immutable", { noFullSync: true, immutable: true }));
 }
@@ -112,5 +120,5 @@ export function getArchives2PublicImmutable(folder: string) {
 }
 export function getArchives2Public(folder: string) {
     let domain = getSafeDomain();
-    return nestBucket(folder, archivesBuilderCache(domain + "-public", { noFullSync: true, public: true, allowedOrigins: getAllowedOrigins(getDomain()) }));
+    return nestBucket(folder, archivesBuilderCache(domain + "-public", { public: true, allowedOrigins: getAllowedOrigins(getDomain()) }));
 }

package/src/deployManager/MachinesPage.tsx

@@ -9,6 +9,7 @@ import { ServiceDetailPage } from "./components/ServiceDetailPage";
 import { MachineDetailPage } from "./components/MachineDetailPage";
 import { Anchor } from "../library-components/ATag";
 import { DeployPage } from "./components/DeployPage";
+import { StoragePage } from "./components/StoragePage";
 
 export class MachinesPage extends qreact.Component {
     private renderTabs() {
@@ -20,6 +21,7 @@ export class MachinesPage extends qreact.Component {
                 { key: "machines", label: "Machines", otherKeys: ["machine-detail"] },
                 { key: "services", label: "Services", otherKeys: ["service-detail"] },
                 { key: "deploy", label: "Deploy", otherKeys: [] },
+                { key: "storage", label: "Storage", otherKeys: [] },
             ].map(tab => {
                 let isActive = currentViewParam.value === tab.key || tab.otherKeys.includes(currentViewParam.value);
                 return <Anchor noStyles key={tab.key}
@@ -47,6 +49,7 @@ export class MachinesPage extends qreact.Component {
                 {currentViewParam.value === "service-detail" && <ServiceDetailPage />}
                 {currentViewParam.value === "machine-detail" && <MachineDetailPage />}
                 {currentViewParam.value === "deploy" && <DeployPage />}
+                {currentViewParam.value === "storage" && <StoragePage />}
             </div>
         </div>;
     }

package/src/deployManager/components/RouteConfigView.tsx

@@ -0,0 +1,349 @@
+import { qreact } from "../../4-dom/qreact";
+import { Querysub } from "../../4-querysub/Querysub";
+import { css } from "typesafecss";
+import { sort } from "socket-function/src/misc";
+import { formatNumber, formatDateTime, formatTime } from "socket-function/src/formatting/format";
+import { parseHostedUrl, parseBackblazeUrl } from "sliftutils/storage/remoteStorage/remoteConfig";
+import { FULL_VALID_WINDOW, FULL_ROUTE } from "sliftutils/storage/IArchives";
+import type { RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig } from "sliftutils/storage/IArchives";
+import { showModal } from "../../5-diagnostics/Modal";
+import { FullscreenModal } from "../../5-diagnostics/FullscreenModal";
+import { Button } from "../../library-components/Button";
+import type { StorageServerBuckets } from "./StoragePage";
+
+module.hotreload = true;
+
+type Source = HostedConfig | BackblazeConfig;
+
+const DEFAULT_HTTPS_PORT = 443;
+const TAG_COLOR = { h: 210, s: 45, l: 88 };
+const WARNING_COLOR = { h: 35, s: 90, l: 85 };
+const SOURCE_BORDER_COLOR = { h: 0, s: 0, l: 75 };
+const ACTIVE_COLOR = { h: 130, s: 55, l: 85 };
+const PAST_COLOR = { h: 0, s: 0, l: 90 };
+const FUTURE_COLOR = { h: 45, s: 80, l: 85 };
+// Windows are days or hours wide, so the countdown only has to be roughly right
+const TIME_REFRESH_INTERVAL = 60 * 1000;
+const WINDOW_HEADER_SIZE = 13;
+const BUCKET_TITLE_SIZE = 15;
+const GROUP_BODY_INDENT = 16;
+const TAG_FONT_SIZE = 11;
+const JSON_INDENT = 4;
+// Stands in for the bucket a config belongs to, so buckets that differ only by their own name group together
+const OWN_BUCKET_PLACEHOLDER = "*";
+
+/** String sources are the shorthand for an always-valid, unsharded backblaze bucket. */
+function normalizeSource(source: RemoteConfigBase): Source {
+    if (typeof source !== "string") return source;
+    return { type: "backblaze", url: source, validWindow: FULL_VALID_WINDOW };
+}
+
+/** The bucket coordinates of a source URL, so the display doesn't have to show the whole routing-file URL. A source pointing at the bucket the config belongs to shows the placeholder instead of repeating the name. */
+function describeUrl(source: Source, ownBucketName: string): string {
+    let maskBucket = (bucketName: string) => bucketName === ownBucketName && OWN_BUCKET_PLACEHOLDER || bucketName;
+    try {
+        if (source.type === "backblaze") {
+            return `b2 ${maskBucket(parseBackblazeUrl(source.url).bucketName)}`;
+        }
+        let { address, port, account, bucketName } = parseHostedUrl(source.url);
+        let host = port === DEFAULT_HTTPS_PORT && address || `${address}:${port}`;
+        return `${host} ${account}/${maskBucket(bucketName)}`;
+    } catch {
+        return source.url;
+    }
+}
+
+function getSourceKey(source: Source, ownBucketName: string): string {
+    return JSON.stringify({ ...source, url: describeUrl(source, ownBucketName) });
+}
+
+/** Identifies a config independently of which bucket it configures, so identical configs on different buckets collapse into one display. */
+function getVariantKey(variant: ConfigVariant, ownBucketName: string): string {
+    return JSON.stringify({ version: variant.version, sources: variant.sources.map(x => getSourceKey(x, ownBucketName)) });
+}
+
+function formatWindowTime(time: number): string {
+    if (time <= FULL_VALID_WINDOW[0]) return "the beginning";
+    if (time >= FULL_VALID_WINDOW[1]) return "forever";
+    return formatDateTime(time);
+}
+
+// A missing version sorts below any explicit one, matching getConfigVersion
+function getSortVersion(variant: ConfigVariant): number {
+    return variant.version ?? -1;
+}
+
+type WindowStatus = {
+    icon: string;
+    text: string;
+    color: { h: number; s: number; l: number };
+};
+
+function getWindowStatus(window: [number, number], now: number): WindowStatus {
+    let [start, end] = window;
+    if (now < start) {
+        return { icon: "⏳", text: `starts in ${formatTime(start - now)}`, color: FUTURE_COLOR };
+    }
+    if (now >= end) {
+        return { icon: "✔", text: `ended ${formatTime(now - end)} ago`, color: PAST_COLOR };
+    }
+    if (end >= FULL_VALID_WINDOW[1]) {
+        return { icon: "●", text: "active", color: ACTIVE_COLOR };
+    }
+    return { icon: "●", text: `active, ends in ${formatTime(end - now)}`, color: ACTIVE_COLOR };
+}
+
+function windowsOverlap(a: [number, number], b: [number, number]): boolean {
+    return a[0] < b[1] && b[0] < a[1];
+}
+
+type WindowCluster = {
+    window: [number, number];
+    sources: Source[];
+};
+
+/** Sources whose valid windows overlap describe one configuration of the bucket, so they merge into a single displayed window. Sources keep their config order within a cluster - the order sources are listed in decides which one shadows which. */
+function getWindowClusters(sources: Source[]): WindowCluster[] {
+    let byStart = [...sources];
+    sort(byStart, x => x.validWindow[0]);
+    let windows: [number, number][] = [];
+    for (let source of byStart) {
+        let last = windows[windows.length - 1];
+        if (last && windowsOverlap(last, source.validWindow)) {
+            last[0] = Math.min(last[0], source.validWindow[0]);
+            last[1] = Math.max(last[1], source.validWindow[1]);
+            continue;
+        }
+        windows.push([...source.validWindow]);
+    }
+    let clusters: WindowCluster[] = windows.map(window => ({ window, sources: [] }));
+    for (let source of sources) {
+        let cluster = clusters.find(x => windowsOverlap(x.window, source.validWindow));
+        if (!cluster) continue;
+        cluster.sources.push(source);
+    }
+    return clusters;
+}
+
+export type ConfigVariant = {
+    servers: string[];
+    sources: Source[];
+    version?: number;
+    rawConfig: RemoteConfig;
+};
+
+export type ConfigConflict = {
+    bucketName: string;
+    variant: ConfigVariant;
+};
+
+export type RouteConfigGroup = {
+    buckets: { bucketName: string; rawConfig: RemoteConfig }[];
+    consensus: ConfigVariant;
+    conflicts: ConfigConflict[];
+};
+
+/** Groups every server's view of each bucket's routing config, taking the most widely reported one as the truth, surfacing the rest as conflicts, then collapsing buckets that share a config. */
+export function getRouteConfigGroups(servers: StorageServerBuckets[]): RouteConfigGroup[] {
+    let byBucket = new Map<string, Map<string, ConfigVariant>>();
+    for (let server of servers) {
+        for (let bucket of server.buckets || []) {
+            let remoteConfig: RemoteConfig | undefined = bucket.config?.remoteConfig;
+            if (!remoteConfig) continue;
+            let sources = remoteConfig.sources.map(normalizeSource);
+            let variants = byBucket.get(bucket.bucketName);
+            if (!variants) {
+                variants = new Map();
+                byBucket.set(bucket.bucketName, variants);
+            }
+            let key = JSON.stringify({ version: remoteConfig.version, sources });
+            let variant = variants.get(key);
+            if (!variant) {
+                variant = { servers: [], sources, version: remoteConfig.version, rawConfig: remoteConfig };
+                variants.set(key, variant);
+            }
+            variant.servers.push(server.url);
+        }
+    }
+
+    let groups = new Map<string, RouteConfigGroup>();
+    for (let [bucketName, variants] of byBucket) {
+        let all = [...variants.values()];
+        // Ties keep insertion order, so the first server to report a config wins
+        let consensus = all.reduce((best, x) => x.servers.length > best.servers.length && x || best);
+        let conflicts = all.filter(x => x !== consensus).map(variant => ({ bucketName, variant }));
+        let key = getVariantKey(consensus, bucketName);
+        let group = groups.get(key);
+        if (!group) {
+            group = { buckets: [], consensus, conflicts: [] };
+            groups.set(key, group);
+        }
+        group.buckets.push({ bucketName, rawConfig: consensus.rawConfig });
+        for (let server of consensus.servers) {
+            if (group.consensus.servers.includes(server)) continue;
+            group.consensus.servers.push(server);
+        }
+        group.conflicts.push(...conflicts);
+    }
+
+    let results = [...groups.values()];
+    for (let group of results) {
+        sort(group.buckets, x => x.bucketName);
+    }
+    sort(results, x => -getSortVersion(x.consensus));
+    return results;
+}
+
+function showConfigModal(bucketName: string, rawConfig: RemoteConfig): void {
+    let close = showModal({
+        content: <FullscreenModal onCancel={() => close.close()}>
+            <div className={css.vbox(10)}>
+                <div className={css.boldStyle}>{bucketName}</div>
+                <div className={css.fontFamily("monospace").whiteSpace("pre").overflow("auto")}>
+                    {JSON.stringify(rawConfig, undefined, JSON_INDENT)}
+                </div>
+            </div>
+        </FullscreenModal>
+    });
+}
+
+class Tag extends qreact.Component<{ icon: string; text: string; warning?: boolean; color?: { h: number; s: number; l: number } }> {
+    render() {
+        let color = this.props.color || this.props.warning && WARNING_COLOR || TAG_COLOR;
+        return <div className={
+            css.hbox(4).pad2(6, 1).fontSize(TAG_FONT_SIZE).whiteSpace("nowrap")
+                .hsl(color.h, color.s, color.l).bord2(color.h, color.s, color.l - 20)
+        }>
+            <span>{this.props.icon}</span>
+            <span>{this.props.text}</span>
+        </div>;
+    }
+}
+
+function getSourceTags(source: Source): { icon: string; text: string }[] {
+    let tags: { icon: string; text: string }[] = [];
+    if (source.public) tags.push({ icon: "🌐", text: "public" });
+    if (source.immutable) tags.push({ icon: "🔒", text: "immutable" });
+    if (source.noFullSync) tags.push({ icon: "⇣", text: "noFullSync" });
+    if (source.intermediate) tags.push({ icon: "⏳", text: "intermediate" });
+    if (source.type === "remote") {
+        if (source.fast) tags.push({ icon: "⚡", text: "fast" });
+        if (source.rawDisk) tags.push({ icon: "💽", text: "rawDisk" });
+        if (source.writeDelay !== undefined) tags.push({ icon: "⏱", text: `writeDelay ${formatTime(source.writeDelay)}` });
+    }
+    if (source.readerDiskLimit !== undefined) tags.push({ icon: "📦", text: `readCache ${formatNumber(source.readerDiskLimit)}B` });
+    return tags;
+}
+
+class SourceRow extends qreact.Component<{ source: Source; window: [number, number]; ownBucketName: string }> {
+    render() {
+        let { source, window, ownBucketName } = this.props;
+        let [routeStart, routeEnd] = source.route || FULL_ROUTE;
+        let [start, end] = source.validWindow;
+        // A source that stops short of the window it's displayed under has a gap where the bucket has no configuration
+        let partial = start > window[0] || end < window[1];
+        return <div className={css.fillWidth}>
+            <div className={
+                css.marginLeft(`${routeStart * 100}%`).width(`${(routeEnd - routeStart) * 100}%`)
+                    .hbox(6).pad2(8, 4).boxSizing("border-box").whiteSpace("nowrap")
+                    .hsl(0, 0, 100).bord2(SOURCE_BORDER_COLOR.h, SOURCE_BORDER_COLOR.s, SOURCE_BORDER_COLOR.l)
+            }>
+                <div className={css.boldStyle}>{describeUrl(source, ownBucketName)}</div>
+                {source.route && <div className={css.fontSize(TAG_FONT_SIZE).colorhsl(0, 0, 40)}>
+                    route {routeStart} – {routeEnd}
+                </div>}
+                {getSourceTags(source).map(tag => <Tag key={tag.text} icon={tag.icon} text={tag.text} />)}
+                {partial && <Tag
+                    icon="⚠"
+                    warning={true}
+                    text={`only valid ${formatWindowTime(start)} → ${formatWindowTime(end)}`}
+                />}
+            </div>
+        </div>;
+    }
+}
+
+class ConflictWarning extends qreact.Component<{ consensus: ConfigVariant; conflict: ConfigConflict }> {
+    render() {
+        let { consensus, conflict } = this.props;
+        let { bucketName, variant } = conflict;
+        let consensusKeys = new Set(consensus.sources.map(x => getSourceKey(x, bucketName)));
+        let conflictKeys = new Set(variant.sources.map(x => getSourceKey(x, bucketName)));
+        let extra = variant.sources.filter(x => !consensusKeys.has(getSourceKey(x, bucketName)));
+        let missing = consensus.sources.filter(x => !conflictKeys.has(getSourceKey(x, bucketName)));
+        let describeDifference = (source: Source, label: string) => <div key={label + getSourceKey(source, bucketName)} className={css.hbox(6).wrap}>
+            <div className={css.boldStyle}>{label}</div>
+            <div>{describeUrl(source, bucketName)}</div>
+            <div className={css.fontSize(TAG_FONT_SIZE)}>
+                route {(source.route || FULL_ROUTE).join(" – ")}, valid {formatWindowTime(source.validWindow[0])} → {formatWindowTime(source.validWindow[1])}
+            </div>
+        </div>;
+        return <div className={css.vbox(6).pad2(10, 8).hsl(WARNING_COLOR.h, WARNING_COLOR.s, WARNING_COLOR.l).bord2(WARNING_COLOR.h, WARNING_COLOR.s, WARNING_COLOR.l - 25)}>
+            <div className={css.boldStyle}>
+                ⚠ {bucketName}: {variant.servers.length} server{variant.servers.length === 1 && "" || "s"} report a different routing config (version {String(variant.version ?? "none")}, consensus is version {String(consensus.version ?? "none")})
+            </div>
+            <div className={css.vbox(2).fontSize(TAG_FONT_SIZE)}>
+                {variant.servers.map(server => <div key={server}>{server}</div>)}
+            </div>
+            {extra.map(source => describeDifference(source, "only there:"))}
+            {missing.map(source => describeDifference(source, "missing there:"))}
+        </div>;
+    }
+}
+
+class RouteConfigGroupView extends qreact.Component<{ group: RouteConfigGroup }> {
+    render() {
+        let { buckets, consensus, conflicts } = this.props.group;
+        let ownBucketName = buckets[0].bucketName;
+        let clusters = getWindowClusters(consensus.sources);
+        let now = Querysub.timeDelayed(TIME_REFRESH_INTERVAL);
+        return <div className={css.vbox(10).fillWidth}>
+            <div className={css.hbox(8).wrap.fontSize(WINDOW_HEADER_SIZE)}>
+                <div className={css.hbox(4).wrap}>
+                    {buckets.map((bucket, index) => <div key={bucket.bucketName} className={css.hbox(4)}>
+                        {index > 0 && <div className={css.colorhsl(0, 0, 60)}>|</div>}
+                        <Button
+                            flavor="tiny"
+                            className={css.fontSize(BUCKET_TITLE_SIZE)}
+                            onClick={() => showConfigModal(bucket.bucketName, bucket.rawConfig)}
+                        >
+                            {bucket.bucketName}
+                        </Button>
+                    </div>)}
+                </div>
+                <div className={css.colorhsl(0, 0, 45)}>
+                    version {String(consensus.version ?? "none")}, from {consensus.servers.length} server{consensus.servers.length === 1 && "" || "s"}
+                </div>
+            </div>
+            <div className={css.vbox(10).fillWidth.paddingLeft(GROUP_BODY_INDENT).boxSizing("border-box")}>
+                {conflicts.map((conflict, index) => <ConflictWarning key={index} consensus={consensus} conflict={conflict} />)}
+                {clusters.map((cluster, index) => {
+                    let status = getWindowStatus(cluster.window, now);
+                    return <div key={index} className={css.vbox(6).fillWidth}>
+                        <div className={css.hbox(6).wrap.colorhsl(0, 0, 35).fontSize(TAG_FONT_SIZE)}>
+                            <div>{formatWindowTime(cluster.window[0])} → {formatWindowTime(cluster.window[1])}</div>
+                            <Tag icon={status.icon} text={status.text} color={status.color} />
+                        </div>
+                        {cluster.sources.map(source => <SourceRow
+                            key={getSourceKey(source, ownBucketName)}
+                            source={source}
+                            window={cluster.window}
+                            ownBucketName={ownBucketName}
+                        />)}
+                    </div>;
+                })}
+            </div>
+        </div>;
+    }
+}
+
+export class RouteConfigView extends qreact.Component<{ servers: StorageServerBuckets[] }> {
+    render() {
+        let groups = getRouteConfigGroups(this.props.servers);
+        if (!groups.length) return undefined;
+        return <div className={css.vbox(20).fillWidth}>
+            <h3>Routing configs</h3>
+            {groups.map((group, index) => <RouteConfigGroupView key={index} group={group} />)}
+        </div>;
+    }
+}

package/src/deployManager/components/StoragePage.tsx

@@ -0,0 +1,265 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { qreact } from "../../4-dom/qreact";
+import { css } from "typesafecss";
+import { formatNumber } from "socket-function/src/formatting/format";
+import { sort } from "socket-function/src/misc";
+import { listServerBuckets } from "sliftutils/storage/remoteStorage/createArchives";
+import type { ServerBucketInfo } from "sliftutils/storage/remoteStorage/storageServerState";
+import { getSyncedController } from "../../library-components/SyncedController";
+import { assertIsManagementUser } from "../../diagnostics/managementPages";
+import { getEffectiveServiceConfigs, getLiveServiceParameters, getMachineTargets, applyCommandTemplate, ServiceParameters } from "../machineSchema";
+import { STORAGE_ACCOUNT } from "../../-a-archives/archives2";
+import { Table } from "../../5-diagnostics/Table";
+import { RouteConfigView } from "./RouteConfigView";
+
+module.hotreload = true;
+
+const STORAGE_COMMAND_PREFIX = "yarn storageserve";
+const URL_ARG_REGEX = /--url\s+(\S+)/g;
+const ERROR_TEXT_LIMIT = 500;
+
+export type StorageServerBuckets = {
+    serviceKey: string;
+    url: string;
+    buckets?: ServerBucketInfo[];
+    error?: string;
+};
+
+function getStorageUrls(parameters: ServiceParameters): string[] {
+    if (!parameters.command.trimStart().startsWith(STORAGE_COMMAND_PREFIX)) return [];
+    let urls: string[] = [];
+    for (let target of getMachineTargets(parameters)) {
+        let command = applyCommandTemplate(parameters.command, target.variables);
+        for (let match of command.matchAll(URL_ARG_REGEX)) {
+            urls.push(match[1]);
+        }
+    }
+    return urls;
+}
+
+class StoragePageControllerBase {
+    public async getStorageBuckets(): Promise<StorageServerBuckets[]> {
+        let configs = await getEffectiveServiceConfigs();
+        let serviceKeyByUrl = new Map<string, string>();
+        for (let config of configs) {
+            let allParameters = [getLiveServiceParameters(config), config.parameters, config.oldParameters];
+            for (let parameters of allParameters) {
+                if (!parameters) continue;
+                for (let url of getStorageUrls(parameters)) {
+                    if (serviceKeyByUrl.has(url)) continue;
+                    serviceKeyByUrl.set(url, parameters.key);
+                }
+            }
+        }
+        return await Promise.all([...serviceKeyByUrl].map(async ([url, serviceKey]) => {
+            try {
+                return { serviceKey, url, buckets: await listServerBuckets({ url, account: STORAGE_ACCOUNT }) };
+            } catch (e) {
+                return { serviceKey, url, error: String((e as Error).stack ?? e).slice(0, ERROR_TEXT_LIMIT) };
+            }
+        }));
+    }
+}
+
+export const StoragePageController = SocketFunction.register(
+    "StoragePageController-4f1c93ab-77e2-4d15-9a30-1c6b8f5d2e04",
+    new StoragePageControllerBase(),
+    () => ({
+        getStorageBuckets: {},
+    }),
+    () => ({
+        hooks: [assertIsManagementUser],
+    }),
+    {
+    }
+);
+
+const StorageSynced = getSyncedController(StoragePageController, {
+    reads: {},
+    writes: {},
+});
+
+type BucketConfig = NonNullable<ServerBucketInfo["config"]>;
+
+type BucketRow = {
+    server: string;
+    bucket: string;
+    state: string;
+    files: string;
+    bytes: string;
+    indexSources: BucketConfig["indexSources"];
+    readerDiskLimit: string;
+    syncing: BucketConfig["syncing"];
+    error: string;
+};
+
+function getBucketRows(servers: StorageServerBuckets[]): BucketRow[] {
+    let rows: BucketRow[] = [];
+    for (let server of servers) {
+        let baseRow: BucketRow = {
+            server: server.url,
+            bucket: "",
+            state: "",
+            files: "",
+            bytes: "",
+            indexSources: undefined,
+            readerDiskLimit: "",
+            syncing: undefined,
+            error: "",
+        };
+        if (server.error) {
+            rows.push({ ...baseRow, error: server.error });
+            continue;
+        }
+        let buckets = [...server.buckets || []];
+        sort(buckets, x => x.bucketName);
+        if (!buckets.length) {
+            rows.push({ ...baseRow, bucket: "No buckets" });
+            continue;
+        }
+        for (let [index, bucket] of buckets.entries()) {
+            let config = bucket.config;
+            rows.push({
+                ...baseRow,
+                server: index === 0 && server.url || "",
+                bucket: bucket.bucketName,
+                state: bucket.active && "active" || "inactive",
+                files: config?.index && formatNumber(config.index.fileCount) || "",
+                bytes: config?.index && formatNumber(config.index.byteCount) + "B" || "",
+                indexSources: config?.indexSources,
+                readerDiskLimit: config?.readerDiskLimit && formatNumber(config.readerDiskLimit) + "B" || "",
+                syncing: config?.syncing,
+                error: bucket.error || "",
+            });
+        }
+    }
+    return rows;
+}
+
+const SOURCE_BAR_COLOR = { h: 210, s: 65, l: 55 };
+const REMOTE_SOURCE_BAR_COLOR = { h: 0, s: 75, l: 55 };
+const SOURCE_BAR_OPACITY = 0.3;
+const SOURCE_ROW_PADDING = { horizontal: 4, vertical: 3 };
+const DISK_SOURCE_TYPE = "disk";
+const REMOTE_SOURCE_TOOLTIP = "Values stored on another server. If this other server goes down, we'll lose access to the data.";
+const SOURCE_TYPE_COLOR = { h: 265, s: 40, l: 88 };
+const SOURCE_PART_COLOR = { h: 0, s: 0, l: 96 };
+const SOURCE_PART_FONT_SIZE = 11;
+
+const DEBUG_NAME_PATTERNS: { regex: RegExp; parts: string[] }[] = [
+    { regex: /^remoteStorage (\S+) account (\S+) bucket (.+)$/, parts: ["remoteStorage"] },
+    { regex: /^localBucket account (\S+) bucket (.+)$/, parts: ["localBucket"] },
+    { regex: /^backblaze (.+)$/, parts: ["backblaze"] },
+    { regex: /^disk (.+)$/, parts: ["disk"] },
+];
+
+/** Splits an IArchives debug name into its type and the identifying pieces after it, so each piece can be boxed separately. */
+function parseSourceDebugName(debugName: string): { type: string; parts: string[] } {
+    for (let pattern of DEBUG_NAME_PATTERNS) {
+        let match = pattern.regex.exec(debugName);
+        if (!match) continue;
+        return { type: pattern.parts[0], parts: match.slice(1) };
+    }
+    return { type: debugName, parts: [] };
+}
+
+class SourceNameParts extends qreact.Component<{ debugName: string }> {
+    render() {
+        let { type, parts } = parseSourceDebugName(this.props.debugName);
+        let partStyle = css.pad2(5, 0).fontSize(SOURCE_PART_FONT_SIZE).whiteSpace("nowrap");
+        return <div className={css.hbox(3).wrap}>
+            <div className={partStyle.hsl(SOURCE_TYPE_COLOR.h, SOURCE_TYPE_COLOR.s, SOURCE_TYPE_COLOR.l).bord2(SOURCE_TYPE_COLOR.h, SOURCE_TYPE_COLOR.s, SOURCE_TYPE_COLOR.l - 20)}>
+                {type}
+            </div>
+            {parts.map(part => <div
+                key={part}
+                className={partStyle.hsl(SOURCE_PART_COLOR.h, SOURCE_PART_COLOR.s, SOURCE_PART_COLOR.l).bord2(SOURCE_PART_COLOR.h, SOURCE_PART_COLOR.s, SOURCE_PART_COLOR.l - 20)}
+            >
+                {part}
+            </div>)}
+        </div>;
+    }
+}
+
+class IndexSourcesCell extends qreact.Component<{ sources: BucketConfig["indexSources"] }> {
+    render() {
+        let sources = [...this.props.sources || []];
+        if (!sources.length) return undefined;
+        sort(sources, x => -x.byteCount);
+        let totalBytes = sources.reduce((sum, x) => sum + x.byteCount, 0);
+        return <div className={css.vbox(2).fillWidth}>
+            {sources.map(source => {
+                let fraction = totalBytes && source.byteCount / totalBytes || 0;
+                // Only the server's own disk holds bytes we can't lose to another machine going down
+                let remote = parseSourceDebugName(source.debugName).type !== DISK_SOURCE_TYPE;
+                let barColor = remote && REMOTE_SOURCE_BAR_COLOR || SOURCE_BAR_COLOR;
+                return <div
+                    key={source.debugName}
+                    className={css.relative.fillWidth.pad2(SOURCE_ROW_PADDING.horizontal, SOURCE_ROW_PADDING.vertical)}
+                    title={remote && REMOTE_SOURCE_TOOLTIP || undefined}
+                >
+                    <div className={
+                        css.absolute.top(0).left(0).size(`${fraction * 100}%`, "100%")
+                            .hsla(barColor.h, barColor.s, barColor.l, SOURCE_BAR_OPACITY)
+                    } />
+                    <div className={css.relative.hbox(6).wrap}>
+                        <div className={css.whiteSpace("nowrap")}>
+                            {formatNumber(source.byteCount)}B, {formatNumber(source.fileCount)} files
+                        </div>
+                        <SourceNameParts debugName={source.debugName} />
+                    </div>
+                </div>;
+            })}
+        </div>;
+    }
+}
+
+export class StoragePage extends qreact.Component {
+    render() {
+        let servers = StorageSynced(SocketFunction.browserNodeId()).getStorageBuckets();
+        if (!servers) return <div>Loading storage servers...</div>;
+        if (!servers.length) {
+            return <div className={css.vbox(16)}>
+                <h2>Storage</h2>
+                <div>No services with a command starting with {JSON.stringify(STORAGE_COMMAND_PREFIX)} were found.</div>
+            </div>;
+        }
+        return <div className={css.vbox(16)}>
+            <div className={css.hbox(12)}>
+                <h2 className={css.flexGrow(1)}>Storage</h2>
+                <button className={css.pad2(12, 8).button.bord2(0, 0, 20).hsl(0, 0, 100)}
+                    onClick={() => {
+                        StorageSynced(SocketFunction.browserNodeId()).getStorageBuckets.resetAll();
+                    }}>
+                    Refresh
+                </button>
+            </div>
+            <Table
+                rows={getBucketRows(servers)}
+                columns={{
+                    server: { title: "Server" },
+                    bucket: { title: "Bucket" },
+                    state: { title: "State" },
+                    files: { title: "Files" },
+                    bytes: { title: "Bytes" },
+                    indexSources: {
+                        title: "Index sources",
+                        formatter: sources => <IndexSourcesCell sources={sources} />
+                    },
+                    readerDiskLimit: { title: "Read cache limit" },
+                    syncing: {
+                        title: "Syncing",
+                        formatter: syncing => <div className={css.vbox(2)}>
+                            {(syncing || []).map((activity, activityIndex) => <div key={activityIndex}>
+                                {activity.type} {activity.sourceDebugName}
+                                {activity.totalFiles !== undefined && ` (${formatNumber(activity.doneFiles || 0)}/${formatNumber(activity.totalFiles)} files)`}
+                            </div>)}
+                        </div>
+                    },
+                    error: { title: "Error" },
+                }}
+            />
+            <RouteConfigView servers={servers} />
+        </div>;
+    }
+}

package/src/deployManager/urlParams.ts

@@ -1,6 +1,6 @@
 import { URLParam } from "../library-components/URLParam";
 
-export type ViewType = "services" | "machines" | "service-detail" | "machine-detail" | "deploy";
+export type ViewType = "services" | "machines" | "service-detail" | "machine-detail" | "deploy" | "storage";
 
 export const currentViewParam = new URLParam<ViewType>("machineview", "machines");
 export const selectedServiceIdParam = new URLParam("serviceId", "");

package/src/misc/random.ts

@@ -39,7 +39,7 @@ export function secureRandom(): number {
     if (!isNode()) {
         window.crypto.getRandomValues(randomData);
     } else {
-        crypto.getRandomValues(randomData);
+        crypto.getRandomValues(randomData as any);
     }
     return randomDataF64[0];
 }