querysub 0.554.0 → 0.555.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.554.0",
+  "version": "0.555.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/deployManager/components/RouteConfigView.tsx

@@ -1,8 +1,9 @@
 import { qreact } from "../../4-dom/qreact";
 import { Querysub } from "../../4-querysub/Querysub";
+import { t } from "../../2-proxy/schema2";
 import { css } from "typesafecss";
 import { sort } from "socket-function/src/misc";
-import { formatNumber, formatDateTime, formatTime } from "socket-function/src/formatting/format";
+import { formatNumber, formatDateTime, formatTime, formatDateTimeDetailed } from "socket-function/src/formatting/format";
 import { parseHostedUrl, parseBackblazeUrl } from "sliftutils/storage/remoteStorage/remoteConfig";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { FULL_VALID_WINDOW, FULL_ROUTE } from "sliftutils/storage/IArchives";
@@ -28,8 +29,13 @@ const TIME_REFRESH_INTERVAL = 60 * 1000;
 const WATCH_POLL_INTERVAL = 60 * 1000;
 const WATCH_ACTIVE_COLOR = { h: 195, s: 60, l: 85 };
 const WINDOW_HEADER_SIZE = 13;
+const WINDOW_TIME_SIZE = 14;
+const INACTIVE_WINDOW_OPACITY = 0.5;
+const EXCESS_COLOR = { h: 265, s: 40, l: 88 };
+const WINDOW_TIME_LIGHTNESS = 12;
 const BUCKET_TITLE_SIZE = 15;
-const GROUP_BODY_INDENT = 16;
+const GROUP_BODY_INDENT = 64;
+const WINDOW_BODY_INDENT = 64;
 const TAG_FONT_SIZE = 11;
 const JSON_INDENT = 4;
 
@@ -82,6 +88,11 @@ function formatWindowTime(time: number): string {
     if (time >= FULL_VALID_WINDOW[1]) return "forever";
     return formatDateTime(time);
 }
+function formatWindowTimeDetailed(time: number): string {
+    if (time <= FULL_VALID_WINDOW[0]) return "the beginning";
+    if (time >= FULL_VALID_WINDOW[1]) return "forever";
+    return formatDateTimeDetailed(time);
+}
 
 // A missing version sorts below any explicit one, matching getConfigVersion
 function getSortVersion(variant: ConfigVariant): number {
@@ -117,27 +128,34 @@ type WindowCluster = {
     sources: Source[];
 };
 
-/** Sources whose valid windows overlap describe one configuration of the bucket, so they merge into a single displayed window. Sources keep their config order within a cluster - the order sources are listed in decides which one shadows which. */
-function getWindowClusters(sources: Source[]): WindowCluster[] {
-    let byStart = [...sources];
-    sort(byStart, x => x.validWindow[0]);
-    let windows: [number, number][] = [];
-    for (let source of byStart) {
-        let last = windows[windows.length - 1];
-        if (last && windowsOverlap(last, source.validWindow)) {
-            last[0] = Math.min(last[0], source.validWindow[0]);
-            last[1] = Math.max(last[1], source.validWindow[1]);
-            continue;
-        }
-        windows.push([...source.validWindow]);
-    }
-    let clusters: WindowCluster[] = windows.map(window => ({ window, sources: [] }));
+/** Every distinct valid window in the config, latest-ending first, each listing every source that overlaps it. A source spanning several windows appears under each of them - what matters is what is valid during a window, not which window a source "belongs" to. Sources keep their config order, since that order decides which one shadows which. */
+function getWindowRanges(sources: Source[]): WindowCluster[] {
+    let byKey = new Map<string, [number, number]>();
     for (let source of sources) {
-        let cluster = clusters.find(x => windowsOverlap(x.window, source.validWindow));
-        if (!cluster) continue;
-        cluster.sources.push(source);
+        let key = source.validWindow.join("|");
+        if (byKey.has(key)) continue;
+        byKey.set(key, [source.validWindow[0], source.validWindow[1]]);
     }
-    return clusters;
+    let windows = [...byKey.values()];
+    // Sorted by start first, so the (stable) sort by end leaves later starts first within one end time
+    sort(windows, x => -x[0]);
+    sort(windows, x => -x[1]);
+    return windows.map(window => ({
+        window,
+        sources: sources.filter(x => windowsOverlap(x.validWindow, window)),
+    }));
+}
+
+/** How far a source reaches past the window it is listed under, on each side. */
+function getWindowExcess(sourceWindow: [number, number], window: [number, number]): { icon: string; text: string }[] {
+    let excess: { icon: string; text: string }[] = [];
+    if (sourceWindow[0] < window[0]) {
+        excess.push({ icon: "←", text: `starts ${formatTime(window[0] - sourceWindow[0])} earlier` });
+    }
+    if (sourceWindow[1] > window[1]) {
+        excess.push({ icon: "→", text: `ends ${formatTime(sourceWindow[1] - window[1])} later` });
+    }
+    return excess;
 }
 
 export type ConfigVariant = {
@@ -218,8 +236,22 @@ type LiveWatch = {
     updatedTime?: number;
 };
 
-// Watches only live as long as the page is open - nothing about them is persisted
-const liveWatchData = Querysub.createLocalSchema<{ watches: { [key: string]: LiveWatch } }>("storageLiveWatches");
+// Watches only live as long as the page is open, so they're plain memory. Only the counter is synchronized - an untyped schema map doesn't report keys being added, which is exactly what the views need to re-render on.
+const liveWatches = new Map<string, LiveWatch>();
+const liveWatchVersion = Querysub.createLocalSchema("storageLiveWatchVersion", {
+    version: t.number,
+});
+
+/** Reading this in a render subscribes it to every watch change. */
+function watchesChanged(): void {
+    Querysub.localCommit(() => liveWatchVersion().version++);
+}
+function readWatches(): [string, LiveWatch][] {
+    // Subscribes the caller, so adding or refreshing a watch re-renders it
+    liveWatchVersion().version;
+    // Copied so the views see a new object identity per change - the stored ones are mutated in place
+    return [...liveWatches].map(([key, watch]) => [key, { ...watch }]);
+}
 
 function getWatchKey(serverUrl: string, bucketName: string): string {
     return `${serverUrl}|${bucketName}`;
@@ -237,18 +269,11 @@ function getServerUrl(source: Source): string | undefined {
 }
 
 async function refreshWatch(key: string): Promise<void> {
-    // Reads outside a synchronized context have to go through localRead, and the plain values are pulled out here so nothing proxied escapes into the async work
-    let target = Querysub.localRead(() => {
-        let watch = liveWatchData().watches[key];
-        if (!watch) return undefined;
-        return { serverUrl: watch.serverUrl, bucketName: watch.bucketName };
-    });
-    if (!target) return;
-    let { serverUrl, bucketName } = target;
-    Querysub.commit(() => {
-        let current = liveWatchData().watches[key];
-        if (current) current.loading = true;
-    });
+    let watch = liveWatches.get(key);
+    if (!watch) return;
+    let { serverUrl, bucketName } = watch;
+    watch.loading = true;
+    watchesChanged();
     try {
         // Proxied through our server, which holds the identity the storage servers authenticate against
         let live = await StorageSynced(SocketFunction.browserNodeId()).getActiveBucket.promise(serverUrl, bucketName);
@@ -260,55 +285,48 @@ async function refreshWatch(key: string): Promise<void> {
         } else {
             result = live.routing;
         }
-        Querysub.commit(() => {
-            let current = liveWatchData().watches[key];
-            // The watch may have been removed while the call was in flight
-            if (!current) return;
+        // The watch may have been removed while the call was in flight
+        let current = liveWatches.get(key);
+        if (current) {
             current.routing = result;
             current.error = error;
             current.updatedTime = Date.now();
-        });
+        }
     } finally {
         // Without this a throw would leave the watch stuck showing "refreshing..."
-        Querysub.commit(() => {
-            let current = liveWatchData().watches[key];
-            if (current) current.loading = false;
-        });
+        let current = liveWatches.get(key);
+        if (current) current.loading = false;
+        watchesChanged();
     }
 }
 
 function isWatched(serverUrl: string, bucketName: string): boolean {
-    return !!liveWatchData().watches[getWatchKey(serverUrl, bucketName)];
+    // Reads the counter so the button re-renders when its own watch is added or removed
+    liveWatchVersion().version;
+    return liveWatches.has(getWatchKey(serverUrl, bucketName));
 }
 
 function toggleWatch(serverUrl: string, bucketName: string): void {
     let key = getWatchKey(serverUrl, bucketName);
-    let existed = Querysub.localRead(() => !!liveWatchData().watches[key]);
-    Querysub.commit(() => {
-        if (existed) {
-            delete liveWatchData().watches[key];
-            return;
-        }
-        liveWatchData().watches[key] = { serverUrl, bucketName, loading: true };
-    });
-    if (existed) return;
+    if (liveWatches.has(key)) {
+        liveWatches.delete(key);
+        watchesChanged();
+        return;
+    }
+    liveWatches.set(key, { serverUrl, bucketName, loading: true });
+    watchesChanged();
     void refreshWatch(key);
 }
 
 function refreshAllWatches(): void {
-    // Fired from a timer, so the key list has to be read through localRead
-    let keys = Querysub.localRead(() => Object.keys(liveWatchData().watches));
-    for (let key of keys) {
+    for (let key of [...liveWatches.keys()]) {
         void refreshWatch(key);
     }
 }
 
 function clearWatches(): void {
-    Querysub.commit(() => {
-        for (let key of Object.keys(liveWatchData().watches)) {
-            delete liveWatchData().watches[key];
-        }
-    });
+    liveWatches.clear();
+    watchesChanged();
 }
 
 function showConfigModal(bucketName: string, rawConfig: RemoteConfig): void {
@@ -324,13 +342,13 @@ function showConfigModal(bucketName: string, rawConfig: RemoteConfig): void {
     });
 }
 
-class Tag extends qreact.Component<{ icon: string; text: string; warning?: boolean; color?: { h: number; s: number; l: number } }> {
+class Tag extends qreact.Component<{ icon: string; text: string; warning?: boolean; color?: { h: number; s: number; l: number }; title?: string; }> {
     render() {
         let color = this.props.color || this.props.warning && WARNING_COLOR || TAG_COLOR;
         return <div className={
             css.hbox(4).pad2(6, 1).fontSize(TAG_FONT_SIZE).whiteSpace("nowrap")
                 .hsl(color.h, color.s, color.l).bord2(color.h, color.s, color.l - 20)
-        }>
+        } title={this.props.title}>
             <span>{this.props.icon}</span>
             <span>{this.props.text}</span>
         </div>;
@@ -370,14 +388,15 @@ class WatchButton extends qreact.Component<{ source: Source; bucketName: string
     }
 }
 
-class SourceRow extends qreact.Component<{ source: Source; window: [number, number]; ownBucketName: string }> {
+class SourceRow extends qreact.Component<{ source: Source; window: [number, number]; ownBucketName: string; now: number }> {
     render() {
-        let { source, window, ownBucketName } = this.props;
+        let { source, window, ownBucketName, now } = this.props;
         let [routeStart, routeEnd] = source.route || FULL_ROUTE;
         let [start, end] = source.validWindow;
-        // A source that stops short of the window it's displayed under has a gap where the bucket has no configuration
-        let partial = start > window[0] || end < window[1];
-        return <div className={css.fillWidth}>
+        let excess = getWindowExcess(source.validWindow, window);
+        // Faded when this source is not valid right now, whichever window it happens to be listed under
+        let active = start <= now && now < end;
+        return <div className={css.fillWidth + (!active && css.opacity(INACTIVE_WINDOW_OPACITY) || "")}>
             <div className={
                 css.marginLeft(`${routeStart * 100}%`).width(`${(routeEnd - routeStart) * 100}%`)
                     .hbox(6).pad2(8, 4).boxSizing("border-box").whiteSpace("nowrap")
@@ -389,16 +408,52 @@ class SourceRow extends qreact.Component<{ source: Source; window: [number, numb
                     route {routeStart} – {routeEnd}
                 </div>}
                 {getSourceTags(source).map(tag => <Tag key={tag.text} icon={tag.icon} text={tag.text} />)}
-                {partial && <Tag
-                    icon="⚠"
-                    warning={true}
-                    text={`only valid ${formatWindowTime(start)} → ${formatWindowTime(end)}`}
-                />}
+                {excess.map(tag => <Tag
+                    key={tag.text}
+                    icon={tag.icon}
+                    text={tag.text}
+                    color={EXCESS_COLOR}
+                    title={`valid ${formatWindowTimeDetailed(start)} → ${formatWindowTimeDetailed(end)}`}
+                />)}
             </div>
         </div>;
     }
 }
 
+/** The config's sources, listed once under every distinct valid window they overlap. */
+class WindowRanges extends qreact.Component<{ sources: Source[]; ownBucketName: string }> {
+    render() {
+        let { sources, ownBucketName } = this.props;
+        let ranges = getWindowRanges(sources);
+        let now = Querysub.timeDelayed(TIME_REFRESH_INTERVAL);
+        return <>
+            {ranges.map((range, index) => {
+                let status = getWindowStatus(range.window, now);
+                return <div key={index} className={css.vbox(6).fillWidth}>
+                    <div className={css.hbox(6).wrap.alignItems("center")}>
+                        <div
+                            className={css.boldStyle.fontSize(WINDOW_TIME_SIZE).colorhsl(0, 0, WINDOW_TIME_LIGHTNESS)}
+                            title={`${formatWindowTimeDetailed(range.window[0])} → ${formatWindowTimeDetailed(range.window[1])}`}
+                        >
+                            {formatWindowTime(range.window[0])} → {formatWindowTime(range.window[1])}
+                        </div>
+                        <Tag icon={status.icon} text={status.text} color={status.color} />
+                    </div>
+                    <div className={css.vbox(6).fillWidth.paddingLeft(WINDOW_BODY_INDENT).boxSizing("border-box")}>
+                        {range.sources.map(source => <SourceRow
+                            key={getSourceKey(source, ownBucketName)}
+                            source={source}
+                            window={range.window}
+                            ownBucketName={ownBucketName}
+                            now={now}
+                        />)}
+                    </div>
+                </div>;
+            })}
+        </>;
+    }
+}
+
 class ConflictWarning extends qreact.Component<{ consensus: ConfigVariant; conflict: ConfigConflict }> {
     render() {
         let { consensus, conflict } = this.props;
@@ -431,15 +486,13 @@ class RouteConfigGroupView extends qreact.Component<{ group: RouteConfigGroup }>
     render() {
         let { buckets, consensus, conflicts } = this.props.group;
         let ownBucketName = buckets[0].bucketName;
-        let clusters = getWindowClusters(consensus.sources);
-        let now = Querysub.timeDelayed(TIME_REFRESH_INTERVAL);
         return <div className={css.vbox(10).fillWidth}>
             <div className={css.hbox(8).wrap.fontSize(WINDOW_HEADER_SIZE)}>
                 <div className={css.hbox(4).wrap}>
                     {buckets.map((bucket, index) => <div key={bucket.bucketName} className={css.hbox(4)}>
                         {index > 0 && <div className={css.colorhsl(0, 0, 60)}>|</div>}
                         <Button
-                            flavor="tiny"
+                            flavor="large"
                             className={css.fontSize(BUCKET_TITLE_SIZE)}
                             onClick={() => showConfigModal(bucket.bucketName, bucket.rawConfig)}
                         >
@@ -453,21 +506,7 @@ class RouteConfigGroupView extends qreact.Component<{ group: RouteConfigGroup }>
             </div>
             <div className={css.vbox(10).fillWidth.paddingLeft(GROUP_BODY_INDENT).boxSizing("border-box")}>
                 {conflicts.map((conflict, index) => <ConflictWarning key={index} consensus={consensus} conflict={conflict} />)}
-                {clusters.map((cluster, index) => {
-                    let status = getWindowStatus(cluster.window, now);
-                    return <div key={index} className={css.vbox(6).fillWidth}>
-                        <div className={css.hbox(6).wrap.colorhsl(0, 0, 35).fontSize(TAG_FONT_SIZE)}>
-                            <div>{formatWindowTime(cluster.window[0])} → {formatWindowTime(cluster.window[1])}</div>
-                            <Tag icon={status.icon} text={status.text} color={status.color} />
-                        </div>
-                        {cluster.sources.map(source => <SourceRow
-                            key={getSourceKey(source, ownBucketName)}
-                            source={source}
-                            window={cluster.window}
-                            ownBucketName={ownBucketName}
-                        />)}
-                    </div>;
-                })}
+                <WindowRanges sources={consensus.sources} ownBucketName={ownBucketName} />
             </div>
         </div>;
     }
@@ -479,11 +518,13 @@ class LiveWatchView extends qreact.Component<{ watchKey: string; watch: LiveWatc
         let { watchKey, watch } = this.props;
         let now = Querysub.timeDelayed(TIME_REFRESH_INTERVAL);
         let sources = (watch.routing?.sources || []).map(normalizeSource);
-        let clusters = getWindowClusters(sources);
         return <div className={css.vbox(10).fillWidth.pad2(10, 8).bord2(WATCH_ACTIVE_COLOR.h, WATCH_ACTIVE_COLOR.s, WATCH_ACTIVE_COLOR.l - 25)}>
             <div className={css.hbox(8).wrap.fontSize(WINDOW_HEADER_SIZE)}>
                 <div className={css.boldStyle}>👁 {watch.bucketName}</div>
                 <div className={css.colorhsl(0, 0, 45)}>live on {watch.serverUrl}</div>
+                {watch.routing && <div className={css.colorhsl(0, 0, 45)}>
+                    version {String(watch.routing.version ?? "none")}
+                </div>}
                 <div className={css.colorhsl(0, 0, 45)}>
                     {watch.loading && "refreshing..."
                         || watch.updatedTime && `updated ${formatTime(now - watch.updatedTime)} ago`
@@ -497,21 +538,7 @@ class LiveWatchView extends qreact.Component<{ watchKey: string; watch: LiveWatc
                 ⚠ {watch.error}
             </div>}
             <div className={css.vbox(10).fillWidth.paddingLeft(GROUP_BODY_INDENT).boxSizing("border-box")}>
-                {clusters.map((cluster, index) => {
-                    let status = getWindowStatus(cluster.window, now);
-                    return <div key={index} className={css.vbox(6).fillWidth}>
-                        <div className={css.hbox(6).wrap.colorhsl(0, 0, 35).fontSize(TAG_FONT_SIZE)}>
-                            <div>{formatWindowTime(cluster.window[0])} → {formatWindowTime(cluster.window[1])}</div>
-                            <Tag icon={status.icon} text={status.text} color={status.color} />
-                        </div>
-                        {cluster.sources.map(source => <SourceRow
-                            key={getSourceKey(source, watch.bucketName)}
-                            source={source}
-                            window={cluster.window}
-                            ownBucketName={watch.bucketName}
-                        />)}
-                    </div>;
-                })}
+                <WindowRanges sources={sources} ownBucketName={watch.bucketName} />
             </div>
         </div>;
     }
@@ -532,7 +559,7 @@ export class RouteConfigView extends qreact.Component<{ servers: StorageServerBu
     }
     render() {
         let groups = getRouteConfigGroups(this.props.servers);
-        let watches = Object.entries(liveWatchData().watches);
+        let watches = readWatches();
         if (!groups.length && !watches.length) return undefined;
         return <div className={css.vbox(20).fillWidth}>
             {watches.length > 0 && <div className={css.vbox(10).fillWidth}>

package/src/deployManager/components/ServiceDetailPage.tsx

@@ -28,7 +28,9 @@ import { getOwnThreadId } from "../../-f-node-discovery/NodeDiscovery";
 import { decodeNodeId } from "sliftutils/misc/https/certs";
 import { showModal } from "../../5-diagnostics/Modal";
 
-
+// Trimmed well below the limit, so trimming is rare - each trim jumps the relative scroll position
+const OUTPUT_BUFFER_LIMIT = 1_000_000;
+const OUTPUT_BUFFER_KEPT = 100_000;
 
 export class ServiceDetailPage extends qreact.Component {
     state = t.state({
@@ -43,6 +45,8 @@ export class ServiceDetailPage extends qreact.Component {
             isWatching: t.type(false),
             data: t.type(""),
             callbackId: t.type(""),
+            // The launch the buffered output belongs to. A new launch is a new process, so its output must not be appended to the previous one's.
+            launchTime: t.number(0),
         }),
         // Milliseconds; 0 means no scheduled time picked yet (use Deploy Now instead)
         switchTime: t.number(0),
@@ -66,37 +70,43 @@ export class ServiceDetailPage extends qreact.Component {
         nodeId: string;
         key: string;
         index: number;
+        launchTime: number;
     }) {
-        const { nodeId, key, index } = config;
+        const { nodeId, key, index, launchTime } = config;
         const outputKey = getPathStr2(key, index + "");
         let callbackId = nextId();
 
+        // Drop the previous watch first: two live callbacks writing to one buffer is what interlaced the output of the old and new processes
+        let previousCallbackId = Querysub.localRead(() => this.state.watchingOutputs[outputKey].callbackId);
+        if (previousCallbackId) {
+            await stopWatchingScreenOutput({ callbackId: previousCallbackId });
+        }
+
         Querysub.commit(() => {
-            this.state.watchingOutputs[outputKey] = { isWatching: true, data: "", callbackId };
+            // Cleared, so the buffer only ever holds the output of the launch it is watching
+            this.state.watchingOutputs[outputKey] = { isWatching: true, data: "", callbackId, launchTime };
         });
 
-        try {
-            await watchScreenOutput({
-                nodeId,
-                key,
-                index,
-                callbackId,
-                onData: async (data: string) => {
-                    Querysub.localCommit(() => {
-                        let fullData = this.state.watchingOutputs[outputKey].data + data;
-                        // Don't trim every time, otherwise the relative scroll position changes by too much
-                        if (fullData.length > 1_000_000) {
-                            fullData = fullData.slice(-100_000);
-                        }
-                        this.state.watchingOutputs[outputKey].data = fullData;
-                    });
-                }
-            });
-        } catch (error: any) {
-            Querysub.localCommit(() => {
-                this.state.watchingOutputs[outputKey].data = `Error watching output: ${error.stack || String(error)}`;
-            });
-        }
+        await watchScreenOutput({
+            nodeId,
+            key,
+            index,
+            callbackId,
+            onData: async (data: string, dataConfig?: { reset?: boolean }) => {
+                Querysub.localCommit(() => {
+                    let watchingState = this.state.watchingOutputs[outputKey];
+                    // A callback that outlived its watch (a restart raced with in-flight data) must not write into the new process's buffer
+                    if (watchingState.callbackId !== callbackId) return;
+                    // The screen's process changed, so what we have belongs to a process that is gone
+                    let fullData = (dataConfig?.reset && "" || watchingState.data) + data;
+                    // Don't trim every time, otherwise the relative scroll position changes by too much
+                    if (fullData.length > OUTPUT_BUFFER_LIMIT) {
+                        fullData = fullData.slice(-OUTPUT_BUFFER_KEPT);
+                    }
+                    watchingState.data = fullData;
+                });
+            }
+        });
     }
 
     private async stopWatchingOutput(key: string, index: number) {
@@ -104,8 +114,11 @@ export class ServiceDetailPage extends qreact.Component {
 
         let callbackId = Querysub.localRead(() => {
             const watchingState = this.state.watchingOutputs[outputKey];
+            let previousCallbackId = watchingState.callbackId;
             watchingState.isWatching = false;
-            return watchingState.callbackId;
+            // Clearing this stops any in-flight data from landing in the buffer, and marks that there is no watch to stop next time
+            watchingState.callbackId = "";
+            return previousCallbackId;
         });
 
         Querysub.onCommitFinished(async () => {
@@ -427,6 +440,8 @@ export class ServiceDetailPage extends qreact.Component {
                         let outputData = this.state.watchingOutputs[outputKey].data;
                         const screenName = getScreenName({ serviceKey: key, index });
 
+                        let launchTime = serviceInfo?.lastLaunchedTime || 0;
+
                         return <div key={machineId}
                             className={css.pad2(12).vbox(10).bord2(0, 0, 20).fillWidth + backgroundColor}
                         >
@@ -505,7 +520,7 @@ export class ServiceDetailPage extends qreact.Component {
                                             if (isWatching) {
                                                 void this.stopWatchingOutput(key, index);
                                             } else {
-                                                void this.startWatchingOutput({ nodeId: applyNodeId, key, index });
+                                                void this.startWatchingOutput({ nodeId: applyNodeId, key, index, launchTime });
                                             }
                                         });
                                     }}

package/src/deployManager/components/StoragePage.tsx

@@ -7,7 +7,7 @@ import preact from "preact";
 import { Querysub } from "../../4-querysub/Querysub";
 import { listServerBuckets, clearServerWriteStats, activateServerBucket, getServerActiveBucket } from "sliftutils/storage/remoteStorage/createArchives";
 import type { ServerBucketInfo, BucketDiskInfo, BucketWriteStats, ActiveBucketInfo } from "sliftutils/storage/remoteStorage/storageServerState";
-import type { ArchivesConfig } from "sliftutils/storage/IArchives";
+import type { ArchivesConfig, SyncActivity } from "sliftutils/storage/IArchives";
 import { UsageBar, getUsageThresholds } from "../../library-components/UsageBar";
 import { getSyncedController } from "../../library-components/SyncedController";
 import { assertIsManagementUser } from "../../diagnostics/managementPages";
@@ -347,6 +347,66 @@ class IndexSourcesCell extends qreact.Component<{ sources: ArchivesConfig["index
     }
 }
 
+const SYNC_PROGRESS_COLOR = { h: 130, s: 60, l: 50 };
+const SYNC_PROGRESS_OPACITY = 0.25;
+const SYNC_TYPE_COLORS: { [type in SyncActivity["type"]]: { h: number; s: number; l: number } } = {
+    metadataScan: { h: 265, s: 40, l: 88 },
+    fullSync: { h: 205, s: 50, l: 85 },
+};
+const SYNC_DETAIL_FONT_SIZE = 11;
+const PERCENT_DECIMALS = 0;
+
+/** How far along a scan is, by files when it knows the file count and by bytes otherwise. Undefined until the total is known - a scan that hasn't finished listing has no denominator yet. */
+function getSyncFraction(activity: SyncActivity): number | undefined {
+    if (activity.totalFiles) return (activity.doneFiles || 0) / activity.totalFiles;
+    if (activity.totalBytes) return (activity.doneBytes || 0) / activity.totalBytes;
+    return undefined;
+}
+
+class SyncingCell extends qreact.Component<{ syncing: ArchivesConfig["syncing"] }> {
+    render() {
+        let syncing = this.props.syncing || [];
+        if (!syncing.length) return undefined;
+        let now = Querysub.nowDelayed(timeInSecond);
+        return <div className={css.vbox(3).fillWidth}>
+            {syncing.map((activity, activityIndex) => {
+                let fraction = getSyncFraction(activity);
+                let elapsed = now - activity.startTime;
+                let typeColor = SYNC_TYPE_COLORS[activity.type];
+                // Only meaningful once something has actually been done, otherwise the rate is a divide by zero
+                let remaining = fraction && fraction > 0 && elapsed * (1 - fraction) / fraction || undefined;
+                return <div key={activityIndex} className={css.relative.fillWidth.pad2(4, 3)}>
+                    {fraction !== undefined && <div className={
+                        css.absolute.top(0).left(0).size(`${fraction * 100}%`, "100%")
+                            .hsla(SYNC_PROGRESS_COLOR.h, SYNC_PROGRESS_COLOR.s, SYNC_PROGRESS_COLOR.l, SYNC_PROGRESS_OPACITY)
+                    } />}
+                    <div className={css.relative.vbox(2)}>
+                        <div className={css.hbox(4).wrap}>
+                            <div className={css.pad2(5, 0).fontSize(SOURCE_PART_FONT_SIZE).whiteSpace("nowrap").hsl(typeColor.h, typeColor.s, typeColor.l).bord2(typeColor.h, typeColor.s, typeColor.l - 20)}>
+                                {activity.type}
+                            </div>
+                            <SourceNameParts debugName={activity.sourceDebugName} />
+                        </div>
+                        <div className={css.hbox(6).wrap.fontSize(SYNC_DETAIL_FONT_SIZE).colorhsl(0, 0, 35)}>
+                            <div className={css.whiteSpace("nowrap")}>running {formatTime(elapsed)}</div>
+                            {fraction !== undefined && <div className={css.whiteSpace("nowrap")}>
+                                {(fraction * 100).toFixed(PERCENT_DECIMALS)}%
+                            </div>}
+                            {remaining !== undefined && <div className={css.whiteSpace("nowrap")}>~{formatTime(remaining)} left</div>}
+                            {activity.totalFiles !== undefined && <div className={css.whiteSpace("nowrap")}>
+                                {formatNumber(activity.doneFiles || 0)}/{formatNumber(activity.totalFiles)} files
+                            </div>}
+                            {activity.totalBytes !== undefined && <div className={css.whiteSpace("nowrap")}>
+                                {formatNumber(activity.doneBytes || 0)}B/{formatNumber(activity.totalBytes)}B
+                            </div>}
+                        </div>
+                    </div>
+                </div>;
+            })}
+        </div>;
+    }
+}
+
 const DEPLOY_PENDING_HUE = { h: 35, s: 80 };
 const DEPLOY_OVERLAP_HUE = { h: 280, s: 60 };
 const DEPLOY_NOTICE_FONT_SIZE = 14;
@@ -498,7 +558,7 @@ export class StoragePage extends qreact.Component {
                     readerDiskLimit: { title: "Read cache limit" },
                     writes: { title: "Writes" },
                     written: { title: "Written" },
-                    writeGain: { title: "Coalescing gain" },
+                    writeGain: { title: "Fast gain" },
                     disk: {
                         title: "Drive",
                         formatter: (disk, context) => {
@@ -514,12 +574,7 @@ export class StoragePage extends qreact.Component {
                     diskError: null,
                     syncing: {
                         title: "Syncing",
-                        formatter: syncing => <div className={css.vbox(2)}>
-                            {(syncing || []).map((activity, activityIndex) => <div key={activityIndex}>
-                                {activity.type} {activity.sourceDebugName}
-                                {activity.totalFiles !== undefined && ` (${formatNumber(activity.doneFiles || 0)}/${formatNumber(activity.totalFiles)} files)`}
-                            </div>)}
-                        </div>
+                        formatter: syncing => <SyncingCell syncing={syncing} />
                     },
                     error: { title: "Error" },
                 }}

package/src/deployManager/machineController.ts

@@ -81,11 +81,12 @@ class MachineControllerBase {
         await streamScreenOutput({
             key: config.key,
             index: config.index,
-            onData: async (data) => {
+            onData: async (data, dataConfig) => {
                 await MachineControllerClient.nodes[caller.nodeId].onScreenOutput({
                     key: config.key,
                     index: config.index,
                     data,
+                    reset: dataConfig?.reset,
                     callbackId: config.callbackId,
                 });
             },
@@ -152,13 +153,14 @@ export const MachineController = getSyncedController(SocketFunction.register(
     reads: {},
 });
 
-let callbacks = new Map<string, (data: string) => Promise<void>>();
+let callbacks = new Map<string, (data: string, config?: { reset?: boolean }) => Promise<void>>();
 export async function watchScreenOutput(config: {
     nodeId: string;
     key: string;
     index: number;
     callbackId: string;
-    onData: (data: string) => Promise<void>;
+    /** reset means the process behind the screen changed: replace what you have with this, don't append it. */
+    onData: (data: string, config?: { reset?: boolean }) => Promise<void>;
 }) {
     let callbackId = config.callbackId;
     callbacks.set(callbackId, config.onData);
@@ -181,6 +183,7 @@ class MachineControllerClientBase {
         key: string;
         index: number;
         data: string;
+        reset?: boolean;
         callbackId: string;
     }): Promise<void> {
         let forwardToNodeId = forwardedCallbacks.get(config.callbackId);
@@ -189,6 +192,7 @@ class MachineControllerClientBase {
                 key: config.key,
                 index: config.index,
                 data: config.data,
+                reset: config.reset,
                 callbackId: config.callbackId,
             });
             return;
@@ -198,7 +202,7 @@ class MachineControllerClientBase {
         if (!callback) {
             throw new Error(`Callback ${config.callbackId} not found (likely removed)`);
         }
-        await callback(config.data);
+        await callback(config.data, { reset: config.reset });
     }
 }
 // NOTE: THis is secure, because callbackId is random.