querysub 0.558.0 → 0.560.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.558.0",
+  "version": "0.560.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -71,7 +71,7 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
-    "sliftutils": "^1.7.40",
+    "sliftutils": "^1.7.41",
     "socket-function": "^1.2.26",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",

package/src/deployManager/components/ProcessesView.tsx

@@ -8,7 +8,7 @@ import { sort, timeInSecond, nextId } from "socket-function/src/misc";
 import { MachineController, watchProcessOutput, stopWatchingProcessOutput } from "../machineController";
 import type { ProcessRecord } from "../processLogs";
 import { Button } from "../../library-components/Button";
-import { parseAnsiColors } from "../../diagnostics/logs/ansiFormat";
+import { parseAnsiColors, rgbToHsl } from "../../diagnostics/logs/ansiFormat";
 
 module.hotreload = true;
 
@@ -17,6 +17,8 @@ const DEAD_COLOR = { h: 0, s: 0, l: 92 };
 const OUTPUT_BUFFER_LIMIT = 1_000_000;
 const OUTPUT_BUFFER_KEPT = 100_000;
 const OUTPUT_MAX_HEIGHT = "40vh";
+const ANSI_SATURATION = 70;
+const ANSI_LIGHTNESS = 70;
 
 /** One process's live output. Mounting starts the stream, unmounting stops it, so the watch lifetime is exactly the time it is on screen. */
 class ProcessOutput extends qreact.Component<{ record: ProcessRecord; machineNodeId: string }> {
@@ -70,7 +72,11 @@ class ProcessOutput extends qreact.Component<{ record: ProcessRecord; machineNod
                 .maxHeight(OUTPUT_MAX_HEIGHT).overflow("auto").pad2(10, 8)
                 .hsl(0, 0, 12).colorhsl(0, 0, 90)
         }>
-            {parseAnsiColors(this.state.data)}
+            {parseAnsiColors(this.state.data).map(({ text, color }) => {
+                if (!color) return <span>{text}</span>;
+                // The pane is dark, so the ansi colour only sets the hue - the lightness has to stay readable against it
+                return <span className={css.colorhsl(rgbToHsl(color).h, ANSI_SATURATION, ANSI_LIGHTNESS)}>{text}</span>;
+            })}
         </div>;
     }
 }

package/src/deployManager/components/RouteConfigView.tsx

@@ -12,6 +12,7 @@ import type { RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig } fr
 import { showModal } from "../../5-diagnostics/Modal";
 import { FullscreenModal } from "../../5-diagnostics/FullscreenModal";
 import { Button } from "../../library-components/Button";
+import { Tag, TAG_FONT_SIZE, TAG_WARNING_COLOR, NO_FULL_SYNC_TAG } from "./Tag";
 import { StorageSynced, type StorageServerBuckets } from "./StoragePage";
 
 module.hotreload = true;
@@ -19,8 +20,7 @@ module.hotreload = true;
 type Source = HostedConfig | BackblazeConfig;
 
 const DEFAULT_HTTPS_PORT = 443;
-const TAG_COLOR = { h: 210, s: 45, l: 88 };
-const WARNING_COLOR = { h: 35, s: 90, l: 85 };
+const WARNING_COLOR = TAG_WARNING_COLOR;
 const NOTICE_COLOR = { h: 205, s: 70, l: 88 };
 const SOURCE_BORDER_COLOR = { h: 0, s: 0, l: 75 };
 const ACTIVE_COLOR = { h: 130, s: 55, l: 85 };
@@ -38,7 +38,6 @@ const WINDOW_TIME_LIGHTNESS = 12;
 const BUCKET_TITLE_SIZE = 15;
 const GROUP_BODY_INDENT = 64;
 const WINDOW_BODY_INDENT = 64;
-const TAG_FONT_SIZE = 11;
 const JSON_INDENT = 4;
 
 /** String sources are the shorthand for an always-valid, unsharded backblaze bucket. */
@@ -130,7 +129,7 @@ type WindowCluster = {
     sources: Source[];
 };
 
-/** Every distinct valid window in the config, latest-ending first, each listing every source that overlaps it. A source spanning several windows appears under each of them - what matters is what is valid during a window, not which window a source "belongs" to. Sources keep their config order, since that order decides which one shadows which. */
+/** Every distinct valid window in the config, latest-starting first, each listing every source that overlaps it. A source spanning several windows appears under each of them - what matters is what is valid during a window, not which window a source "belongs" to. Sources keep their config order, since that order decides which one shadows which. */
 function getWindowRanges(sources: Source[]): WindowCluster[] {
     let byKey = new Map<string, [number, number]>();
     for (let source of sources) {
@@ -139,9 +138,9 @@ function getWindowRanges(sources: Source[]): WindowCluster[] {
         byKey.set(key, [source.validWindow[0], source.validWindow[1]]);
     }
     let windows = [...byKey.values()];
-    // Sorted by start first, so the (stable) sort by end leaves later starts first within one end time
-    sort(windows, x => -x[0]);
+    // Sorted by end first, so the (stable) sort by start leaves later ends first within one start time
     sort(windows, x => -x[1]);
+    sort(windows, x => -x[0]);
     return windows.map(window => ({
         window,
         sources: sources.filter(x => windowsOverlap(x.validWindow, window)),
@@ -344,24 +343,11 @@ function showConfigModal(bucketName: string, rawConfig: RemoteConfig): void {
     });
 }
 
-class Tag extends qreact.Component<{ icon: string; text: string; warning?: boolean; color?: { h: number; s: number; l: number }; title?: string; }> {
-    render() {
-        let color = this.props.color || this.props.warning && WARNING_COLOR || TAG_COLOR;
-        return <div className={
-            css.hbox(4).pad2(6, 1).fontSize(TAG_FONT_SIZE).whiteSpace("nowrap")
-                .hsl(color.h, color.s, color.l).bord2(color.h, color.s, color.l - 20)
-        } title={this.props.title}>
-            <span>{this.props.icon}</span>
-            <span>{this.props.text}</span>
-        </div>;
-    }
-}
-
 function getSourceTags(source: Source): { icon: string; text: string }[] {
     let tags: { icon: string; text: string }[] = [];
     if (source.public) tags.push({ icon: "🌐", text: "public" });
     if (source.immutable) tags.push({ icon: "🔒", text: "immutable" });
-    if (source.noFullSync) tags.push({ icon: "⇣", text: "noFullSync" });
+    if (source.noFullSync) tags.push(NO_FULL_SYNC_TAG);
     if (source.intermediate) tags.push({ icon: "⏳", text: "intermediate" });
     if (source.type === "remote") {
         if (source.fast) tags.push({ icon: "⚡", text: "fast" });

package/src/deployManager/components/StoragePage.tsx

@@ -7,7 +7,8 @@ import preact from "preact";
 import { Querysub } from "../../4-querysub/Querysub";
 import { listServerBuckets, clearServerWriteStats, activateServerBucket, getServerActiveBucket } from "sliftutils/storage/remoteStorage/createArchives";
 import type { ServerBucketInfo, BucketDiskInfo, BucketWriteStats, ActiveBucketInfo } from "sliftutils/storage/remoteStorage/storageServerState";
-import type { ArchivesConfig, SyncActivity } from "sliftutils/storage/IArchives";
+import type { ArchivesConfig, SyncActivity, RemoteConfig, HostedConfig } from "sliftutils/storage/IArchives";
+import { parseHostedUrl } from "sliftutils/storage/remoteStorage/remoteConfig";
 import { UsageBar, getUsageThresholds } from "../../library-components/UsageBar";
 import { getSyncedController } from "../../library-components/SyncedController";
 import { assertIsManagementUser } from "../../diagnostics/managementPages";
@@ -16,6 +17,7 @@ import { isDefined } from "../../misc";
 import { STORAGE_ACCOUNT } from "../../-a-archives/archives2";
 import { Table } from "../../5-diagnostics/Table";
 import { RouteConfigView } from "./RouteConfigView";
+import { Tag, NO_FULL_SYNC_TAG } from "./Tag";
 
 module.hotreload = true;
 
@@ -132,6 +134,7 @@ type BucketRow = {
     state: string;
     files: string;
     bytes: string;
+    noFullSync: boolean;
     indexSources: ArchivesConfig["indexSources"];
     readerDiskLimit: string;
     writes: string;
@@ -144,10 +147,35 @@ type BucketRow = {
 };
 
 const GAIN_DECIMALS = 1;
+const DEFAULT_HTTPS_PORT = 443;
 const DISK_BAR_TYPE = "DISK";
 const ACTIVE_STATE = "active";
 const INACTIVE_STATE = "inactive";
 
+/** Whether this server only caches what is read of the bucket instead of keeping a full copy, read off its own entry in the bucket's routing config. */
+function getNoFullSync(serverUrl: string, bucketName: string, remoteConfig: RemoteConfig | undefined): boolean {
+    let host: URL;
+    try {
+        host = new URL(serverUrl);
+    } catch {
+        return false;
+    }
+    let selfEntries: HostedConfig[] = [];
+    for (let source of remoteConfig?.sources || []) {
+        if (typeof source === "string" || source.type !== "remote") continue;
+        try {
+            let parsed = parseHostedUrl(source.url);
+            if (parsed.address !== host.hostname) continue;
+            if (String(parsed.port) !== (host.port || String(DEFAULT_HTTPS_PORT))) continue;
+            if (parsed.bucketName !== bucketName) continue;
+            selfEntries.push(source);
+        } catch {
+            // A malformed url in the config is not this column's problem
+        }
+    }
+    return selfEntries.some(x => x.noFullSync);
+}
+
 /** How much fast-mode coalescing saved: every accepted write over the ones that actually reached a source. */
 function getWriteGain(stats: BucketWriteStats | undefined): string {
     if (!stats) return "";
@@ -167,6 +195,7 @@ function getBucketRows(servers: StorageServerBuckets[]): BucketRow[] {
             state: "",
             files: "",
             bytes: "",
+            noFullSync: false,
             indexSources: undefined,
             readerDiskLimit: "",
             writes: "",
@@ -196,6 +225,7 @@ function getBucketRows(servers: StorageServerBuckets[]): BucketRow[] {
                 state: bucket.active && ACTIVE_STATE || INACTIVE_STATE,
                 files: config?.index && formatNumber(config.index.fileCount) || "",
                 bytes: config?.index && formatNumber(config.index.byteCount) + "B" || "",
+                noFullSync: getNoFullSync(server.url, bucket.bucketName, config?.remoteConfig),
                 indexSources: config?.indexSources,
                 readerDiskLimit: config?.readerDiskLimit && formatNumber(config.readerDiskLimit) + "B" || "",
                 writes: bucket.writeStats && formatNumber(bucket.writeStats.originalWrites) || "",
@@ -551,6 +581,11 @@ export class StoragePage extends qreact.Component {
                     },
                     files: { title: "Files" },
                     bytes: { title: "Bytes" },
+                    noFullSync: {
+                        title: "Sync",
+                        // Nothing to show when a full copy is kept - that is the default, and the tag is the flag
+                        formatter: noFullSync => noFullSync && <Tag icon={NO_FULL_SYNC_TAG.icon} text={NO_FULL_SYNC_TAG.text} /> || ""
+                    },
                     indexSources: {
                         title: "Index sources",
                         formatter: sources => <IndexSourcesCell sources={sources} />

package/src/deployManager/components/Tag.tsx

@@ -0,0 +1,26 @@
+import { qreact } from "../../4-dom/qreact";
+import { css } from "typesafecss";
+
+module.hotreload = true;
+
+export type TagColor = { h: number; s: number; l: number };
+
+export const TAG_COLOR: TagColor = { h: 210, s: 45, l: 88 };
+export const TAG_WARNING_COLOR: TagColor = { h: 35, s: 90, l: 85 };
+export const TAG_FONT_SIZE = 11;
+
+/** A source flag, shown wherever that flag matters. The tag is only ever rendered when the flag is on - an absent tag is the default, so there is nothing to show for it. */
+export const NO_FULL_SYNC_TAG = { icon: "⇣", text: "noFullSync" };
+
+export class Tag extends qreact.Component<{ icon: string; text: string; warning?: boolean; color?: TagColor; title?: string; }> {
+    render() {
+        let color = this.props.color || this.props.warning && TAG_WARNING_COLOR || TAG_COLOR;
+        return <div className={
+            css.hbox(4).pad2(6, 1).fontSize(TAG_FONT_SIZE).whiteSpace("nowrap")
+                .hsl(color.h, color.s, color.l).bord2(color.h, color.s, color.l - 20)
+        } title={this.props.title}>
+            <span>{this.props.icon}</span>
+            <span>{this.props.text}</span>
+        </div>;
+    }
+}

package/src/deployManager/machineApplyMainCode.ts

@@ -26,7 +26,7 @@ import { PromiseObj } from "../promise";
 import path from "path";
 import { fsExistsAsync } from "../fs";
 import { ALIVE_WINDOW_FOREVER, ParametersTimelineEntry, syncParametersTimelineFiles } from "./parametersTimeline";
-import { getScreenName, getFutureScreenName, getTmuxPrefix, getScreenState, isScreenRunningProcess, getProcessStartTime, runScreenCommand, killScreen, streamProcessOutput } from "./processManager";
+import { getScreenName, getFutureScreenName, getTmuxPrefix, getScreenState, isScreenRunningProcess, getProcessStartTime, ensureProcessRecord, runScreenCommand, killScreen, streamProcessOutput } from "./processManager";
 import { ProcessRecord, syncProcessRecords, writeProcessRecord, listProcessRecords, getProcessLogPath } from "./processLogs";
 
 
@@ -288,6 +288,19 @@ async function ensureFutureStarted(config: FutureConfig) {
     let existing = config.screenStateMap.get(futureScreenName);
     if (existing?.isProcessRunning) {
         config.screenNamesUsed.add(futureScreenName);
+        // Started before we restarted, so record it rather than only ever recording processes we launched ourselves
+        await ensureProcessRecord({
+            screenName: futureScreenName,
+            folder: os.homedir() + "/" + SERVICE_FOLDER + canonicalScreenName + "/",
+            panePid: existing.pid,
+            record: {
+                serviceId: config.serviceId,
+                serviceKey: config.next.key,
+                index: config.index,
+                machineId: config.machineId,
+                parameters: config.next,
+            },
+        });
         console.log(green(`Verified future instance ${magenta(futureScreenName)} is running`));
         return;
     }
@@ -515,6 +528,22 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
                 if (isPrepTime) {
                     // From prep time on, the canonical (old) screen is left completely untouched — the folder holds the NEW code and parameters by now, so the normal compare/sync logic below must not run against the old process
                     if (screenStateMap.get(screenName)?.isProcessRunning) {
+                        // The old instance keeps running untouched through the overlap, but it still needs a record - it may have been launched before we restarted
+                        let panePid = screenStateMap.get(screenName)?.pid;
+                        if (panePid) {
+                            await ensureProcessRecord({
+                                screenName,
+                                folder,
+                                panePid,
+                                record: {
+                                    serviceId: config.serviceId,
+                                    serviceKey: config.parameters.key,
+                                    index: i,
+                                    machineId,
+                                    parameters: instanceParameters,
+                                },
+                            });
+                        }
                         await syncTimeline(screenStateMap.get(screenName)?.pid);
                         let nodePathId = folder + SERVICE_NODE_FILE_NAME;
                         if (await fsExistsAsync(nodePathId)) {
@@ -573,6 +602,22 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
                     if (prevParameters !== newParametersString) {
                         await fs.promises.writeFile(parameterPath, newParametersString);
                     }
+                    // We did not launch this process (we restarted, or it predates process records), so make sure it is recorded before we leave it alone
+                    let panePid = screenStateMap.get(screenName)?.pid;
+                    if (panePid) {
+                        await ensureProcessRecord({
+                            screenName,
+                            folder,
+                            panePid,
+                            record: {
+                                serviceId: config.serviceId,
+                                serviceKey: config.parameters.key,
+                                index: i,
+                                machineId,
+                                parameters: instanceParameters,
+                            },
+                        });
+                    }
                     await syncTimeline(screenStateMap.get(screenName)?.pid);
                     console.log(green(`Verified ${magenta(screenName)} is running`));
                     continue;

package/src/deployManager/processManager.ts

@@ -11,7 +11,7 @@ import { forceRemoveNode } from "../-f-node-discovery/NodeDiscovery";
 import { fsExistsAsync } from "../fs";
 import { PromiseObj } from "../promise";
 import { SERVICE_FOLDER, SERVICE_NODE_FILE_NAME } from "./machineSchema";
-import { ProcessRecord, createLaunchId, getLogPipeScript, getLogTailScript, getProcessLogPath, getPipeScriptPath, getTailScriptPath, writeProcessRecord } from "./processLogs";
+import { ProcessRecord, createLaunchId, getLogPipeScript, getLogTailScript, getProcessLogPath, getPipeScriptPath, getTailScriptPath, listProcessRecords, writeProcessRecord } from "./processLogs";
 
 // Running, inspecting and killing the tmux screens services run in. This layer only knows "here is a configuration, run it" - which version should be running when is the deploy logic's problem.
 
@@ -244,6 +244,38 @@ ${config.command}
     return launchId;
 });
 
+/** Links an already-running screen to a process record, creating one when this process has never been recorded - it was launched before we restarted, or before process records existed at all. Idempotent: a process we already know about is left alone, so the pipe is only attached the first time we adopt it.
+ *
+ * The screen identifies itself (pane pid + the start time the OS reports for it), so adoption is just naming what is already there - there is nothing to reconcile and nothing to guess. */
+export async function ensureProcessRecord(config: {
+    screenName: string;
+    folder: string;
+    panePid: string;
+    record: Omit<ProcessRecord, "launchId" | "folder" | "screenName" | "startTime" | "pid">;
+}): Promise<void> {
+    let startTime = await getProcessStartTime(config.panePid);
+    if (!startTime) return;
+    let launchId = createLaunchId(config.panePid, startTime);
+    let existing = (await listProcessRecords()).find(x => x.launchId === launchId && x.folder === config.folder);
+    if (existing) {
+        // A takeover renames the session, so the record follows the name it now runs under
+        if (existing.screenName === config.screenName) return;
+        await writeProcessRecord({ ...existing, screenName: config.screenName });
+        return;
+    }
+    console.log(`Adopting already-running process ${launchId} on screen ${config.screenName}, which has no record yet`);
+    await writeProcessRecord({
+        ...config.record,
+        launchId,
+        folder: config.folder,
+        screenName: config.screenName,
+        pid: parseInt(config.panePid) || undefined,
+        startTime,
+    });
+    // Its output was going to whatever the previous incarnation pointed at (or nowhere), so point it at its own log from here on
+    await setupPipePane({ screenName: config.screenName, folder: config.folder, launchId });
+}
+
 // Points the screen's pipe-pane at this launch's own log file.
 async function setupPipePane(config: { screenName: string; folder: string; launchId: string }) {
     let prefix = getTmuxPrefix();