querysub 0.634.0 → 0.636.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.634.0",
+  "version": "0.636.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -79,7 +79,7 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
-    "sliftutils": "^1.7.105",
+    "sliftutils": "^1.7.106",
     "socket-function": "^1.2.31",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -532,7 +532,8 @@ async function runServerSyncLoops() {
 
     console.log(magenta(`Node discovery is loaded`));
 
-    // Delayed by a bit as it might take a little bit for our heartbeat to show up on startup, especially if we are the storage node that is supposed to store our own heartbeat. 
+    await runInfinitePollCallAtStart(HEARTBEAT_INTERVAL, writeHeartbeat);
+    // Delayed by a bit as it might take a little bit for our heartbeat to show up on startup, especially if we are the storage node that is supposed to store our own heartbeat.
     delay(HEARTBEAT_INTERVAL * 5).then(() => runInfinitePoll(HEARTBEAT_INTERVAL, async function nodeDiscoverHeartbeat() {
         // If we waited too long, other nodes might think we are dead. In which case, we SHOULD terminate.
         if (!isNoNetwork()) {
@@ -556,8 +557,6 @@ async function runServerSyncLoops() {
                 ignoreErrors(NodeDiscoveryController.nodes[nodeId].addNode(getOwnNodeId()));
             }
         }
-
-        await writeHeartbeat();
     }));
 }
 

package/src/5-diagnostics/Table.tsx

@@ -63,7 +63,8 @@ export class Table<RowT extends RowType> extends qreact.Component<TableType<RowT
         return (
             <table class={css.borderCollapse("collapse") + this.props.class}>
                 <tr class={
-                    css.position("sticky").top(0)
+                    // Above the rows, as they hold positioned content (bars, palettes) which otherwise paints over the sticky header as it scrolls under it
+                    css.position("sticky").top(0).zIndex(1)
                     + (this.props.dark && css.hsla(0, 0, 10, 0.9))
                     + (!this.props.dark && css.hsla(0, 0, 95, 0.9))
                 }>

package/src/deployManager/components/ProcessesView.tsx

@@ -17,6 +17,8 @@ const DEAD_COLOR = { h: 0, s: 0, l: 92 };
 const OUTPUT_BUFFER_LIMIT = 1_000_000;
 const OUTPUT_BUFFER_KEPT = 100_000;
 const OUTPUT_MAX_HEIGHT = "40vh";
+// Within this of the bottom still counts as "at the bottom", as scroll positions land a few pixels off the exact end
+const OUTPUT_PIN_THRESHOLD_PX = 40;
 const DETAIL_FONT_SIZE = 12;
 const ANSI_SATURATION = 70;
 const ANSI_LIGHTNESS = 70;
@@ -68,12 +70,27 @@ class ProcessOutput extends qreact.Component<{ record: ProcessRecord; machineNod
             await stopWatchingProcessOutput({ callbackId });
         });
     }
+    private outputDiv: HTMLDivElement | undefined;
+    // The pane opens showing the newest output and follows it as it streams; scrolling up to read releases the pin, scrolling back down re-engages it
+    private pinnedToBottom = true;
+    componentDidUpdate() {
+        if (this.pinnedToBottom && this.outputDiv) {
+            this.outputDiv.scrollTop = this.outputDiv.scrollHeight;
+        }
+    }
     render() {
-        return <div className={
-            css.fontFamily("monospace").whiteSpace("pre-wrap").fillWidth
-                .maxHeight(OUTPUT_MAX_HEIGHT).overflow("auto").pad2(10, 8)
-                .hsl(0, 0, 12).colorhsl(0, 0, 90)
-        }>
+        return <div
+            className={
+                css.fontFamily("monospace").whiteSpace("pre-wrap").fillWidth
+                    .maxHeight(OUTPUT_MAX_HEIGHT).overflow("auto").pad2(10, 8)
+                    .hsl(0, 0, 12).colorhsl(0, 0, 90)
+            }
+            ref={element => this.outputDiv = element as HTMLDivElement || undefined}
+            onScroll={element => {
+                let div = element.currentTarget;
+                this.pinnedToBottom = div.scrollHeight - div.scrollTop - div.clientHeight < OUTPUT_PIN_THRESHOLD_PX;
+            }}
+        >
             {parseAnsiColors(this.state.data).map(({ text, color }) => {
                 if (!color) return <span>{text}</span>;
                 // The pane is dark, so the ansi colour only sets the hue - the lightness has to stay readable against it

package/src/deployManager/components/storage/StorageLogsPage.tsx

@@ -10,20 +10,36 @@ import { timeInSecond } from "socket-function/src/misc";
 import type { LogFileInfo } from "sliftutils/storage/StreamingLogs";
 import { MachineServiceController } from "../../machineSchema";
 import { getStorageServers } from "./storageServers";
-import { Table } from "../../../5-diagnostics/Table";
+import { Table, TableType } from "../../../5-diagnostics/Table";
+import { showModal } from "../../../5-diagnostics/Modal";
+import { FullscreenModal } from "../../../5-diagnostics/FullscreenModal";
 import { URLParam } from "../../../library-components/URLParam";
 import { InputLabelURL } from "../../../library-components/InputLabel";
 import { Button } from "../../../library-components/Button";
+import { atomic } from "../../../2-proxy/PathValueProxyWatcher";
 import { TimeRangeSelector, getTimeRange } from "../../../diagnostics/logs/TimeRangeSelector";
 import { StorageLogsSynced } from "./storageLogsController";
-import { searchStorageLogs, invalidateStorageLogSearchCache } from "./storageLogSearch";
+import { searchStorageLogs, invalidateStorageLogSearchCache, FailedLogFile } from "./storageLogSearch";
 
 module.hotreload = true;
 
 export const storageLogSearchParam = new URLParam("storageLogSearch", "");
+// true adds a field as a column, false removes a default - same shape as the log viewer's selectedFields
+const storageLogFieldsParam = new URLParam("storageLogFields", {} as Record<string, boolean>);
+
+const defaultSelectedFields = {
+    time: true,
+    path: true,
+    kind: true,
+    bucketName: true,
+};
 
 // Fixed columns first so every search reads the same way; the entries' own fields follow in the order they appear.
 const LEADING_COLUMNS = ["server", "file", "time", "kind"];
+const FIELD_CHIP_COLOR = { h: 0, s: 0, l: 92 };
+const FIELD_CHIP_ACTIVE_COLOR = { h: 210, s: 60, l: 78 };
+const FIELD_COUNT_COLOR = { h: 0, s: 0, l: 40 };
+const FIELD_COUNT_FONT_SIZE = 12;
 const ERROR_COLOR = { h: 0, s: 60, l: 40 };
 const AGE_COLOR = { h: 0, s: 0, l: 40 };
 const AGE_FONT_SIZE = 13;
@@ -36,6 +52,67 @@ type ServerLogListing = {
     fetchTime?: number;
 };
 
+const MODAL_FIELD_NAME_MIN_WIDTH = 140;
+const VIEW_BUTTON_ACTIVE_HUE = 210;
+
+// Shared between the modal and the table's View buttons, so the button of the row on display highlights and arrow keys move the selection
+const viewedRowState = Querysub.createLocalSchema("storageLogsViewedRow", {
+    open: t.boolean,
+    index: t.number,
+});
+// The rows the open modal navigates. Plain memory: the modal re-renders on index changes, this is just the data those changes index into.
+let viewedRows: Record<string, unknown>[] = [];
+
+class RowViewModal extends qreact.Component<{ onClose: () => void }> {
+    componentDidMount() {
+        document.addEventListener("keydown", this.onKeyDown);
+    }
+    componentWillUnmount() {
+        document.removeEventListener("keydown", this.onKeyDown);
+    }
+    private onKeyDown = (e: KeyboardEvent) => {
+        if (e.key !== "ArrowUp" && e.key !== "ArrowDown") return;
+        e.preventDefault();
+        let delta = e.key === "ArrowDown" ? 1 : -1;
+        // A document listener is not a Querysub-dispatched event, so the synced write needs an explicit commit
+        Querysub.commit(() => {
+            let index = viewedRowState().index + delta;
+            if (index < 0 || index >= viewedRows.length) return;
+            viewedRowState().index = index;
+        });
+    };
+    render() {
+        let index = viewedRowState().index;
+        let row = viewedRows[index];
+        if (!row) return undefined;
+        return <FullscreenModal onCancel={this.props.onClose}>
+            <div className={css.vbox(6).pad2(16)}>
+                <div className={css.colorhsl(0, 0, 45)}>Row {index + 1} of {formatNumber(viewedRows.length)} (↑/↓ to move)</div>
+                {Object.entries(row).map(([field, value]) => <div key={field} className={css.hbox(10)}>
+                    <div className={css.boldStyle.minWidth(MODAL_FIELD_NAME_MIN_WIDTH)}>{field}</div>
+                    <div className={css.fontFamily("monospace").whiteSpace("pre-wrap")}>
+                        {field === "time" && typeof value === "number" && `${formatDateTime(value)} (${value})`
+                            || typeof value === "string" && value
+                            || JSON.stringify(value)}
+                    </div>
+                </div>)}
+            </div>
+        </FullscreenModal>;
+    }
+}
+
+function showRowModal(rows: Record<string, unknown>[], index: number): void {
+    viewedRows = rows;
+    viewedRowState().open = true;
+    viewedRowState().index = index;
+    let close = showModal({
+        content: <RowViewModal onClose={() => {
+            viewedRowState().open = false;
+            close.close();
+        }} />
+    });
+}
+
 /** Whether a log file could hold entries in the range - a live file (no endTime) runs to now. */
 function fileOverlapsRange(file: LogFileInfo, startTime: number, endTime: number): boolean {
     let fileEnd = file.endTime ?? Date.now();
@@ -53,7 +130,8 @@ export class StorageLogsPage extends qreact.Component {
         /** The oldest fetch time among the results being displayed */
         resultsFetchTime: t.number,
         rows: t.atomic<Record<string, unknown>[]>([]),
-        columns: t.atomic<string[]>([]),
+        fields: t.atomic<{ field: string; count: number }[]>([]),
+        failedFiles: t.atomic<FailedLogFile[]>([]),
         error: t.string,
     });
 
@@ -91,7 +169,7 @@ export class StorageLogsPage extends qreact.Component {
         let listings = this.getListings();
         let files = this.getSelectedFiles(listings).map(file => ({ url: file.url, name: file.name }));
         let query = storageLogSearchParam.value;
-        let { startTime, endTime } = getTimeRange();
+        let { startTime, endTime, searchFromStart } = getTimeRange();
         this.state.searching = true;
         this.state.error = "";
         this.state.doneFiles = 0;
@@ -111,25 +189,30 @@ export class StorageLogsPage extends qreact.Component {
                     },
                 });
                 let rows: Record<string, unknown>[] = [];
-                let columns: string[] = [...LEADING_COLUMNS];
                 for (let file of result.files) {
                     for (let entry of file.entries) {
                         let fields: Record<string, unknown> = entry && typeof entry === "object" ? entry as Record<string, unknown> : { value: String(entry) };
                         // The file selection is only as granular as whole files, so the entries themselves still need the range applied
                         let time = fields.time;
                         if (typeof time === "number" && (time < startTime || time > endTime)) continue;
-                        for (let key of Object.keys(fields)) {
-                            if (!columns.includes(key)) {
-                                columns.push(key);
-                            }
-                        }
                         rows.push({ server: file.url, file: file.name, ...fields });
                     }
                 }
-                rows.sort((a, b) => (typeof a.time === "number" && a.time || 0) - (typeof b.time === "number" && b.time || 0));
+                // Ordered the way the time range UI says: from the end (newest first) by default, oldest first when searching from the start
+                let direction = searchFromStart ? 1 : -1;
+                rows.sort((a, b) => direction * ((typeof a.time === "number" && a.time || 0) - (typeof b.time === "number" && b.time || 0)));
+                // Seeded so the fixed fields lead the chip row; ones no entry has are dropped below
+                let fieldCounts = new Map<string, number>(LEADING_COLUMNS.map(field => [field, 0]));
+                for (let row of rows) {
+                    for (let key of Object.keys(row)) {
+                        fieldCounts.set(key, (fieldCounts.get(key) || 0) + 1);
+                    }
+                }
+                let fields = [...fieldCounts.entries()].filter(([, count]) => count > 0).map(([field, count]) => ({ field, count }));
                 Querysub.commit(() => {
                     this.state.rows = rows;
-                    this.state.columns = columns;
+                    this.state.fields = fields;
+                    this.state.failedFiles = result.failures;
                     this.state.durationTime = result.durationTime;
                     this.state.resultsFetchTime = result.oldestFetchTime;
                 });
@@ -153,8 +236,47 @@ export class StorageLogsPage extends qreact.Component {
         let oldestListingTime = loadedListings.length && Math.min(...loadedListings.map(listing => listing.fetchTime || 0)) || 0;
         let now = Querysub.nowDelayed(timeInSecond);
 
-        let columnsConfig: { [column: string]: { title?: string; formatter?: (value: unknown) => preact.ComponentChild } } = {};
-        for (let column of this.state.columns) {
+        // Defaults show unless explicitly toggled off, plus whatever was explicitly toggled on
+        let selectedSet = new Set<string>();
+        for (let field of Object.keys(defaultSelectedFields)) {
+            if (atomic(storageLogFieldsParam.value[field]) === undefined) {
+                selectedSet.add(field);
+            }
+        }
+        for (let [key, value] of Object.entries(storageLogFieldsParam.value)) {
+            if (value) {
+                selectedSet.add(key);
+            }
+        }
+        // Columns follow the chip row's order; selections with no matching field this search (ex, persisted from an earlier one) go last
+        let selectedFields = this.state.fields.map(x => x.field).filter(field => selectedSet.has(field));
+        for (let field of selectedSet) {
+            if (!selectedFields.includes(field)) {
+                selectedFields.push(field);
+            }
+        }
+
+        // Object identity map, as the formatter only receives the row itself and highlighting needs its index
+        let rows = this.state.rows;
+        let rowIndexes = new Map(rows.map((row, index) => [row, index]));
+        let columnsConfig: TableType<Record<string, unknown>>["columns"] = {};
+        columnsConfig["view"] = {
+            title: "View",
+            formatter: (x, context) => {
+                let row = context?.row;
+                if (!row) return undefined;
+                let index = rowIndexes.get(row) ?? -1;
+                let viewed = viewedRowState().open && viewedRowState().index === index;
+                return <Button
+                    flavor="tiny"
+                    hue={viewed ? VIEW_BUTTON_ACTIVE_HUE : undefined}
+                    onClick={() => showRowModal(rows, index)}
+                >
+                    View
+                </Button>;
+            },
+        };
+        for (let column of selectedFields) {
             if (column === "time") {
                 columnsConfig[column] = {
                     title: "Time",
@@ -175,7 +297,8 @@ export class StorageLogsPage extends qreact.Component {
                         StorageLogsSynced(SocketFunction.browserNodeId()).getServerLogFiles.resetAll();
                         invalidateStorageLogSearchCache();
                         this.state.rows = [];
-                        this.state.columns = [];
+                        this.state.fields = [];
+                        this.state.failedFiles = [];
                         this.state.resultsFetchTime = 0;
                         this.state.durationTime = 0;
                     }}
@@ -188,7 +311,7 @@ export class StorageLogsPage extends qreact.Component {
                 <InputLabelURL
                     flavor="large"
                     fillWidth
-                    label="Search (a | b & c - or of ands, empty downloads everything in range)"
+                    label="Search"
                     url={storageLogSearchParam}
                     onKeyDown={e => {
                         if (e.key === "Enter") {
@@ -228,7 +351,34 @@ export class StorageLogsPage extends qreact.Component {
                 </div>
             </div>
             {this.state.error && <div className={css.colorhsl(ERROR_COLOR.h, ERROR_COLOR.s, ERROR_COLOR.l).whiteSpace("pre-wrap")}>{this.state.error}</div>}
-            {!!this.state.rows.length && <div>
+            {!!this.state.failedFiles.length && <div className={css.vbox(2).colorhsl(ERROR_COLOR.h, ERROR_COLOR.s, ERROR_COLOR.l)}>
+                <div className={css.boldStyle}>{formatNumber(this.state.failedFiles.length)} files failed to search (their entries are missing from the results):</div>
+                {this.state.failedFiles.map(failure => <div key={`${failure.url}|${failure.name}`} className={css.ellipsis} title={failure.error}>
+                    {failure.url} {failure.name}
+                </div>)}
+            </div>}
+            {!!this.state.rows.length && <div className={css.vbox(8)}>
+                <div className={css.hbox(6, 4).wrap.alignItems("center")}>
+                    {this.state.fields.map(({ field, count }) => {
+                        let selected = selectedSet.has(field);
+                        let color = selected ? FIELD_CHIP_ACTIVE_COLOR : FIELD_CHIP_COLOR;
+                        return <div
+                            key={field}
+                            className={
+                                css.hbox(5).alignItems("center").pad2(8, 4).button.whiteSpace("nowrap")
+                                    .hsl(color.h, color.s, color.l).bord2(color.h, color.s, color.l - 20)
+                            }
+                            onClick={() => {
+                                let newValues = { ...storageLogFieldsParam.value };
+                                newValues[field] = !selected;
+                                storageLogFieldsParam.value = newValues;
+                            }}
+                        >
+                            <span className={css.boldStyle}>{field}</span>
+                            <span className={css.fontSize(FIELD_COUNT_FONT_SIZE).colorhsl(FIELD_COUNT_COLOR.h, FIELD_COUNT_COLOR.s, FIELD_COUNT_COLOR.l)}>{formatNumber(count)}</span>
+                        </div>;
+                    })}
+                </div>
                 <Table
                     rows={this.state.rows}
                     columns={columnsConfig}

package/src/deployManager/components/storage/storageLogSearch.ts

@@ -60,13 +60,20 @@ export type StorageLogSearchProgress = {
     currentFile: string;
 };
 
-/** Searches the query across the given files (per-file, so progress is per-file and each file's result caches on its own). */
+export type FailedLogFile = {
+    url: string;
+    name: string;
+    error: string;
+};
+
+/** Searches the query across the given files (per-file, so progress is per-file and each file's result caches on its own). A file that fails costs only its own results - it lands in failures and the rest of the search continues. */
 export async function searchStorageLogs(config: {
     files: { url: string; name: string }[];
     query: string;
     onProgress: (progress: StorageLogSearchProgress) => void;
 }): Promise<{
     files: SearchedLogFile[];
+    failures: FailedLogFile[];
     durationTime: number;
     /** The oldest fetch time among the results used, so the UI can say how stale the display is. */
     oldestFetchTime: number;
@@ -74,16 +81,24 @@ export async function searchStorageLogs(config: {
     let startTime = Date.now();
     let doneFiles = 0;
     let results: SearchedLogFile[] = [];
+    let failures: FailedLogFile[] = [];
     let runFile = runInParallel({ parallelCount: PARALLEL_FILE_SEARCHES }, async (file: { url: string; name: string }) => {
         config.onProgress({ doneFiles, totalFiles: config.files.length, currentFile: file.name });
-        let result = await searchFile(file.url, file.name, config.query);
-        results.push({ url: file.url, name: file.name, entries: result.entries, fetchTime: result.fetchTime });
+        try {
+            let result = await searchFile(file.url, file.name, config.query);
+            results.push({ url: file.url, name: file.name, entries: result.entries, fetchTime: result.fetchTime });
+        } catch (e) {
+            let error = (e as Error).stack ?? String(e);
+            console.error(`Searching log file ${file.name} on ${file.url} failed, continuing with the other files:`, error);
+            failures.push({ url: file.url, name: file.name, error });
+        }
         doneFiles++;
         config.onProgress({ doneFiles, totalFiles: config.files.length, currentFile: file.name });
     });
     await Promise.all(config.files.map(runFile));
     return {
         files: results,
+        failures,
         durationTime: Date.now() - startTime,
         oldestFetchTime: results.length && Math.min(...results.map(x => x.fetchTime)) || Date.now(),
     };