querysub 0.635.0 → 0.637.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.635.0",
+  "version": "0.637.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -79,7 +79,7 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
-    "sliftutils": "^1.7.105",
+    "sliftutils": "^1.7.107",
     "socket-function": "^1.2.31",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",

package/src/deployManager/components/storage/StorageLogsPage.tsx

@@ -6,24 +6,40 @@ import { Querysub } from "../../../4-querysub/Querysub";
 import { t } from "../../../2-proxy/schema2";
 import { isDefined } from "../../../misc";
 import { formatNumber, formatTime, formatDateTime } from "socket-function/src/formatting/format";
-import { timeInSecond } from "socket-function/src/misc";
+import { timeInMinute, timeInSecond } from "socket-function/src/misc";
 import type { LogFileInfo } from "sliftutils/storage/StreamingLogs";
 import { MachineServiceController } from "../../machineSchema";
 import { getStorageServers } from "./storageServers";
-import { Table } from "../../../5-diagnostics/Table";
+import { Table, TableType } from "../../../5-diagnostics/Table";
+import { showModal } from "../../../5-diagnostics/Modal";
+import { FullscreenModal } from "../../../5-diagnostics/FullscreenModal";
 import { URLParam } from "../../../library-components/URLParam";
 import { InputLabelURL } from "../../../library-components/InputLabel";
 import { Button } from "../../../library-components/Button";
+import { atomic } from "../../../2-proxy/PathValueProxyWatcher";
 import { TimeRangeSelector, getTimeRange } from "../../../diagnostics/logs/TimeRangeSelector";
 import { StorageLogsSynced } from "./storageLogsController";
-import { searchStorageLogs, invalidateStorageLogSearchCache } from "./storageLogSearch";
+import { searchStorageLogs, invalidateStorageLogSearchCache, FailedLogFile } from "./storageLogSearch";
 
 module.hotreload = true;
 
 export const storageLogSearchParam = new URLParam("storageLogSearch", "");
+// true adds a field as a column, false removes a default - same shape as the log viewer's selectedFields
+const storageLogFieldsParam = new URLParam("storageLogFields", {} as Record<string, boolean>);
+
+const defaultSelectedFields = {
+    time: true,
+    path: true,
+    kind: true,
+    bucketName: true,
+};
 
 // Fixed columns first so every search reads the same way; the entries' own fields follow in the order they appear.
 const LEADING_COLUMNS = ["server", "file", "time", "kind"];
+const FIELD_CHIP_COLOR = { h: 0, s: 0, l: 92 };
+const FIELD_CHIP_ACTIVE_COLOR = { h: 210, s: 60, l: 78 };
+const FIELD_COUNT_COLOR = { h: 0, s: 0, l: 40 };
+const FIELD_COUNT_FONT_SIZE = 12;
 const ERROR_COLOR = { h: 0, s: 60, l: 40 };
 const AGE_COLOR = { h: 0, s: 0, l: 40 };
 const AGE_FONT_SIZE = 13;
@@ -36,6 +52,67 @@ type ServerLogListing = {
     fetchTime?: number;
 };
 
+const MODAL_FIELD_NAME_MIN_WIDTH = 140;
+const VIEW_BUTTON_ACTIVE_HUE = 210;
+
+// Shared between the modal and the table's View buttons, so the button of the row on display highlights and arrow keys move the selection
+const viewedRowState = Querysub.createLocalSchema("storageLogsViewedRow", {
+    open: t.boolean,
+    index: t.number,
+});
+// The rows the open modal navigates. Plain memory: the modal re-renders on index changes, this is just the data those changes index into.
+let viewedRows: Record<string, unknown>[] = [];
+
+class RowViewModal extends qreact.Component<{ onClose: () => void }> {
+    componentDidMount() {
+        document.addEventListener("keydown", this.onKeyDown);
+    }
+    componentWillUnmount() {
+        document.removeEventListener("keydown", this.onKeyDown);
+    }
+    private onKeyDown = (e: KeyboardEvent) => {
+        if (e.key !== "ArrowUp" && e.key !== "ArrowDown") return;
+        e.preventDefault();
+        let delta = e.key === "ArrowDown" ? 1 : -1;
+        // A document listener is not a Querysub-dispatched event, so the synced write needs an explicit commit
+        Querysub.commit(() => {
+            let index = viewedRowState().index + delta;
+            if (index < 0 || index >= viewedRows.length) return;
+            viewedRowState().index = index;
+        });
+    };
+    render() {
+        let index = viewedRowState().index;
+        let row = viewedRows[index];
+        if (!row) return undefined;
+        return <FullscreenModal onCancel={this.props.onClose}>
+            <div className={css.vbox(6).pad2(16)}>
+                <div className={css.colorhsl(0, 0, 45)}>Row {index + 1} of {formatNumber(viewedRows.length)} (↑/↓ to move)</div>
+                {Object.entries(row).map(([field, value]) => <div key={field} className={css.hbox(10)}>
+                    <div className={css.boldStyle.minWidth(MODAL_FIELD_NAME_MIN_WIDTH)}>{field}</div>
+                    <div className={css.fontFamily("monospace").whiteSpace("pre-wrap")}>
+                        {field === "time" && typeof value === "number" && `${formatDateTime(value)} (${value})`
+                            || typeof value === "string" && value
+                            || JSON.stringify(value)}
+                    </div>
+                </div>)}
+            </div>
+        </FullscreenModal>;
+    }
+}
+
+function showRowModal(rows: Record<string, unknown>[], index: number): void {
+    viewedRows = rows;
+    viewedRowState().open = true;
+    viewedRowState().index = index;
+    let close = showModal({
+        content: <RowViewModal onClose={() => {
+            viewedRowState().open = false;
+            close.close();
+        }} />
+    });
+}
+
 /** Whether a log file could hold entries in the range - a live file (no endTime) runs to now. */
 function fileOverlapsRange(file: LogFileInfo, startTime: number, endTime: number): boolean {
     let fileEnd = file.endTime ?? Date.now();
@@ -53,7 +130,8 @@ export class StorageLogsPage extends qreact.Component {
         /** The oldest fetch time among the results being displayed */
         resultsFetchTime: t.number,
         rows: t.atomic<Record<string, unknown>[]>([]),
-        columns: t.atomic<string[]>([]),
+        fields: t.atomic<{ field: string; count: number }[]>([]),
+        failedFiles: t.atomic<FailedLogFile[]>([]),
         error: t.string,
     });
 
@@ -85,19 +163,51 @@ export class StorageLogsPage extends qreact.Component {
         return selected;
     }
 
-    private search() {
+    /** fresh drops the log file listings and the per-file search cache first, so everything searched is refetched. */
+    private search(fresh?: boolean) {
         if (this.state.searching) return;
-        // State and synced reads are captured HERE, in the tracked part, before the async work
+        // State and synced reads are captured HERE, in the tracked part, before the async work (and before the invalidation below, which would turn the listings back into loading)
         let listings = this.getListings();
-        let files = this.getSelectedFiles(listings).map(file => ({ url: file.url, name: file.name }));
+        let cachedFiles = this.getSelectedFiles(listings).map(file => ({ url: file.url, name: file.name }));
+        let urls = listings.map(listing => listing.url);
         let query = storageLogSearchParam.value;
         let { startTime, endTime, searchFromStart } = getTimeRange();
+        let synced = StorageLogsSynced(SocketFunction.browserNodeId());
+        if (fresh) {
+            invalidateStorageLogSearchCache();
+            synced.getServerLogFiles.resetAll();
+        }
         this.state.searching = true;
         this.state.error = "";
         this.state.doneFiles = 0;
-        this.state.totalFiles = files.length;
+        this.state.totalFiles = fresh ? 0 : cachedFiles.length;
         this.state.currentFile = "";
+        // Cleared up front, so anything on screen during the search is never stale results from the previous one
+        this.state.rows = [];
+        this.state.fields = [];
+        this.state.failedFiles = [];
+        this.state.durationTime = 0;
+        this.state.resultsFetchTime = 0;
         Querysub.onCommitFinished(async () => {
+            let files = cachedFiles;
+            if (fresh) {
+                // The selection has to wait for the fresh listings; a server that fails costs only its own files (its error also shows on its listing row, which refetches the same way)
+                files = [];
+                await Promise.all(urls.map(async url => {
+                    try {
+                        let listing = await synced.getServerLogFiles.promise(url);
+                        for (let file of listing.files) {
+                            if (!fileOverlapsRange(file, startTime, endTime)) continue;
+                            files.push({ url, name: file.name });
+                        }
+                    } catch (e) {
+                        console.error(`Listing log files on ${url} failed, continuing with the other servers:`, (e as Error).stack ?? e);
+                    }
+                }));
+                Querysub.commit(() => {
+                    this.state.totalFiles = files.length;
+                });
+            }
             try {
                 let result = await searchStorageLogs({
                     files,
@@ -111,27 +221,30 @@ export class StorageLogsPage extends qreact.Component {
                     },
                 });
                 let rows: Record<string, unknown>[] = [];
-                let columns: string[] = [...LEADING_COLUMNS];
                 for (let file of result.files) {
                     for (let entry of file.entries) {
                         let fields: Record<string, unknown> = entry && typeof entry === "object" ? entry as Record<string, unknown> : { value: String(entry) };
                         // The file selection is only as granular as whole files, so the entries themselves still need the range applied
                         let time = fields.time;
                         if (typeof time === "number" && (time < startTime || time > endTime)) continue;
-                        for (let key of Object.keys(fields)) {
-                            if (!columns.includes(key)) {
-                                columns.push(key);
-                            }
-                        }
                         rows.push({ server: file.url, file: file.name, ...fields });
                     }
                 }
                 // Ordered the way the time range UI says: from the end (newest first) by default, oldest first when searching from the start
                 let direction = searchFromStart ? 1 : -1;
                 rows.sort((a, b) => direction * ((typeof a.time === "number" && a.time || 0) - (typeof b.time === "number" && b.time || 0)));
+                // Seeded so the fixed fields lead the chip row; ones no entry has are dropped below
+                let fieldCounts = new Map<string, number>(LEADING_COLUMNS.map(field => [field, 0]));
+                for (let row of rows) {
+                    for (let key of Object.keys(row)) {
+                        fieldCounts.set(key, (fieldCounts.get(key) || 0) + 1);
+                    }
+                }
+                let fields = [...fieldCounts.entries()].filter(([, count]) => count > 0).map(([field, count]) => ({ field, count }));
                 Querysub.commit(() => {
                     this.state.rows = rows;
-                    this.state.columns = columns;
+                    this.state.fields = fields;
+                    this.state.failedFiles = result.failures;
                     this.state.durationTime = result.durationTime;
                     this.state.resultsFetchTime = result.oldestFetchTime;
                 });
@@ -153,10 +266,49 @@ export class StorageLogsPage extends qreact.Component {
         let selectedBytes = selectedFiles.reduce((sum, file) => sum + file.size, 0);
         let loadedListings = listings.filter(listing => listing.fetchTime);
         let oldestListingTime = loadedListings.length && Math.min(...loadedListings.map(listing => listing.fetchTime || 0)) || 0;
-        let now = Querysub.nowDelayed(timeInSecond);
+        let now = Querysub.nowDelayed(timeInMinute);
+
+        // Defaults show unless explicitly toggled off, plus whatever was explicitly toggled on
+        let selectedSet = new Set<string>();
+        for (let field of Object.keys(defaultSelectedFields)) {
+            if (atomic(storageLogFieldsParam.value[field]) === undefined) {
+                selectedSet.add(field);
+            }
+        }
+        for (let [key, value] of Object.entries(storageLogFieldsParam.value)) {
+            if (value) {
+                selectedSet.add(key);
+            }
+        }
+        // Columns follow the chip row's order; selections with no matching field this search (ex, persisted from an earlier one) go last
+        let selectedFields = this.state.fields.map(x => x.field).filter(field => selectedSet.has(field));
+        for (let field of selectedSet) {
+            if (!selectedFields.includes(field)) {
+                selectedFields.push(field);
+            }
+        }
 
-        let columnsConfig: { [column: string]: { title?: string; formatter?: (value: unknown) => preact.ComponentChild } } = {};
-        for (let column of this.state.columns) {
+        // Object identity map, as the formatter only receives the row itself and highlighting needs its index
+        let rows = this.state.rows;
+        let rowIndexes = new Map(rows.map((row, index) => [row, index]));
+        let columnsConfig: TableType<Record<string, unknown>>["columns"] = {};
+        columnsConfig["view"] = {
+            title: "View",
+            formatter: (x, context) => {
+                let row = context?.row;
+                if (!row) return undefined;
+                let index = rowIndexes.get(row) ?? -1;
+                let viewed = viewedRowState().open && viewedRowState().index === index;
+                return <Button
+                    flavor="tiny"
+                    hue={viewed ? VIEW_BUTTON_ACTIVE_HUE : undefined}
+                    onClick={() => showRowModal(rows, index)}
+                >
+                    View
+                </Button>;
+            },
+        };
+        for (let column of selectedFields) {
             if (column === "time") {
                 columnsConfig[column] = {
                     title: "Time",
@@ -177,7 +329,8 @@ export class StorageLogsPage extends qreact.Component {
                         StorageLogsSynced(SocketFunction.browserNodeId()).getServerLogFiles.resetAll();
                         invalidateStorageLogSearchCache();
                         this.state.rows = [];
-                        this.state.columns = [];
+                        this.state.fields = [];
+                        this.state.failedFiles = [];
                         this.state.resultsFetchTime = 0;
                         this.state.durationTime = 0;
                     }}
@@ -207,6 +360,15 @@ export class StorageLogsPage extends qreact.Component {
                 >
                     {this.state.searching && `Searching ${this.state.doneFiles}/${this.state.totalFiles}...` || "Search"}
                 </Button>
+                <Button
+                    flavor="large"
+                    hue={120}
+                    title="Invalidates the log file listings and the cached search results, then searches"
+                    onClick={() => this.search(true)}
+                    className={this.state.searching && css.opacity(0.5) || undefined}
+                >
+                    Fresh Search
+                </Button>
             </div>
             <div className={css.vbox(4)}>
                 {listings.map(listing => {
@@ -230,7 +392,34 @@ export class StorageLogsPage extends qreact.Component {
                 </div>
             </div>
             {this.state.error && <div className={css.colorhsl(ERROR_COLOR.h, ERROR_COLOR.s, ERROR_COLOR.l).whiteSpace("pre-wrap")}>{this.state.error}</div>}
-            {!!this.state.rows.length && <div>
+            {!!this.state.failedFiles.length && <div className={css.vbox(2).colorhsl(ERROR_COLOR.h, ERROR_COLOR.s, ERROR_COLOR.l)}>
+                <div className={css.boldStyle}>{formatNumber(this.state.failedFiles.length)} files failed to search (their entries are missing from the results):</div>
+                {this.state.failedFiles.map(failure => <div key={`${failure.url}|${failure.name}`} className={css.ellipsis} title={failure.error}>
+                    {failure.url} {failure.name}
+                </div>)}
+            </div>}
+            {!!this.state.rows.length && <div className={css.vbox(8)}>
+                <div className={css.hbox(6, 4).wrap.alignItems("center")}>
+                    {this.state.fields.map(({ field, count }) => {
+                        let selected = selectedSet.has(field);
+                        let color = selected ? FIELD_CHIP_ACTIVE_COLOR : FIELD_CHIP_COLOR;
+                        return <div
+                            key={field}
+                            className={
+                                css.hbox(5).alignItems("center").pad2(8, 4).button.whiteSpace("nowrap")
+                                    .hsl(color.h, color.s, color.l).bord2(color.h, color.s, color.l - 20)
+                            }
+                            onClick={() => {
+                                let newValues = { ...storageLogFieldsParam.value };
+                                newValues[field] = !selected;
+                                storageLogFieldsParam.value = newValues;
+                            }}
+                        >
+                            <span className={css.boldStyle}>{field}</span>
+                            <span className={css.fontSize(FIELD_COUNT_FONT_SIZE).colorhsl(FIELD_COUNT_COLOR.h, FIELD_COUNT_COLOR.s, FIELD_COUNT_COLOR.l)}>{formatNumber(count)}</span>
+                        </div>;
+                    })}
+                </div>
                 <Table
                     rows={this.state.rows}
                     columns={columnsConfig}

package/src/deployManager/components/storage/storageLogSearch.ts

@@ -60,13 +60,20 @@ export type StorageLogSearchProgress = {
     currentFile: string;
 };
 
-/** Searches the query across the given files (per-file, so progress is per-file and each file's result caches on its own). */
+export type FailedLogFile = {
+    url: string;
+    name: string;
+    error: string;
+};
+
+/** Searches the query across the given files (per-file, so progress is per-file and each file's result caches on its own). A file that fails costs only its own results - it lands in failures and the rest of the search continues. */
 export async function searchStorageLogs(config: {
     files: { url: string; name: string }[];
     query: string;
     onProgress: (progress: StorageLogSearchProgress) => void;
 }): Promise<{
     files: SearchedLogFile[];
+    failures: FailedLogFile[];
     durationTime: number;
     /** The oldest fetch time among the results used, so the UI can say how stale the display is. */
     oldestFetchTime: number;
@@ -74,16 +81,24 @@ export async function searchStorageLogs(config: {
     let startTime = Date.now();
     let doneFiles = 0;
     let results: SearchedLogFile[] = [];
+    let failures: FailedLogFile[] = [];
     let runFile = runInParallel({ parallelCount: PARALLEL_FILE_SEARCHES }, async (file: { url: string; name: string }) => {
         config.onProgress({ doneFiles, totalFiles: config.files.length, currentFile: file.name });
-        let result = await searchFile(file.url, file.name, config.query);
-        results.push({ url: file.url, name: file.name, entries: result.entries, fetchTime: result.fetchTime });
+        try {
+            let result = await searchFile(file.url, file.name, config.query);
+            results.push({ url: file.url, name: file.name, entries: result.entries, fetchTime: result.fetchTime });
+        } catch (e) {
+            let error = (e as Error).stack ?? String(e);
+            console.error(`Searching log file ${file.name} on ${file.url} failed, continuing with the other files:`, error);
+            failures.push({ url: file.url, name: file.name, error });
+        }
         doneFiles++;
         config.onProgress({ doneFiles, totalFiles: config.files.length, currentFile: file.name });
     });
     await Promise.all(config.files.map(runFile));
     return {
         files: results,
+        failures,
         durationTime: Date.now() - startTime,
         oldestFetchTime: results.length && Math.min(...results.map(x => x.fetchTime)) || Date.now(),
     };