querysub 0.632.0 → 0.633.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.632.0",
+  "version": "0.633.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -79,7 +79,7 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
-    "sliftutils": "^1.7.102",
+    "sliftutils": "^1.7.103",
     "socket-function": "^1.2.30",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",

package/src/deployManager/MachinesPage.tsx

@@ -10,6 +10,7 @@ import { MachineDetailPage } from "./components/MachineDetailPage";
 import { Anchor } from "../library-components/ATag";
 import { DeployPage } from "./components/DeployPage";
 import { StoragePage } from "./components/storage/StoragePage";
+import { StorageLogsPage } from "./components/storage/StorageLogsPage";
 import { RoutingTablePage } from "../diagnostics/misc-pages/RoutingTablePage";
 
 export class MachinesPage extends qreact.Component {
@@ -23,6 +24,7 @@ export class MachinesPage extends qreact.Component {
                 { key: "services", label: "Services", otherKeys: ["service-detail"] },
                 { key: "deploy", label: "Deploy", otherKeys: [] },
                 { key: "storage", label: "Storage", otherKeys: [] },
+                { key: "storage-logs", label: "Storage Logs", otherKeys: [] },
                 { key: "routing", label: "PathValues / Functions Routing", otherKeys: [] },
             ].map(tab => {
                 let isActive = currentViewParam.value === tab.key || tab.otherKeys.includes(currentViewParam.value);
@@ -52,6 +54,7 @@ export class MachinesPage extends qreact.Component {
                 {currentViewParam.value === "machine-detail" && <MachineDetailPage />}
                 {currentViewParam.value === "deploy" && <DeployPage />}
                 {currentViewParam.value === "storage" && <StoragePage />}
+                {currentViewParam.value === "storage-logs" && <StorageLogsPage />}
                 {currentViewParam.value === "routing" && <RoutingTablePage />}
             </div>
         </div>;

package/src/deployManager/components/storage/StorageLogsPage.tsx

@@ -0,0 +1,251 @@
+import preact from "preact";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { qreact } from "../../../4-dom/qreact";
+import { css } from "typesafecss";
+import { Querysub } from "../../../4-querysub/Querysub";
+import { isDefined } from "../../../misc";
+import { formatNumber, formatTime, formatDateTime } from "socket-function/src/formatting/format";
+import { timeInSecond } from "socket-function/src/misc";
+import type { LogFileInfo } from "sliftutils/storage/StreamingLogs";
+import { MachineServiceController } from "../../machineSchema";
+import { getStorageServers } from "./storageServers";
+import { Table } from "../../../5-diagnostics/Table";
+import { URLParam } from "../../../library-components/URLParam";
+import { InputLabelURL } from "../../../library-components/InputLabel";
+import { Button } from "../../../library-components/Button";
+import { TimeRangeSelector, getTimeRange } from "../../../diagnostics/logs/TimeRangeSelector";
+import { StorageLogsSynced } from "./storageLogsController";
+import { searchStorageLogs, invalidateStorageLogSearchCache } from "./storageLogSearch";
+
+module.hotreload = true;
+
+export const storageLogSearchParam = new URLParam("storageLogSearch", "");
+
+// Fixed columns first so every search reads the same way; the entries' own fields follow in the order they appear.
+const LEADING_COLUMNS = ["server", "file", "time", "kind"];
+const ERROR_COLOR = { h: 0, s: 60, l: 40 };
+const AGE_COLOR = { h: 0, s: 0, l: 40 };
+const AGE_FONT_SIZE = 13;
+
+type ServerLogListing = {
+    url: string;
+    loading?: boolean;
+    error?: string;
+    files?: LogFileInfo[];
+    fetchTime?: number;
+};
+
+type SearchState = {
+    searching: boolean;
+    doneFiles: number;
+    totalFiles: number;
+    currentFile: string;
+    /** How long the last search took */
+    durationTime: number;
+    /** The oldest fetch time among the results being displayed */
+    resultsFetchTime: number;
+    rows: Record<string, unknown>[];
+    columns: string[];
+    error: string;
+};
+
+/** Whether a log file could hold entries in the range - a live file (no endTime) runs to now. */
+function fileOverlapsRange(file: LogFileInfo, startTime: number, endTime: number): boolean {
+    let fileEnd = file.endTime ?? Date.now();
+    return file.startTime <= endTime && fileEnd >= startTime;
+}
+
+export class StorageLogsPage extends qreact.Component {
+    state: SearchState = {
+        searching: false,
+        doneFiles: 0,
+        totalFiles: 0,
+        currentFile: "",
+        durationTime: 0,
+        resultsFetchTime: 0,
+        rows: [],
+        columns: [],
+        error: "",
+    };
+
+    private getListings(): ServerLogListing[] {
+        let machineController = MachineServiceController(SocketFunction.browserNodeId());
+        let serviceList = machineController.getServiceList();
+        let configs = (serviceList || []).map(serviceId => machineController.getServiceConfig(serviceId)).filter(isDefined);
+        return getStorageServers(configs).map(server => {
+            // Every other server still has something worth showing, so one that cannot be read becomes its own error line instead of an empty page
+            try {
+                let listing = StorageLogsSynced(SocketFunction.browserNodeId()).getServerLogFiles(server.url);
+                if (!listing) return { url: server.url, loading: true };
+                return { url: server.url, files: listing.files, fetchTime: listing.fetchTime };
+            } catch (e: any) {
+                return { url: server.url, error: e.stack ?? String(e) };
+            }
+        });
+    }
+
+    private getSelectedFiles(listings: ServerLogListing[]): { url: string; name: string; size: number }[] {
+        let { startTime, endTime } = getTimeRange();
+        let selected: { url: string; name: string; size: number }[] = [];
+        for (let listing of listings) {
+            for (let file of listing.files || []) {
+                if (!fileOverlapsRange(file, startTime, endTime)) continue;
+                selected.push({ url: listing.url, name: file.name, size: file.size });
+            }
+        }
+        return selected;
+    }
+
+    private search() {
+        if (this.state.searching) return;
+        // State and synced reads are captured HERE, in the tracked part, before the async work
+        let listings = this.getListings();
+        let files = this.getSelectedFiles(listings).map(file => ({ url: file.url, name: file.name }));
+        let query = storageLogSearchParam.value;
+        let { startTime, endTime } = getTimeRange();
+        this.state.searching = true;
+        this.state.error = "";
+        this.state.doneFiles = 0;
+        this.state.totalFiles = files.length;
+        this.state.currentFile = "";
+        Querysub.onCommitFinished(async () => {
+            try {
+                let result = await searchStorageLogs({
+                    files,
+                    query,
+                    onProgress: progress => {
+                        Querysub.commit(() => {
+                            this.state.doneFiles = progress.doneFiles;
+                            this.state.totalFiles = progress.totalFiles;
+                            this.state.currentFile = progress.currentFile;
+                        });
+                    },
+                });
+                let rows: Record<string, unknown>[] = [];
+                let columns: string[] = [...LEADING_COLUMNS];
+                for (let file of result.files) {
+                    for (let entry of file.entries) {
+                        let fields = entry && typeof entry === "object" && entry as Record<string, unknown> || { value: String(entry) };
+                        // The file selection is only as granular as whole files, so the entries themselves still need the range applied
+                        let time = fields.time;
+                        if (typeof time === "number" && (time < startTime || time > endTime)) continue;
+                        for (let key of Object.keys(fields)) {
+                            if (!columns.includes(key)) {
+                                columns.push(key);
+                            }
+                        }
+                        rows.push({ server: file.url, file: file.name, ...fields });
+                    }
+                }
+                rows.sort((a, b) => (typeof a.time === "number" && a.time || 0) - (typeof b.time === "number" && b.time || 0));
+                Querysub.commit(() => {
+                    this.state.rows = rows;
+                    this.state.columns = columns;
+                    this.state.durationTime = result.durationTime;
+                    this.state.resultsFetchTime = result.oldestFetchTime;
+                });
+            } catch (e: any) {
+                Querysub.commit(() => {
+                    this.state.error = e.stack ?? String(e);
+                });
+            } finally {
+                Querysub.commit(() => {
+                    this.state.searching = false;
+                });
+            }
+        });
+    }
+
+    render() {
+        let listings = this.getListings();
+        let selectedFiles = this.getSelectedFiles(listings);
+        let selectedBytes = selectedFiles.reduce((sum, file) => sum + file.size, 0);
+        let loadedListings = listings.filter(listing => listing.fetchTime);
+        let oldestListingTime = loadedListings.length && Math.min(...loadedListings.map(listing => listing.fetchTime || 0)) || 0;
+        let now = Querysub.nowDelayed(timeInSecond);
+
+        let columnsConfig: { [column: string]: { title?: string; formatter?: (value: unknown) => preact.ComponentChild } } = {};
+        for (let column of this.state.columns) {
+            if (column === "time") {
+                columnsConfig[column] = {
+                    title: "Time",
+                    formatter: value => typeof value === "number" && <span className={css.whiteSpace("nowrap")}>{formatDateTime(value)}</span> || value as preact.ComponentChild,
+                };
+            } else {
+                columnsConfig[column] = {};
+            }
+        }
+
+        return <div className={css.vbox(16)}>
+            <div className={css.hbox(12).alignItems("center")}>
+                <h2 className={css.flexGrow(1)}>Storage logs</h2>
+                <button
+                    className={css.pad2(12, 8).button.bord2(0, 0, 20).hsl(0, 0, 100)}
+                    title="Drops both the cached log file listings and the cached search results"
+                    onClick={() => {
+                        StorageLogsSynced(SocketFunction.browserNodeId()).getServerLogFiles.resetAll();
+                        invalidateStorageLogSearchCache();
+                        this.state.rows = [];
+                        this.state.columns = [];
+                        this.state.resultsFetchTime = 0;
+                        this.state.durationTime = 0;
+                    }}
+                >
+                    Invalidate
+                </button>
+            </div>
+            <TimeRangeSelector />
+            <div className={css.hbox(10).fillWidth}>
+                <InputLabelURL
+                    flavor="large"
+                    fillWidth
+                    label="Search (a | b & c - or of ands, empty downloads everything in range)"
+                    url={storageLogSearchParam}
+                    onKeyDown={e => {
+                        if (e.key === "Enter") {
+                            storageLogSearchParam.value = e.currentTarget.value;
+                            this.search();
+                        }
+                    }}
+                />
+                <Button
+                    flavor="large"
+                    hue={210}
+                    onClick={() => this.search()}
+                    className={this.state.searching && css.opacity(0.5) || undefined}
+                >
+                    {this.state.searching && `Searching ${this.state.doneFiles}/${this.state.totalFiles}...` || "Search"}
+                </Button>
+            </div>
+            <div className={css.vbox(4)}>
+                {listings.map(listing => {
+                    let files = listing.files || [];
+                    let totalBytes = files.reduce((sum, file) => sum + file.size, 0);
+                    return <div key={listing.url} className={css.hbox(8).wrap.alignItems("center")}>
+                        <b>{listing.url}</b>
+                        {listing.loading && <span>Loading log files...</span>}
+                        {listing.error && <span className={css.colorhsl(ERROR_COLOR.h, ERROR_COLOR.s, ERROR_COLOR.l).ellipsis} title={listing.error}>{listing.error}</span>}
+                        {listing.files && <span>{formatNumber(files.length)} log files, {formatNumber(totalBytes)}B</span>}
+                    </div>;
+                })}
+                <div>
+                    Selected range covers <b>{formatNumber(selectedFiles.length)}</b> files, <b>{formatNumber(selectedBytes)}B</b> (compressed) to search
+                </div>
+                <div className={css.hbox(12).wrap.fontSize(AGE_FONT_SIZE).colorhsl(AGE_COLOR.h, AGE_COLOR.s, AGE_COLOR.l)}>
+                    {!!oldestListingTime && <span>File listing from {formatTime(now - oldestListingTime)} ago</span>}
+                    {!!this.state.resultsFetchTime && <span>Results from {formatTime(now - this.state.resultsFetchTime)} ago</span>}
+                    {!!this.state.durationTime && <span>Search took {formatTime(this.state.durationTime)}</span>}
+                    {this.state.searching && !!this.state.currentFile && <span>Searching {this.state.currentFile}</span>}
+                </div>
+            </div>
+            {this.state.error && <div className={css.colorhsl(ERROR_COLOR.h, ERROR_COLOR.s, ERROR_COLOR.l).whiteSpace("pre-wrap")}>{this.state.error}</div>}
+            {!!this.state.rows.length && <div>
+                <Table
+                    rows={this.state.rows}
+                    columns={columnsConfig}
+                />
+            </div>}
+            {!this.state.rows.length && !this.state.searching && !!this.state.durationTime && <div>No matching log entries.</div>}
+        </div>;
+    }
+}

package/src/deployManager/components/storage/storageLogSearch.ts

@@ -0,0 +1,90 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { runInParallel } from "socket-function/src/batching";
+import { StorageLogsController } from "./storageLogsController";
+
+module.hotreload = true;
+
+// searchServerLogs only ANDs its search strings, so the `|` level of a query has to be decomposed on our side: one server-side search per or-clause, unioned. The union check uses the entry's own JSON (the same text the server searched), so it never collapses genuinely duplicate log lines the way a dedupe-by-content would.
+
+const PARALLEL_FILE_SEARCHES = 4;
+
+/** A query's or-clauses, each the list of terms that must all appear - the same `|` / `&` semantics matchFilter (misc.ts) applies locally, minus its lowercasing, as the server search is case-sensitive. */
+export function parseSearchClauses(query: string): string[][] {
+    return query.split("|")
+        .map(clause => clause.split("&").map(x => x.trim()).filter(x => x))
+        .filter(clause => clause.length > 0);
+}
+
+export type SearchedLogFile = {
+    url: string;
+    name: string;
+    entries: unknown[];
+    /** When these entries were actually fetched - older than the search when they came from the cache. */
+    fetchTime: number;
+};
+
+const searchCache = new Map<string, { entries: unknown[]; fetchTime: number }>();
+export function invalidateStorageLogSearchCache(): void {
+    searchCache.clear();
+}
+
+async function searchFile(url: string, name: string, query: string): Promise<{ entries: unknown[]; fetchTime: number }> {
+    let cacheKey = `${url}|${name}|${query}`;
+    let cached = searchCache.get(cacheKey);
+    if (cached) return cached;
+    let controller = StorageLogsController.nodes[SocketFunction.browserNodeId()];
+    let clauses = parseSearchClauses(query);
+    let entries: unknown[];
+    if (clauses.length <= 1) {
+        entries = await controller.searchLogFile(url, name, clauses[0] || []);
+    } else {
+        // Later clauses drop entries an earlier clause already returned, which unions the or-clauses without collapsing real duplicates
+        entries = [];
+        for (let [index, clause] of clauses.entries()) {
+            let clauseEntries = await controller.searchLogFile(url, name, clause);
+            for (let entry of clauseEntries) {
+                let line = JSON.stringify(entry);
+                if (clauses.slice(0, index).some(prev => prev.every(term => line.includes(term)))) continue;
+                entries.push(entry);
+            }
+        }
+    }
+    let result = { entries, fetchTime: Date.now() };
+    searchCache.set(cacheKey, result);
+    return result;
+}
+
+export type StorageLogSearchProgress = {
+    doneFiles: number;
+    totalFiles: number;
+    currentFile: string;
+};
+
+/** Searches the query across the given files (per-file, so progress is per-file and each file's result caches on its own). */
+export async function searchStorageLogs(config: {
+    files: { url: string; name: string }[];
+    query: string;
+    onProgress: (progress: StorageLogSearchProgress) => void;
+}): Promise<{
+    files: SearchedLogFile[];
+    durationTime: number;
+    /** The oldest fetch time among the results used, so the UI can say how stale the display is. */
+    oldestFetchTime: number;
+}> {
+    let startTime = Date.now();
+    let doneFiles = 0;
+    let results: SearchedLogFile[] = [];
+    let runFile = runInParallel({ parallelCount: PARALLEL_FILE_SEARCHES }, async (file: { url: string; name: string }) => {
+        config.onProgress({ doneFiles, totalFiles: config.files.length, currentFile: file.name });
+        let result = await searchFile(file.url, file.name, config.query);
+        results.push({ url: file.url, name: file.name, entries: result.entries, fetchTime: result.fetchTime });
+        doneFiles++;
+        config.onProgress({ doneFiles, totalFiles: config.files.length, currentFile: file.name });
+    });
+    await Promise.all(config.files.map(runFile));
+    return {
+        files: results,
+        durationTime: Date.now() - startTime,
+        oldestFetchTime: results.length && Math.min(...results.map(x => x.fetchTime)) || Date.now(),
+    };
+}

package/src/deployManager/components/storage/storageLogsController.ts

@@ -0,0 +1,54 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { listServerLogFiles, getServerLogs, searchServerLogs } from "sliftutils/storage/remoteStorage/createArchives";
+import type { LogFileInfo } from "sliftutils/storage/StreamingLogs";
+import { getSyncedController } from "../../../library-components/SyncedController";
+import { assertIsManagementUser } from "../../../diagnostics/managementPages";
+import { STORAGE_ACCOUNT } from "../../../-a-archives/archives2";
+
+// The registered controller and its synced wrapper have to keep a stable identity, so this file is a full-refresh boundary rather than a hot-reload one (matching SyncedController itself).
+module.hotreload = false;
+
+export type ServerLogFiles = {
+    files: LogFileInfo[];
+    /** When this listing was fetched, so the UI can say how stale a cached listing is. */
+    fetchTime: number;
+};
+
+/** Every storage call is proxied through here: they authenticate with this machine's identity CA, which the browser has no access to. */
+class StorageLogsControllerBase {
+    /** One call per server, so each caches and refreshes on its own. */
+    public async getServerLogFiles(url: string): Promise<ServerLogFiles> {
+        return {
+            files: await listServerLogFiles({ url, account: STORAGE_ACCOUNT }),
+            fetchTime: Date.now(),
+        };
+    }
+    /** One file per call, so the client can report per-file progress. Empty searches downloads and decodes the whole file (searchServerLogs requires at least one search string). */
+    public async searchLogFile(url: string, name: string, searches: string[]): Promise<unknown[]> {
+        if (!searches.length) {
+            let results = await getServerLogs({ url, account: STORAGE_ACCOUNT, names: [name] });
+            return results[0]?.entries || [];
+        }
+        let results = await searchServerLogs({ url, account: STORAGE_ACCOUNT, names: [name], searches });
+        return results[0]?.entries || [];
+    }
+}
+
+export const StorageLogsController = SocketFunction.register(
+    "StorageLogsController-8b3f52c1-6a4e-4c0d-9d71-2f8e4a5b6c93",
+    new StorageLogsControllerBase(),
+    () => ({
+        getServerLogFiles: {},
+        searchLogFile: {},
+    }),
+    () => ({
+        hooks: [assertIsManagementUser],
+    }),
+    {
+    }
+);
+
+export const StorageLogsSynced = getSyncedController(StorageLogsController, {
+    reads: {},
+    writes: {},
+});

package/src/deployManager/urlParams.ts

@@ -1,6 +1,6 @@
 import { URLParam } from "../library-components/URLParam";
 
-export type ViewType = "services" | "machines" | "service-detail" | "machine-detail" | "deploy" | "storage" | "routing";
+export type ViewType = "services" | "machines" | "service-detail" | "machine-detail" | "deploy" | "storage" | "storage-logs" | "routing";
 
 export const currentViewParam = new URLParam<ViewType>("machineview", "machines");
 export const selectedServiceIdParam = new URLParam("serviceId", "");