querysub 0.632.0 → 0.634.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.632.0",
+  "version": "0.634.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -79,8 +79,8 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
-    "sliftutils": "^1.7.102",
-    "socket-function": "^1.2.30",
+    "sliftutils": "^1.7.105",
+    "socket-function": "^1.2.31",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",
     "typesafecss": "^0.32.0",

package/src/-c-identity/IdentityController.ts

@@ -6,7 +6,7 @@
 
 import debugbreak from "debugbreak";
 import { SocketFunction } from "socket-function/SocketFunction";
-import { CallerContext } from "socket-function/SocketFunctionTypes";
+import { CallerContext, ClientHookContext } from "socket-function/SocketFunctionTypes";
 import { cache, cacheWeak, lazy } from "socket-function/src/caching";
 import { getClientNodeId, getNodeId, getNodeIdDomain, getNodeIdIP, getNodeIdLocation, isClientNodeId } from "socket-function/src/nodeCache";
 import { decodeNodeId, getCommonName, getIdentityCA, getMachineId, getOwnMachineId, getPublicIdentifier, getThreadKeyCert, parseCert, sign, validateCertificate, verify } from "sliftutils/misc/https/certs";
@@ -252,7 +252,7 @@ const changeIdentityOnce = cacheWeak(async function changeIdentityOnce(connectio
     );
 });
 let startupTime = Date.now();
-SocketFunction.addGlobalClientHook(async function identityHook(context) {
+async function identityHook(context: ClientHookContext) {
     if (context.call.classGuid === IdentityController._classGuid) return;
     // This is for US to tell them our identity. And if they established the connection the identity will come from their original connection url (that they used to connect to us), and they validated it either being a real certificate, or they added the cert from the trusted backblaze bucket. If it just from a real certificate it means we identified them, but they might not have network trust. But that's fine, as IdentityController is JUST for identification, and if it's a real certificate we know who they are! (Which doesn't mean we trust them).
     if (isClientNodeId(context.call.nodeId)) {
@@ -265,4 +265,7 @@ SocketFunction.addGlobalClientHook(async function identityHook(context) {
     if (duration > 200 && now - startupTime > timeInMinute) {
         console.log(red(`IdentityHook took ${formatTime(duration)} for ${context.connectionId.nodeId} ${context.call.classGuid}.${context.call.functionName}`));
     }
-});
+}
+// Don't measure this, as oftentimes it blocks on other servers, and so it isn't interesting to measure it. 
+identityHook.noMeasure = true;
+SocketFunction.addGlobalClientHook(identityHook);

package/src/deployManager/MachinesPage.tsx

@@ -10,6 +10,7 @@ import { MachineDetailPage } from "./components/MachineDetailPage";
 import { Anchor } from "../library-components/ATag";
 import { DeployPage } from "./components/DeployPage";
 import { StoragePage } from "./components/storage/StoragePage";
+import { StorageLogsPage } from "./components/storage/StorageLogsPage";
 import { RoutingTablePage } from "../diagnostics/misc-pages/RoutingTablePage";
 
 export class MachinesPage extends qreact.Component {
@@ -23,6 +24,7 @@ export class MachinesPage extends qreact.Component {
                 { key: "services", label: "Services", otherKeys: ["service-detail"] },
                 { key: "deploy", label: "Deploy", otherKeys: [] },
                 { key: "storage", label: "Storage", otherKeys: [] },
+                { key: "storage-logs", label: "Storage Logs", otherKeys: [] },
                 { key: "routing", label: "PathValues / Functions Routing", otherKeys: [] },
             ].map(tab => {
                 let isActive = currentViewParam.value === tab.key || tab.otherKeys.includes(currentViewParam.value);
@@ -52,6 +54,7 @@ export class MachinesPage extends qreact.Component {
                 {currentViewParam.value === "machine-detail" && <MachineDetailPage />}
                 {currentViewParam.value === "deploy" && <DeployPage />}
                 {currentViewParam.value === "storage" && <StoragePage />}
+                {currentViewParam.value === "storage-logs" && <StorageLogsPage />}
                 {currentViewParam.value === "routing" && <RoutingTablePage />}
             </div>
         </div>;

package/src/deployManager/components/ServiceMeasureBars.tsx

@@ -13,6 +13,8 @@ import { assertIsManagementUser } from "../../diagnostics/managementPages";
 const NODE_CALL_TIMEOUT = timeInSecond * 10;
 const BAR_HEIGHT_PX = 6;
 const MIN_BAR_WIDTH_PX = 30;
+// A percentage width would be against the bar's own content-sized wrapper, which has no width, so it resolves to zero and every bar sits at the minimum. The width is instead this many pixels at 100% CPU.
+const FULL_BAR_WIDTH_PX = 300;
 // Green at idle, red at fully busy.
 const BAR_IDLE_HUE = 120;
 const BAR_FLASH_THRESHOLD = 0.6;
@@ -73,7 +75,7 @@ export class ServiceMeasureBars extends qreact.Component<{
                     <div
                         className={
                             css.height(BAR_HEIGHT_PX)
-                                .width(`${fraction * 100}%`)
+                                .width(Math.round(clampedFraction * FULL_BAR_WIDTH_PX))
                                 .minWidth(MIN_BAR_WIDTH_PX)
                                 .button
                                 .hsl(hue, 70, isExpanded ? 40 : 55)

package/src/deployManager/components/storage/StorageLogsPage.tsx

@@ -0,0 +1,240 @@
+import preact from "preact";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { qreact } from "../../../4-dom/qreact";
+import { css } from "typesafecss";
+import { Querysub } from "../../../4-querysub/Querysub";
+import { t } from "../../../2-proxy/schema2";
+import { isDefined } from "../../../misc";
+import { formatNumber, formatTime, formatDateTime } from "socket-function/src/formatting/format";
+import { timeInSecond } from "socket-function/src/misc";
+import type { LogFileInfo } from "sliftutils/storage/StreamingLogs";
+import { MachineServiceController } from "../../machineSchema";
+import { getStorageServers } from "./storageServers";
+import { Table } from "../../../5-diagnostics/Table";
+import { URLParam } from "../../../library-components/URLParam";
+import { InputLabelURL } from "../../../library-components/InputLabel";
+import { Button } from "../../../library-components/Button";
+import { TimeRangeSelector, getTimeRange } from "../../../diagnostics/logs/TimeRangeSelector";
+import { StorageLogsSynced } from "./storageLogsController";
+import { searchStorageLogs, invalidateStorageLogSearchCache } from "./storageLogSearch";
+
+module.hotreload = true;
+
+export const storageLogSearchParam = new URLParam("storageLogSearch", "");
+
+// Fixed columns first so every search reads the same way; the entries' own fields follow in the order they appear.
+const LEADING_COLUMNS = ["server", "file", "time", "kind"];
+const ERROR_COLOR = { h: 0, s: 60, l: 40 };
+const AGE_COLOR = { h: 0, s: 0, l: 40 };
+const AGE_FONT_SIZE = 13;
+
+type ServerLogListing = {
+    url: string;
+    loading?: boolean;
+    error?: string;
+    files?: LogFileInfo[];
+    fetchTime?: number;
+};
+
+/** Whether a log file could hold entries in the range - a live file (no endTime) runs to now. */
+function fileOverlapsRange(file: LogFileInfo, startTime: number, endTime: number): boolean {
+    let fileEnd = file.endTime ?? Date.now();
+    return file.startTime <= endTime && fileEnd >= startTime;
+}
+
+export class StorageLogsPage extends qreact.Component {
+    state = t.state({
+        searching: t.boolean,
+        doneFiles: t.number,
+        totalFiles: t.number,
+        currentFile: t.string,
+        /** How long the last search took */
+        durationTime: t.number,
+        /** The oldest fetch time among the results being displayed */
+        resultsFetchTime: t.number,
+        rows: t.atomic<Record<string, unknown>[]>([]),
+        columns: t.atomic<string[]>([]),
+        error: t.string,
+    });
+
+    private getListings(): ServerLogListing[] {
+        let machineController = MachineServiceController(SocketFunction.browserNodeId());
+        let serviceList = machineController.getServiceList();
+        let configs = (serviceList || []).map(serviceId => machineController.getServiceConfig(serviceId)).filter(isDefined);
+        return getStorageServers(configs).map(server => {
+            // Every other server still has something worth showing, so one that cannot be read becomes its own error line instead of an empty page
+            try {
+                let listing = StorageLogsSynced(SocketFunction.browserNodeId()).getServerLogFiles(server.url);
+                if (!listing) return { url: server.url, loading: true };
+                return { url: server.url, files: listing.files, fetchTime: listing.fetchTime };
+            } catch (e: any) {
+                return { url: server.url, error: e.stack ?? String(e) };
+            }
+        });
+    }
+
+    private getSelectedFiles(listings: ServerLogListing[]): { url: string; name: string; size: number }[] {
+        let { startTime, endTime } = getTimeRange();
+        let selected: { url: string; name: string; size: number }[] = [];
+        for (let listing of listings) {
+            for (let file of listing.files || []) {
+                if (!fileOverlapsRange(file, startTime, endTime)) continue;
+                selected.push({ url: listing.url, name: file.name, size: file.size });
+            }
+        }
+        return selected;
+    }
+
+    private search() {
+        if (this.state.searching) return;
+        // State and synced reads are captured HERE, in the tracked part, before the async work
+        let listings = this.getListings();
+        let files = this.getSelectedFiles(listings).map(file => ({ url: file.url, name: file.name }));
+        let query = storageLogSearchParam.value;
+        let { startTime, endTime } = getTimeRange();
+        this.state.searching = true;
+        this.state.error = "";
+        this.state.doneFiles = 0;
+        this.state.totalFiles = files.length;
+        this.state.currentFile = "";
+        Querysub.onCommitFinished(async () => {
+            try {
+                let result = await searchStorageLogs({
+                    files,
+                    query,
+                    onProgress: progress => {
+                        Querysub.commit(() => {
+                            this.state.doneFiles = progress.doneFiles;
+                            this.state.totalFiles = progress.totalFiles;
+                            this.state.currentFile = progress.currentFile;
+                        });
+                    },
+                });
+                let rows: Record<string, unknown>[] = [];
+                let columns: string[] = [...LEADING_COLUMNS];
+                for (let file of result.files) {
+                    for (let entry of file.entries) {
+                        let fields: Record<string, unknown> = entry && typeof entry === "object" ? entry as Record<string, unknown> : { value: String(entry) };
+                        // The file selection is only as granular as whole files, so the entries themselves still need the range applied
+                        let time = fields.time;
+                        if (typeof time === "number" && (time < startTime || time > endTime)) continue;
+                        for (let key of Object.keys(fields)) {
+                            if (!columns.includes(key)) {
+                                columns.push(key);
+                            }
+                        }
+                        rows.push({ server: file.url, file: file.name, ...fields });
+                    }
+                }
+                rows.sort((a, b) => (typeof a.time === "number" && a.time || 0) - (typeof b.time === "number" && b.time || 0));
+                Querysub.commit(() => {
+                    this.state.rows = rows;
+                    this.state.columns = columns;
+                    this.state.durationTime = result.durationTime;
+                    this.state.resultsFetchTime = result.oldestFetchTime;
+                });
+            } catch (e: any) {
+                Querysub.commit(() => {
+                    this.state.error = e.stack ?? String(e);
+                });
+            } finally {
+                Querysub.commit(() => {
+                    this.state.searching = false;
+                });
+            }
+        });
+    }
+
+    render() {
+        let listings = this.getListings();
+        let selectedFiles = this.getSelectedFiles(listings);
+        let selectedBytes = selectedFiles.reduce((sum, file) => sum + file.size, 0);
+        let loadedListings = listings.filter(listing => listing.fetchTime);
+        let oldestListingTime = loadedListings.length && Math.min(...loadedListings.map(listing => listing.fetchTime || 0)) || 0;
+        let now = Querysub.nowDelayed(timeInSecond);
+
+        let columnsConfig: { [column: string]: { title?: string; formatter?: (value: unknown) => preact.ComponentChild } } = {};
+        for (let column of this.state.columns) {
+            if (column === "time") {
+                columnsConfig[column] = {
+                    title: "Time",
+                    formatter: value => typeof value === "number" && <span className={css.whiteSpace("nowrap")}>{formatDateTime(value)}</span> || value as preact.ComponentChild,
+                };
+            } else {
+                columnsConfig[column] = {};
+            }
+        }
+
+        return <div className={css.vbox(16)}>
+            <div className={css.hbox(12).alignItems("center")}>
+                <h2 className={css.flexGrow(1)}>Storage logs</h2>
+                <button
+                    className={css.pad2(12, 8).button.bord2(0, 0, 20).hsl(0, 0, 100)}
+                    title="Drops both the cached log file listings and the cached search results"
+                    onClick={() => {
+                        StorageLogsSynced(SocketFunction.browserNodeId()).getServerLogFiles.resetAll();
+                        invalidateStorageLogSearchCache();
+                        this.state.rows = [];
+                        this.state.columns = [];
+                        this.state.resultsFetchTime = 0;
+                        this.state.durationTime = 0;
+                    }}
+                >
+                    Invalidate
+                </button>
+            </div>
+            <TimeRangeSelector />
+            <div className={css.hbox(10).fillWidth}>
+                <InputLabelURL
+                    flavor="large"
+                    fillWidth
+                    label="Search (a | b & c - or of ands, empty downloads everything in range)"
+                    url={storageLogSearchParam}
+                    onKeyDown={e => {
+                        if (e.key === "Enter") {
+                            storageLogSearchParam.value = e.currentTarget.value;
+                            this.search();
+                        }
+                    }}
+                />
+                <Button
+                    flavor="large"
+                    hue={210}
+                    onClick={() => this.search()}
+                    className={this.state.searching && css.opacity(0.5) || undefined}
+                >
+                    {this.state.searching && `Searching ${this.state.doneFiles}/${this.state.totalFiles}...` || "Search"}
+                </Button>
+            </div>
+            <div className={css.vbox(4)}>
+                {listings.map(listing => {
+                    let files = listing.files || [];
+                    let totalBytes = files.reduce((sum, file) => sum + file.size, 0);
+                    return <div key={listing.url} className={css.hbox(8).wrap.alignItems("center")}>
+                        <b>{listing.url}</b>
+                        {listing.loading && <span>Loading log files...</span>}
+                        {listing.error && <span className={css.colorhsl(ERROR_COLOR.h, ERROR_COLOR.s, ERROR_COLOR.l).ellipsis} title={listing.error}>{listing.error}</span>}
+                        {listing.files && <span>{formatNumber(files.length)} log files, {formatNumber(totalBytes)}B</span>}
+                    </div>;
+                })}
+                <div>
+                    Selected range covers <b>{formatNumber(selectedFiles.length)}</b> files, <b>{formatNumber(selectedBytes)}B</b> (compressed) to search
+                </div>
+                <div className={css.hbox(12).wrap.fontSize(AGE_FONT_SIZE).colorhsl(AGE_COLOR.h, AGE_COLOR.s, AGE_COLOR.l)}>
+                    {!!oldestListingTime && <span>File listing from {formatTime(now - oldestListingTime)} ago</span>}
+                    {!!this.state.resultsFetchTime && <span>Results from {formatTime(now - this.state.resultsFetchTime)} ago</span>}
+                    {!!this.state.durationTime && <span>Search took {formatTime(this.state.durationTime)}</span>}
+                    {this.state.searching && !!this.state.currentFile && <span>Searching {this.state.currentFile}</span>}
+                </div>
+            </div>
+            {this.state.error && <div className={css.colorhsl(ERROR_COLOR.h, ERROR_COLOR.s, ERROR_COLOR.l).whiteSpace("pre-wrap")}>{this.state.error}</div>}
+            {!!this.state.rows.length && <div>
+                <Table
+                    rows={this.state.rows}
+                    columns={columnsConfig}
+                />
+            </div>}
+            {!this.state.rows.length && !this.state.searching && !!this.state.durationTime && <div>No matching log entries.</div>}
+        </div>;
+    }
+}

package/src/deployManager/components/storage/bucketRows.ts

@@ -1,7 +1,7 @@
 import { formatNumber } from "socket-function/src/formatting/format";
 import { sort } from "socket-function/src/misc";
 import type { ArchivesConfig, RemoteConfig, HostedConfig } from "sliftutils/storage/IArchives";
-import type { BucketWriteStats } from "sliftutils/storage/remoteStorage/storageServerState";
+import type { BucketWriteStats } from "sliftutils/storage/remoteStorage/accessStats";
 import type { BucketDiskInfo } from "sliftutils/storage/remoteStorage/bucketDisk";
 import { parseHostedUrl } from "sliftutils/storage/remoteStorage/remoteConfig";
 import { SourceTag, getSourceTags } from "./Tag";

package/src/deployManager/components/storage/storageLogSearch.ts

@@ -0,0 +1,90 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { runInParallel } from "socket-function/src/batching";
+import { StorageLogsController } from "./storageLogsController";
+
+module.hotreload = true;
+
+// searchServerLogs only ANDs its search strings, so the `|` level of a query has to be decomposed on our side: one server-side search per or-clause, unioned. The union check uses the entry's own JSON (the same text the server searched), so it never collapses genuinely duplicate log lines the way a dedupe-by-content would.
+
+const PARALLEL_FILE_SEARCHES = 4;
+
+/** A query's or-clauses, each the list of terms that must all appear - the same `|` / `&` semantics matchFilter (misc.ts) applies locally, minus its lowercasing, as the server search is case-sensitive. */
+export function parseSearchClauses(query: string): string[][] {
+    return query.split("|")
+        .map(clause => clause.split("&").map(x => x.trim()).filter(x => x))
+        .filter(clause => clause.length > 0);
+}
+
+export type SearchedLogFile = {
+    url: string;
+    name: string;
+    entries: unknown[];
+    /** When these entries were actually fetched - older than the search when they came from the cache. */
+    fetchTime: number;
+};
+
+const searchCache = new Map<string, { entries: unknown[]; fetchTime: number }>();
+export function invalidateStorageLogSearchCache(): void {
+    searchCache.clear();
+}
+
+async function searchFile(url: string, name: string, query: string): Promise<{ entries: unknown[]; fetchTime: number }> {
+    let cacheKey = `${url}|${name}|${query}`;
+    let cached = searchCache.get(cacheKey);
+    if (cached) return cached;
+    let controller = StorageLogsController.nodes[SocketFunction.browserNodeId()];
+    let clauses = parseSearchClauses(query);
+    let entries: unknown[];
+    if (clauses.length <= 1) {
+        entries = await controller.searchLogFile(url, name, clauses[0] || []);
+    } else {
+        // Later clauses drop entries an earlier clause already returned, which unions the or-clauses without collapsing real duplicates
+        entries = [];
+        for (let [index, clause] of clauses.entries()) {
+            let clauseEntries = await controller.searchLogFile(url, name, clause);
+            for (let entry of clauseEntries) {
+                let line = JSON.stringify(entry);
+                if (clauses.slice(0, index).some(prev => prev.every(term => line.includes(term)))) continue;
+                entries.push(entry);
+            }
+        }
+    }
+    let result = { entries, fetchTime: Date.now() };
+    searchCache.set(cacheKey, result);
+    return result;
+}
+
+export type StorageLogSearchProgress = {
+    doneFiles: number;
+    totalFiles: number;
+    currentFile: string;
+};
+
+/** Searches the query across the given files (per-file, so progress is per-file and each file's result caches on its own). */
+export async function searchStorageLogs(config: {
+    files: { url: string; name: string }[];
+    query: string;
+    onProgress: (progress: StorageLogSearchProgress) => void;
+}): Promise<{
+    files: SearchedLogFile[];
+    durationTime: number;
+    /** The oldest fetch time among the results used, so the UI can say how stale the display is. */
+    oldestFetchTime: number;
+}> {
+    let startTime = Date.now();
+    let doneFiles = 0;
+    let results: SearchedLogFile[] = [];
+    let runFile = runInParallel({ parallelCount: PARALLEL_FILE_SEARCHES }, async (file: { url: string; name: string }) => {
+        config.onProgress({ doneFiles, totalFiles: config.files.length, currentFile: file.name });
+        let result = await searchFile(file.url, file.name, config.query);
+        results.push({ url: file.url, name: file.name, entries: result.entries, fetchTime: result.fetchTime });
+        doneFiles++;
+        config.onProgress({ doneFiles, totalFiles: config.files.length, currentFile: file.name });
+    });
+    await Promise.all(config.files.map(runFile));
+    return {
+        files: results,
+        durationTime: Date.now() - startTime,
+        oldestFetchTime: results.length && Math.min(...results.map(x => x.fetchTime)) || Date.now(),
+    };
+}

package/src/deployManager/components/storage/storageLogsController.ts

@@ -0,0 +1,54 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { listServerLogFiles, getServerLogs, searchServerLogs } from "sliftutils/storage/remoteStorage/createArchives";
+import type { LogFileInfo } from "sliftutils/storage/StreamingLogs";
+import { getSyncedController } from "../../../library-components/SyncedController";
+import { assertIsManagementUser } from "../../../diagnostics/managementPages";
+import { STORAGE_ACCOUNT } from "../../../-a-archives/archives2";
+
+// The registered controller and its synced wrapper have to keep a stable identity, so this file is a full-refresh boundary rather than a hot-reload one (matching SyncedController itself).
+module.hotreload = false;
+
+export type ServerLogFiles = {
+    files: LogFileInfo[];
+    /** When this listing was fetched, so the UI can say how stale a cached listing is. */
+    fetchTime: number;
+};
+
+/** Every storage call is proxied through here: they authenticate with this machine's identity CA, which the browser has no access to. */
+class StorageLogsControllerBase {
+    /** One call per server, so each caches and refreshes on its own. */
+    public async getServerLogFiles(url: string): Promise<ServerLogFiles> {
+        return {
+            files: await listServerLogFiles({ url, account: STORAGE_ACCOUNT }),
+            fetchTime: Date.now(),
+        };
+    }
+    /** One file per call, so the client can report per-file progress. Empty searches downloads and decodes the whole file (searchServerLogs requires at least one search string). */
+    public async searchLogFile(url: string, name: string, searches: string[]): Promise<unknown[]> {
+        if (!searches.length) {
+            let results = await getServerLogs({ url, account: STORAGE_ACCOUNT, names: [name] });
+            return results[0]?.entries || [];
+        }
+        let results = await searchServerLogs({ url, account: STORAGE_ACCOUNT, names: [name], searches });
+        return results[0]?.entries || [];
+    }
+}
+
+export const StorageLogsController = SocketFunction.register(
+    "StorageLogsController-8b3f52c1-6a4e-4c0d-9d71-2f8e4a5b6c93",
+    new StorageLogsControllerBase(),
+    () => ({
+        getServerLogFiles: {},
+        searchLogFile: {},
+    }),
+    () => ({
+        hooks: [assertIsManagementUser],
+    }),
+    {
+    }
+);
+
+export const StorageLogsSynced = getSyncedController(StorageLogsController, {
+    reads: {},
+    writes: {},
+});

package/src/deployManager/urlParams.ts

@@ -1,6 +1,6 @@
 import { URLParam } from "../library-components/URLParam";
 
-export type ViewType = "services" | "machines" | "service-detail" | "machine-detail" | "deploy" | "storage" | "routing";
+export type ViewType = "services" | "machines" | "service-detail" | "machine-detail" | "deploy" | "storage" | "storage-logs" | "routing";
 
 export const currentViewParam = new URLParam<ViewType>("machineview", "machines");
 export const selectedServiceIdParam = new URLParam("serviceId", "");