querysub 0.46.0 → 0.50.0

package/src/5-diagnostics/GenericFormat.tsx

@@ -4,6 +4,8 @@ import { canHaveChildren } from "socket-function/src/types";
 import { qreact } from "../4-dom/qreact";
 import preact, { ContextType } from "preact";
 import { errorMessage, warnMessage } from "../library-components/colors";
+import { URLParam } from "../library-components/URLParam";
+import { Querysub } from "../4-querysub/QuerysubController";
 
 export { errorMessage, warnMessage };
 
@@ -26,6 +28,7 @@ type StringFormatters = (
     | "timeSpan" | "date"
     | "error" | "link"
     | "toSpaceCase"
+    | "<Selector>"
 );
 
 function d(value: unknown, formattedValue: preact.ComponentChild) {
@@ -44,6 +47,7 @@ let formatters: { [formatter in StringFormatters]: (value: unknown) => preact.Co
     date: (value) => d(value, formatVeryNiceDateTime(Number(value))),
     error: (value) => d(value, <span class={errorMessage}>{String(value)}</span>),
     toSpaceCase: (value) => d(value, toSpaceCase(String(value))),
+    "<Selector>": (value) => d(value, <Selector {...JSON.parse(String(value).slice("<Selector>".length))} />),
     link: (value) => {
         if (value === undefined || value === null) {
             return "";
@@ -102,6 +106,12 @@ let formatters: { [formatter in StringFormatters]: (value: unknown) => preact.Co
         if (typeof value === "string" && value.startsWith("https://")) {
             return formatters.link(value);
         }
+        if (typeof value === "string" && value.startsWith("<Selector>")) {
+            let configParts = value.slice("<Selector>".length);
+            if (isJSON(configParts)) {
+                return formatters["<Selector>"](value);
+            }
+        }
         // Don't render nested objects, etc, otherwise passing large arrays
         //  could just in us blowing the output up, when their intention
         //  is to just show "Array()"
@@ -111,6 +121,13 @@ let formatters: { [formatter in StringFormatters]: (value: unknown) => preact.Co
         return formatters.string(value);
     },
 };
+function isJSON(value: unknown) {
+    try {
+        JSON.parse(value as string);
+        return true;
+    } catch { }
+    return false;
+}
 function formatVArray(value: unknown[], formatter: StringFormatters) {
     if (!Array.isArray(value)) {
         return <span class={errorMessage}>Expected array, got {typeof value}</span>;
@@ -164,4 +181,78 @@ export function toSpaceCase(text: string) {
         // Convert multiple spaces to a single space
         .replace(/\s+/g, " ");
     ;
-}
+}
+
+export interface SelectorProps {
+    options: { value: string, urlValue: string, title: string }[];
+    urlKey: string;
+}
+class Selector extends qreact.Component<SelectorProps> {
+    state = {
+        expanded: false,
+    };
+    componentDidMount(): void {
+        document.body.addEventListener("click", this.onBodyClick);
+        qreact.onDispose(this.onBodyClick);
+    }
+
+    justSetExpanded = false;
+    onBodyClick = () => {
+        if (this.justSetExpanded) {
+            this.justSetExpanded = false;
+            return;
+        }
+        Querysub.commit(() => {
+            this.state.expanded = false;
+        });
+    };
+    render() {
+        const { options, urlKey } = this.props;
+        if (options.length === 0) {
+            return "";
+        }
+        let urlParam = new URLParam(urlKey, options[0].urlValue);
+        let selectedOption = options.find(x => x.urlValue === urlParam.value);
+        let component = qreact.getRenderingComponentAssert();
+        return (
+            <div
+                className={css.relative.zIndex(1)}
+                onClick={() => {
+                    if (this.state.expanded) return;
+
+                    this.state.expanded = true;
+                    this.justSetExpanded = true;
+                }}
+            >
+                <div
+                    className={css.hsla(0, 0, 50, 0.3).pad2(6, 2).button}
+                    title={selectedOption?.title}
+                >
+                    {formatValue(selectedOption?.value)}
+                </div>
+
+                {this.state.expanded && (
+                    <div
+                        className={css.absolute.pos(0, 0).width("100vw").vbox0}
+                    >
+                        <div className={css.vbox(2).hsla(0, 0, 80, 1).pad2(6)}>
+                            {options.map(x =>
+                                <div
+                                    className={css.pad2(6, 2).hsla(0, 0, 50, 0.5).button}
+                                    onClick={() => {
+                                        urlParam.value = x.urlValue;
+                                    }}
+                                    title={x.title}
+                                >({x.title}) {formatValue(x.value)}</div>
+                            )}
+                        </div>
+                    </div>
+                )}
+            </div>
+        );
+    }
+}
+
+export function encodeFormattedSelector(config: SelectorProps) {
+    return `<Selector>${JSON.stringify(config)}`;
+}

package/src/5-diagnostics/Table.tsx

@@ -102,7 +102,12 @@ export class Table<RowT extends RowType> extends qreact.Component<TableType<RowT
                                 }
                                 innerContent = props.children as any;
                             }
-                            return <td {...attributes}>
+                            return <td
+                                {...attributes}
+                                data-column={columnName}
+                                // If it ever becomes non-table cell, it breaks the table
+                                style={{ display: "table-cell" }}
+                            >
                                 {innerContent}
                             </td>;
                         })}
@@ -170,14 +175,4 @@ function renderTrimmed(config: {
                 let close = showModal({
                     content: <FullscreenModal onCancel={() => {
                         close.close();
-                    }}>
-                        <div class={css.whiteSpace("pre-wrap")}>
-                            {content}
-                        </div>
-                    </FullscreenModal>
-                });
-            }
-        },
-        innerContent: contentStr
-    };
-}
+         

package/src/5-diagnostics/TimeGrouper.tsx

@@ -1,6 +1,7 @@
 import debugbreak from "debugbreak";
 import { formatTime } from "socket-function/src/formatting/format";
 import { isDefined } from "../misc";
+import { encodeFormattedSelector } from "./GenericFormat";
 
 const SUB_BUCKET_COUNT = 100;
 
@@ -71,13 +72,14 @@ export class TimeGrouper {
         }
     }
     public getSummary() {
+
         let buckets = this.buckets.map(bucket => {
             // Reduce subBuckets within each bucket
             let sub = bucket.subBuckets;
             return {
                 name: bucket.name,
                 duration: bucket.duration,
-                recordedDuration: sub.at(-1)!.startTime - sub[0].endTime,
+                recordedDuration: (sub.at(-1)?.startTime ?? 0) - (sub.at(0)?.endTime ?? 0),
                 min: sub.map(x => x.min).reduce((a, b) => Math.min(a, b), Number.POSITIVE_INFINITY),
                 max: sub.map(x => x.max).reduce((a, b) => Math.max(a, b), Number.NEGATIVE_INFINITY),
                 sum: sub.map(x => x.sum).reduce((a, b) => a + b, 0),
@@ -95,22 +97,29 @@ export class TimeGrouper {
 
     /** For sampled values, ex, % CPU usage */
     public getStateSummary(format: (value: number) => string): string {
-        return this.getSummary().map(bucket => {
-            if (bucket.duration === 0 || bucket.min === bucket.max) {
-                if (bucket.min === 0) return undefined;
-                return `(${bucket.name}) ${format(bucket.min)}`;
-            }
-            return `(${bucket.name}) ${format(bucket.min)} to ${format(bucket.max)}`;
-        }).filter(isDefined).join("\n");
+        return encodeFormattedSelector({
+            // Always use the same urlKey, so we can change all displays at once
+            urlKey: "timegroupergroup",
+            options: this.getSummary().map(bucket => {
+                if (bucket.duration === 0 || bucket.min === bucket.max) {
+                    if (bucket.min === 0) return undefined;
+                    return { value: `${format(bucket.min)}`, title: bucket.name, urlValue: bucket.name };
+                }
+                return { value: `${format(bucket.min)} to ${format(bucket.max)}`, title: bucket.name, urlValue: bucket.name };
+            }).filter(isDefined)
+        });
     }
     /** Sums up the values. Ex, for time waiting. */
     public getEventSummary(format: (value: number) => string): string {
-        return this.getSummary().map(bucket => {
-            if (bucket.duration === 0) {
-                if (bucket.sum === 0) return undefined;
-                return `(${bucket.name}) ${format(bucket.sum)}`;
-            }
-            return `(${bucket.name}) ${format(bucket.sum)}`;
-        }).filter(isDefined).join("\n");
+        return encodeFormattedSelector({
+            urlKey: "timegroupergroup",
+            options: this.getSummary().map(bucket => {
+                if (bucket.duration === 0) {
+                    if (bucket.sum === 0) return undefined;
+                    return { value: `${format(bucket.sum)}`, title: bucket.name, urlValue: bucket.name };
+                }
+                return { value: `${format(bucket.sum)}`, title: bucket.name, urlValue: bucket.name };
+            }).filter(isDefined)
+        });
     }
 }

package/src/5-diagnostics/memoryValueAudit.ts

@@ -3,7 +3,7 @@ import { PathValueController } from "../0-path-value-core/PathValueController";
 import { PathValue, authorityStorage, pathWatcher } from "../0-path-value-core/pathValueCore";
 import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
 import { measureFnc } from "socket-function/src/profiling/measure";
-import { isNode, list, sort, timeInMinute, timeInSecond } from "socket-function/src/misc";
+import { QueueLimited, isNode, list, sort, timeInMinute, timeInSecond } from "socket-function/src/misc";
 import { getSingleSizeEstimate } from "./shared";
 import { devDebugbreak, isDevDebugbreak } from "../config";
 import { nodePathAuthority } from "../0-path-value-core/NodePathAuthorities";
@@ -172,12 +172,11 @@ function sampleRandom(config: {
     }
 }
 
-const AUDIT_WINDOW_SIZE = 10;
-let lastAuditsSizes: {
+let lastAuditsSizes = new QueueLimited<{
     count: number;
     size: number;
     time: number;
-}[] = [];
+}>(10);
 
 async function auditSpecificPaths(config: {
     paths: string[];
@@ -211,9 +210,10 @@ async function auditSpecificPathsBase(config: {
     //      audit fewer paths, up to just auditing a single path).
     //  Use getSingleSizeEstimate to get sizes
     // If forceAudit, don't limit the paths
-    if (lastAuditsSizes.length > 0 && !config.forceAudit) {
-        let auditWindowTotalSize = lastAuditsSizes.map(v => v.size).reduce((a, b) => a + b, 0);
-        let auditWindowTime = Date.now() - lastAuditsSizes[0].time;
+    let oldest = lastAuditsSizes.getOldest();
+    if (oldest && !config.forceAudit) {
+        let auditWindowTotalSize = lastAuditsSizes.getAllUnordered().map(v => v.size).reduce((a, b) => a + b, 0);
+        let auditWindowTime = Date.now() - oldest.time;
         let auditWindowRate = auditWindowTotalSize / auditWindowTime;
         let fractionToReduce = Math.min(1, auditWindowRate / AUDIT_MAX_BYTES_PER_SECOND);
         let count = Math.max(1, Math.floor(paths.length * fractionToReduce));
@@ -229,9 +229,6 @@ async function auditSpecificPathsBase(config: {
             + v.path.length * 2
         ).reduce((a, b) => a + b, 0);
         lastAuditsSizes.push({ count: paths.length, size, time: Date.now() });
-        while (lastAuditsSizes.length > AUDIT_WINDOW_SIZE) {
-            lastAuditsSizes.shift();
-        }
     }
 
     const THRESHOLD_TIME = Date.now() - MAX_ASSUMED_PROPAGATION_DELAY;

package/src/5-diagnostics/nodeMetadata.ts

@@ -11,9 +11,21 @@ import { formatNumber, formatPercent, formatTime } from "socket-function/src/for
 import os from "os";
 import { isNode } from "typesafecss";
 import { diskLog } from "../diagnostics/logs/diskLogger";
+import { devDebugbreak } from "../config";
+import { addStatPeriodic, addStatSumPeriodic, addTimeProfileDistribution, onTimeProfile, logNodeStats, logNodeStateStats, registerNodeMetadata } from "../-0-hooks/hooks";
+import { magenta } from "socket-function/src/formatting/logColors";
 
 
-const POLL_INTERVAL = timeInMinute * 5;
+let POLL_INTERVAL = timeInMinute * 5;
+
+//todonext
+//  Updating didn't work?
+//  - Reboot, then try do-update again, then if it still isn't up, ssh in and see what's wrong
+
+//todonext;
+// Testing
+POLL_INTERVAL = 10 * 1000;
+
 
 // Undefined means we infer the column
 // Null means the column is removed
@@ -37,10 +49,10 @@ export type ExtraMetadata = {
     getValue: () => MaybePromise<unknown>;
 };
 let extraMetadatas: ExtraMetadata[] = [];
-export function registerNodeMetadata(getter: ExtraMetadata) {
+registerNodeMetadata.declare(function registerNodeMetadata(getter: ExtraMetadata) {
     //console.log(`Registering metadata: ${getter.columnName}`);
     extraMetadatas.push(getter);
-}
+});
 
 // IMPORTANT! We no longer diskLog here! diskLogging is not ideal for metrics like these. You almost only
 //      care about the current metrics OR past trends. A single value in the logs from the past is just
@@ -76,21 +88,24 @@ let eventValues = cache((title: string, format: (value: number) => string): { (v
 });
 
 /** For values that should be summed to get the value. */
-export function logNodeStats(title: string, format: (value: number) => string): { (value: number): void } {
-    return eventValues(title, format);
-}
+logNodeStats.declare(function registerLogNodeStats(title: string, format: (value: number) => string, value: number) {
+    eventValues(title, format)(value);
+});
 /** For state values (ex, % CPU usage), for which summing wouldn't make sense. */
-export function logNodeStateStats(title: string, format: (value: number) => string): { (value: number): void } {
-    return stateValues(title, format);
-}
+logNodeStateStats.declare(function registerLogNodeStateStats(title: string, format: (value: number) => string, value: number) {
+    stateValues(title, format)(value);
+});
 
 let existingPeriodic = new Set<string>();
 /** For state values (ex, % CPU usage), for which summing wouldn't make sense. */
-export function addStatPeriodic(
-    title: string,
-    getValue: () => number,
-    format: (value: number) => string = formatNumber,
+addStatPeriodic.declare(function addStatPeriodic(
+    config: {
+        title: string,
+        getValue: () => number,
+        format?: (value: number) => string,
+    }
 ) {
+    const { title, getValue, format = formatNumber } = config;
     let hash = JSON.stringify({ title, getValue: getValue.toString(), format: format.toString() });
     // Ignoring duplicates, otherwise we end up logging way too much
     if (globalThis.isHotReloading?.() && existingPeriodic.has(hash)) return;
@@ -106,13 +121,16 @@ export function addStatPeriodic(
             return timeGrouper.getStateSummary(format);
         },
     });
-}
+});
 /** For values that should be summed to get the value. */
-export function addStatSumPeriodic(
-    title: string,
-    getValue: () => number,
-    format: (value: number) => string = formatNumber,
+addStatSumPeriodic.declare(function addStatSumPeriodic(
+    config: {
+        title: string,
+        getValue: () => number,
+        format?: ((value: number) => string),
+    }
 ) {
+    const { title, getValue, format = formatNumber } = config;
     let hash = JSON.stringify({ title, getValue: getValue.toString(), format: format.toString() });
     // Ignoring duplicates, otherwise we end up logging way too much
     if (globalThis.isHotReloading?.() && existingPeriodic.has(hash)) return;
@@ -128,10 +146,10 @@ export function addStatSumPeriodic(
             return timeGrouper.getEventSummary(format);
         },
     });
-}
+});
 
 let pendingTimeProfiles = new Map<string, { start: number; end: number }[]>();
-export function onTimeProfile(title: string, startTime: number) {
+onTimeProfile.declare(function onTimeProfile(title: string, startTime: number) {
     let profiles = pendingTimeProfiles.get(title);
     if (!profiles) {
         profiles = [];
@@ -143,7 +161,7 @@ export function onTimeProfile(title: string, startTime: number) {
         });
     }
     profiles.push({ start: startTime, end: Date.now() });
-}
+});
 
 /** Ex, for tracking the time distribution of API calls, or watchFunction evaluation, etc.
  *      - Specifically for time, so we can handle overlap cases (such as waiting for 10 calls at once),
@@ -155,7 +173,7 @@ export function onTimeProfile(title: string, startTime: number) {
  *      you need to harvest values from an external library, in which case, you will need
  *      this function.
  */
-export function addTimeProfileDistribution(
+addTimeProfileDistribution.declare(function addTimeProfileDistribution(
     title: string,
     /** It's expected that this will return all values since the last call to harvestValues
      *      (basically, copy your value buffer and clear the old one).
@@ -166,8 +184,16 @@ export function addTimeProfileDistribution(
     // Ignoring duplicates, otherwise we end up logging way too much
     if (globalThis.isHotReloading?.() && existingPeriodic.has(hash)) return;
 
+    // TODO: Because we know the time ranges, we can actually provide better state values. As in,
+    //  within a single call we can determine the maximum and minimum instantaneous rates. However...
+    //  this is more difficult, and might not be useful (as one really fast time group shouldn't result
+    //  in a spike of an infinite rate, it's better to smooth it out a bit).
+
     let fractionTimeWaitingGroup = new TimeGrouper();
     let ratePerSecondGroup = new TimeGrouper();
+    let timePerValueGroup = new TimeGrouper();
+    let parallelFactorGroup = new TimeGrouper();
+    let maxEstRateGroup = new TimeGrouper();
     let lastPollTime = Date.now();
     runInfinitePoll(POLL_INTERVAL, () => {
         let originalStartTime = lastPollTime;
@@ -175,39 +201,61 @@ export function addTimeProfileDistribution(
         values = values.slice();
         sort(values, x => x.start);
         let freeTime = 0;
-        for (let { start, end } of values) {
+        let totalTime = 0;
+
+        console.log(magenta(`${title} Time: ${values.length}`));
+
+        for (let obj of values) {
+            if (!obj) {
+                devDebugbreak();
+            }
+            let { start, end } = obj;
             if (start > lastPollTime) {
                 freeTime += start - lastPollTime;
             }
             lastPollTime = end;
+            totalTime += end - start;
         }
         let now = Date.now();
         freeTime += now - lastPollTime;
         lastPollTime = now;
 
         let duration = lastPollTime - originalStartTime;
-        let waitingTime = duration - freeTime;
+        let activeTime = duration - freeTime;
         let ratePerSecond = values.length / (duration / 1000);
-
-        fractionTimeWaitingGroup.onValueChanged(waitingTime / duration);
+        let averageTimePerValue = totalTime / values.length;
+        let fracActive = activeTime / duration;
+        fractionTimeWaitingGroup.onValueChanged(fracActive);
         ratePerSecondGroup.onValueChanged(ratePerSecond);
-    });
+        if (values.length > 0) {
+            timePerValueGroup.onValueChanged(averageTimePerValue);
+        }
 
-    registerNodeMetadata({
-        columnName: `% Time in ${title}`,
-        column: {},
-        getValue() {
-            return fractionTimeWaitingGroup.getStateSummary(formatPercent);
-        },
+        let parallelFactor = 0;
+        let totalTimeOverlapped = values.map(x => x.end - x.start).reduce((a, b) => a + b, 0);
+        if (totalTimeOverlapped > 0) {
+            parallelFactor = totalTimeOverlapped / activeTime;
+        }
+        parallelFactorGroup.onValueChanged(parallelFactor);
+
+        let maxEstRate = fracActive === 0 ? 0 : ratePerSecond / fracActive;
+        maxEstRateGroup.onValueChanged(maxEstRate);
     });
+
     registerNodeMetadata({
-        columnName: `${title}/S`,
+        columnName: `${title} Time`,
         column: {},
         getValue() {
-            return ratePerSecondGroup.getStateSummary(formatNumber);
+            return [
+                fractionTimeWaitingGroup.getStateSummary(formatPercent),
+                timePerValueGroup.getStateSummary(x => formatTime(x) + "/per"),
+                ratePerSecondGroup.getStateSummary(x => formatNumber(x) + "/s"),
+                parallelFactorGroup.getStateSummary(x => formatNumber(x) + "X"),
+                maxEstRateGroup.getStateSummary(x => `est max(${formatNumber(x)}/s)`),
+            ];
         },
     });
-}
+});
 
 if (isNode()) {
     void runInfinitePollCallAtStart(POLL_INTERVAL, async () => {
@@ -216,24 +264,24 @@ if (isNode()) {
         const free = os.freemem();
         const total = os.totalmem();
 
-        logNodeStateStats("System|Memory Free", x => formatNumber(x) + "B")(free);
-        logNodeStateStats("System|Memory Free %", formatPercent)(free / total);
+        logNodeStateStats("System|Memory Free", x => formatNumber(x) + "B", free);
+        logNodeStateStats("System|Memory Free %", formatPercent, free / total);
 
         let constrainedMemory = process.constrainedMemory();
         if (constrainedMemory !== undefined && constrainedMemory < 2 ** 48) {
-            logNodeStateStats("System|Constrained Memory", x => formatNumber(x) + "B")(constrainedMemory);
+            logNodeStateStats("System|Constrained Memory", x => formatNumber(x) + "B", constrainedMemory);
         }
 
         let usedCPU = process.cpuUsage();
-        logNodeStateStats("System|CPU User", formatTime)(usedCPU.user * 1000);
-        logNodeStateStats("System|CPU System", formatTime)(usedCPU.system * 1000);
+        logNodeStateStats("System|CPU User", formatTime, usedCPU.user * 1000);
+        logNodeStateStats("System|CPU System", formatTime, usedCPU.system * 1000);
 
         let cpuCount = os.cpus().length;
-        logNodeStateStats("System|CPU Count", formatNumber)(cpuCount);
+        logNodeStateStats("System|CPU Count", formatNumber, cpuCount);
 
         let load = os.loadavg();
-        logNodeStateStats("System|Average Load", formatPercent)(load[0]);
-        logNodeStateStats("System|Average Free Cores", formatPercent)((os.cpus().length - load[0]));
+        logNodeStateStats("System|Average Load", formatPercent, load[0]);
+        logNodeStateStats("System|Average Free Cores", formatPercent, (os.cpus().length - load[0]));
 
     });
 }

package/src/5-diagnostics/synchronousLagTracking.ts

@@ -3,7 +3,7 @@ import { formatTime } from "socket-function/src/formatting/format";
 import { registerMeasureInfo } from "socket-function/src/profiling/measure";
 import { isNode } from "typesafecss";
 import { monitorEventLoopDelay } from "perf_hooks";
-import { registerNodeMetadata } from "./nodeMetadata";
+import { registerNodeMetadata } from "../-0-hooks/hooks";
 import { getOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
 
 const POLL_INTERVAL = 350;

package/src/diagnostics/NodeViewer.tsx

@@ -18,7 +18,7 @@ import { requiresNetworkTrustHook } from "../../src/-d-trust/NetworkTrust2";
 import { getDomain, isNoNetwork } from "../../src/config";
 import { NodeMetadataController } from "../../src/5-diagnostics/nodeMetadata";
 import { isDefined } from "../../src/misc";
-import { errorMessage, formatValue, toSpaceCase } from "../../src/5-diagnostics/GenericFormat";
+import { encodeFormattedSelector, errorMessage, formatValue, toSpaceCase } from "../../src/5-diagnostics/GenericFormat";
 import { ValueAuditController } from "../../src/5-diagnostics/memoryValueAudit";
 import { getExternalIP } from "../../src/misc/networking";
 import dns from "dns";
@@ -458,7 +458,6 @@ class NodeViewerControllerBase {
             wrapAddTableValue("port", {}, async () => {
                 return nodeId.split(":").at(-1);
             }),
-            // 
             wrapAddTableValue("threadId", {
                 formatter: "link:/?page=user&showingmanagement&managementpage=LogViewer&filterThreadIds=$VALUE$<Thread Errors $VALUE$>"
             }, async () => {
@@ -493,8 +492,8 @@ class NodeViewerControllerBase {
             }
         } catch (e: any) {
             // Any errors almost certainly mean they aren't exposing any extra metadata, which is fine.
-            // columns["extraMetadatasError"] = { formatter: "error" };
-            // row["extraMetadatasError"] = "Error: " + e.stack;
+            columns["extraMetadatasError"] = { formatter: "error" };
+            row["extraMetadatasError"] = "Error: " + e.stack;
         }
         return {
             columns,