querysub 0.40.0 → 0.42.0

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "querysub",
-    "version": "0.40.0",
+    "version": "0.42.0",
     "main": "index.js",
     "license": "MIT",
     "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -24,7 +24,7 @@
         "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
         "pako": "^2.1.0",
         "preact": "^10.11.3",
-        "socket-function": "^0.38.0",
+        "socket-function": "^0.39.0",
         "terser": "^5.31.0",
         "typesafecss": "^0.6.3",
         "yaml": "^2.5.0",

package/src/3-path-functions/PathFunctionRunner.ts

@@ -18,6 +18,7 @@ import { parseArgs } from "./PathFunctionHelpers";
 import { PERMISSIONS_FUNCTION_ID, getExportPath } from "./syncSchema";
 import { formatTime } from "socket-function/src/formatting/format";
 import { set_debug_getFunctionRunnerShards } from "../-g-core-values/NodeCapabilities";
+import { diskLog } from "../diagnostics/logs/diskLogger";
 
 export const functionSchema = rawSchema<{
     [domainName: string]: {
@@ -630,6 +631,7 @@ export class PathFunctionRunner {
         if (PathFunctionRunner.DEBUG_CALLS) {
             console.log(`FINISHED${nooped ? " (skipped)" : ""} ${getDebugName(callPath, functionSpec, true)}, writes: ${finalWrites?.length}`);
         }
+        diskLog(`Finished FunctionRunner funciton`, { ...callPath, argsEncoded: "", functionSpec });
         PathFunctionRunner.RUN_FINISH_COUNT++;
     }
 }

package/src/diagnostics/NodeViewer.tsx

@@ -185,7 +185,7 @@ export class NodeViewer extends qreact.Component {
         }
 
         let builtinGroups = {
-            "Default": ["buttons", "devToolsURL", "nodeId", "ip", "uptime", "loadTime", "Heap", "Buffers", "All Memory", "Blocking Lag", "port", "threadId", "machineId", "apiError", "live_entryPoint"],
+            "Default": ["buttons", "devToolsURL", "nodeId", "ip", "uptime", "loadTime", "Est % Usage", "Heap", "Buffers", "All Memory", "Blocking Lag", "port", "threadId", "machineId", "apiError", "live_entryPoint"],
         };
         // Column => group
         let builtInGroupsLookup = new Map<string, string>();
@@ -279,7 +279,7 @@ export class NodeViewer extends qreact.Component {
                                     },
                                 },
                                 ip: {},
-                                loadTime: { formatter: "timeSpan" },
+                                loadTime: { title: "Meta Time", formatter: "timeSpan" },
                                 ...x.table?.columns,
                                 //capabilities: null,
                                 apiError: { formatter: "error" },

package/src/diagnostics/logs/BrowserLargeFileCache.ts

@@ -0,0 +1,64 @@
+import { SocketFunctionClientHook } from "socket-function/SocketFunctionTypes";
+import { cache, lazy } from "socket-function/src/caching";
+import { decodeCborx, encodeCborx } from "../../misc/cloneHelpers";
+import { sha256 } from "js-sha256";
+import { errorToUndefined } from "../../errors";
+import { isNode } from "typesafecss";
+
+let getRootDirectory = lazy(async () => {
+    await navigator.storage.persist();
+    return navigator.storage.getDirectory();
+});
+
+export const getBrowserLargeFileCache = cache((name: string) => new BrowserLargeFileCache(name));
+
+export class BrowserLargeFileCache {
+    constructor(private name: string) { }
+
+    private getDir = lazy(async () => {
+        let root = await getRootDirectory();
+        return await root.getDirectoryHandle(this.name, { create: true });
+    });
+
+    public async set(key: string, value: Buffer) {
+        let dir = await this.getDir();
+        let file = await dir.getFileHandle(key, { create: true });
+        let writable = await file.createWritable();
+        await writable.write(value);
+        await writable.close();
+    }
+    public async get(key: string): Promise<Buffer | undefined> {
+        let dir = await this.getDir();
+        try {
+            let file = await dir.getFileHandle(key, { create: false });
+            let readable = await file.getFile();
+            return Buffer.from(await readable.arrayBuffer());
+        } catch {
+            return undefined;
+        }
+    }
+}
+
+/** Cache key = [args, functionName, classGuid]
+ *      - Not nodeId, as that change so frequently, so caching based on server is not usually useful.
+ *          - If you want to cache based on nodeId, pass it as an unused arg.
+ * 
+ *  Uses a file per key, and never cleans them up. So... basically, this sucks, but, it should
+ *      work for a few specific functions with large and consistent values.
+ */
+export const cacheCalls: SocketFunctionClientHook = async config => {
+    if (isNode()) return;
+
+    let { args, functionName, classGuid } = config.call;
+    let bucket = sha256(JSON.stringify({ functionName, classGuid }));
+    let cache = getBrowserLargeFileCache(bucket);
+    let key = sha256(encodeCborx(args));
+    let cachedValue = await cache.get(key);
+    if (cachedValue) {
+        config.overrideResult = decodeCborx(cachedValue);
+        return;
+    }
+    config.onResult.push(async (result) => {
+        await errorToUndefined(cache.set(key, encodeCborx(result)));
+    });
+};

package/src/diagnostics/logs/DiskLoggerPage.tsx

@@ -10,7 +10,7 @@ import { getSyncedController } from "../../library-components/SyncedController";
 import { getBrowserUrlNode } from "../../-f-node-discovery/NodeDiscovery";
 import { formatDateTime, formatNiceDateTime, formatNumber, formatTime } from "socket-function/src/formatting/format";
 import { Anchor } from "../../library-components/ATag";
-import { cache, cacheLimited, cacheShallowConfigArgEqual } from "socket-function/src/caching";
+import { cache, cacheLimited, cacheShallowConfigArgEqual, lazy } from "socket-function/src/caching";
 import { InputLabel, InputLabelURL } from "../../library-components/InputLabel";
 import { DropdownSelector } from "../../library-components/DropdownSelector";
 import { Table, TableType } from "../../5-diagnostics/Table";
@@ -30,6 +30,7 @@ import { getNodeIdLocation } from "socket-function/src/nodeCache";
 import { decodeNodeId, encodeNodeId, getMachineId } from "../../-a-auth/certs";
 import { Button } from "../../library-components/Button";
 import { TimeRangeSelector } from "../../library-components/TimeRangeSelector";
+import { BrowserLargeFileCache, cacheCalls } from "./BrowserLargeFileCache";
 
 // TODO: Realtime log mode, by reading from the previous length forward, to add buffers
 //  to what we already read.
@@ -550,10 +551,15 @@ export const DiskLoggerController = SocketFunction.register(
     new DiskLoggerControllerBase(),
     () => ({
         getRemoteLogFiles: {},
-        getRemoteLogBuffer: {},
+        getRemoteLogBuffer: {
+            compress: true,
+            clientHooks: [cacheCalls]
+        },
 
         getLogFiles: {},
-        getLogBuffer: {},
+        getLogBuffer: {
+            compress: true,
+        },
     }),
     () => ({
         hooks: [assertIsManagementUser],

package/src/diagnostics/logs/diskLogger.ts

@@ -44,8 +44,8 @@ export type LogObj = {
 
 // NOTE: This is visible, otherwise it's easy to accidentally copy it, and not know why
 //  the text is behaving strangely (not === other seemingly equal text, etc).
+// NOTE: Also hardcoded in measure.ts (in socket-function)
 export const noDiskLogPrefix = "█ ";
-
 export const diskLog = logDisk;
 export function logDisk(...args: unknown[]) {
     if (!isNode()) return;

package/src/diagnostics/watchdog.ts

@@ -1,13 +1,40 @@
 import { runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
-import { FormattedMeasureTable, logMeasureTable, startMeasure } from "socket-function/src/profiling/measure";
+import { FormattedMeasureTable, MeasureProfile, logMeasureTable, startMeasure } from "socket-function/src/profiling/measure";
 import { logErrors } from "../errors";
-import { isNode } from "socket-function/src/misc";
+import { isNode, sort } from "socket-function/src/misc";
 import debugbreak from "debugbreak";
 import { registerPeriodic } from "./periodic";
 import { getOwnMachineId } from "../-a-auth/certs";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { logDisk } from "./logs/diskLogger";
+import { registerNodeMetadata } from "../5-diagnostics/nodeMetadata";
+import { formatPercent } from "socket-function/src/formatting/format";
 
+let lastProfile: MeasureProfile | undefined;
+
+registerNodeMetadata({
+    columnName: "Est Recent % Usage",
+    getValue() {
+        if (!lastProfile) return "";
+        let entries = Object.values(lastProfile.entries);
+        if (entries.length === 0) return "";
+        const timeProfiled = lastProfile.endTime - lastProfile.startTime;
+        let timeActive = entries.map(x => x.ownTime.sum).reduce((a, b) => a + b, 0);
+        let frac = timeActive / timeProfiled;
+        let rootSums = new Map<string, number>();
+        for (let entry of entries) {
+            let name = entry.name.split("|")[0];
+            let sum = rootSums.get(name) ?? 0;
+            rootSums.set(name, sum + entry.ownTime.sum);
+        }
+        let most = sort(Array.from(rootSums.entries()), x => -x[1])[0];
+        return `${formatPercent(frac)} (${formatPercent(most[1] / timeProfiled)} ${endEllipsis(most[0], 26)})`;
+    },
+});
+function endEllipsis(str: string, maxLength: number) {
+    if (str.length <= maxLength) return str;
+    return "..." + str.slice(-(maxLength - 3));
+}
 
 let measureObj = startMeasure();
 function logProfileMeasuresTimingsNow() {
@@ -29,6 +56,8 @@ function logProfileMeasuresTimingsNow() {
         mergeDepth: 1,
         minTimeToLog: 250,
     }));
+
+    lastProfile = profile;
     // if (isNode()) {
     //     if (SocketFunction.mountedNodeId) {
     //         console.log("Mounted as " + SocketFunction.mountedNodeId);