querysub 0.403.0 → 0.405.0

package/src/config2.ts

@@ -1,48 +1,23 @@
-import { deepCloneJSON, isNode } from "socket-function/src/misc";
-import { hasArchivesPermissions } from "./-a-archives/archives";
-import { authorityRaw, baseIsClient, getDomain } from "./config";
-import { JSONLACKS } from "socket-function/src/JSONLACKS/JSONLACKS";
-import { AuthorityPath } from "./0-path-value-core/NodePathAuthorities";
-import { rootPathStr, prependToPathStr } from "./path";
-import fs from "fs";
-
-let isInCall = false;
-export function isClient() {
-    isInCall = true;
-    try {
-        return !isNode() || baseIsClient() || !isInCall && !hasArchivesPermissions();
-    } finally {
-        isInCall = false;
-    }
-}
-
-export function isServer() {
-    return !isClient();
-}
-
-export function getOurAuthorities() {
-    let authorities: AuthorityPath[] = [];
-    let authority = authorityRaw();
-
-    let baseAuthorityPaths: AuthorityPath[] = [{
-        pathPrefix: rootPathStr
-    }];
-    if (authority) {
-        if (fs.existsSync(authority)) {
-            baseAuthorityPaths = JSONLACKS.parse(fs.readFileSync(authority, "utf8"));
-        } else {
-            baseAuthorityPaths = JSONLACKS.parse(Buffer.from(authority, "base64").toString("utf8"));
-        }
-    }
-
-    let domain = getDomain();
-    for (let base of baseAuthorityPaths) {
-        base = deepCloneJSON(base);
-        base.pathPrefix = prependToPathStr(base.pathPrefix, domain);
-        if (base.excludedChildren) {
-            base.excludedChildren = base.excludedChildren.map(x => prependToPathStr(x, domain));
-        }
-        authorities.push(base);
-    }
-    return authorities;
-}
+import { deepCloneJSON, isNode } from "socket-function/src/misc";
+import { hasArchivesPermissions } from "./-a-archives/archives";
+import { baseIsClient, getDomain } from "./config";
+import { JSONLACKS } from "socket-function/src/JSONLACKS/JSONLACKS";
+import { rootPathStr, prependToPathStr, getPathDepth } from "./path";
+import fs from "fs";
+import { AuthoritySpec } from "./0-path-value-core/PathRouter";
+
+let isInCall = false;
+export function isClient() {
+    isInCall = true;
+    try {
+        return !isNode() || baseIsClient() || !isInCall && !hasArchivesPermissions();
+    } finally {
+        isInCall = false;
+    }
+}
+
+export function isServer() {
+    return !isClient();
+}
+
+// NOTE: getOurAuthorities moved to PathRouterHashOverrides, and renamed to getOurAuthoritySpec

package/src/deployManager/components/DeployPage.tsx

@@ -257,7 +257,10 @@ export class DeployPage extends qreact.Component {
     render() {
         let controller = MachineServiceController(SocketFunction.browserNodeId());
         let gitInfo = controller.getGitInfo();
-        let pendingFunctions = controller.getPendingFunctions() || [];
+        let pendingFunctions = controller.getPendingFunctions() || {
+            functionSpecs: [],
+            prefixes: [],
+        };
         let liveFunctions = controller.getLiveFunctions() || [];
 
         let liveGitRef = mostCommon(liveFunctions.map(x => x.gitRef)) || "";
@@ -285,7 +288,8 @@ export class DeployPage extends qreact.Component {
 
                         try {
                             await deployFunctionsWithProgress({
-                                functionSpecs: pendingFunctions,
+                                functionSpecs: pendingFunctions.functionSpecs,
+                                prefixes: pendingFunctions.prefixes,
                                 // NOTE: Old code should always be compatible with new code, so a long delay is fine. We don't want to have deal with bug reports today for bugs we fixed tomorrow, but we also don't want to force users to refresh, so this time is longer than the expected maximum session length.
                                 // NOTE: Even if we want to deploy now, we can't deploy IMMEDIATELY. Give users at least 10 minutes to accept that that page will forcefully refresh.
                                 notifyRefreshDelay: forceDeployNowURL.value ? timeInMinute * 10 : timeInHour * 12,
@@ -379,7 +383,7 @@ export class DeployPage extends qreact.Component {
                     ✅ Everything is up to date - no changes to deploy
                 </div>
             ) || (
-                    FunctionDiffView({ pendingFunctions, liveFunctions })
+                    FunctionDiffView({ pendingFunctions: pendingFunctions.functionSpecs, liveFunctions })
                 )}
         </div>;
     }

package/src/deployManager/machineSchema.ts

@@ -118,7 +118,7 @@ async function registerNodeForMachineCleanup(nodeId: string) {
         console.log(green(`Registering node for machine cleanup at ${nodeIdPath}`));
         await fs.promises.writeFile(nodeIdPath, nodeId);
     } else {
-        console.log(`Not registering node for machine cleanup because we are not in the service folder: ${currentPath}`);
+        console.log(`Not registering node for machine cleanup because we are not in the service folder (${SERVICE_FOLDER_NAME}), and are instead in: ${currentPath}`);
     }
 }
 
@@ -358,6 +358,7 @@ export class MachineServiceControllerBase {
 
     public async deployFunctions(config: {
         functionSpecs: FunctionSpec[];
+        prefixes: string[];
         notifyRefreshDelay: number;
         deployOnlyCode?: boolean;
         deployOnlyUI?: boolean;
@@ -368,6 +369,7 @@ export class MachineServiceControllerBase {
         };
         await deployFunctions({
             functionSpecs: config.functionSpecs,
+            prefixes: config.prefixes,
             notifyRefreshDelay: config.notifyRefreshDelay,
             deployOnlyCode: config.deployOnlyCode,
             deployOnlyUI: config.deployOnlyUI,
@@ -410,6 +412,7 @@ const DeployProgressController = SocketFunction.register(
 );
 export async function deployFunctionsWithProgress(config: {
     functionSpecs: FunctionSpec[];
+    prefixes: string[];
     notifyRefreshDelay: number;
     deployOnlyCode?: boolean;
     deployOnlyUI?: boolean;

package/src/diagnostics/ActionsHistory.ts

@@ -1,13 +1,8 @@
-import { cache } from "socket-function/src/caching";
-import { blue, green, red } from "socket-function/src/formatting/logColors";
-import { getStorageDir, getSubFolder } from "../fs";
+import { blue, green } from "socket-function/src/formatting/logColors";
 import { ClientWatcher } from "../1-path-client/pathValueClientWatcher";
-import { PathValueProxyWatcher } from "../2-proxy/PathValueProxyWatcher";
-import { debugPathValuePath, PathValue, PathWatcherCallback, ReadLock, WatchConfig, WriteState } from "../0-path-value-core/pathValueCore";
-import fs from "fs";
-import { measureWrap } from "socket-function/src/profiling/measure";
+import { debugPathValuePath, PathValue } from "../0-path-value-core/pathValueCore";
 import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
-import { LOCAL_DOMAIN_PATH } from "../0-path-value-core/NodePathAuthorities";
+import { LOCAL_DOMAIN_PATH } from "../0-path-value-core/PathRouter";
 
 
 

package/src/diagnostics/AuditLogPage.tsx

@@ -5,8 +5,6 @@ import { css } from "typesafecss";
 import { getSyncedController } from "../library-components/SyncedController";
 import { DebugLog, DebugLogController, getLogHistoryEquals, getLogHistoryIncludes } from "../0-path-value-core/auditLogs";
 import { remoteWatcher } from "../1-path-client/RemoteWatcher";
-import { pathValueAuthority2 } from "../0-path-value-core/NodePathAuthorities";
-import { getParentPathStr } from "../path";
 import { getBrowserUrlNode, getOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
 import { nextId, sort } from "socket-function/src/misc";
 import { t } from "../2-proxy/schema2";
@@ -14,6 +12,7 @@ import { InputLabelURL } from "../library-components/InputLabel";
 import { URLParam } from "../library-components/URLParam";
 import { Table } from "../5-diagnostics/Table";
 import { ObjectDisplay } from "./logs/ObjectDisplay";
+import { PathRouter } from "../0-path-value-core/PathRouter";
 
 /**
 TODO:
@@ -92,7 +91,7 @@ class AuditLogControllerBase {
         let { path, includeDescendants } = config;
         let authorityNodeId = remoteWatcher.getExistingWatchRemoteNodeId(path);
         if (!authorityNodeId) {
-            authorityNodeId = await pathValueAuthority2.getSingleReadNodeSync(path);
+            authorityNodeId = await PathRouter.getReadyAuthority(path)?.nodeId;
         }
         let logs: DebugLog[] = [];
         let remoteLogs: DebugLog[] = [];

package/src/diagnostics/FunctionCallInfo.tsx

@@ -0,0 +1,141 @@
+import { qreact } from "../4-dom/qreact";
+import { css } from "../4-dom/css";
+import { t } from "../2-proxy/schema2";
+import { callState, ensureSubscribed } from "./FunctionCallInfoState";
+import { formatNumber, formatPercent, formatTime } from "socket-function/src/formatting/format";
+import { measureBlock } from "socket-function/src/profiling/measure";
+import { StatsValue, getStatsTop } from "socket-function/src/profiling/stats";
+import { isCurrentUserSuperUser } from "../user-implementation/userData";
+
+module.hotreload = true;
+
+function formatTimeStats(stats: StatsValue): string {
+    if (stats.count === 0) return formatTime(0);
+
+    let avgTime = stats.sum / stats.count;
+    let top = getStatsTop(stats);
+
+    if (!top.topHeavy) {
+        return formatTime(avgTime);
+    }
+
+    let bottomAvg = (stats.sum - top.value) / (stats.count - top.count) || 0;
+    let topAvg = top.value / top.count;
+    let topPercent = formatPercent(top.countFraction);
+
+    return `${formatTime(bottomAvg)} (top ${topPercent} = ${formatTime(topAvg)})`;
+}
+
+export class FunctionCallInfo extends qreact.Component {
+    showDetailedStats() {
+        if (!isCurrentUserSuperUser()) return undefined;
+        let state = callState();
+
+        let avgInternalReruns = state.totalCalls > 0 && state.totalInternalReruns / state.totalCalls || 0;
+        let avgFullReruns = state.totalCalls > 0 && state.totalFullReruns / state.totalCalls || 0;
+        let percentMultipleInternalRuns = state.totalCalls > 0 && state.callsWithMultipleInternalRuns / state.totalCalls || 0;
+        let percentCascadingRuns = state.totalCalls > 0 && state.callsWithCascadingRuns / state.totalCalls || 0;
+        let avgCascadingReruns = state.callsWithCascadingRuns > 0 && state.totalInternalRerunsForCascading / state.callsWithCascadingRuns || 0;
+        let percentMultipleFullRuns = state.totalCalls > 0 && state.callsWithMultipleFullRuns / state.totalCalls || 0;
+
+        console.log("=== Function Call Statistics ===");
+        console.log([
+            `${formatNumber(state.totalCalls)} ⚡`,
+            [
+                `${formatNumber(avgInternalReruns)} ♻️`,
+                percentMultipleInternalRuns > 0 && `(${formatPercent(percentMultipleInternalRuns)} 🔦)`,
+                percentCascadingRuns > 0 && `(${formatPercent(percentCascadingRuns)} / ${formatNumber(avgCascadingReruns)} / MAX ${state.maxInternalReruns} 📈)`
+            ].filter(v => v).join(" "),
+            percentMultipleFullRuns > 0 && `${formatNumber(avgFullReruns)} (${formatPercent(percentMultipleFullRuns)}) ⚠️`,
+            `${formatTimeStats(state.evalTimeStats)} 💫 / ${formatTimeStats(state.totalTimeStats)} 🕒`
+        ].filter(v => v).join(" | "));
+        console.log("");
+        console.log("=== Per-Function Statistics ===");
+
+        let perFunctionStats = state.perFunctionStats;
+        if (perFunctionStats) {
+            for (let functionId in perFunctionStats) {
+                let stats = perFunctionStats[functionId];
+                let avgInternal = stats.totalInternalReruns / stats.totalCalls;
+                let avgFull = stats.totalFullReruns / stats.totalCalls;
+                let pctInternal = stats.callsWithMultipleInternalRuns / stats.totalCalls;
+                let pctCascading = stats.callsWithCascadingRuns / stats.totalCalls;
+                let avgCascading = stats.callsWithCascadingRuns > 0 && stats.totalInternalRerunsForCascading / stats.callsWithCascadingRuns || 0;
+                let pctFull = stats.callsWithMultipleFullRuns / stats.totalCalls;
+
+                console.log([
+                    `${functionId}: ${formatNumber(stats.totalCalls)} ⚡`,
+                    [
+                        `${formatNumber(avgInternal)} ♻️`,
+                        pctInternal > 0 && `(${formatPercent(pctInternal)} 🔦)`,
+                        pctCascading > 0 && `(${formatPercent(pctCascading)} / ${formatNumber(avgCascading)} / MAX ${stats.maxInternalReruns} 📈)`
+                    ].filter(v => v).join(" "),
+                    pctFull > 0 && `${formatNumber(avgFull)} (${formatPercent(pctFull)}) ⚠️`,
+                    `${formatTimeStats(stats.evalTimeStats)} 💫 / ${formatTimeStats(stats.totalTimeStats)} 🕒`
+                ].filter(v => v).join(" | "));
+            }
+        }
+    }
+
+    render() {
+        ensureSubscribed();
+        let state = callState();
+
+        let avgInternalReruns = state.totalCalls > 0 && state.totalInternalReruns / state.totalCalls || 0;
+        let avgFullReruns = state.totalCalls > 0 && state.totalFullReruns / state.totalCalls || 0;
+        let percentMultipleInternalRuns = state.totalCalls > 0 && state.callsWithMultipleInternalRuns / state.totalCalls || 0;
+        let percentCascadingRuns = state.totalCalls > 0 && state.callsWithCascadingRuns / state.totalCalls || 0;
+        let avgCascadingReruns = state.callsWithCascadingRuns > 0 && state.totalInternalRerunsForCascading / state.callsWithCascadingRuns || 0;
+        let percentMultipleFullRuns = state.totalCalls > 0 && state.callsWithMultipleFullRuns / state.totalCalls || 0;
+
+        return <div className={css.button.vbox(4).pad2(4).alignItems("end")} onClick={() => this.showDetailedStats()}>
+            <div className={css.hbox(10)}>
+                <span title="Committed synced function calls">
+                    {formatNumber(state.totalCalls)}⚡
+                </span>
+                <div className={css.hbox(7)}>
+                    <span title="Number of function evaluations">
+                        {formatNumber(avgInternalReruns)}♻️
+                    </span>
+                    {percentMultipleInternalRuns > 0 && <span title="Cases where the first call did not have all the synced data">
+                        (🔦{formatPercent(percentMultipleInternalRuns)})
+                    </span>}
+                    {percentCascadingRuns > 0 && <span title="The initial sync did not provide all the data, requiring cascading syncing.">
+                        (📈{formatPercent(percentCascadingRuns)} / {formatNumber(avgCascadingReruns)} / MAX {state.maxInternalReruns})
+                    </span>}
+                </div>
+                {percentMultipleFullRuns > 0 && <span title="Rejected calls requiring full reruns. Unless you are changing data another user is actively changing, this is a bug!" className={css.hsl(60, 100, 80).pad2(2, 1).hslcolor(0, 0, 0)}>
+                    {formatNumber(avgFullReruns)} ({formatPercent(percentMultipleFullRuns)}) ⚠️
+                </span>}
+            </div>
+            <div>
+                <span title="Average time to evaluate the application function (excluding proxy overhead)">
+                    {formatTimeStats(state.evalTimeStats)}💫
+                </span>
+                {" / "}
+                <span title="Average time to evaluate from start to finish, including rejections, etc. Top % shows the top % count, and if top % is shown the other amount has the top values removed.">
+                    {formatTimeStats(state.totalTimeStats)}🕒
+                </span>
+            </div>
+            <div className={css.hbox(20)}>
+                <div className={css.hbox(5)}>
+                    <span title="Remote watched paths (syncing from other nodes)">
+                        {formatNumber(state.remotePathCount)}🔥
+                    </span>
+                    {" / "}
+                    <span title="Local watched paths">
+                        {formatNumber(state.localPathCount)}🏠
+                    </span>
+                    {" / "}
+                    <span title="Total values stored in memory">
+                        {formatNumber(state.totalValueCount)}📦
+                    </span>
+                    {" / "}
+                    <span title="Watchers">
+                        {formatNumber(state.proxyWatcherCount)}🕵🏻‍♀️
+                    </span>
+                </div>
+            </div>
+        </div>;
+    }
+}

package/src/diagnostics/FunctionCallInfoState.ts

@@ -0,0 +1,162 @@
+import { Querysub } from "../4-querysub/QuerysubController";
+import { onPredictionFinished } from "../4-querysub/querysubPrediction";
+import { lazy } from "socket-function/src/caching";
+import { t } from "../2-proxy/schema2";
+import { StatsValue, createStatsValue, addToStatsValue } from "socket-function/src/profiling/stats";
+import { authorityStorage } from "../0-path-value-core/pathValueCore";
+import { pathWatcher } from "../0-path-value-core/PathWatcher";
+import { proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
+
+interface CallStatsData {
+    totalCalls: number;
+    totalInternalReruns: number;
+    totalFullReruns: number;
+    callsWithMultipleInternalRuns: number;
+    callsWithCascadingRuns: number;
+    totalInternalRerunsForCascading: number;
+    maxInternalReruns: number;
+    callsWithMultipleFullRuns: number;
+    evalTimeStats: StatsValue;
+    timeTakenStats: StatsValue;
+    totalTimeStats: StatsValue;
+    localPathCount: number;
+    remotePathCount: number;
+    totalValueCount: number;
+    proxyWatcherCount: number;
+    perFunctionStats: {
+        [functionId: string]: {
+            totalCalls: number;
+            totalInternalReruns: number;
+            totalFullReruns: number;
+            callsWithMultipleInternalRuns: number;
+            callsWithCascadingRuns: number;
+            totalInternalRerunsForCascading: number;
+            maxInternalReruns: number;
+            callsWithMultipleFullRuns: number;
+            evalTimeStats: StatsValue;
+            timeTakenStats: StatsValue;
+            totalTimeStats: StatsValue;
+        }
+    };
+}
+
+let statsData: CallStatsData = {
+    totalCalls: 0,
+    totalInternalReruns: 0,
+    totalFullReruns: 0,
+    callsWithMultipleInternalRuns: 0,
+    callsWithCascadingRuns: 0,
+    totalInternalRerunsForCascading: 0,
+    maxInternalReruns: 0,
+    callsWithMultipleFullRuns: 0,
+    evalTimeStats: createStatsValue(),
+    timeTakenStats: createStatsValue(),
+    totalTimeStats: createStatsValue(),
+    localPathCount: 0,
+    remotePathCount: 0,
+    totalValueCount: 0,
+    proxyWatcherCount: 0,
+    perFunctionStats: {},
+};
+
+let callStateSchema = Querysub.createLocalSchema("callState", {
+    sequenceNumber: t.number,
+    localPathCount: t.number,
+    remotePathCount: t.number,
+    totalValueCount: t.number,
+    proxyWatcherCount: t.number,
+});
+
+export function callState(): CallStatsData {
+    callStateSchema().sequenceNumber;
+    return statsData;
+}
+
+export let ensureSubscribed = lazy(() => {
+    onPredictionFinished((data) => {
+        statsData.totalCalls++;
+        statsData.totalInternalReruns += data.result.totalInternalLoopCount;
+        statsData.totalFullReruns += data.result.outerLoopCount;
+        addToStatsValue(statsData.evalTimeStats, data.result.evalTime);
+        addToStatsValue(statsData.timeTakenStats, data.result.timeTaken);
+        addToStatsValue(statsData.totalTimeStats, data.result.totalTime);
+
+        if (data.result.totalInternalLoopCount > 1) {
+            statsData.callsWithMultipleInternalRuns++;
+        }
+        if (data.result.totalInternalLoopCount > 2) {
+            statsData.callsWithCascadingRuns++;
+            statsData.totalInternalRerunsForCascading += data.result.totalInternalLoopCount;
+            if (data.result.totalInternalLoopCount > statsData.maxInternalReruns) {
+                statsData.maxInternalReruns = data.result.totalInternalLoopCount;
+            }
+        }
+        if (data.result.outerLoopCount > 1) {
+            statsData.callsWithMultipleFullRuns++;
+        }
+
+        if (!(data.functionId in statsData.perFunctionStats)) {
+            statsData.perFunctionStats[data.functionId] = {
+                totalCalls: 0,
+                totalInternalReruns: 0,
+                totalFullReruns: 0,
+                callsWithMultipleInternalRuns: 0,
+                callsWithCascadingRuns: 0,
+                totalInternalRerunsForCascading: 0,
+                maxInternalReruns: 0,
+                callsWithMultipleFullRuns: 0,
+                evalTimeStats: createStatsValue(),
+                timeTakenStats: createStatsValue(),
+                totalTimeStats: createStatsValue(),
+            };
+        }
+
+        let fnStats = statsData.perFunctionStats[data.functionId];
+        fnStats.totalCalls++;
+        fnStats.totalInternalReruns += data.result.totalInternalLoopCount;
+        fnStats.totalFullReruns += data.result.outerLoopCount;
+        addToStatsValue(fnStats.evalTimeStats, data.result.evalTime);
+        addToStatsValue(fnStats.timeTakenStats, data.result.timeTaken);
+        addToStatsValue(fnStats.totalTimeStats, data.result.totalTime);
+
+        if (data.result.totalInternalLoopCount > 1) {
+            fnStats.callsWithMultipleInternalRuns++;
+        }
+        if (data.result.totalInternalLoopCount > 2) {
+            fnStats.callsWithCascadingRuns++;
+            fnStats.totalInternalRerunsForCascading += data.result.totalInternalLoopCount;
+            if (data.result.totalInternalLoopCount > fnStats.maxInternalReruns) {
+                fnStats.maxInternalReruns = data.result.totalInternalLoopCount;
+            }
+        }
+        if (data.result.outerLoopCount > 1) {
+            fnStats.callsWithMultipleFullRuns++;
+        }
+
+        Querysub.commit(() => {
+            let schema = callStateSchema();
+            schema.sequenceNumber = schema.sequenceNumber + 1;
+        });
+    });
+
+    function updateSyncStats() {
+        let watcherStats = pathWatcher.getWatcherStats();
+        let valueCount = authorityStorage.getValueCount();
+        let proxyWatcherCount = proxyWatcher.getAllWatchers().size;
+
+        statsData.localPathCount = watcherStats.localPathCount;
+        statsData.remotePathCount = watcherStats.remotePathCount;
+        statsData.totalValueCount = valueCount;
+        statsData.proxyWatcherCount = proxyWatcherCount;
+
+        Querysub.commit(() => {
+            let schema = callStateSchema();
+            schema.localPathCount = watcherStats.localPathCount;
+            schema.remotePathCount = watcherStats.remotePathCount;
+            schema.totalValueCount = valueCount;
+            schema.proxyWatcherCount = proxyWatcherCount;
+        });
+    }
+    updateSyncStats();
+    setInterval(updateSyncStats, 15000);
+});

package/src/diagnostics/MachineThreadInfo.tsx

@@ -246,7 +246,7 @@ class MachineThreadInfoBase {
     }
 
 }
-const MachineThreadInfoController = getSyncedController(SocketFunction.register(
+export const MachineThreadInfoController = getSyncedController(SocketFunction.register(
     "MachineThreadInfoController-019c88cb-a16f-7219-9a5e-08919bd1475b",
     new MachineThreadInfoBase(),
     () => ({