querysub 0.403.0 → 0.405.0

package/src/4-deploy/deployFunctions.ts

@@ -1,18 +1,24 @@
 import { magenta } from "socket-function/src/formatting/logColors";
 import { FunctionSpec, preloadFunctions } from "../3-path-functions/PathFunctionRunner";
 import { Querysub } from "../4-querysub/QuerysubController";
-import { errorify } from "../errors";
+import { errorify, timeoutToError } from "../errors";
 import { runPromise } from "../functional/runCommand";
 import path from "path";
 import { preloadUI } from "./edgeNodes";
 import { proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
 import { replaceFunctions } from "../3-path-functions/PathFunctionHelpers";
 import { setLiveDeployedHash } from "./deploySchema";
+import { timeInMinute } from "socket-function/src/misc";
+import { setShardPrefixes } from "../0-path-value-core/ShardPrefixes";
 
-export async function deployGetFunctions(): Promise<FunctionSpec[]> {
+export async function deployGetFunctions(): Promise<{
+    functionSpecs: FunctionSpec[];
+    prefixes: string[];
+}> {
     let innerPath = path.resolve(__dirname + "/deployGetFunctionsInner.ts").replaceAll("\\", "/");
-    let result = await runPromise(`yarn typenode ${innerPath}`);
+    let result = await timeoutToError(timeInMinute * 7, runPromise(`yarn typenode ${innerPath}`), () => new Error("deployGetFunctions timed out"));
     let functionSpecs: FunctionSpec[] = [];
+    let prefixes: string[] = [];
     for (let line of result.split("\n")) {
         if (line.startsWith("ERROR:")) {
             throw errorify(line.slice("ERROR:".length));
@@ -21,8 +27,11 @@ export async function deployGetFunctions(): Promise<FunctionSpec[]> {
             let spec = JSON.parse(line.slice("FUNCTION_SPEC:".length));
             functionSpecs.push(spec);
         }
+        if (line.startsWith("PREFIX:")) {
+            prefixes.push(line.slice("PREFIX:".length));
+        }
     }
-    return functionSpecs;
+    return { functionSpecs, prefixes };
 }
 
 export type DeployProgress = {
@@ -31,6 +40,7 @@ export type DeployProgress = {
 
 export async function deployFunctions(config: {
     functionSpecs: FunctionSpec[];
+    prefixes: string[];
     notifyRefreshDelay: number;
     deployOnlyCode?: boolean;
     deployOnlyUI?: boolean;
@@ -75,5 +85,10 @@ export async function deployFunctions(config: {
     });
     config.progress?.({ section: "Deploying Functions", progress: 1 });
 
+    config.progress?.({ section: "Deploying Prefixes", progress: 0 });
+    // NOTE: This is only used by the path value server, and if it's out of sync with the code, it doesn't really matter. The path value server only uses the prefixes on startup, and it only needs prefixes at least newer than the ones last used by the gc code. And the gc code gets it's prefixes from the same source, so... We can technically write this anywhere we want, or just randomly not write it, and it would be fine (If we never update it, then the client accesses will get more and more inefficient and start accessing every single path value server, but it won't break anything).
+    await setShardPrefixes(config.prefixes);
+    config.progress?.({ section: "Deploying Prefixes", progress: 1 });
+
     console.log(magenta(`FINISHED DEPLOY (${gitRef})`));
 }

package/src/4-deploy/deployGetFunctionsInner.ts

@@ -3,12 +3,13 @@ import { deployBlock } from "./deployBlock";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { getThreadKeyCert } from "../-a-auth/certs";
 import { getDomain, isPublic } from "../config";
-import { LOCAL_DOMAIN } from "../0-path-value-core/PathController";
-import { getSchemaObject, getModuleRelativePath, PERMISSIONS_FUNCTION_ID, getExportPath } from "../3-path-functions/syncSchema";
+import { getSchemaObject, getModuleRelativePath, PERMISSIONS_FUNCTION_ID, getExportPath, getPrefixesForDeploy } from "../3-path-functions/syncSchema";
 import { getPathStr2 } from "../path";
 import { FunctionSpec } from "../3-path-functions/PathFunctionRunner";
 import fs from "fs";
 import { getGitRefLive, getGitURLLive } from "./git";
+import { LOCAL_DOMAIN } from "../0-path-value-core/PathRouter";
+import { delay } from "socket-function/src/batching";
 
 async function main() {
     let domainName = getDomain();
@@ -24,6 +25,7 @@ async function main() {
     let folderRoot = path.resolve(".").replaceAll("\\", "/");
     const deployPath = path.resolve("./deploy.ts");
     await import(deployPath);
+    await delay(1);
 
     const querysubRoot = path.resolve(__dirname + "/../").replaceAll("\\", "/");
 
@@ -91,6 +93,10 @@ async function main() {
             console.log(`FUNCTION_SPEC:${JSON.stringify(spec)}`);
         }
     }
+
+    for (let prefix of getPrefixesForDeploy()) {
+        console.log(`PREFIX:${prefix}`);
+    }
 }
 
 main().catch(e => console.error("ERROR:" + String(e?.stack || e).replaceAll("\n", "\\n"))).finally(() => process.exit(0));

package/src/4-deploy/deployMain.ts

@@ -1,69 +1,52 @@
-import "../inject";
-
-import { FunctionSpec, preloadFunctions } from "../3-path-functions/PathFunctionRunner";
-import path from "path";
-import { getExportPath, getSchemaObject, PERMISSIONS_FUNCTION_ID } from "../3-path-functions/syncSchema";
-import { getModuleRelativePath } from "../3-path-functions/syncSchema";
-import fs from "fs";
-import { SocketFunction } from "socket-function/SocketFunction";
-import { getThreadKeyCert } from "../-a-auth/certs";
-import { replaceFunctions } from "../3-path-functions/PathFunctionHelpers";
-import { getPathStr2 } from "../path";
-import { setIsDeploy } from "./deployCheck";
-
-import yargs from "yargs";
-import { getDomain, isPublic } from "../config";
-import { deployBlock } from "./deployBlock";
-import { LOCAL_DOMAIN } from "../0-path-value-core/PathController";
-import { blue, magenta, red } from "socket-function/src/formatting/logColors";
-import { getGitRefLive, getGitURLLive } from "./git";
-import { setLiveDeployedHash } from "./deploySchema";
-import { proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
-import debugbreak from "debugbreak";
-import { timeInHour } from "socket-function/src/misc";
-import { preloadUI } from "./edgeNodes";
-import { shutdown } from "../diagnostics/periodic";
-import { delay } from "socket-function/src/batching";
-import { waitForImportBlockers } from "../3-path-functions/pathFunctionLoader";
-import { deployFunctions, deployGetFunctions } from "./deployFunctions";
-import { Querysub } from "../4-querysub/QuerysubController";
-
-let yargObj = yargs(process.argv)
-    .option("deployonlycode", { type: "boolean", desc: "Only deploy code, not ui" })
-    .option("deployonlyui", { type: "boolean", desc: "Only deploy ui, not code" })
-    .option("notifyrefreshdelay", { type: "number", default: timeInHour * 12, desc: "The time clients have to refresh their UI after a deploy." })
-    .option("dryrun", { type: "boolean", desc: "Dry run, don't actually deploy" })
-    .argv
-    ;
-
-setIsDeploy();
-
-export async function deployMain() {
-    //SocketFunction.logMessages = true;
-    //quietCoreMode();
-    //ClientWatcher.DEBUG_READS = true;
-    //ClientWatcher.DEBUG_WRITES = true;
-
-    await Querysub.hostService("deployMain");
-
-    const currentFunctions = await deployGetFunctions();
-    for (let fnc of currentFunctions) {
-        console.log(blue(`${fnc.DomainName}.${fnc.ModuleId}.${fnc.FunctionId}`));
-    }
-    if (yargObj.dryrun) {
-        console.log(red("Dry run, not actually deploying"));
-        return;
-    }
-
-    await deployFunctions({
-        functionSpecs: currentFunctions,
-        notifyRefreshDelay: yargObj.notifyrefreshdelay,
-        deployOnlyCode: yargObj.deployonlycode,
-        deployOnlyUI: yargObj.deployonlyui,
-    });
-
-    await shutdown();
-}
-
-
+import "../inject";
+
+import { setIsDeploy } from "./deployCheck";
+
+import yargs from "yargs";
+import { blue, red } from "socket-function/src/formatting/logColors";
+import { timeInHour } from "socket-function/src/misc";
+import { shutdown } from "../diagnostics/periodic";
+import { deployFunctions, deployGetFunctions } from "./deployFunctions";
+import { Querysub } from "../4-querysub/QuerysubController";
+import { setShardPrefixes } from "../0-path-value-core/ShardPrefixes";
+
+let yargObj = yargs(process.argv)
+    .option("deployonlycode", { type: "boolean", desc: "Only deploy code, not ui" })
+    .option("deployonlyui", { type: "boolean", desc: "Only deploy ui, not code" })
+    .option("notifyrefreshdelay", { type: "number", default: timeInHour * 12, desc: "The time clients have to refresh their UI after a deploy." })
+    .option("dryrun", { type: "boolean", desc: "Dry run, don't actually deploy" })
+    .argv
+    ;
+
+setIsDeploy();
+
+export async function deployMain() {
+    //SocketFunction.logMessages = true;
+    //quietCoreMode();
+    //ClientWatcher.DEBUG_READS = true;
+    //ClientWatcher.DEBUG_WRITES = true;
+
+    await Querysub.hostService("deployMain");
+
+    const { functionSpecs, prefixes } = await deployGetFunctions();
+    for (let fnc of functionSpecs) {
+        console.log(blue(`${fnc.DomainName}.${fnc.ModuleId}.${fnc.FunctionId}`));
+    }
+    if (yargObj.dryrun) {
+        console.log(red("Dry run, not actually deploying"));
+        return;
+    }
+
+    await deployFunctions({
+        functionSpecs: functionSpecs,
+        notifyRefreshDelay: yargObj.notifyrefreshdelay,
+        deployOnlyCode: yargObj.deployonlycode,
+        deployOnlyUI: yargObj.deployonlyui,
+        prefixes: prefixes,
+    });
+
+    await shutdown();
+}
+
+
 deployMain().catch(e => console.error(e)).finally(() => process.exit());

package/src/4-deploy/edgeClientWatcher.tsx

@@ -6,7 +6,6 @@ import { throttleFunction, timeInMinute, timeInSecond } from "socket-function/sr
 import { isNode } from "typesafecss";
 import { logErrors, timeoutToError } from "../errors";
 import { blue, red, yellow } from "socket-function/src/formatting/logColors";
-import { showModal } from "../5-diagnostics/Modal";
 import { qreact } from "../4-dom/qreact";
 import { liveHashOverrideURL } from "./edgeBootstrap";
 import { css } from "typesafecss";
@@ -17,6 +16,11 @@ import { lazy } from "socket-function/src/caching";
 import { Button } from "../library-components/Button";
 import { updateRootDiscoveryLocation } from "../-f-node-discovery/NodeDiscovery";
 
+setImmediate(() => {
+    // Modal
+    import("../5-diagnostics/Modal");
+});
+
 const SWITCH_SERVER_TIMEOUT = timeInSecond * 15;
 const MAX_DISPLAY_INTERVAL = timeInMinute * 5;
 const FINAL_DELAY = timeInSecond * 30;
@@ -78,6 +82,7 @@ function onLiveHashChange(liveHash: string, refreshThresholdTime: number) {
 
     // Start notification loop
     void (async () => {
+        const { showModal } = await import("../5-diagnostics/Modal");
         // Show notifications at intervals
         for (let delayTime of delays) {
             await delayTime;

package/src/4-deploy/edgeNodes.ts

@@ -116,7 +116,7 @@ export async function registerEdgeNode(config: {
 
     await waitForFirstTimeSync();
 
-    if (!isLocal() && !noSyncing()) {
+    if (isPublic() && !noSyncing()) {
         let loadedHash = "";
         let firstLoad = new PromiseObj<void>();
         Querysub.createWatcher(() => {
@@ -342,7 +342,7 @@ class EdgeNodeControllerBase {
     public async preloadUI(config: {
         hash: string;
     }) {
-        if (isLocal()) return;
+        if (!isPublic()) return;
         const { hash } = config;
         if (!registeredEdgeNode || !canHaveChildren(registeredEdgeNode)) {
             throw new Error(`Somehow we have no registered edge node. How did we even expose this controller?`);

package/src/4-dom/qreact.tsx

@@ -1,9 +1,6 @@
 import type preact from "preact";
 import { compareArray, isNode, sort, timeInSecond } from "socket-function/src/misc";
-import { isDeploy } from "../4-deploy/deployCheck";
-import { PermissionsCheck } from "../4-querysub/permissions";
 import { ClientWatcher, clientWatcher } from "../1-path-client/pathValueClientWatcher";
-import { LOCAL_DOMAIN } from "../0-path-value-core/PathController";
 import { rawSchema } from "../2-proxy/pathDatabaseProxyBase";
 import { SyncWatcher, atomic, atomicObjectWrite, atomicObjectWriteNoFreeze, doProxyOptions, doUnatomicWrites, proxyWatcher, registerSchemaPrefix, unregisterSchemaPrefix } from "../2-proxy/PathValueProxyWatcher";
 import { createPathValueProxy, getProxyPath, isValueProxy, isValueProxy2 } from "../2-proxy/pathValueProxy";
@@ -2798,4 +2795,5 @@ function triggerGlobalAfterRenderWatch(component: QRenderClass) {
 // NOTE: Import Querysub at the end, so we can export qreact before we require it. That way Querysub
 //      can statically access qreact.
 import { Querysub } from "../4-querysub/Querysub";
-import { logDisk } from "../diagnostics/logs/diskLogger";
+import { logDisk } from "../diagnostics/logs/diskLogger"; import { LOCAL_DOMAIN } from "../0-path-value-core/PathRouter";
+

package/src/4-dom/qreactTest.tsx

@@ -1,8 +1,8 @@
-import { disableMeasurements, measureBlock, measureCode, measureFnc } from "socket-function/src/profiling/measure";
+import { disableMeasurements, measureBlock, measureCode } from "socket-function/src/profiling/measure";
 disableMeasurements();
 
 import preact from "preact";
-import { isNode, list, timeInHour } from "socket-function/src/misc";
+import { isNode, list } from "socket-function/src/misc";
 import { ClientWatcher, clientWatcher } from "../1-path-client/pathValueClientWatcher";
 import { isDeploy } from "../4-deploy/deployCheck";
 import { logErrors } from "../errors";
@@ -10,18 +10,12 @@ import { watchFilesAndTriggerHotReloading } from "socket-function/hot/HotReloadC
 import { qreact } from "./qreact";
 import { Querysub } from "../4-querysub/QuerysubController";
 import { PermissionsCheck } from "../4-querysub/permissions";
-import { PathValueProxyWatcher, proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
-import { setFlag } from "socket-function/require/compileFlags";
-import { rawSchema } from "../2-proxy/pathDatabaseProxyBase";
-import { LOCAL_DOMAIN } from "../0-path-value-core/PathController";
-import { batchFunction, delay } from "socket-function/src/batching";
+import { delay } from "socket-function/src/batching";
 import { formatTime } from "socket-function/src/formatting/format";
-import { pathValueAuthority2 } from "../0-path-value-core/NodePathAuthorities";
-import { schema } from "../3-path-functions/tests/rejectTest";
-import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
 import { getPathFromStr } from "../path";
-import { PromiseObj } from "../promise";
 import { css } from "./css";
+import { isSuperUserPERMISSIONS } from "../user-implementation/userData";
+import { authorityLookup } from "../0-path-value-core/AuthorityLookup";
 
 
 
@@ -264,7 +258,7 @@ async function browserMain() {
 
     //*
     await new Promise(resolve => setTimeout(resolve, 0));
-    await pathValueAuthority2.waitUntilRoutingIsReady();
+    await authorityLookup.startSyncing();
 
     let benchmarkComponents = [
         // ONLY run with devtools closed AND disableMeasurements (at the top of the file)! Otherwise the timings will be wrong.
@@ -334,7 +328,7 @@ let { data, functions } = Querysub.createSchema<{
     moduleId: "qreactTest",
     functionMetadata: {},
     permissions: {
-        PERMISSIONS: () => true
+        PERMISSIONS: isSuperUserPERMISSIONS
     }
 });
 

package/src/4-querysub/Querysub.ts

@@ -18,8 +18,7 @@ import { RequireController, setRequireBootRequire } from "socket-function/requir
 import { cache, cacheLimited, lazy } from "socket-function/src/caching";
 import { getOwnMachineId, getOwnThreadId, getThreadKeyCert, verifyMachineIdForPublicKey } from "../-a-auth/certs";
 import { getHostedIP, getSNICerts, publishMachineARecords } from "../-e-certs/EdgeCertController";
-import { LOCAL_DOMAIN, nodePathAuthority } from "../0-path-value-core/NodePathAuthorities";
-import { debugCoreMode, registerGetCompressNetwork, encodeParentFilter, authorityStorage } from "../0-path-value-core/pathValueCore";
+import { debugCoreMode, registerGetCompressNetwork, authorityStorage } from "../0-path-value-core/pathValueCore";
 import { clientWatcher, ClientWatcher } from "../1-path-client/pathValueClientWatcher";
 import { SyncWatcher, proxyWatcher, specialObjectWriteValue, isSynced, PathValueProxyWatcher, atomic, doAtomicWrites, noAtomicSchema, undeleteFromLookup, registerSchemaPrefix, WatcherOptions, doProxyOptions } from "../2-proxy/PathValueProxyWatcher";
 import { isInProxyDatabase, rawSchema } from "../2-proxy/pathDatabaseProxyBase";
@@ -178,7 +177,6 @@ export class Querysub {
     public static WILDCARD_NUM = "*" as any as number;
     public static createLocalSchema = createLocalSchema;
     public static flushDelayedFunctions = () => flushDelayedFunctions();
-    public static LOCAL_DOMAIN = LOCAL_DOMAIN;
     public static DEBUG_CALLS = false;
     public static DEBUG_PREDICTIONS = false;
     public static PREDICT_CALLS = true;
@@ -201,6 +199,11 @@ export class Querysub {
      */
     public static MAX_FUTURE_CALL_TIME = 10_000;
 
+    /** NOTE: Is required for the function runner to work. Eventually, we should allow you to request the full history on just specific paths.
+     *      - It's not *actually* the full history, As the underlying path value storage will delete values that are past the golden threshold anyways.
+     */
+    public static SEND_FULL_HISTORY_ON_INITIAL_SYNC = false;
+
 
     public static COMPRESS_NETWORK = true;
 
@@ -210,6 +213,7 @@ export class Querysub {
 
     public static getCallTime = getSyncedTime;
     public static getFunctionCallTime = getSyncedTime;
+    // NOTE: This is just an ID inside of a call that will be unique. It's not the call ID that the function uses when it's running. That's something that's different, and that's defined in PathFunctionHelpers
     public static getCallId = () => Querysub.getCallerMachineId() + "_" + Querysub.getCallTime();
     // Returns a random value between 0 and 1. The value depends on the callId, and index (being identical for
     //      the same callid and index).
@@ -284,7 +288,7 @@ export class Querysub {
         await measureBlock(async () => {
             await waitForFirstTimeSync();
         }, "waitForFirstTimeSync");
-        await nodePathAuthority.waitUntilRoutingIsReady();
+        await authorityLookup.startSyncing();
     }
 
     public static createWatcher(watcher: (obj: SyncWatcher) => void, options?: Partial<WatcherOptions<unknown>>): {
@@ -788,6 +792,8 @@ export class Querysub {
 
 
         RequireController.injectHTMLBeforeStartup(async () => {
+            const { getEdgeBootstrapScript } = await import("../4-deploy/edgeBootstrap");
+            const { getEdgeNodeConfigURL } = await import("../4-deploy/edgeNodes");
             let edgeBootstrapFile = await getEdgeBootstrapScript({
                 edgeNodeConfigURL: await getEdgeNodeConfigURL(),
             });
@@ -830,6 +836,7 @@ export class Querysub {
         let { ip, ipDomain } = await publishMachineARecords();
 
         if (!isBootstrapOnly()) {
+            const { registerEdgeNode } = await import("../4-deploy/edgeNodes");
             await registerEdgeNode({
                 host: ipDomain + ":" + port,
                 entryPaths,
@@ -1050,9 +1057,14 @@ export class Querysub {
          *      (otherwise likely does filtering serverside, which is still somewhat efficient).
          */
         startFraction?: number;
-        /** Value between 0 and 1 */
+        /** Value between 0 and 1.
+         *  BOTH start and end must be provided, otherwise if only one is provided we will ignore it.
+         */
         endFraction?: number;
     }): (keyof T)[] {
+        if (!isNode()) {
+            console.warn(`keys() is unlikely to work clientside. If we need to emulate how the server does the filtering, we should add an additional function to help with that. But there are issues with how query sub proxies watches, which make proxying partial key watches too difficult. ALSO, this is mostly for watching very large data streams with dynamic sharding (in FunctionRunner), and so shouldn't be needed clientside (the application can handle filtering at a schema level if it really wants something similar to this).`);
+        }
         let { startFraction, endFraction } = config || {};
         if (startFraction === undefined || endFraction === undefined) {
             return Object.keys(obj);
@@ -1284,6 +1296,7 @@ setImmediate(() => {
 setImmediate(async () => {
     // Import, so it registers addStatPeriodic
     await import("../5-diagnostics/nodeMetadata");
+    await import("../diagnostics/pathAuditer");
 });
 
 setImmediate(async () => {
@@ -1300,10 +1313,7 @@ registerGetCompressNetwork(() => Querysub.COMPRESS_NETWORK);
 import "../diagnostics/watchdog";
 import "../diagnostics/trackResources";
 import "../diagnostics/benchmark";
-import { getEdgeNodeConfigURL, registerEdgeNode } from "../4-deploy/edgeNodes";
-import { getEdgeBootstrapScript } from "../4-deploy/edgeBootstrap";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
-import { css } from "../4-dom/css";
 import { getCountPerPaint } from "../functional/onNextPaint";
 import { addEpsilons } from "../bits";
 import { blue, magenta } from "socket-function/src/formatting/logColors";
@@ -1312,4 +1322,7 @@ import { getRecords, setRecord } from "../-b-authorities/dnsAuthority";
 import { testTCPIsListening } from "socket-function/src/networking";
 import { getNodeId, getNodeIdLocation } from "socket-function/src/nodeCache";
 import { onAllPredictionsFinished } from "../-0-hooks/hooks";
+import { LOCAL_DOMAIN } from "../0-path-value-core/PathRouter";
+import { authorityLookup } from "../0-path-value-core/AuthorityLookup";
+import { encodeParentFilter } from "../0-path-value-core/hackedPackedPathParentFiltering";
 

package/src/4-querysub/QuerysubController.ts

@@ -1,22 +1,23 @@
 import { SocketFunction } from "socket-function/SocketFunction";
 import { cache, lazy } from "socket-function/src/caching";
-import { appendToPathStr, getPathDepth, getPathFromStr, getPathIndex, getPathStr1, getPathStr2, getPathStr3, hack_getPackedPathSuffix, removePathLastPart } from "../path";
-import { CHILD_CHECK_PREFIX, FunctionMetadata } from "../3-path-functions/syncSchema";
-import { RemoteWatcher, remoteWatcher } from "../1-path-client/RemoteWatcher";
+import { appendToPathStr, getPathDepth, getPathStr1, getPathStr3 } from "../path";
+import { FunctionMetadata } from "../3-path-functions/syncSchema";
+import { RemoteWatcher } from "../1-path-client/RemoteWatcher";
 import { getProxyPath } from "../2-proxy/pathValueProxy";
-import { atomic, atomicObjectWrite, proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
-import { CallSpec, DEPTH_TO_DATA, FunctionSpec, commitCall, debugCallSpec, functionSchema, getDevFunctionSpecFromCall } from "../3-path-functions/PathFunctionRunner";
-import { PathValueController } from "../0-path-value-core/PathValueController";
-import { epochTime, MAX_ACCEPTED_CHANGE_AGE, PathValue, pathWatcher, Time, WatchConfig, compareTime, debugTime, getNextTime, MAX_CHANGE_AGE } from "../0-path-value-core/pathValueCore";
+import { atomic, proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
+import { CallSpec, DEPTH_TO_DATA, commitCall, debugCallSpec, functionSchema, getDevFunctionSpecFromCall } from "../3-path-functions/PathFunctionRunner";
+import { PathValueController, PathValueControllerBase } from "../0-path-value-core/PathValueController";
+import { epochTime, MAX_ACCEPTED_CHANGE_AGE, PathValue, WatchConfig, debugTime, MAX_CHANGE_AGE } from "../0-path-value-core/pathValueCore";
+import { pathWatcher } from "../0-path-value-core/PathWatcher";
 import { IdentityController_getMachineId, IdentityController_getPubKeyShort, IdentityController_getSecureIP } from "../-c-identity/IdentityController";
 import { getControllerNodeId } from "../-g-core-values/NodeCapabilities";
 import { getModuleFromConfig, watchModuleHotreloads } from "../3-path-functions/pathFunctionLoader";
 import { errorToUndefined, ignoreErrors, logErrors } from "../errors";
 import { blue, green, magenta, red } from "socket-function/src/formatting/logColors";
 import { PermissionsCheck } from "./permissions";
-import { parseArgs, writeCall } from "../3-path-functions/PathFunctionHelpers";
+import { writeCall } from "../3-path-functions/PathFunctionHelpers";
 import { delay } from "socket-function/src/batching";
-import { isNode, isNodeTrue, timeInMinute, timeInSecond } from "socket-function/src/misc";
+import { isNodeTrue } from "socket-function/src/misc";
 import { registerResource } from "../diagnostics/trackResources";
 
 // Always whitelist preact, as most of the time we want it clientside
@@ -24,27 +25,21 @@ import "preact";
 import { setFlag } from "socket-function/require/compileFlags";
 import { CallerContextBase } from "socket-function/SocketFunctionTypes";
 import { isTrustedByNode } from "../-d-trust/NetworkTrust2";
-import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
 import { Querysub, id } from "./Querysub";
-import { measureBlock } from "socket-function/src/profiling/measure";
-import debugbreak from "debugbreak";
 import { isDefined } from "../misc";
 import { isClient, isServer } from "../config2";
 import { PromiseObj } from "../promise";
 import { LoggingClient } from "../0-path-value-core/LoggingClient";
 import * as prediction from "./querysubPrediction";
-import { getCallResultPath } from "./querysubPrediction";
-import { nodePathAuthority, pathValueAuthority2 } from "../0-path-value-core/NodePathAuthorities";
-import { assertIsManagementUser } from "../diagnostics/managementPages";
-import { getBrowserUrlNode } from "../-f-node-discovery/NodeDiscovery";
 setFlag(require, "preact", "allowclient", true);
 
 import yargs from "yargs";
 import { mergeFilterables, parseFilterable, serializeFilterable } from "../misc/filterable";
 import { isManagementUser, onAllPredictionsFinished } from "../-0-hooks/hooks";
-import { getDomain, isBootstrapOnly, isLocal } from "../config";
-import { logDisk } from "../diagnostics/logs/diskLogger";
+import { getDomain, isBootstrapOnly } from "../config";
 import { flushPredictionQueueBase, runInPredictionQueue, syncHasPendingPredictionsBase } from "./predictionQueue";
+import { PathRouter } from "../0-path-value-core/PathRouter";
+import { authorityLookup } from "../0-path-value-core/AuthorityLookup";
 
 let yargObj = isNodeTrue() && yargs(process.argv)
     .option("fncfilter", { type: "string", default: "", desc: `Sets the filterable state for function calls, causing them to target specific FunctionRunners. If no FunctionRunner matches, all functions will fail to run. For example: "devtestserver" will match a FunctionRunner that uses the "devtestserver" filter. Merges with the existing filterable state if a client sets it explicitly.` })
@@ -59,7 +54,7 @@ export { Querysub, id };
 // TODO: Eventually this will be split by domain, which required nodeIds to be split by domain, so that getControllerNodeId can check nodeIds on a specific domain.
 export async function querysubNodeId() {
     if (isClient()) {
-        return nodePathAuthority.getSingleReadNodePromise(getPathStr1(getDomain()));
+        return PathRouter.getReadyAuthority(getPathStr1(getDomain()))?.nodeId;
     } else {
         return getControllerNodeId(QuerysubController);
     }
@@ -482,12 +477,14 @@ export class QuerysubControllerBase {
 
                 console.info("Disallowing PathValue watches due to disallowed permissions", { count: newPathsNotAllowed.length, callerId });
 
-                ignoreErrors(pathValueSerializer.serialize(undefinedValues, { compress: Querysub.COMPRESS_NETWORK }).then(buffers =>
-                    PathValueController.nodes[callerId].forwardWrites(
-                        buffers,
-                        Array.from(newParentPathsNotAllowed),
-                    )
-                ));
+                ignoreErrors(PathValueControllerBase.sendValues({
+                    nodeId: callerId,
+                    pathValues: undefinedValues,
+                    initialTriggers: {
+                        values: new Set(undefinedValues.map(value => value.path)),
+                        parentPaths: new Set(newParentPathsNotAllowed)
+                    },
+                }));
             }
 
             if (removedPaths.size > 0) {
@@ -497,8 +494,10 @@ export class QuerysubControllerBase {
             if (allowedPaths.size > 0) {
                 let [newParentPathsAllowed, newPathsAllowed] = splitParentPaths(allowedPaths);
                 let watchConfig: WatchConfig = { paths: newPathsAllowed, parentPaths: newParentPathsAllowed };
-                pathWatcher.watchPath({ callback: callerId, ...watchConfig });
-                remoteWatcher.watchLatest(watchConfig);
+                pathWatcher.watchPath({
+                    nodeId: callerId,
+                    ...watchConfig
+                });
             }
         };
     }
@@ -541,6 +540,13 @@ export class QuerysubControllerBase {
         }
         Querysub.assertDomainAllowed(getPathStr1(call.DomainName));
 
+        console.info(`Querysub starting to add function call`, {
+            callId: call.CallId,
+            timeId: call.runAtTime.time,
+            functionId: call.FunctionId,
+            moduleId: call.ModuleId,
+        });
+
         let caller = SocketFunction.getCaller();
         let callerMachineId = IdentityController_getMachineId(caller);
         let callerCreatorId = IdentityController_getPubKeyShort(caller);
@@ -608,6 +614,9 @@ export class QuerysubControllerBase {
             throw new Error(`Caller does not have permission to call ${call.DomainName}.${call.ModuleId}`);
         }
 
+        console.info(`Querysub committing call`, {
+            callId: call.CallId,
+        });
         commitCall(call);
         // NOTE: We don't wait for our writes to actually commit, because... there isn't any reason they shouldn't
         //  except the server being down, in which case, the client will gracefully timeout when it doesn't receive the confirmation
@@ -654,11 +663,17 @@ export class QuerysubControllerBase {
 
     // NOTE: Maybe remove this? It's very useful for debugging though...
     public async debugGetSingleReadNode(path: string) {
-        return pathValueAuthority2.getSingleReadNodeSync(path);
+        return PathRouter.getReadyAuthority(path)?.nodeId;
     }
 
     public async debugGetPathAuthorities(nodeId: string) {
-        return pathValueAuthority2.getAuthorityPaths(nodeId);
+        let authorities = await authorityLookup.getTopology();
+        return authorities.find(x => x.nodeId === nodeId)?.authoritySpec;
+    }
+
+    public async debugGetTopologyEntry(nodeId: string) {
+        let topology = await authorityLookup.getTopology();
+        return topology.find(x => x.nodeId === nodeId);
     }
 }
 
@@ -672,6 +687,7 @@ export const QuerysubController = SocketFunction.register(
         addCall: { compress: true, },
         debugGetSingleReadNode: {},
         debugGetPathAuthorities: {},
+        debugGetTopologyEntry: {},
         getModulePath: {},
         getDevFunctionSpecFromCall: {},
     }),

package/src/4-querysub/permissions.ts

@@ -13,7 +13,7 @@ import { isClient } from "../config2";
 import { sort } from "socket-function/src/misc";
 import { Querysub } from "./QuerysubController";
 import { CALL_PERMISSIONS_KEY } from "./permissionsShared";
-import { getDomain, isLocal } from "../config";
+import { getDomain, isLocal, isPublic } from "../config";
 
 function watchModule(config: FunctionSpec): NodeJS.Module | undefined {
     let module = getModuleFromConfig(config);
@@ -52,7 +52,7 @@ export class PermissionsCheck {
     });
 
     private getModulePermissions(domainName: string) {
-        if (isLocal() && domainName === getDomain()) {
+        if (!isPublic() && domainName === getDomain()) {
             return function (moduleId: string): {
                 schema: SchemaObject;
                 fnc: FunctionSpec;