querysub 0.2.0

package/src/4-querysub/Querysub.ts

@@ -0,0 +1,877 @@
+// NOTE: This should be imported at the top of all entrypoints. BUT, if it is missed,
+//  this should ensure it is at least included once (even though including it after
+//  a file prevents it from transforming it, so we'll miss a lot of files doing it this late).
+import "../inject";
+
+import { isNode, timeInMinute } from "socket-function/src/misc";
+
+import { SocketFunction } from "socket-function/SocketFunction";
+import { isHotReloading, onHotReload, watchFilesAndTriggerHotReloading } from "socket-function/hot/HotReloadController";
+import { RequireController, SerializedModule, setRequireBootRequire } from "socket-function/require/RequireController";
+import { cache, cacheLimited, lazy } from "socket-function/src/caching";
+import { getOwnMachineId, getThreadKeyCert, verifyMachineIdForPublicKey } from "../-a-auth/certs";
+import { getSNICerts, publishMachineARecords } from "../-e-certs/EdgeCertController";
+import { LOCAL_DOMAIN, nodePathAuthority } from "../0-path-value-core/NodePathAuthorities";
+import { debugCoreMode, registerGetCompressNetwork, encodeParentFilter, registerGetCompressDisk } from "../0-path-value-core/pathValueCore";
+import { clientWatcher, ClientWatcher } from "../1-path-client/pathValueClientWatcher";
+import { SyncWatcher, proxyWatcher, specialObjectWriteValue, isSynced, PathValueProxyWatcher, atomic, doAtomicWrites, noAtomicSchema, undeleteFromLookup, registerSchemaPrefix } from "../2-proxy/PathValueProxyWatcher";
+import { isInProxyDatabase, rawSchema } from "../2-proxy/pathDatabaseProxyBase";
+import { isValueProxy2, getProxyPath } from "../2-proxy/pathValueProxy";
+import { getCurrentCallAllowUndefined, getCurrentCall, CallSpec } from "../3-path-functions/PathFunctionRunner";
+import { listenOnDebugger } from "../diagnostics/listenOnDebugger";
+import { logErrors } from "../errors";
+import { getLastPathPart, getPathIndexAssert, getPathStr2, hack_setPackedPathSuffix } from "../path";
+import { QuerysubController, flushDelayedFunctions, onCallPredict, waitUntilAllPredictionsFinish } from "./QuerysubController";
+import { PermissionsCheck } from "./permissions";
+import { inlineNestedCalls, syncSchema } from "../3-path-functions/syncSchema";
+import type { identityStorageKey, IdentityStorageType } from "../-a-auth/certs";
+
+import "../diagnostics/watchdog";
+import "../diagnostics/trackResources";
+import "../diagnostics/benchmark";
+import { qreact } from "../4-dom/qreact";
+import { configRootDiscoveryLocation, getOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
+import { pathValueCommitter } from "../0-path-value-core/PathValueCommitter";
+import debugbreak from "debugbreak";
+import { extractPublicKey, verifyED25519 } from "../-a-auth/ed25519";
+import { registerDynamicResource, registerResource } from "../diagnostics/trackResources";
+import { registerPeriodic } from "../diagnostics/periodic";
+import { sha256 } from "js-sha256";
+import { green, red } from "socket-function/src/formatting/logColors";
+import { minify_sync } from "terser";
+import { isClient } from "../config2";
+import { remoteWatcher } from "../1-path-client/RemoteWatcher";
+import { waitForFirstTimeSync } from "socket-function/time/trueTimeShim";
+import { measureFnc } from "socket-function/src/profiling/measure";
+import { delay } from "socket-function/src/batching";
+import { MaybePromise } from "socket-function/src/types";
+import { devDebugbreak, getDomain, isDevDebugbreak, isNoNetwork, isPublic } from "../config";
+import { hookErrors } from "../diagnostics/errorLogs/hookErrors";
+import { Schema2, Schema2T, t } from "../2-proxy/schema2";
+import { CALL_PERMISSIONS_KEY } from "./permissionsShared";
+
+export { t };
+
+
+export type MachineSourceCheck<T = unknown> = {
+    /** NOTE: The source for this is added inline, and so it cannot use any external variables. */
+    getExtraDataClientsideInline?: () => T | "nosource";
+    shouldMachineIdSeeSource: (config: { machineId: string, extra: T }) => Promise<boolean>;
+}
+
+export function id(obj: unknown) {
+    let path = isValueProxy2(obj);
+    if (!path) {
+        return "";
+    }
+    return getLastPathPart(path);
+}
+
+export class Querysub {
+    public static syncSchema = syncSchema;
+    public static qreact = qreact;
+    public static Component = qreact.Component;
+    public static render = qreact.render;
+    public static context = qreact.context;
+    public static createElement = qreact.createElement;
+    public static Fragment = qreact.Fragment;
+    public static t = t;
+    /**
+        IMPORTANT! New schemas must be deployed by calling `yarn deploy`
+
+        const { data } = syncSchema<{
+            width: number;
+            height: number;
+        }>()({
+            // If domainName is LOCAL_DOMAIN, the values are kept in memory in the current process.
+            domainName: getDomain(),
+            functions: {
+                setWidth,
+            },
+            module,
+            moduleId: "sharedParams",
+        });
+        function setWidth(width: number) {
+            data().width = width;
+        }
+    */
+    public static createSchema = syncSchema;
+    public static noAtomicSchema = <T>(code: () => T) => noAtomicSchema(code);
+    /** Undelete is a bit dangerous, and races with garbage collection. Using gcDelay
+     *      helps reduce this race, but it is still possible to undelete, and then
+     *      have the value disappear after a few minutes.
+     *      - The race occurs if an undelete happens less than ~5 * MAX_CHANGE_AGE
+     *          before we start to GC.
+     *      - This race is more of a propagation delay. The value is already past the point
+     *          of no return, and so the undeletion cannot work. IF you set a large enough gcDelay
+     *          (a month), then not only is this unlikely to happen, but it means it will only happen
+     *          if the user waits a month to undelete it. Because undeletions aren't random, and are
+     *          more likely to occur shorter after the deletion, this means the chance is really less
+     *          than 5 minutes / 1 month (< 1/100K).
+     */
+    public static undeleteFromLookup = undeleteFromLookup;
+    public static WILDCARD = "*";
+    public static WILDCARD_NUM = "*" as any as number;
+    public static createLocalSchema = createLocalSchema;
+    public static flushDelayedFunctions = () => flushDelayedFunctions();
+    public static LOCAL_DOMAIN = LOCAL_DOMAIN;
+    public static DEBUG_CALLS = false;
+    public static DEBUG_PREDICTIONS = false;
+    public static PREDICT_CALLS = true;
+    public static AUDIT_PREDICTIONS = true;
+    public static SIMULATE_LAG = 0;
+
+    /** Delay used when functions are specified as delayCommit */
+    public static DELAY_COMMIT_DELAY = 1000 * 3;
+
+    /** Time predictions last for, after which point we remove them irregardless of if we have received a response.
+     *  - NOTE: If this is too short, predictions stop before the server can finish. If it's too long,
+     *      server issues (such as the FunctionRunner being missing) won't be noticed. So...
+     *      we SHOULD increase this when our setup is more stable (60 seconds is probably good).
+    */
+    public static PREDICTION_MAX_LIFESPAN = 1000 * 15;
+
+    /** Fairly lax, to handle lag during initial connection setup.
+     *      AND THEN, VERY lax, to handle server lag.
+     */
+    public static MAX_FUTURE_CALL_TIME = 10_000;
+
+    /** Maximum amount of synchronous time spend handling triggered operations until we
+     *      abort them (with an error).
+     */
+    public static SET_MAX_TRIGGER_TIME = (value: number) => ClientWatcher.MAX_TRIGGER_TIME = value;
+
+    /** Compression makes serialization about 2X slower, but reduces the size by about 2X (more or less if your
+     *      data size is dominated by value size instead of key size). */
+    public static COMPRESS_DISK = false;
+    public static COMPRESS_NETWORK = true;
+
+    public static now = getSyncedTime;
+    public static time = getSyncedTime;
+    public static getCallTime = getSyncedTime;
+    public static getFunctionCallTime = getSyncedTime;
+    public static getCallId = () => Querysub.getCallerMachineId() + "_" + Querysub.getCallTime();
+    // Returns a random value between 0 and 1. The value depends on the callId, and index (being identical for
+    //      the same callid and index).
+    public static callRandom = (index: number) => hashRandom(Querysub.getCallId(), index);
+
+    public static configRootDiscoveryLocation = configRootDiscoveryLocation;
+
+
+    public static nowDelayed = timeDelayed;
+    public static timeDelayed = timeDelayed;
+
+    public static getCallerMachineId() {
+        let call = getCurrentCallAllowUndefined();
+        if (!call) return getOwnMachineId();
+        return getCurrentCall().callerMachineId;
+    }
+    public static getCallerIP = () => getCurrentCall().callerIP;
+    public static getCallerIPAllowUndefined = () => getCurrentCallAllowUndefined()?.callerIP;
+    public static CALL_PERMISSIONS_KEY = CALL_PERMISSIONS_KEY;
+
+    public static watchCurrentFnc = () => PathValueProxyWatcher.BREAK_ON_CALL.add(
+        getPathStr2(getCurrentCall().ModuleId, getCurrentCall().FunctionId)
+    );
+    public static unwatchCurrentFnc = () => PathValueProxyWatcher.BREAK_ON_CALL.delete(
+        getPathStr2(getCurrentCall().ModuleId, getCurrentCall().FunctionId)
+    );
+
+    public static watchUnsyncedComponents = () => qreact.watchUnsyncedComponents();
+    public static watchAnyUnsyncedComponents = () => qreact.watchUnsyncedComponents().size > 0;
+
+    public static doAtomicWrites = <T>(callback: () => T): T => doAtomicWrites(callback);
+
+    public static trustedDomains = new Set<string>();
+
+    /** Wait for some startup initialization. Not necessary, but can ensure any initial calls
+     *      are in a better state.
+     *      - Useful for client service scripts.
+     *      NOTE: waitForFirstTimeSync is called in Socket.mount, so this function
+     *          isn't presently necessary in most serverside scripts.
+     */
+    @measureFnc
+    public static async optionalStartupWait() {
+        await waitForFirstTimeSync();
+        await nodePathAuthority.waitUntilRoutingIsReady();
+    }
+
+    public static createWatcher(watcher: (obj: SyncWatcher) => void): {
+        dispose: () => void;
+        explicitlyTrigger: () => void;
+    } {
+        return proxyWatcher.createWatcher({
+            debugName: watcher.name,
+            canWrite: true,
+            watchFunction: () => {
+                return watcher(proxyWatcher.getTriggeredWatcher());
+            },
+        });
+    }
+    public static createWriteWatcher(watcher: (obj: SyncWatcher) => void) {
+        return this.createWatcher(watcher);
+    }
+    public static undoDelete(holder: { [key: string]: unknown }, key: string) {
+        holder[key] = specialObjectWriteValue;
+    }
+
+    public static localWriteWrapper<Args>(fnc: (...args: Args[]) => void): (...args: Args[]) => void {
+        return (...args: Args[]) => Querysub.serviceWriteDetached(() => fnc(...args));
+    }
+    public static runSynced = Querysub.serviceWriteDetached;
+    public static commit = Querysub.serviceWriteDetached;
+    public static write = Querysub.serviceWriteDetached;
+    public static serviceWriteDetached(fnc: () => unknown) {
+        logErrors(proxyWatcher.commitFunction({
+            canWrite: true,
+            watchFunction: fnc
+        }));
+    }
+
+    public static exit = Querysub.gracefulTerminateProcess;
+    public static async gracefulTerminateProcess() {
+        await pathValueCommitter.waitForValuesToCommit();
+        if (isNode()) {
+            process.exit();
+        }
+    };
+
+    public static syncedCommit = Querysub.serviceWrite;
+    public static commitSynced = Querysub.serviceWrite;
+    public static async serviceWrite<T>(fnc: () => T) {
+        return await proxyWatcher.commitFunction({
+            canWrite: true,
+            watchFunction: fnc
+        });
+    }
+    public static localCommit = Querysub.localRead;
+    public static commitLocal = Querysub.localRead;
+    public static localRead<T>(fnc: () => T) {
+        return proxyWatcher.runOnce({
+            watchFunction: fnc
+        });
+    }
+
+    public static async unsafeInlineFunctionsServiceWrite<T>(code: () => T) {
+        return await Querysub.serviceWrite(() => {
+            return inlineNestedCalls(code);
+        });
+    }
+
+    public static undefinedIfAnyUnsynced<T>(get: () => T) {
+        let count = { value: 0 };
+        let result = proxyWatcher.countUnsynced(count, get);
+        if (count.value > 0) return undefined;
+        return result;
+    }
+
+    public static ignorePermissionsChecks = <T>(code: () => T) => PermissionsCheck.skipPermissionsChecks(code);
+    public static skipPermissionsChecks = <T>(code: () => T) => PermissionsCheck.skipPermissionsChecks(code);
+
+    public static anyUnsynced() {
+        return !Querysub.allSynced();
+    }
+    public static allSynced() {
+        return proxyWatcher.isAllSynced();
+    }
+    public static fullySynced = Querysub.allSynced;
+    public static isFullySynced = Querysub.allSynced;
+    public static isAllSynced = Querysub.allSynced;
+    public static isAnyUnsynced = Querysub.anyUnsynced;
+
+    public static pathHasAnyWatchers(get: () => unknown) {
+        return clientWatcher.pathHasAnyWatchers(getProxyPath(get));
+    }
+
+    public static onCommitFinished(callback: () => void) {
+        proxyWatcher.getTriggeredWatcher().onInnerDisposed.push(callback);
+    }
+
+    public static onCallPredict = (x: CallSpec | undefined) => onCallPredict(x);
+    /** NOTE: USUALLY you don't have to wait for predictions, as functions will run in order anyways.
+     *      BUT, if you run a synced function in the UI, then somewhere unrelated try to read values
+     *      for offload writing (which isn't ideal, but sometimes is necessary), then waiting
+     *      for predictions to finish can help in the rare cases where predictions are slow
+     *      (such as if code is still loading).
+     */
+    public static waitUntilAllPredictionsFinished = () => waitUntilAllPredictionsFinish();
+    public static onCommitPredictFinished = this.onCallPredict;
+
+    public static getOwnMachineId = getOwnMachineId;
+    public static getSelfMachineId = getOwnMachineId;
+
+    public static getOwnNodeId = getOwnNodeId;
+    public static getSelfNodeId = getOwnMachineId;
+
+    /** Set ClientWatcher.DEBUG_SOURCES to true for to be populated */
+    public static getTriggerReason() {
+        return proxyWatcher.getTriggeredWatcher().triggeredByChanges;
+    }
+
+    public static isInSyncedCall() {
+        return proxyWatcher.inWatcher() && isInProxyDatabase();
+    }
+
+    public static isSynced(value: unknown) {
+        return isSynced(value);
+    }
+
+    public static watchWrites(get: () => unknown) {
+        let path = getProxyPath(get);
+        if (!path) throw new Error(`Failed to get path`);
+        PathValueProxyWatcher.LOG_WRITES_INCLUDES.add(path);
+    }
+
+    public static assertDomainAllowed(path: string) {
+        let domain = getPathIndexAssert(path, 0);
+        if (!Querysub.trustedDomains.has(domain)) {
+            throw new Error(`Domain ${domain} is not trusted on this server`);
+        }
+    }
+
+    private static pendingLazyCommit: undefined | {
+        fncs: (() => unknown)[];
+    };
+    private static getLazyCommit(delayTime: number = 100) {
+        const TARGET_FRACTION_TIME = 0.1;
+        if (this.pendingLazyCommit) {
+            return this.pendingLazyCommit;
+        }
+        this.pendingLazyCommit = {
+            fncs: [],
+        };
+        let obj = this.pendingLazyCommit;
+        setTimeout(async () => {
+            let fncs = obj.fncs;
+            obj.fncs = [];
+            // Go back to non-active mode if there are no triggers
+            if (fncs.length === 0) {
+                this.pendingLazyCommit = undefined;
+                return;
+            }
+
+            let startTime = Date.now();
+            //console.log(`Batched ${fncs.length} functions in lazy commit`);
+            await Querysub.commitSynced(() => {
+                for (let fnc of fncs) {
+                    try {
+                        fnc();
+                    } catch (e) {
+                        console.error(`Error in lazy commit`, e);
+                    }
+                }
+            });
+            await clientWatcher.waitForTriggerFinished();
+            //await delay("afterPaint");
+            let totalTimeCaused = Date.now() - startTime;
+            totalTimeCaused = Math.max(16, totalTimeCaused);
+
+            let delayTime = totalTimeCaused / TARGET_FRACTION_TIME;
+            delayTime = Math.min(5000, delayTime);
+            //console.log(`Lazy commit took ${totalTimeCaused}ms, setting delay time to ${delayTime}ms`);
+            let queuedWhileCommitting = obj.fncs;
+            this.pendingLazyCommit = undefined;
+            // Create a new commit with our new delay time
+            let newCommit = this.getLazyCommit(delayTime);
+            newCommit.fncs.push(...queuedWhileCommitting);
+        }, delayTime);
+        return obj;
+    }
+    /** Waits to commit
+     *      - Might reorder other types of commit calls, but WILL NOT reorder commitLazy calls
+     *      - Batches fncs into single functions (which might result in extra dependencies /
+     *         extra rejections of unrelated functions).  
+     *      - Tries to track triggered time from these commits, and keep it below 10%,
+     *          batching more and more to make sure this is the case.
+     */
+    public static commitLazy(
+        fnc: () => unknown,
+    ) {
+        this.getLazyCommit().fncs.push(fnc);
+    }
+
+    private static synchronousStartTime = 0;
+    /** To be called in a loop, frequently, and when enough time has passed without a paint,
+     *      it will return a promise which delays long enough until we can paint again.
+     *      - Works in conjunctions with other yieldForPaint calls, so if many loops are
+     *          running we won't unnecessarily wait in all of them.
+     *  @param yieldAfterTime, if there are multiple concurrent callers the first one will
+     *      determine the used time, and both will use that time.
+     */
+    public static yieldForPaint(
+        yieldAfterTime = 1000
+    ): MaybePromise<void> {
+        // NOTE: We even wait serverside, as the server doesn't want to get locked up
+        //      on synchronous processing either!
+
+        // NOTE: Painting can easily take 32ms, and even more if we are rendering,
+        //  so... we can't wait too frequently, otherwise slow processing scripts (the
+        //  only reason to use this function) will be significantly slower.
+        if (!this.synchronousStartTime) {
+            this.synchronousStartTime = Date.now();
+            void delay("paintLoop").finally(() => {
+                this.synchronousStartTime = 0;
+            });
+        }
+        let timeSinceStart = Date.now() - this.synchronousStartTime;
+        if (timeSinceStart > yieldAfterTime) {
+            console.log(`Yielded for paint after ${timeSinceStart}ms`);
+            return delay("paintLoop");
+        }
+    }
+
+    public static debugMax() {
+        this.debug();
+        ClientWatcher.DEBUG_READS = true;
+        SocketFunction.silent = false;
+        PathValueProxyWatcher.TRACE = true;
+        debugCoreMode();
+    }
+    public static debug() {
+        console.log(`We are ${Querysub.getSelfMachineId()}`);
+        ClientWatcher.DEBUG_WRITES = true;
+        Querysub.DEBUG_CALLS = true;
+    }
+
+    private static hostCalled = false;
+    private static afterBootImports: string[] = [];
+    /** NOTE: Imports occur one at a time, after the "clientsideEntryPoint", with each import waiting for the previous to finish.
+     */
+    public static registerAdditionalRoot(path: string) {
+        if (Querysub.hostCalled) {
+            throw new Error(`All "Querysub.registerAfterBootImport" calls MUST be called before "Querysub.hostServer" (to prevent servers from existing in partially loaded states).`);
+        }
+        Querysub.afterBootImports.push(path);
+    }
+    public static async hostServer(config: {
+        // Domains we will run functions from. If this domains have any malicious code deploy (to the FunctionRunner data schema),
+        //  we are vulnerable to running this code when we check permissions for syncing paths.
+        trustedDomains: string[];
+        // The root of your project, likely where your package.json and .git are.
+        rootPath: string;
+        // Ex, `./src/client.ts`
+        //  - Relative to rootPath
+        clientsideEntryPoint: string;
+        hotReload: boolean;
+        /** You can also set `port: 0` if any port is fine. */
+        port: number;
+        justHTTP?: boolean;
+        debugName?: string;
+
+        sourceCheck?: MachineSourceCheck<any>;
+    }) {
+        if (isClient()) {
+            throw new Error(`--client processes cannot host a service. Either stop passing --client and keep the process on the network and trusted, or stop calling hostServer and call Querysub.configRootDiscoveryLocation instead.`);
+        }
+        Querysub.hostCalled = true;
+        for (let domain of config.trustedDomains) {
+            Querysub.trustedDomains.add(domain);
+        }
+
+        if (config.hotReload) {
+            watchFilesAndTriggerHotReloading();
+        }
+
+        listenOnDebugger(config.debugName ?? "Querysub");
+        //SocketFunction.silent = false;
+        //PermissionsCheck.DEBUG = true;
+        //ClientWatcher.DEBUG_WRITES = true;
+
+        await this.addSourceMapCheck(config);
+
+        // NOTE: This is just static hosting, and unrelated to querysub (sort of). Any site with some kind of transpiler
+        //  and way to serve transpiled files can server our code.
+        SocketFunction.expose(RequireController);
+        setRequireBootRequire(config.rootPath);
+        SocketFunction.setDefaultHTTPCall(RequireController, "requireHTML", {
+            requireCalls: [
+                config.clientsideEntryPoint,
+                ...Querysub.afterBootImports,
+            ]
+        });
+
+        if (!config.justHTTP) {
+            SocketFunction.expose(QuerysubController);
+        }
+
+        let allowHostnames: string[] = [];
+        for (let domain of Querysub.trustedDomains) {
+            allowHostnames.push(domain);
+        }
+        allowHostnames.push("127-0-0-1." + getDomain());
+        await SocketFunction.mount({
+            public: isPublic(),
+            port: config.port,
+            autoForwardPort: true,
+            ...await getThreadKeyCert(),
+            SNICerts: {
+                ...getSNICerts({
+                    publicPort: config.port,
+                }),
+            },
+            allowHostnames,
+        });
+
+        await publishMachineARecords();
+    }
+    private static async addSourceMapCheck(config: {
+        sourceCheck?: MachineSourceCheck;
+    }) {
+        const sourceCheck = config.sourceCheck;
+        if (!sourceCheck) return;
+        let { getExtraDataClientsideInline, shouldMachineIdSeeSource, } = sourceCheck;
+
+        let ed25519 = await import("../-a-auth/ed25519");
+        const ed25519Module = require.cache[require.resolve("../-a-auth/ed25519")];
+        if (!ed25519Module) {
+            throw new Error(`Failed to find imported ed25519 module in require.cache`);
+        }
+        let moduleContents = ed25519Module.moduleContents;
+        if (!moduleContents) {
+            throw new Error(`ed25519 module was missing moduleContents`);
+        }
+        moduleContents = moduleContents.replace(/\/\/# sourceMappingURL=.*\n/g, "");
+
+        type SignedIdentity = {
+            payload: {
+                machineId: string;
+                publicKeyB64: string;
+                time: number;
+                extra: ReturnType<Exclude<MachineSourceCheck["getExtraDataClientsideInline"], undefined>>;
+            };
+            signatureB64: string;
+        };
+
+        const getExtraDataClientsideSource = (getExtraDataClientsideInline || (() => { })).toString();
+        function signWrapper(moduleContents: string, getExtraDataClientsideSource: string) {
+            let exports = {} as typeof ed25519;
+            let module = { exports };
+            function require() { throw new Error("require not supported"); }
+            let fnc = `(function ed25519(exports, require, module, __filename, __dirname, importDynamic) {${moduleContents}\n })`;
+            eval(fnc)(exports, require, module, "inlined", "inlined", require);
+
+            if (!getExtraDataClientsideSource.startsWith("function") && !getExtraDataClientsideSource.split("\n").includes("=>")) {
+                getExtraDataClientsideSource = "function " + getExtraDataClientsideSource;
+            }
+            const getExtraDataClientside = eval(`(${getExtraDataClientsideSource})`) as Exclude<MachineSourceCheck["getExtraDataClientsideInline"], undefined>;
+
+            globalThis.remapImportRequestsClientside = globalThis.remapImportRequestsClientside || [];
+            globalThis.remapImportRequestsClientside.push(async (args) => {
+                try {
+                    let key: typeof identityStorageKey = "machineCA_6";
+                    let storageValueJSON = localStorage.getItem(key);
+                    if (!storageValueJSON) return args;
+                    let storageValue = JSON.parse(storageValueJSON) as IdentityStorageType;
+                    let machineId = storageValue.domain;
+
+                    let pem = Buffer.from(storageValue.keyB64, "base64");
+                    let privateKey = exports.extractRawED25519PrivateKey(pem);
+                    let publicKey = await exports.extractPublicKey(privateKey);
+
+                    let extra = getExtraDataClientside();
+                    if (extra === "nosource") return args;
+                    let payload: SignedIdentity["payload"] = {
+                        machineId,
+                        time: Date.now(),
+                        extra,
+                        publicKeyB64: publicKey.toString("base64"),
+                    };
+                    let signature = await exports.signWithPEM({
+                        pem,
+                        data: payload,
+                    });
+                    let signedIdentity: SignedIdentity = {
+                        payload,
+                        signatureB64: signature.toString("base64"),
+                    };
+
+                    args[2] = args[2] || {};
+                    let config = args[2] as any;
+                    config.signedIdentity = signedIdentity;
+                    return args;
+                } catch (e) {
+                    console.error(`Error signing request, code won't include sourcesmaps`, e);
+                    return args;
+                }
+            });
+        }
+        moduleContents = `(${signWrapper.toString()})(${JSON.stringify(moduleContents)}, ${JSON.stringify(getExtraDataClientsideSource)})`;
+
+        RequireController.injectHTMLBeforeStartup(`
+        <script>
+        ${moduleContents}
+        </script>
+        `);
+
+        async function isAllowedToSeeSource(signedIdentity: SignedIdentity | undefined): Promise<boolean> {
+            if (!signedIdentity) return false;
+            let { signatureB64, payload } = signedIdentity;
+            let { publicKeyB64, machineId, time, extra } = payload;
+            let publicKey = Buffer.from(publicKeyB64, "base64");
+            if (!verifyMachineIdForPublicKey({ machineId, publicKey })) return false;
+
+            // Requests shouldn't take TOO long, and if they do, it is likely due to cache, and permissions should be denied.
+            let expireTime = Date.now() - timeInMinute * 5;
+            if (time < expireTime) return false;
+
+            return await shouldMachineIdSeeSource({ machineId, extra });
+        }
+        const stripSourceBase = cacheLimited(100_000, function stripSourceBase(source: string | undefined) {
+            if (!source) return source;
+
+            const USE_TERSER = true;
+
+            if (USE_TERSER) {
+                // NOTE: Using terser results in a bundle of 638KB, instead of 815KB (a 20% reduction),
+                //      which is just barely good enough to justify using a library. It is a lot slower too,
+                //      but as it is cached, it should be fine...
+                //  - It takes about 5s to minify our entire code base, which again, is barely good enough.
+                try {
+                    return minify_sync(
+                        { file: source },
+                        {
+                            mangle: {
+                                module: true
+                            }
+                        }
+                    ).code;
+                } catch {
+                    return source;
+                }
+            }
+
+            // Strip inline sourcemaps
+            source = source.replace(/\/\/# sourceMappingURL=.*\n/g, "");
+            // Strip comments
+            let inSingleLineComment = false;
+            let inMultiLineComment = false;
+            let output = "";
+            for (let i = 0; i < source.length; i++) {
+                let char = source[i];
+                if (inSingleLineComment) {
+                    if (char === "\n") {
+                        inSingleLineComment = false;
+                        output += "\n";
+                    }
+                    continue;
+                }
+                if (inMultiLineComment) {
+                    if (char === "*" && source[i + 1] === "/") {
+                        inMultiLineComment = false;
+                        i++;
+                    }
+                    continue;
+                }
+                if (char === "/") {
+                    // HACK: To prevent needing to handle strings, we only strip comment at the beginning of the line
+                    if (source[i + 1] === "/") {
+                        // Find line start
+                        let lineStart = source.lastIndexOf("\n", i);
+                        if (!source.slice(lineStart, i).trim()) {
+                            inSingleLineComment = true;
+                            i++;
+                            continue;
+                        }
+                    }
+                    if (source[i + 1] === "*") {
+                        inMultiLineComment = true;
+                        i++;
+                        continue;
+                    }
+                }
+                output += char;
+            }
+            return output;
+        });
+        function stripSource(module: SerializedModule) {
+            module = { ...module };
+            module.source = stripSourceBase(module.source);
+            return module;
+        }
+
+        RequireController.addMapGetModules(async (result, args) => {
+            let configObj = args[2] as { signedIdentity: SignedIdentity | undefined } | undefined;
+            if (!await isAllowedToSeeSource(configObj?.signedIdentity)) {
+                await isAllowedToSeeSource(configObj?.signedIdentity);
+                //console.log(red(`Not allowed to see source`));
+                for (let [key, value] of Object.entries(result.modules)) {
+                    result.modules[key] = stripSource(value);
+                }
+            } else {
+                //console.log(green(`Allowed to see source`));
+            }
+            return result;
+        });
+    }
+
+    public static async hostService(name: string, port = 0) {
+        if (isClient()) {
+            throw new Error(`--client processes cannot host a service. Either stop passing --client and keep the process on the network and trusted, or stop calling hostServer and call Querysub.configRootDiscoveryLocation instead.`);
+        }
+
+        listenOnDebugger(name);
+        await SocketFunction.mount({
+            public: isPublic(),
+            port,
+            ...await getThreadKeyCert(),
+            // WAIT! Why were services getting SNI CERTS!!!??? This is used to set the
+            //  HTTPS cloudflare pool ips, so... services should DEFINITELY not use this!
+            // SNICerts: {
+            //     ...getSNICerts({
+            //         publicPort: port,
+            //     }),
+            // }
+        });
+
+        await publishMachineARecords();
+
+        await Querysub.optionalStartupWait();
+    }
+
+    public static keys<T extends { [key: string | number]: unknown }>(obj: T, config?: {
+        /** Value between 0 and 1, used with endFraction to try return a subset of keys.
+         *  - Tries to only talk to the necessary servers, being highly efficient if the servers are sharded
+         *      (otherwise likely does filtering serverside, which is still somewhat efficient).
+         */
+        startFraction?: number;
+        /** Value between 0 and 1 */
+        endFraction?: number;
+    }): (keyof T)[] {
+        let { startFraction, endFraction } = config || {};
+        if (startFraction === undefined || endFraction === undefined) {
+            return Object.keys(obj);
+        }
+        let path = isValueProxy2(obj);
+        if (!path) {
+            return Object.keys(obj);
+        }
+        let packedPath = encodeParentFilter({ path, startFraction, endFraction });
+        return proxyWatcher.getKeys(packedPath);
+    }
+
+    // TODO: Maybe expose checkPermissions(getValue: () => unknown)?
+    //  - It would be easy, if we every need to explicitly check if we have permissions. Although, it seems
+    //      like just relying on the automatic checking is better?
+}
+
+let localSchemaNames = new Set<string>();
+function createLocalSchema<T>(name: string): () => T;
+function createLocalSchema<SchemaDef extends Schema2>(name: string, schema: SchemaDef): () => Schema2T<SchemaDef>;
+function createLocalSchema<T>(name: string, schema2?: Schema2): () => T {
+    if (localSchemaNames.has(name) && !isHotReloading()) {
+        throw new Error(`Already have local schema with name ${JSON.stringify(name)}`);
+    }
+    if (schema2) {
+        registerSchemaPrefix({
+            schema: schema2,
+            prefixPathStr: getPathStr2(LOCAL_DOMAIN, name),
+        });
+    }
+    localSchemaNames.add(name);
+    let schema = rawSchema<{
+        [LOCAL_DOMAIN]: {
+            [name: string]: T;
+        }
+    }>();
+    return () => {
+        return schema()[LOCAL_DOMAIN][name];
+    };
+}
+
+const timeData = createLocalSchema<{
+    time: number;
+    intervals: {
+        [interval: number]: number;
+    };
+}>("time");
+const initTimeLoop = lazy(() => {
+    function onNextTime() {
+        requestAnimationFrame(onNextTime);
+        // TODO: We should really stop the loop, and start it again when someone is watching
+        //  our values. Which we can either do via getSyncedTime, OR, we could technically
+        //  get pathValueClientWatcher to tell us when someone is watching it (but using getSyncedTime
+        //  is probably better).
+        if (clientWatcher.pathHasAnyWatchers(getProxyPath(() => timeData().time))) {
+            Querysub.serviceWriteDetached(() => {
+                timeData().time = Date.now();
+            });
+        }
+    }
+    onNextTime();
+});
+let initInterval = cache((interval: number) => {
+    setInterval(() => {
+        if (clientWatcher.pathHasAnyWatchers(getProxyPath(() => timeData().intervals[interval]))) {
+            Querysub.serviceWriteDetached(() => {
+                timeData().intervals[interval] = Date.now();
+            });
+        }
+    }, interval);
+});
+function getSyncedTime() {
+    let call = getCurrentCallAllowUndefined();
+    if (call) {
+        return call.runAtTime.time;
+    }
+    if (isNode()) {
+        return Date.now();
+    }
+    initTimeLoop();
+    return atomic(timeData().time) || Date.now();
+}
+
+
+function timeDelayed(interval: number) {
+    let call = getCurrentCallAllowUndefined();
+    if (call) {
+        return call.runAtTime.time;
+    }
+    initInterval(interval);
+    atomic(timeData().intervals[interval]);
+    return Date.now();
+}
+
+// Returns a value between 0 and 1
+function hashRandom(base: string, offset: number): number {
+    let hashBuf = Buffer.from(sha256(base + "_" + offset), "hex");
+    let value0 = hashBuf.readUInt32BE(0);
+    let value1 = hashBuf.readUInt16BE(4);
+    return value0 / (2 ** 32) + value1 / (2 ** 48);
+}
+
+// NOTE: Use `SocketFunction.logMessages = true` to diagnose what specifically is using the bandwidth.
+setImmediate(() => {
+    let uploadLatest = 0;
+    let downloadLatest = 0;
+    let uploadTotal = 0;
+    let downloadTotal = 0;
+    registerDynamicResource("network|TOTAL_UPLOAD", () => uploadTotal);
+    registerDynamicResource("network|TOTAL_DOWNLOAD", () => downloadTotal);
+    registerDynamicResource("network|LATEST_UPLOAD", () => {
+        let value = uploadLatest;
+        uploadLatest = 0;
+        return value;
+    });
+    registerDynamicResource("network|LATEST_DOWNLOAD", () => {
+        let value = downloadLatest;
+        downloadLatest = 0;
+        return value;
+    });
+    SocketFunction.trackMessageSizes.upload.push(size => {
+        uploadLatest += size;
+        uploadTotal += size;
+    });
+    SocketFunction.trackMessageSizes.download.push(size => {
+        downloadLatest += size;
+        downloadTotal += size;
+    });
+});
+
+setImmediate(() => {
+    hookErrors();
+});
+
+registerGetCompressNetwork(() => Querysub.COMPRESS_NETWORK);
+registerGetCompressDisk(() => Querysub.COMPRESS_DISK);
+
+(global as any).Querysub = Querysub;