querysub 0.403.0 → 0.405.0

package/src/2-proxy/schema2.ts

@@ -246,6 +246,24 @@ export class Schema2Fncs {
     public static schema2ToTypeString(schema: Schema2): string {
         return schema2ToTypeString(schema);
     }
+
+    /** Uses * for wildcards */
+    public static getLookups(schema: Schema2): string[][] {
+        let internalType = typeDefCached(schema);
+        let lookupPaths = new Set<string>();
+
+        for (let def of internalType.defs) {
+            for (let i = 0; i < def.path.length; i++) {
+                if (def.path[i] === WildCard) {
+                    let lookupPath = def.path.slice(0, i);
+                    let stringPath = lookupPath.map(p => p === WildCard ? "*" : p as string);
+                    lookupPaths.add(JSON.stringify(stringPath));
+                }
+            }
+        }
+
+        return Array.from(lookupPaths).map(p => JSON.parse(p));
+    }
 }
 
 /*
@@ -699,7 +717,7 @@ function isDefParentOrEqualMatch(defPath: SchemaPath, path: string[]) {
     return true;
 }
 
-// let x = t.lookup({
+// let x = ({
 //     name: t.string,
 //     num: t.number,
 //     defaultedNum: t.number(1),
@@ -720,6 +738,8 @@ function isDefParentOrEqualMatch(defPath: SchemaPath, path: string[]) {
 //         }),
 //     }
 // });
+// let lookups = Schema2Fncs.getLookups(x);
+// console.log(lookups);
 // console.log(typeDefCached(x));
 
 // console.log("name", Schema2Fncs.isAtomic(x, "read", ["x", "name"]));

package/src/3-path-functions/PathFunctionHelpers.ts

@@ -1,181 +1,207 @@
-import { getOwnMachineId } from "../-a-auth/certs";
-import { atomicObjectWrite, proxyWatcher, specialObjectWriteSymbol } from "../2-proxy/PathValueProxyWatcher";
-import { pathValueCommitter } from "../0-path-value-core/PathValueController";
-import { getNextTime } from "../0-path-value-core/pathValueCore";
-import { FunctionSpec, functionSchema, CallSpec, commitCall } from "./PathFunctionRunner";
-import { logErrors } from "../errors";
-import { FunctionMetadata } from "./syncSchema";
-import debugbreak from "debugbreak";
-import cborx from "cbor-x";
-import { lazy } from "socket-function/src/caching";
-import { secureRandom } from "../misc/random";
-import { isDefined } from "../misc";
-import { blue, green, red } from "socket-function/src/formatting/logColors";
-import { getPathStr2 } from "../path";
-import { isNode, sort } from "socket-function/src/misc";
-import { decodeCborx, encodeCborx } from "../misc/cloneHelpers";
-import { parseFilterable } from "../misc/filterable";
-import { interceptCalls } from "../-0-hooks/hooks";
-
-// NOTE: We could deploy single functions, but... we will almost always be updating all functions at
-//      once, because keeping everything on the same git hash reduces a lot of potential bugs.
-export async function replaceFunctions(config: {
-    functions: FunctionSpec[];
-}) {
-    await proxyWatcher.commitFunction({
-        debugName: "replaceFunctions",
-        watchFunction() {
-            function debugFunction(func: FunctionSpec) {
-                return `${func.DomainName}:${func.FilePath}:${func.ModuleId}:${func.FunctionId}`;
-            }
-
-            let { functions } = config;
-            let domainsNames = new Set(functions.map(x => x.DomainName));
-            if (domainsNames.size !== 1) {
-                throw new Error(`Tried to deploy functions with multiple domains, ${JSON.stringify(domainsNames)}`);
-            }
-            let domainName = Array.from(domainsNames)[0];
-
-            let base = functionSchema()[domainName].PathFunctionRunner;
-            let previousFunctions = Object.values(base).flatMap(x => Object.values(x.Sources)).filter(isDefined);
-
-            function getFunctionName(func: FunctionSpec) {
-                return getPathStr2(func.ModuleId, func.FunctionId);
-            }
-
-            let currentFunctions = functions;
-
-            let previousHashes = new Set(previousFunctions.map(x => JSON.stringify(x)));
-            let currentHashes = new Set(currentFunctions.map(x => JSON.stringify(x)));
-
-            let prevFunctionNames = new Set(previousFunctions.map(getFunctionName));
-            let currentFunctionNames = new Set(currentFunctions.map(getFunctionName));
-
-            let messages: {
-                text: string;
-                order: number;
-            }[] = [];
-
-            for (let previous of previousFunctions) {
-                if (!currentFunctionNames.has(getFunctionName(previous))) {
-                    messages.push({ text: `Removing\t ${red(debugFunction(previous))}`, order: 3 });
-                    base[previous.ModuleId].Sources[previous.FunctionId] = undefined;
-                }
-            }
-
-            console.log();
-            for (let func of currentFunctions) {
-                if (previousHashes.has(JSON.stringify(func))) {
-                    messages.push({ text: `Unchanged\t ${debugFunction(func)}`, order: 0 });
-                    continue;
-                }
-                if (!prevFunctionNames.has(getFunctionName(func))) {
-                    messages.push({ text: `Deploying new\t ${green(debugFunction(func))}`, order: 2 });
-                } else {
-                    messages.push({ text: `Deploying\t ${blue(debugFunction(func))}`, order: 1 });
-                }
-                base[func.ModuleId] = specialObjectWriteSymbol as any;
-                base[func.ModuleId].Sources[func.FunctionId] = atomicObjectWrite(func);
-            }
-
-            if (proxyWatcher.isAllSynced()) {
-                sort(messages, x => -x.order);
-                for (let message of messages) {
-                    console.log(message.text);
-                }
-            }
-        }
-    });
-
-    await pathValueCommitter.waitForValuesToCommit();
-}
-
-export const writeCall = {
-    value: async (callSpec: CallSpec, metadata: FunctionMetadata) => {
-        commitCall(callSpec);
-        await pathValueCommitter.waitForValuesToCommit();
-    }
-};
-
-export async function runCall(call: CallSpec, metadata: FunctionMetadata) {
-    return await writeCall.value(call, metadata);
-}
-
-
-export interface CallInterceptor<T> {
-    onCall: (call: CallSpec, metadata: FunctionMetadata) => void;
-    code: () => T;
-}
-
-let curInterceptor: CallInterceptor<unknown> | undefined = undefined;
-export function interceptCallsBase<T>(
-    interceptor: CallInterceptor<T>
-) {
-    let prev = curInterceptor;
-    curInterceptor = interceptor;
-    try {
-        return interceptor.code();
-    } finally {
-        curInterceptor = prev;
-    }
-}
-interceptCalls.declare(interceptCallsBase);
-
-/** Writes the function call allowing interceptors to capture it (vs runCall which just runs it). */
-export function writeFunctionCall(config: {
-    domainName: string;
-    moduleId: string;
-    functionId: string;
-    args: unknown[];
-    metadata: FunctionMetadata,
-}) {
-    let { domainName, moduleId, functionId, args, metadata } = config;
-
-    let now = Date.now();
-    // IMPORTANT! CallId MUST be secure, otherwise other users can guess it, and read our calls
-    //  (which could contain secret data).
-    //  secureRandom() is pretty good... combined with date it should be hard enough to guess
-    //      - date adds about 10 bits of randomness (an attacker might be able to guess within a second
-    //          the time a specific call was made)
-    //      - secureRandom adds at more 64 bits of randomess, but in reality it is probably less
-    //      - ALSO, if they don't guess the call within the event remove window... it will be be deleted,
-    //          which gives them about a minute to guess it...
-    let callId = now + "_" + secureRandom();
-    let argsEncoded = encodeArgs(args);
-    let callSpec: CallSpec = {
-        argsEncoded,
-        ModuleId: moduleId,
-        CallId: callId,
-        DomainName: domainName,
-        FunctionId: functionId,
-        runAtTime: getNextTime(),
-        callerMachineId: getOwnMachineId(),
-        // Will be updated by Querysub to be correct (unless we are directly writing, then...
-        //  this is fine).
-        callerIP: "127.0.0.1",
-    };
-    if (!isNode()) {
-        // Get the "setfncfilter" querystring parameter
-        let url = new URL(window.location.href);
-        let setfncfilter = url.searchParams.get("setfncfilter");
-        if (setfncfilter) {
-            callSpec.filterable = parseFilterable(setfncfilter);
-        }
-    }
-
-    if (curInterceptor) {
-        curInterceptor.onCall(callSpec, metadata);
-    } else {
-        logErrors(writeCall.value(callSpec, metadata));
-    }
-
-    return callSpec;
-}
-
-export function parseArgs(call: CallSpec): unknown[] {
-    return decodeCborx(Buffer.from(call.argsEncoded, "base64"));
-}
-
-export function encodeArgs(args: unknown[]) {
-    return encodeCborx(args).toString("base64");
+import { getOwnMachineId } from "../-a-auth/certs";
+import { atomicObjectWrite, proxyWatcher, specialObjectWriteSymbol } from "../2-proxy/PathValueProxyWatcher";
+import { pathValueCommitter } from "../0-path-value-core/PathValueController";
+import { getNextTime, getTimeUnique } from "../0-path-value-core/pathValueCore";
+import { FunctionSpec, functionSchema, CallSpec, commitCall } from "./PathFunctionRunner";
+import { logErrors } from "../errors";
+import { FunctionMetadata, getCallIdOverride } from "./syncSchema";
+import debugbreak from "debugbreak";
+import cborx from "cbor-x";
+import { cache, lazy } from "socket-function/src/caching";
+import { secureRandom } from "../misc/random";
+import { isDefined } from "../misc";
+import { blue, green, red } from "socket-function/src/formatting/logColors";
+import { getPathStr2 } from "../path";
+import { isNode, sort } from "socket-function/src/misc";
+import { decodeCborx, encodeCborx } from "../misc/cloneHelpers";
+import { parseFilterable } from "../misc/filterable";
+import { interceptCalls } from "../-0-hooks/hooks";
+import { createRoutingOverrideKey } from "../0-path-value-core/PathRouterRouteOverride";
+import { PathRouter } from "../0-path-value-core/PathRouter";
+import { getPathFromProxy } from "../2-proxy/pathValueProxy";
+import { getDomain } from "../config";
+
+// NOTE: We could deploy single functions, but... we will almost always be updating all functions at
+//      once, because keeping everything on the same git hash reduces a lot of potential bugs.
+export async function replaceFunctions(config: {
+    functions: FunctionSpec[];
+}) {
+    await proxyWatcher.commitFunction({
+        debugName: "replaceFunctions",
+        watchFunction() {
+            function debugFunction(func: FunctionSpec) {
+                return `${func.DomainName}:${func.FilePath}:${func.ModuleId}:${func.FunctionId}`;
+            }
+
+            let { functions } = config;
+            let domainsNames = new Set(functions.map(x => x.DomainName));
+            if (domainsNames.size !== 1) {
+                throw new Error(`Tried to deploy functions with multiple domains, ${JSON.stringify(domainsNames)}`);
+            }
+            let domainName = Array.from(domainsNames)[0];
+
+            let base = functionSchema()[domainName].PathFunctionRunner;
+            let previousFunctions = Object.values(base).flatMap(x => Object.values(x.Sources)).filter(isDefined);
+
+            function getFunctionName(func: FunctionSpec) {
+                return getPathStr2(func.ModuleId, func.FunctionId);
+            }
+
+            let currentFunctions = functions;
+
+            let previousHashes = new Set(previousFunctions.map(x => JSON.stringify(x)));
+            let currentHashes = new Set(currentFunctions.map(x => JSON.stringify(x)));
+
+            let prevFunctionNames = new Set(previousFunctions.map(getFunctionName));
+            let currentFunctionNames = new Set(currentFunctions.map(getFunctionName));
+
+            let messages: {
+                text: string;
+                order: number;
+            }[] = [];
+
+            for (let previous of previousFunctions) {
+                if (!currentFunctionNames.has(getFunctionName(previous))) {
+                    messages.push({ text: `Removing\t ${red(debugFunction(previous))}`, order: 3 });
+                    base[previous.ModuleId].Sources[previous.FunctionId] = undefined;
+                }
+            }
+
+            console.log();
+            for (let func of currentFunctions) {
+                if (previousHashes.has(JSON.stringify(func))) {
+                    messages.push({ text: `Unchanged\t ${debugFunction(func)}`, order: 0 });
+                    continue;
+                }
+                if (!prevFunctionNames.has(getFunctionName(func))) {
+                    messages.push({ text: `Deploying new\t ${green(debugFunction(func))}`, order: 2 });
+                } else {
+                    messages.push({ text: `Deploying\t ${blue(debugFunction(func))}`, order: 1 });
+                }
+                base[func.ModuleId] = specialObjectWriteSymbol as any;
+                base[func.ModuleId].Sources[func.FunctionId] = atomicObjectWrite(func);
+            }
+
+            if (proxyWatcher.isAllSynced()) {
+                sort(messages, x => -x.order);
+                for (let message of messages) {
+                    console.log(message.text);
+                }
+            }
+        }
+    });
+
+    await pathValueCommitter.waitForValuesToCommit();
+}
+
+export const writeCall = {
+    value: async (callSpec: CallSpec, metadata: FunctionMetadata) => {
+        commitCall(callSpec);
+        await pathValueCommitter.waitForValuesToCommit();
+    }
+};
+
+export async function runCall(call: CallSpec, metadata: FunctionMetadata) {
+    return await writeCall.value(call, metadata);
+}
+
+
+export interface CallInterceptor<T> {
+    onCall: (call: CallSpec, metadata: FunctionMetadata) => void;
+    code: () => T;
+}
+
+let curInterceptor: CallInterceptor<unknown> | undefined = undefined;
+export function interceptCallsBase<T>(
+    interceptor: CallInterceptor<T>
+) {
+    let prev = curInterceptor;
+    curInterceptor = interceptor;
+    try {
+        return interceptor.code();
+    } finally {
+        curInterceptor = prev;
+    }
+}
+interceptCalls.declare(interceptCallsBase);
+
+
+let getFunctionRunnerPrefix = cache((moduleId: string) => {
+    // We really want to remap it to the data prefix. However, if the user hasn't provided this to us, I guess we can just remap to whoever's watching the call. There has to be a server watching that anyway. 
+    let path = getPathFromProxy(functionSchema()[getDomain()].PathFunctionRunner[moduleId].Calls);
+    if (!path) throw new Error(`getPathFromProxy was used wrong in getFunctionRunnerPrefix?`);
+    return path;
+});
+
+/** Writes the function call allowing interceptors to capture it (vs runCall which just runs it). */
+export function writeFunctionCall(config: {
+    domainName: string;
+    moduleId: string;
+    functionId: string;
+    args: unknown[];
+    metadata: FunctionMetadata,
+}) {
+    let { domainName, moduleId, functionId, args, metadata } = config;
+
+    // NOTE: Having this be an always-increasing time means that at least if it's the same client making the call ID, the numbers will always be increasing. So we can go from the call ID to determine the order of the calls, which is extremely useful. 
+    let now = getTimeUnique();
+    // IMPORTANT! CallId MUST be secure, otherwise other users can guess it, and read our calls
+    //  (which could contain secret data).
+    //  secureRandom() is pretty good... combined with date it should be hard enough to guess
+    //      - date adds about 10 bits of randomness (an attacker might be able to guess within a second
+    //          the time a specific call was made)
+    //      - secureRandom adds at more 64 bits of randomess, but in reality it is probably less
+    //      - ALSO, if they don't guess the call within the event remove window... it will be be deleted,
+    //          which gives them about a minute to guess it...
+    let callId = now + "_" + secureRandom();
+
+    // We HAVE to override the routing id, otherwise PathFunctionRunner secondary sharding won't work
+    callId = getCallIdOverride({
+        moduleId,
+        functionId,
+        callId,
+        args,
+    }) || createRoutingOverrideKey({
+        originalKey: callId,
+        routeKey: callId,
+        remappedPrefix: getFunctionRunnerPrefix(moduleId),
+    });
+
+    let argsEncoded = encodeArgs(args);
+    let callSpec: CallSpec = {
+        argsEncoded,
+        ModuleId: moduleId,
+        CallId: callId,
+        DomainName: domainName,
+        FunctionId: functionId,
+        runAtTime: getNextTime(),
+        callerMachineId: getOwnMachineId(),
+        // Will be updated by Querysub to be correct (unless we are directly writing, then...
+        //  this is fine).
+        callerIP: "127.0.0.1",
+    };
+    if (!isNode()) {
+        // Get the "setfncfilter" querystring parameter
+        let url = new URL(window.location.href);
+        let setfncfilter = url.searchParams.get("setfncfilter");
+        if (setfncfilter) {
+            callSpec.filterable = parseFilterable(setfncfilter);
+        }
+    }
+
+    if (curInterceptor) {
+        curInterceptor.onCall(callSpec, metadata);
+    } else {
+        logErrors(writeCall.value(callSpec, metadata));
+    }
+
+    return callSpec;
+}
+
+export function parseArgs(call: CallSpec): unknown[] {
+    return decodeCborx(Buffer.from(call.argsEncoded, "base64"));
+}
+
+export function encodeArgs(args: unknown[]) {
+    return encodeCborx(args).toString("base64");
 }