querysub 0.2.0

package/src/3-path-functions/PathFunctionRunnerMain.ts

@@ -0,0 +1,67 @@
+// import { disableMeasurements } from "socket-function/src/profiling/measure";
+// disableMeasurements();
+
+import "../inject";
+import yargs from "yargs";
+let yargObj = yargs(process.argv)
+    .option("shardcenter", { type: "number", default: 0.5, desc: "Center of sharding fraction from 0 to 1" })
+    .option("shardsize", { type: "number", default: 1, desc: "Size of sharding fraction from 0 to 1 (ex, shardsize = 0.35 and shardcenter = 0.5 results in sharding from 0.15 to 0.85)" })
+    .option("secondarysize", { type: "number", default: 0, desc: "Also shards this size HOWEVER, any values in this range (which are not matched in shardsize) are delayed." })
+    .argv
+    ;
+
+import { logErrors } from "../errors";
+import { PathFunctionRunner } from "./PathFunctionRunner";
+import { authorityStorage, debugCoreMode, MAX_CHANGE_AGE, quietCoreMode } from "../0-path-value-core/pathValueCore";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { getThreadKeyCert } from "../-a-auth/certs";
+import { ClientWatcher } from "../1-path-client/pathValueClientWatcher";
+import { listenOnDebugger } from "../diagnostics/listenOnDebugger";
+import { PathValueProxyWatcher } from "../2-proxy/PathValueProxyWatcher";
+import { PermissionsCheck } from "../4-querysub/permissions";
+import { Querysub } from "../4-querysub/QuerysubController";
+import { timeInMinute } from "socket-function/src/misc";
+import { getDomain, isPublic } from "../config";
+import { publishMachineARecords } from "../-e-certs/EdgeCertController";
+import { green } from "socket-function/src/formatting/logColors";
+
+async function main() {
+
+    listenOnDebugger("FunctionRunner");
+    Error.stackTraceLimit = 20;
+
+    //ActionsHistory.LOG_ACTION_HISTORY = "runner";
+
+    // ClientWatcher.DEBUG_READS = true;
+    ClientWatcher.DEBUG_WRITES = true;
+    // ClientWatcher.DEBUG_TRIGGERS = "heavy";
+    // authorityStorage.DEBUG_UNWATCH = true;
+
+    PathFunctionRunner.DEBUG_CALLS = true;
+    // debugCoreMode();
+
+    let keyPair = await getThreadKeyCert();
+    await SocketFunction.mount({ port: 0, ...keyPair, public: isPublic() });
+    await publishMachineARecords();
+
+    // Use a fairly high stick time (the default is 10s), because having wait to sync data is very slow,
+    //  and the function runner SHOULD have more memory than the clients, and much faster network speeds
+    //  (it should be on the same local network as the path value authorities).
+    ClientWatcher.WATCH_STICK_TIME = timeInMinute * 5;
+
+    let shardStart = yargObj.shardcenter - yargObj.shardsize / 2;
+    let shardEnd = yargObj.shardcenter + yargObj.shardsize / 2;
+    let secondaryStart = yargObj.shardcenter - yargObj.secondarysize / 2;
+    let secondaryEnd = yargObj.shardcenter + yargObj.secondarysize / 2;
+
+    console.log(green(`Sharding from ${shardStart} to ${shardEnd}. Fallback sharding (dead function recovery) from ${secondaryStart} to ${secondaryEnd}`));
+
+    new PathFunctionRunner({
+        domainName: getDomain(),
+        shardRange: { startFraction: shardStart, endFraction: shardEnd },
+        secondaryShardRange: yargObj.secondarysize ? { startFraction: secondaryStart, endFraction: secondaryEnd } : undefined,
+        // TODO: Maybe abstract this out even more, so anything can plug in permissions checks?
+        PermissionsChecker: PermissionsCheck,
+    });
+}
+logErrors(main());

package/src/3-path-functions/deployBlock.ts

@@ -0,0 +1,10 @@
+let promises: Promise<void>[] = [];
+/** Blocks the deploy until this promise finishes. Only check before the deploy starts
+ *      (essentially is just a way to register async startup code).
+ */
+export function registerDeployBlocker(promise: Promise<void>) {
+    promises.push(promise);
+}
+export async function deployBlock() {
+    await Promise.all(promises);
+}

package/src/3-path-functions/deployCheck.ts

@@ -0,0 +1,7 @@
+let isDeployValue = false;
+export function setIsDeploy() {
+    isDeployValue = true;
+}
+export function isDeploy() {
+    return isDeployValue;
+}

package/src/3-path-functions/deployMain.ts

@@ -0,0 +1,160 @@
+import "../inject";
+
+import { errorToUndefined } from "../errors";
+import { functionSchema, FunctionSpec } from "./PathFunctionRunner";
+import path from "path";
+import { getExportPath, getSchemaObject, PERMISSIONS_FUNCTION_ID } from "./syncSchema";
+import { getModuleRelativePath } from "./syncSchema";
+import * as child_process from "child_process";
+import { proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
+import { green, red } from "socket-function/src/formatting/logColors";
+import { isDefined } from "../misc";
+import fs from "fs";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { getThreadKeyCert } from "../-a-auth/certs";
+import { deployFunction, undeployFunction } from "./PathFunctionHelpers";
+import { pathValueCommitter } from "../0-path-value-core/PathValueController";
+import { getPathStr2 } from "../path";
+import { cacheJSONArgsEqual } from "socket-function/src/caching";
+import { setIsDeploy } from "./deployCheck";
+import { ClientWatcher } from "../1-path-client/pathValueClientWatcher";
+import debugbreak from "debugbreak";
+
+import yargs from "yargs";
+import { isPublic } from "../config";
+import { deployBlock } from "./deployBlock";
+let yargObj = yargs(process.argv)
+    .option("domain", { type: "string", required: true, desc: "Domain to deploy to" })
+    .option("nogit", { type: "boolean", default: false, desc: `Disable git operations. Useful for debugging, when using --local` })
+    .argv
+    ;
+
+setIsDeploy();
+
+export async function deployMain() {
+    //SocketFunction.logMessages = true;
+    //quietCoreMode();
+    //ClientWatcher.DEBUG_READS = true;
+    //ClientWatcher.DEBUG_WRITES = true;
+    await deployBlock();
+
+    // Mount, so we can write directly to the database
+    await SocketFunction.mount({ port: 0, ...await getThreadKeyCert(), public: isPublic() });
+
+    const domainName = yargObj.domain;
+
+    let folderRoot = path.resolve(".").replaceAll("\\", "/");
+    const deployPath = path.resolve("./deploy.ts");
+    require(deployPath);
+
+    const srcRoot = path.resolve(__dirname + "/../").replaceAll("\\", "/");
+
+    let currentFunctions: FunctionSpec[] = [];
+    function debugFunction(func: FunctionSpec) {
+        return `${func.DomainName}:${func.FilePath}:${func.FunctionId}`;
+    }
+
+    let gitDir = folderRoot;
+    if (!fs.existsSync(gitDir + "/.git")) {
+        throw new Error(`No .git folder found at ${JSON.stringify(gitDir + "/.git")}`);
+    }
+    let gitURL = getGitURL(gitDir);
+    if (!gitURL) {
+        throw new Error(`Git repo has no remote url, at ${JSON.stringify(gitDir + "/.git")}`);
+    }
+    let gitRef = getGitRef(gitDir);
+
+    let usedModules = new Map<string, string>();
+
+    for (let module of Object.values(require.cache)) {
+        if (!module) continue;
+
+        // NOTE: We don't want dependencies to deploy schemas. This would result in using an
+        //  API also deploying it.
+        //  - Also, this add more security, by preventing 3rd party libraries from adding apis
+        //      (maybe even just innocent, for diagnostics), which might have vulnerabilities.
+        let isAllowedToDeploy = (
+            !module.filename.includes("node_modules")
+            && module.filename.startsWith(folderRoot)
+        );
+
+        // We DO allow shard to deploy, so we can deploy diagnostic functions, such as
+        //  for the managment interfaces. It only works if the user opts in (by calling
+        //  registerManagementPages2), so it is actually quite safe to do this.
+        if (module.filename.startsWith(srcRoot)) {
+            isAllowedToDeploy = true;
+        }
+
+        if (!isAllowedToDeploy) continue;
+        let schema = getSchemaObject(module);
+
+        if (schema) {
+            let filePath = getModuleRelativePath(module);
+
+            let moduleId = getPathStr2(domainName, filePath);
+            let otherUsed = usedModules.get(moduleId);
+            if (otherUsed) {
+                throw new Error(`Overlapping module ids for ${domainName}.${filePath}, at ${otherUsed} and ${filePath}`);
+            }
+            usedModules.set(moduleId, filePath);
+
+            const functionIds = Object.keys(schema.rawFunctions);
+            functionIds.push(PERMISSIONS_FUNCTION_ID);
+            for (let functionId of functionIds) {
+                const exportPathStr = getExportPath(functionId);
+                currentFunctions.push({
+                    DomainName: domainName,
+                    ModuleId: schema.moduleId,
+                    FilePath: filePath,
+                    FunctionId: functionId,
+                    exportPathStr,
+                    gitURL,
+                    gitRef,
+                });
+            }
+        }
+    }
+
+    let previousFunctions = await proxyWatcher.commitFunction({
+        debugName: "readPreviousFunctions",
+        watchFunction(): FunctionSpec[] {
+            let base = functionSchema()[domainName].PathFunctionRunner;
+            return Object.values(base).flatMap(x => Object.values(x.Sources)).filter(isDefined);
+        },
+    });
+
+    let previousHashes = new Set(previousFunctions.map(x => JSON.stringify(x)));
+    let currentHashes = new Set(currentFunctions.map(x => JSON.stringify(x)));
+    // BUG: This is incorrect, and breaks when you accidentally reuse a modelId
+    for (let func of currentFunctions) {
+        if (previousHashes.has(JSON.stringify(func))) {
+            console.log(`Unchanged ${debugFunction(func)}`);
+        }
+    }
+
+    for (let previous of previousFunctions) {
+        let hash = JSON.stringify(previous);
+        if (currentHashes.has(hash)) continue;
+        console.log(`Removing  ${red(debugFunction(previous))}`);
+        await undeployFunction(previous);
+    }
+
+    console.log();
+    for (let func of currentFunctions) {
+        if (previousHashes.has(JSON.stringify(func))) continue;
+        console.log(`Deploying ${green(debugFunction(func))}`);
+        await deployFunction(func);
+    }
+
+    await pathValueCommitter.waitForValuesToCommit();
+}
+function getGitURL(gitDir: string) {
+    if (yargObj.nogit) return "git@github.com:nogit/nogit.git";
+    return child_process.execSync(`git remote get-url origin`, { cwd: gitDir }).toString().trim();
+}
+function getGitRef(gitDir: string) {
+    if (yargObj.nogit) return "nogit";
+    return child_process.execSync(`git rev-parse HEAD`, { cwd: gitDir }).toString().trim();
+}
+
+deployMain().catch(e => console.error(e)).finally(() => process.exit());

package/src/3-path-functions/pathFunctionLoader.ts

@@ -0,0 +1,282 @@
+import { getSubFolder } from "../fs";
+import { FunctionSpec } from "./PathFunctionRunner";
+import child_process from "child_process";
+import fs from "fs";
+import { blue, magenta, red } from "socket-function/src/formatting/logColors";
+import debugbreak from "debugbreak";
+import { cache, lazy } from "socket-function/src/caching";
+import { batchFunction, runInfinitePoll } from "socket-function/src/batching";
+import { logErrors } from "../errors";
+import crypto from "crypto";
+import { MaybePromise } from "socket-function/src/types";
+import yargs from "yargs";
+import { SyncWatcher } from "../2-proxy/PathValueProxyWatcher";
+import { MAX_CHANGE_AGE } from "../0-path-value-core/pathValueCore";
+import { isNode, isNodeTrue } from "socket-function/src/misc";
+import { getPathStr2, getPathStr3 } from "../path";
+import { consistentHash } from "../misc/hash";
+import { setExternalHotReloading } from "socket-function/hot/HotReloadController";
+import { isPublic } from "../config";
+
+// Get localPathRemappings using yargs, so it is easy to configure in multiple entry points
+let yargObj = isNodeTrue() && yargs(process.argv)
+    .option("local", { type: "boolean", desc: `If true, uses the local directory instead of the remote git repo. Also hotreloads from disk. Determines the repo to replace through the package.json "repository" property.` })
+    .argv || {}
+    ;
+
+let watchers = new Set<SyncWatcher>();
+let ensureDisposingDeadWatchers = lazy(() => {
+    runInfinitePoll(MAX_CHANGE_AGE, function disposeDeadWatchers() {
+        for (let watcher of watchers) {
+            if (watcher.disposed) {
+                watchers.delete(watcher);
+            }
+        }
+    });
+});
+export function watchModuleHotreloads(watcher: SyncWatcher) {
+    watchers.add(watcher);
+    ensureDisposingDeadWatchers();
+}
+
+const getLocalPathRemapping = lazy((): { [gitUrl: string]: string } => {
+    let localPaths: string[] = [];
+    if (yargObj.local) {
+        localPaths.push(process.cwd().replaceAll(/\\/g, "/") + "/");
+    }
+    if (!localPaths.length) return {};
+    let mapping: { [gitUrl: string]: string } = Object.create(null);
+    for (let path of localPaths) {
+        if (!path.endsWith("/")) {
+            path += "/";
+        }
+        let packageJSON = fs.readFileSync(path + "package.json", "utf8");
+        let packageJSONObj = JSON.parse(packageJSON);
+        let repo = packageJSONObj.repository as { type: string, url: string } | undefined;
+        if (!repo) throw new Error(`No "repository" property in package.json at ${path}`);
+        if (repo.type !== "git") {
+            throw new Error(`Repository type ${JSON.stringify(repo.type)} is not supported yet. Only "git" is presently supported.`);
+        }
+        mapping[repo.url] = path;
+        // Also, use the git@ version
+        // Replace "https://*/" with "git@*:"
+        let gitAtUrl = repo.url.replace(/^https:\/\/([^\/]+)\//, "git@$1:");
+        mapping[gitAtUrl] = path;
+    }
+    return mapping;
+});
+
+// If we have so many different functions that we can't cache them all... then we are going
+//  to have a problem! Also... this cache is only the function, not the require.cache, which
+//  isn't being cleared anyways, so THIS isn't really where we would leak.
+export function getModuleFromSpec(spec: FunctionSpec): MaybePromise<NodeJS.Module> {
+    return getModuleFromConfig(spec);
+}
+let moduleCache = new Map<string, {
+    result: MaybePromise<NodeJS.Module>;
+    error: Error | undefined;
+}>();
+function getSpecKey(spec: FunctionSpec) {
+    // Only include these specific keys AND order them, so the hash hits no matter how this function is called
+    spec = {
+        FilePath: spec.FilePath,
+        gitURL: spec.gitURL,
+        gitRef: spec.gitRef,
+        DomainName: spec.DomainName,
+        exportPathStr: spec.exportPathStr,
+        FunctionId: spec.FunctionId,
+        ModuleId: spec.ModuleId,
+    };
+    return JSON.stringify(spec);
+}
+export function getModuleFromConfig(spec: FunctionSpec): MaybePromise<NodeJS.Module> {
+    let key = getSpecKey(spec);
+    let value = moduleCache.get(key);
+    if (!value) {
+        let promise = getModuleFromSpecBase(spec);
+        value = { result: promise, error: undefined };
+        moduleCache.set(key, value);
+        promise.then(
+            result => {
+                moduleCache.set(key, { result, error: undefined });
+            },
+            error => {
+                moduleCache.set(key, { result: null as any, error });
+            }
+        );
+    }
+    if (value.error) throw value.error;
+    return value.result;
+}
+
+let gitURLRefMappings = new Map<string, string>();
+export function setGitURLMapping(config: {
+    spec: FunctionSpec;
+    resolvedPath: string;
+}) {
+    gitURLRefMappings.set(getSpecKey(config.spec), config.resolvedPath);
+}
+
+/** spec => path that we can use with require */
+let moduleResolver = async (spec: FunctionSpec) => {
+    //todonext;
+    //  I think the cloning isn't working?
+    debugbreak(2);
+    debugger;
+
+    let gitURL = spec.gitURL;
+    let urlForPath = gitURL;
+
+    if (urlForPath.startsWith("http")) {
+        // Switch to a git url... incase the repo is private? Or something?
+        // TODO: We should only do this selectively, as I think we can't sync this
+        //  way without owning the repo, or... something?
+
+        let url = new URL(urlForPath);
+        gitURL = "git@" + url.host + ":" + url.pathname.slice(1);
+        urlForPath = gitURL;
+    }
+    urlForPath = urlForPath.replaceAll("@", "/");
+    urlForPath = urlForPath.replaceAll(":", "/");
+    if (urlForPath.endsWith(".git")) {
+        urlForPath = urlForPath.slice(0, -".git".length);
+    }
+    let repoPath = getSubFolder("synced_repos") + urlForPath + "/" + spec.gitRef;
+    if (!fs.existsSync(repoPath)) {
+        await executeCommand("git", ["clone", gitURL, repoPath]);
+        await executeCommand("git", ["reset", "--hard", spec.gitRef], { cwd: repoPath });
+    }
+    if (!fs.existsSync(repoPath + "node_modules")) {
+        // NOTE: `--ignore-scripts` to make installing faster, as presently a slow install step will cause functions
+        //  after a new deploy to timeout for a bit.
+        await executeCommand("yarn", ["install", "--ignore-scripts"], { cwd: repoPath });
+    }
+    return repoPath;
+};
+
+
+async function getModuleFromSpecBase(
+    spec: FunctionSpec
+): Promise<NodeJS.Module> {
+    let hotReloadPackagePath = "";
+    let path = gitURLRefMappings.get(getSpecKey(spec));
+    if (!path) {
+        // Sync the git repo, `yarn install --ignore-scripts`, require the path, get the export, and then return that function.
+        let packagePath = getLocalPathRemapping()[spec.gitURL];
+        hotReloadPackagePath = packagePath;
+        if (!packagePath) {
+            packagePath = await moduleResolver(spec);
+        }
+
+        if (!packagePath.endsWith("/")) {
+            packagePath += "/";
+        }
+
+        let specFilePath = spec.FilePath;
+        if (specFilePath.startsWith("/")) {
+            specFilePath = specFilePath.slice(1);
+        }
+        path = packagePath + specFilePath;
+    }
+    console.log(blue(`require(${JSON.stringify(path)})`));
+    try {
+        // NOTE: The true tells require to not warn about the async loading
+        await (require as any)(path, true);
+    } catch (e: any) {
+        throw new Error(`Error when loading function for ${JSON.stringify(path)}\n${e.stack}`);
+    }
+    let moduleId = require.resolve(path) || path;
+    let module = require.cache[moduleId];
+    if (!module) {
+        debugbreak(2);
+        debugger;
+        throw new Error(`Module not found: ${moduleId}`);
+    }
+
+    //todonext
+    //  Expose a function that goes from NodeJS.Module to FunctionSpec,
+    //      via a lookup we control?
+    //      - Hmm... we COULD change the module. This is nice, as it is more discoverable.
+    //          But... eh... I don't know. Just a lookup is safer...
+
+    if (hotReloadPackagePath) {
+        hotReloadUnderPath(hotReloadPackagePath);
+    }
+
+    return module;
+}
+
+// Hot reload at or under the path
+const hotReloadUnderPath = cache((path: string) => {
+    console.log(magenta(`Hot reloading under path: ${path}`));
+    path = path.replace(/\\/g, "/");
+    for (let module of Object.values(require.cache)) {
+        if (!module) continue;
+        let modulePath = module.filename.replace(/\\/g, "/");
+        if (modulePath.startsWith(path)) {
+            hotReloadModule(module);
+        }
+    }
+});
+const hotReloadModule = cache((module: NodeJS.Module) => {
+    fs.watch(module.filename, () => {
+        logErrors(hotreloadIfChanged(module));
+    });
+});
+const hotreloadIfChanged = batchFunction({ delay: 100, name: "hotreloadIfChanged" }, async (modules: NodeJS.Module[]) => {
+    let changedModules = new Set<NodeJS.Module>();
+    for (let module of modules) {
+        let newContents = await fs.promises.readFile(module.filename, "utf8");
+        let newSHA256 = crypto.createHash("sha256").update(newContents).digest("hex");
+        if (newSHA256 !== module.sourceSHA256) {
+            changedModules.add(module);
+        }
+    }
+    console.log(magenta(`Hot reloading changed modules:`));
+    for (let module of changedModules) {
+        if (module.updateContents) {
+            let justContents = module.hotreload === false || isNode() && module.noserverhotreload;
+            console.log(magenta(`        ${module.filename} ${justContents && "(no re-evaluate)" || ""}`));
+            setExternalHotReloading(true);
+            try {
+                try {
+                    module.updateContents();
+                } catch (e) {
+                    console.error(red(`Error when loading module contents ${module.filename}`));
+                    console.error(e);
+                }
+                if (!justContents) {
+                    module.loaded = false;
+                    try {
+                        module.load(module.id);
+                    } catch (e) {
+                        module.load(module.id);
+                        console.error(red(`Error when hot reloading ${module.filename}`));
+                        console.error(e);
+                    }
+                }
+            } finally {
+                setExternalHotReloading(false);
+            }
+        } else {
+            console.log(red(`        (skipping due to missing updateContents functions) ${module.filename}`));
+        }
+    }
+
+    for (let watcher of watchers) {
+        if (watcher.disposed) {
+            watcher.dispose();
+        } else {
+            watcher.explicitlyTrigger();
+        }
+    }
+});
+
+async function executeCommand(command: string, args: string[], options?: {
+    cwd?: string;
+}) {
+    child_process.execSync(`${command} ${args.join(" ")}`, {
+        stdio: "inherit",
+        cwd: options?.cwd
+    });
+}