querysub 0.2.0

package/src/diagnostics/logs/injectFileLocationToConsole.ts

@@ -0,0 +1,50 @@
+import { css, isNode } from "typesafecss";
+import { compileTransformBefore } from "typenode";
+// IMPORTANT! Don't add any imports BEFORE we add compileTransform
+if (isNode()) {
+    // NOTE: While mapping line numbers would be nice, just adding the file is easy,
+    //  and provides a huge benefit. And... you should be able to trace down the specific
+    //  line from the message (and if not, you should make the message more specific).
+    compileTransformBefore((contents, path, module) => {
+        function shimConsole() {
+            let baseConsole = globalThis.console;
+            if (typeof window !== "undefined") return baseConsole;
+
+            let name = __filename.replaceAll("\\", "/").split("/").pop();
+            let dirName = __dirname.replaceAll("\\", "/").split("/").pop();
+            let context = {
+                __FILE__: __filename,
+                __DIR__: __dirname,
+                __NAME__: name,
+                __DIRNAME__: dirName
+            };
+            let newConsole = {
+                ...baseConsole
+            };
+            for (let shim of ["log", "info", "warn", "error"] as const) {
+                newConsole[shim] = (...args: unknown[]) => {
+                    args.push(context);
+                    baseConsole[shim](...args);
+                };
+            }
+            (newConsole as any).__context = context;
+            return newConsole;
+        }
+
+        // Use line numbers BEFORE we inject the compile transform. The line numbers are for vs code,
+        //  so this makes them accurate for original source.
+        contents = contents.split("\n").map((line, index) => {
+            return line
+                .replaceAll(`(0, diskLogger_1.diskLog)(`, `(0, diskLogger_1.diskLog)(console.__context, { __LINE__: ${index} }, `)
+                .replaceAll(`  diskLog(`, ` diskLog(console.__context, { __LINE__: ${index} },`)
+                .replaceAll(`(0, diskLogger_1.logDisk)(`, `(0, diskLogger_1.logDisk)(console.__context, { __LINE__: ${index} }, `)
+                .replaceAll(`  logDisk(`, ` logDisk(console.__context, { __LINE__: ${index} },`)
+                ;
+        }).join("\n");
+        return `var console = (${shimConsole.toString()})();\n${contents}`;
+    });
+
+    // We want to call shimConsoleLogs early. Before we call it, our additional context isn't
+    //      screened out, which makes the terminal hard to read.
+    void import("./diskShimConsoleLogs").then(x => x.shimConsoleLogs());
+}

package/src/diagnostics/logs/logGitHashes.ts

@@ -0,0 +1,30 @@
+import child_process from "child_process";
+import { isNode } from "socket-function/src/misc";
+
+export async function logGitHashes() {
+    let shardHash = await getGitHashFromDir(__dirname);
+    let appHash = await getGitHashFromDir(".");
+
+    console.log("Git hashes", { shardHash, appHash });
+}
+
+async function getGitHashFromDir(dir: string) {
+    if (!isNode()) return "CLIENTSIDE";
+    try {
+        return await execPromise(`git rev-parse HEAD`, { cwd: dir });
+    } catch {
+        return "ERROR";
+    }
+}
+
+async function execPromise(command: string, options: child_process.ExecOptions) {
+    return new Promise<string>((resolve, reject) => {
+        child_process.exec(command, options, (err, stdout, stderr) => {
+            if (err) {
+                reject(err);
+            } else {
+                resolve(stdout.trim());
+            }
+        });
+    });
+}

package/src/diagnostics/managementPages.tsx

@@ -0,0 +1,289 @@
+import { Querysub } from "../../src/4-querysub/QuerysubController";
+import { isClient, isServer } from "../../src/config2";
+import { logErrors } from "../../src/errors";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { css, isNode } from "typesafecss";
+import preact from "preact"; import { qreact } from "../../src/4-dom/qreact";
+import { getBrowserUrlNode, onNodeDiscoveryReady } from "../../src/-f-node-discovery/NodeDiscovery";
+import { URLParam } from "../../src/library-components/URLParam";
+import { ATag } from "../../src/library-components/ATag";
+import { lazy } from "socket-function/src/caching";
+import { t } from "../../src/2-proxy/schema2";
+
+import { SocketRegistered } from "socket-function/SocketFunctionTypes";
+import { nextId } from "socket-function/src/misc";
+import { IdentityController_getSecureIP, IdentityController_getMachineId } from "../-c-identity/IdentityController";
+import { LOCAL_DOMAIN } from "../0-path-value-core/PathController";
+import { getNextTime } from "../0-path-value-core/pathValueCore";
+import { encodeArgs } from "../3-path-functions/PathFunctionHelpers";
+import { CallSpec, getCurrentCall, getCurrentCallAllowUndefined, overrideCurrentCall } from "../3-path-functions/PathFunctionRunner";
+import { createLazyComponent } from "../library-components/LazyComponent";
+import { LogNotify } from "./errorLogs/LogNotify";
+import { isTrusted } from "../-d-trust/NetworkTrust2";
+import { getDomain } from "../config";
+import { getCallWrites } from "../4-querysub/querysubPrediction";
+
+
+export const managementPageURL = new URLParam("managementpage", "");
+export const showingManagementURL = new URLParam("showingmanagement", false);
+
+let pages: {
+    componentClass: preact.ComponentClass;
+    componentName: string;
+    title: string;
+}[] = [];
+
+let internalIsManagementUser: () => boolean = () => {
+    throw new Error("registerManagementPages2 must be called during deploy for management functions to be exposed");
+};
+
+// NOTE: We don't store the UI in the database (here, or anywhere else, at least
+//  not yet), but we do want to store the permission function call, as we may
+//  need to update it immediately, across all servers.
+const schema = Querysub.createSchema()({
+    module,
+    moduleId: "managementPages",
+    functions: {
+        schemaIsManagementUser,
+    },
+});
+function schemaIsManagementUser() {
+    return internalIsManagementUser();
+}
+
+export async function registerManagementPages2(config: {
+    isManagementUser: () => boolean;
+    pages: {
+        componentName: string;
+        controllerName?: string;
+        getModule: () => Promise<any>;
+
+        title?: string;
+    }[];
+}) {
+    baseIsManagementUser = config.isManagementUser;
+    let inputPages = config.pages.slice();
+
+    inputPages.push({
+        title: "Nodes",
+        componentName: "NodeViewer",
+        controllerName: "NodeViewerController",
+        getModule: () => import("./NodeViewer"),
+    });
+    inputPages.push({
+        title: "Logs",
+        componentName: "DiskLoggerPage",
+        controllerName: "DiskLoggerController",
+        getModule: () => import("./logs/DiskLoggerPage"),
+    });
+    inputPages.push({
+        title: "Error Notifications",
+        componentName: "LogViewer",
+        controllerName: "LogViewerController",
+        getModule: () => import("./errorLogs/LogViewer"),
+    });
+    inputPages.push({
+        title: "Error Classes",
+        componentName: "LogClassifiers",
+        getModule: () => import("./errorLogs/LogClassifiers"),
+    });
+    inputPages.push({
+        title: "Time",
+        componentName: "TimeDebug",
+        controllerName: "",
+        getModule: () => import("./TimeDebug"),
+    });
+
+    if (isServer()) {
+        for (let page of inputPages) {
+            // NOTE: If we split this into a module for component/controller, we need to make sure we
+            //  import both serverside, so we can whitelist them for import clientside.
+            let mod = await page.getModule();
+            if (!page.controllerName) continue;
+            if (!(page.controllerName in mod)) {
+                console.error(`Controller ${page.controllerName} not found in module`, mod);
+                throw new Error(`Controller ${page.controllerName} not found in module`);
+            }
+            let controller = mod[page.controllerName] as SocketRegistered;
+            if (!controller) {
+                throw new Error(`Controller ${page.controllerName} not found in module`);
+            }
+            if (!controller._classGuid) {
+                throw new Error(`Controller ${page.controllerName} does not have a class guid`);
+            }
+            SocketFunction.expose(controller);
+        }
+    } else {
+        for (let page of inputPages) {
+            pages.push({
+                title: page.title ?? page.componentName,
+                componentName: page.componentName,
+                componentClass: createLazyComponent(page.getModule)(page.componentName),
+            });
+        }
+    }
+}
+
+let baseIsManagementUser: () => boolean = () => {
+    // If we are clientside, all calls come from the server, and so are allowed.
+    if (isClient()) return true;
+    throw new Error("baseIsManagementUser not set");
+};
+
+export async function isManagementUser() {
+    let caller = SocketFunction.getCaller();
+    let callerMachineId = IdentityController_getMachineId(caller);
+
+    // Always trust other servers. They can read/write any data anyways. This helps
+    //  when clients are given callbacks by servers, for management apis which
+    //  have every endpoint protected by isManagementUser.
+    if (await isTrusted(callerMachineId)) return true;
+
+    let testCall: CallSpec = {
+        callerIP: IdentityController_getSecureIP(caller),
+        callerMachineId,
+        argsEncoded: encodeArgs([]),
+        CallId: nextId(),
+        DomainName: getDomain(),
+        ModuleId: "managementPages",
+        FunctionId: "schemaIsManagementUser",
+        runAtTime: getNextTime(),
+    };
+
+    let writes = await getCallWrites({
+        debugName: "isManagementUser",
+        call: testCall,
+    });
+    return !!writes.result;
+}
+
+export async function assertIsManagementUser() {
+    if (!await isManagementUser()) {
+        throw new Error(`User is not a management user, and so does not have permissions for this endpoint.`);
+    }
+}
+
+let populateIsManagementUser = lazy(() => {
+    setImmediate(async () => {
+        let result = await ManagementController.nodes[getBrowserUrlNode()].isManagementUser();
+        Querysub.commit(() => {
+            isManagementUserValue().value = result;
+        });
+    });
+});
+
+let isManagementUserValue = Querysub.createLocalSchema("isManagementUser", {
+    value: t.boolean
+});
+export function renderIsManagementUser() {
+    if (!isClient()) throw new Error(`renderIsManagementUser must be called clientside. Instead, call isManagementUser() serverside`);
+    populateIsManagementUser();
+    return isManagementUserValue().value;
+}
+
+class ManagementRoot extends qreact.Component {
+    state = {
+        ready: false
+    };
+    componentDidMount() {
+        document.addEventListener("keydown", (e) => {
+            if (
+                e.target instanceof HTMLInputElement
+                || e.target instanceof HTMLTextAreaElement
+                || !(e.target instanceof HTMLElement)
+                || e.target.getAttribute("role") === "textbox"
+                || e.target.hasAttribute("data-input")
+            ) return;
+            if (e.key === "`") {
+                Querysub.commit(() => {
+                    showingManagementURL.value = !showingManagementURL.value;
+                });
+            }
+        });
+        // Have to wait for node discovery, which handles detecting the localhost optimization
+        logErrors(onNodeDiscoveryReady().finally(async () => {
+            Querysub.localCommit(() => this.state.ready = true);
+        }));
+    }
+    render() {
+        if (!showingManagementURL.value) return undefined;
+        if (!this.state.ready) return undefined;
+
+        let currentPage: preact.ComponentChild | undefined = undefined;
+
+        let urlValue = managementPageURL.value;
+        let ComponentClass = pages.find(a => a.componentName === urlValue)?.componentClass;
+        if (ComponentClass) {
+            currentPage = <ComponentClass />;
+        }
+
+        return (
+            <>
+                <div class={
+                    css
+                        .fixed.pos(0, 0).size("100vw", "100vh").zIndex(10).vbox0
+                        .color("white")
+                        .color("hsl(0, 0%, 7%)")
+                }>
+                    <style>
+                        {`
+                            .ManagementRoot-page a {
+                                color: hsl(210, 75%, 50%);
+                            }
+                        `}
+                    </style>
+                    <div class={css.fillWidth.hbox(30).wrap.hsl(245, 25, 60).pad2(10)}>
+                        <LogNotify />
+                        {pages.map(page =>
+                            <ATag values={[{ param: managementPageURL, value: page.componentName }]}>{page.title}</ATag>
+                        )}
+                    </div>
+                    {currentPage &&
+                        <div
+                            class={
+                                css.fillBoth.vbox0.overflowAuto.hsl(145, 50, 75)
+                                + " ManagementRoot-page"
+                            }
+                            onKeyDown={e => {
+                                if (e.currentTarget === e.target) {
+                                    // Ignore keyboard events
+                                    e.preventDefault();
+                                }
+                            }}
+                        >
+                            {currentPage}
+                        </div>
+                    }
+                </div>
+
+            </>
+        );
+    }
+}
+
+class ManagementControllerBase {
+    public async isManagementUser() {
+        return await isManagementUser();
+    }
+}
+const ManagementController = SocketFunction.register(
+    "ManagementControllerBase-fa482fbb-ed3b-4377-bf8e-847357d17244",
+    new ManagementControllerBase(),
+    () => ({
+        isManagementUser: {},
+    }),
+    () => ({
+
+    }),
+);
+
+async function clientMain() {
+    let root = document.createElement("div");
+    root.id = "managementRoot";
+    document.body.appendChild(root);
+    Querysub.render(<ManagementRoot />, root);
+}
+
+if (!isNode()) {
+    logErrors(clientMain());
+}

package/src/diagnostics/periodic.ts

@@ -0,0 +1,89 @@
+import { runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { isNode } from "socket-function/src/misc";
+import { logErrors } from "../errors";
+import debugbreak from "debugbreak";
+import { nodeDiscoveryShutdown } from "../-f-node-discovery/NodeDiscovery";
+import { authorityStorage } from "../0-path-value-core/pathValueCore";
+
+let periodicFncs: (() => void)[] = [];
+export function registerPeriodic(fnc: () => void) {
+    periodicFncs.push(fnc);
+}
+
+let shutdownHandlers: (() => Promise<void>)[] = [];
+// NOTE: Won't be always called, but at least makes it possible to gracefully shutdown.
+export function registerShutdownHandler(fnc: () => Promise<void>) {
+    shutdownHandlers.push(fnc);
+}
+
+function logAll() {
+    for (let fnc of periodicFncs) {
+        fnc();
+    }
+}
+
+logErrors(runInfinitePollCallAtStart(1000 * 60, logAll));
+
+async function shutdown() {
+    const { authorityStorage } = await import("../0-path-value-core/pathValueCore");
+    try {
+        await authorityStorage.onShutdown();
+        await nodeDiscoveryShutdown();
+    } catch (e) {
+        console.log("Error on shutdown", e);
+    }
+    for (let fnc of shutdownHandlers) {
+        try {
+            await fnc();
+        } catch (e) {
+            console.log(`Error on shutdown handler ${fnc.name}`, e);
+        }
+    }
+    process.exit();
+}
+
+if (isNode()) {
+    let lineBuffer = "";
+    process.stdin.on("data", data => {
+        lineBuffer += data.toString();
+        let lines = lineBuffer.split("\r");
+        lineBuffer = lines.pop()!;
+        for (let line of lines) {
+            if (line === "SHUTDOWN_NOW_MULTIRUN") {
+                logErrors(shutdown());
+            }
+        }
+
+        if (data.toString().includes("\r")) {
+            logAll();
+        }
+    });
+
+
+    // NOTE: This extra code is required to actual capture ctrl+c
+    if (process.platform === "win32") {
+        var rl = require("readline").createInterface({
+            input: process.stdin,
+            output: process.stdout
+        });
+
+        rl.on("SIGINT", function () {
+            process.emit("SIGINT");
+        });
+    }
+    let killCount = 0;
+    function doShutdown() {
+        killCount++;
+        console.log(`Caught interrupt signal. Attempt number ${killCount}`);
+        if (killCount >= 3) {
+            console.log("Force exit");
+            process.exit();
+        }
+
+        logErrors(shutdown());
+    }
+    process.on("SIGINT", doShutdown);
+
+}
+(globalThis as any).logAll = logAll;
+(globalThis as any).logNow = logAll;