querysub 0.189.0 → 0.191.0

package/bin/addsuperuser.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+
+require("typenode");
+require("../src/user-implementation/addSuperUser");

package/bin/deploy.js

@@ -1,4 +1,7 @@
 #!/usr/bin/env node
 
+// Always local, as we want to always use the local code? Might not be needed anymore?
+process.argv.push("--local");
+
 require("typenode");
 require("../src/4-deploy/deployMain");

package/bin/function-public.js

@@ -0,0 +1,7 @@
+#!/usr/bin/env node
+
+process.argv.push("--nobreak");
+process.argv.push("--public");
+
+require("typenode");
+require("../src/3-path-functions/PathFunctionRunnerMain");

package/bin/function.js

@@ -1,4 +1,9 @@
 #!/usr/bin/env node
 
+process.argv.push("--local");
+// Only dev. If we listen to all functions it will start picking up production functions calls, which breaks things (unlike the path value server, where we are unreachable, so nothing breaks).
+process.argv.push("--filter");
+process.argv.push("dev");
+
 require("typenode");
-require("../src/3-path-functions/PathFunctionRunnerMain");
+require("../src/3-path-functions/PathFunctionRunnerMain");

package/bin/gc-watch-public.js

@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+
+process.argv.push("--watch");
+process.argv.push("--nobreak");
+process.argv.push("--public");
+
+require("typenode");
+require("../src/archiveapps/archiveGCEntry");

package/bin/gc-watch.js

@@ -0,0 +1,6 @@
+#!/usr/bin/env node
+
+process.argv.push("--watch");
+
+require("typenode");
+require("../src/archiveapps/archiveGCEntry");

package/bin/gc.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+
+require("typenode");
+require("../src/archiveapps/archiveGCEntry");

package/bin/join-public.js

@@ -0,0 +1,7 @@
+#!/usr/bin/env node
+
+process.argv.push("--nobreak");
+process.argv.push("--public");
+
+require("typenode");
+require("../src/archiveapps/archiveJoinEntry");

package/bin/join.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+
+require("typenode");
+require("../src/archiveapps/archiveGCEntry");

package/bin/machine.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+
+require("typenode");
+require("../src/deployManager/machineApplyMain");

package/bin/server-public.js

@@ -0,0 +1,7 @@
+#!/usr/bin/env node
+
+process.argv.push("--nobreak");
+process.argv.push("--public");
+
+require("typenode");
+require("../src/server.ts");

package/bin/server.js

@@ -1,4 +1,6 @@
 #!/usr/bin/env node
 
+process.argv.push("--local");
+
 require("typenode");
-require("../src/server.ts");
+require("../src/server.ts");

package/bin/setemailkey.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+
+require("typenode");
+require("../src/user-implementation/setEmailKey");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.189.0",
+  "version": "0.191.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -39,12 +39,24 @@
     "type": "yarn tsc --noEmit",
     "depend": "yarn --silent depcruise src --include-only \"^src\" --config --output-type dot | dot -T svg > dependency-graph.svg",
     "test": "yarn typenode ./src/test/test.tsx --local",
-    "test2": "yarn typenode ./src/4-dom/qreactTest.tsx --local"
+    "test2": "yarn typenode ./src/4-dom/qreactTest.tsx --local",
+    "machine-apply": "yarn typenode ./src/deployManager/deployApplyMain.ts"
   },
   "bin": {
-    "querysub-deploy": "./bin/deploy.js",
-    "querysub-server": "./bin/server.js",
-    "querysub-function": "./bin/function.js"
+    "deploy": "./bin/deploy.js",
+    "server": "./bin/server.js",
+    "server-public": "./bin/server-public.js",
+    "function": "./bin/function.js",
+    "function-public": "./bin/function-public.js",
+    "machine": "./bin/machine.js",
+    "gc": "./bin/gc.js",
+    "gc-watch": "./bin/gc-watch.js",
+    "gc-watch-public": "./bin/gc-watch-public.js",
+    "join": "./bin/join.js",
+    "join-public": "./bin/join-public.js",
+    "merge": "./bin/merge.js",
+    "addsuperuser": "./bin/addsuperuser.js",
+    "setemailkey": "./bin/setemailkey.js"
   },
   "devDependencies": {
     "dependency-cruiser": "^12.11.0",

package/src/-a-archives/archivesJSONT.ts

@@ -0,0 +1,46 @@
+import { lazy } from "socket-function/src/caching";
+import { Archives } from "./archives";
+
+export type ArchiveJSONT<T> = {
+    get(key: string): Promise<T | undefined>;
+    set(key: string, value: T): Promise<void>;
+    delete(key: string): Promise<void>;
+    keys(): Promise<string[]>;
+    values(): Promise<T[]>;
+    entries(): Promise<[string, T][]>;
+    [Symbol.asyncIterator](): AsyncIterator<[string, T]>;
+};
+
+export function archiveJSONT<T>(archives: () => Archives): ArchiveJSONT<T> {
+    archives = lazy(archives);
+    async function get(key: string) {
+        let buffer = await archives().get(key);
+        if (!buffer) return undefined;
+        return JSON.parse(buffer.toString());
+    }
+    async function set(key: string, value: T) {
+        await archives().set(key, Buffer.from(JSON.stringify(value)));
+    }
+    async function deleteFnc(key: string) {
+        await archives().del(key);
+    }
+    async function keys() {
+        return (await archives().find("")).map(value => value.toString());
+    }
+    async function values() {
+        let keysArray = await keys();
+        return Promise.all(keysArray.map(key => get(key)));
+    }
+    async function entries(): Promise<[string, T][]> {
+        let keysArray = await keys();
+        return Promise.all(keysArray.map(async key => [key, await get(key)]));
+    }
+    async function* asyncIterator(): AsyncIterator<[string, T]> {
+        for (let [key, value] of await entries()) {
+            yield [key, value];
+        }
+    }
+    return {
+        get, set, delete: deleteFnc, keys, values, entries, [Symbol.asyncIterator]: asyncIterator
+    };
+}

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -267,7 +267,7 @@ async function syncArchives() {
 async function runHeartbeatAuditLoop() {
     await getAllNodeIds();
     let deadCount = new Map<string, number>();
-    // 90% of the normal interval, so we don't run at the same tmie as the other audit
+    // 90% of the normal interval, so we don't run at the same time as the other audit
     await runInfinitePollCallAtStart(CHECK_INTERVAL * 0.9, async () => {
         if (shutdown) return;
         // Wait a bit longer, to try to prevent all nodes from synchronizing their audit times.

package/src/2-proxy/PathValueProxyWatcher.ts

@@ -435,6 +435,7 @@ export function undeleteFromLookup<T>(lookup: { [key: string]: T }, key: string)
 
 const syncedSymbol = Symbol.for("syncedSymbol");
 // HACK: This should probably be somewhere else, but... it is just so useful for PathFunctionRunner...
+/** @deprecated this is not very accurate (it breaks for schema accesses). Only use it for low level places that can't call the more accurate Querysub.isSynced) */
 export function isSynced(obj: unknown): boolean {
     // If it is a primitive, then it must be synced!
     if (!canHaveChildren(obj)) return true;

package/src/3-path-functions/PathFunctionRunnerMain.ts

@@ -20,7 +20,7 @@ import { PermissionsCheck } from "../4-querysub/permissions";
 import { timeInMinute } from "socket-function/src/misc";
 import { getDomain, isLocal, isPublic } from "../config";
 import { publishMachineARecords } from "../-e-certs/EdgeCertController";
-import { green } from "socket-function/src/formatting/logColors";
+import { green, magenta } from "socket-function/src/formatting/logColors";
 import { parseFilterSelector } from "../misc/filterable";
 import path from "path";
 
@@ -69,5 +69,9 @@ async function main() {
         let deployPath = path.resolve("./deploy.ts");
         await import(deployPath);
     }
+
+    if (yargObj.filter) {
+        console.log(magenta(`Only running functions that match the filter: ${yargObj.filter}. Use ?setfncfilter=${yargObj.filter} in the browser to trigger functions that match this filter.`));
+    }
 }
 logErrors(main());

package/src/4-dom/qreact.tsx

@@ -379,6 +379,7 @@ function nextLocalId() {
 export type ExternalRenderClass = {
     data(): QComponent;
     getParent(): ExternalRenderClass | undefined;
+    isDisposed(): boolean;
     readonly renderWatcher: SyncWatcher;
     readonly VNodeWatcher: SyncWatcher;
     readonly mountDOMWatcher: SyncWatcher;
@@ -1719,6 +1720,9 @@ class QRenderClass {
 
     private disposing = false;
     public disposed = false;
+    public isDisposed(): boolean {
+        return this.disposed;
+    }
     private dispose() {
         if (this.disposed) return;
         this.disposing = true;

package/src/4-querysub/Querysub.ts

@@ -15,7 +15,7 @@ import { cache, cacheLimited, lazy } from "socket-function/src/caching";
 import { getOwnMachineId, getThreadKeyCert, verifyMachineIdForPublicKey } from "../-a-auth/certs";
 import { getSNICerts, publishMachineARecords } from "../-e-certs/EdgeCertController";
 import { LOCAL_DOMAIN, nodePathAuthority } from "../0-path-value-core/NodePathAuthorities";
-import { debugCoreMode, registerGetCompressNetwork, encodeParentFilter, registerGetCompressDisk } from "../0-path-value-core/pathValueCore";
+import { debugCoreMode, registerGetCompressNetwork, encodeParentFilter, registerGetCompressDisk, authorityStorage } from "../0-path-value-core/pathValueCore";
 import { clientWatcher, ClientWatcher } from "../1-path-client/pathValueClientWatcher";
 import { SyncWatcher, proxyWatcher, specialObjectWriteValue, isSynced, PathValueProxyWatcher, atomic, doAtomicWrites, noAtomicSchema, undeleteFromLookup, registerSchemaPrefix, WatcherOptions } from "../2-proxy/PathValueProxyWatcher";
 import { isInProxyDatabase, rawSchema } from "../2-proxy/pathDatabaseProxyBase";
@@ -285,7 +285,7 @@ export class Querysub {
     public static createWatcher(watcher: (obj: SyncWatcher) => void, options?: Partial<WatcherOptions<unknown>>): {
         dispose: () => void;
         explicitlyTrigger: () => void;
-    } {
+    } & SyncWatcher {
         return proxyWatcher.createWatcher({
             debugName: watcher.name,
             canWrite: true,
@@ -462,9 +462,18 @@ export class Querysub {
         // onCommitFinished prevents duplicates, as well as only running when we are actually done
         Querysub.onCommitFinished(async () => {
             await clientWatcher.waitForTriggerFinished();
+            // No idea why this work, but... we do need to wait for promises... twice. Maybe we are waiting for some promise stacks to unwind? But twice? Bizarre
+            //  - Needed for ReactEditor:updateSelectionRects AND LazyRenderList...
+            await Promise.resolve();
+            await Promise.resolve();
             callback();
         });
     }
+    public static async afterAllRendersFinishedPromise() {
+        return new Promise<void>(r => {
+            Querysub.afterAllRendersFinished(r);
+        });
+    }
 
     /** Solely for use to prevent local writes from occuring before predictions. We MIGHT make this a framework thing,
      *      or just not bother with it (as after a prediction is run once the code will be loaded allowing it to always
@@ -510,8 +519,10 @@ export class Querysub {
         return proxyWatcher.inWatcher() && isInProxyDatabase();
     }
 
-    public static isSynced(value: unknown) {
-        return isSynced(value);
+    public static isSynced(value: unknown | (() => unknown)) {
+        let path = Querysub.getPath(value);
+        if (!path) return true;
+        return authorityStorage.isSynced(path);
     }
 
     public static ignoreWatches<T>(code: () => T) {

package/src/5-diagnostics/FullscreenModal.tsx

@@ -42,7 +42,7 @@ export class FullscreenModal extends qreact.Component<{
                     display: "flex",
                     flexDirection: "column",
                     gap: 10,
-                    maxHeight: "calc(100% - 200px)",
+                    maxHeight: "calc(100vh - 200px)",
                     overflow: "auto",
                     ...this.props.style
                 }}

package/src/5-diagnostics/Modal.tsx

@@ -95,6 +95,9 @@ export function closeAllModals() {
         }
     });
 }
+export function isShowingModal() {
+    return Object.values(data().modals).length > 0;
+}
 
 
 export class ModalHolder extends qreact.Component {

package/src/config.ts

@@ -3,6 +3,8 @@ import yargs from "yargs";
 import debugbreak from "debugbreak";
 import { MaybePromise } from "socket-function/src/types";
 import { parseArgsFactory } from "./misc/rawParams";
+import { lazy } from "socket-function/src/caching";
+import fs from "fs";
 
 export const serverPort = 11748;
 
@@ -11,7 +13,6 @@ let yargObj = parseArgsFactory()
     .option("domain", { type: "string", desc: `Sets the domain` })
     .option("client", { type: "boolean", desc: `Drops permissions, acting as an unauthenticated node` })
     .option("authority", { type: "string", desc: `Defines the base paths we are an authority on (the domain is prepended to them). Either a file path to a JSON(AuthorityPath[]), or a base64 representation of the JSON(AuthorityPath[]).` })
-    .option("debugbreak", { type: "boolean", desc: "Break and show a popup when there are fatal errors" })
     .option("nobreak", { type: "boolean", desc: "Do not break on errors. Safer to set this than to just not set debugbreak, as some places might break without checking debugbreak, but nobreak works at a level where it is always used." })
     .option("public", { type: "boolean", desc: "Expose on public ports." })
     .option("local", { type: "boolean", desc: `If true, uses the local directory instead of the remote git repo. Also hotreloads from disk. Determines the repo to replace through the package.json "repository" property.` })
@@ -20,10 +21,21 @@ let yargObj = parseArgsFactory()
     .argv
     ;
 
-if (!isNode()) {
-    // TODO: Get yargs running in the browser, instead of this hack processing
-    yargObj.local = process.argv.includes("--local");
-}
+let querysubConfig = lazy((): {
+    domain?: string;
+} => {
+    if (!isNode()) throw new Error("querysubConfig is only available on the server");
+    const path = "./querysub.json";
+    if (!fs.existsSync(path)) {
+        return {};
+    }
+    try {
+        return JSON.parse(fs.readFileSync(path, "utf8"));
+    } catch (e) {
+        console.error("Error parsing querysub.json", e);
+        return {};
+    }
+});
 
 export function isNoNetwork() {
     return yargObj.nonetwork;
@@ -33,7 +45,7 @@ export function getDomain() {
     if (!isNode()) {
         return location.hostname.split(".").slice(-2).join(".");
     }
-    return yargObj.domain || "querysub.com";
+    return yargObj.domain || querysubConfig().domain || "querysub.com";
 }
 
 export function baseIsClient() {
@@ -49,13 +61,13 @@ export function devDebugbreak() {
         debugger;
         return;
     }
-    if (!yargObj.debugbreak) return;
+    if (yargObj.nobreak) return;
     debugbreak(2);
     debugger;
 }
 let debuggedEnabled = false;
 export function isDevDebugbreak() {
-    return yargObj.debugbreak || !isNode() && location.hostname.startsWith("127-0-0-1.") || debuggedEnabled;
+    return !yargObj.nobreak || !isNode() && location.hostname.startsWith("127-0-0-1.") || debuggedEnabled;
 }
 export function enableDebugging() {
     debuggedEnabled = true;

package/src/deployManager/DeployPage.tsx

@@ -0,0 +1,52 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { qreact } from "../../src/4-dom/qreact";
+import { DeployController } from "./deploySchema";
+import { css } from "typesafecss";
+
+export class DeployPage extends qreact.Component {
+    render() {
+        let controller = DeployController(SocketFunction.browserNodeId());
+        return <div className={css.vbox(10)}>
+            <h1>DeployPage</h1>
+            <div className={css.whiteSpace("pre-wrap")}>
+                {JSON.stringify(controller.getAllInfo(), null, 4)}
+            </div>
+        </div>;
+    }
+}
+/*
+
+todonext
+
+# Domain config file
+- Have the domain configured via a file in the repo, so we don't have to set it on every single script
+    - This also makes it possible to use binary scripts directly in querysub without having to add a script in our package.json to set the domain parameter
+        - FORWARD the main scripts
+            function
+            function-public
+            server
+            server-public
+            deploy
+            ... anything that uses node_modules, which is a lot...
+        - And remove all --domain arguments (which is a lot as well)
+    - ALSO, support setting arbitrary config parameter in this config file?
+        - Maybe on a per script basis, giving scripts names.
+
+5) synced controller loading indicator
+    - Probably which wataches all controllers (on all nodes), for any pending, shown globally
+6) Validate that our loading indicator work, as well as our notifications on changed configuration, etc
+    - Test with errors, repeated crashes, etc.
+2) Web interface to setup configuration
+    - Needs to be able to run with NO PathValueServer, etc, just running locally
+        - I think we have a flag to recovery which allows anyone on localhost to be a superuser. Test this, as we might need it...
+    - A regular management page
+    - Shows servers
+    - Shows services
+    - Allows changing services (configuration, adding, removing, etc)
+        - Both writes to the backblaze file AND explicitly tells each service
+        - Use an editor which allows directly editting it as text?
+            - Monaco?
+            - BUT, which applies a type to it (I think monaco supports this, like with tsconfig.json).
+        - Copy the repo from the last changed service config
+    - Gives command for our setup script, in case we forget it (basically just instructions)
+*/

package/src/deployManager/deploySchema.ts

@@ -0,0 +1,170 @@
+import { isNodeTrue } from "socket-function/src/misc";
+import { nestArchives } from "../-a-archives/archives";
+import { getArchivesBackblaze } from "../-a-archives/archivesBackBlaze";
+import { getDomain } from "../config";
+import { archiveJSONT } from "../-a-archives/archivesJSONT";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { assertIsManagementUser } from "../diagnostics/managementPages";
+import { getCallObj } from "socket-function/src/nodeProxy";
+import { getSyncedController } from "../library-components/SyncedController";
+import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
+
+export type ServiceConfig = {
+    // Just a random id to manage the service
+    serviceId: string;
+    // When parameters update, we restart it (and when info updates, we do not)
+    parameters: {
+        // MUST be unique, and clean enough to be used as the screen/tmux name, and folder name
+        key: string;
+
+        // Run multiple services, as `${key}-${index}-dply`, each with their own screen and folder
+        count: number;
+
+        repoUrl: string;
+        gitRef: string;
+        command: string;
+        // Allows forcing an update
+        poke?: number;
+        // If once we don't restart it on exit, only running it once. However we will run it again on boot, so it's not once globally, it's just running it once for now.
+        once?: boolean;
+
+        // Not set by default, so we can setup the configuration before deploying it (or so we can undeploy easily without deleting it)
+        deploy?: boolean;
+
+
+        // TODO:
+        //rollingWindow?: number
+    };
+    info: {
+        title: string;
+        notes: string;
+        lastUpdatedTime: number;
+    };
+};
+export type MachineConfig = {
+    machineId: string;
+
+    services: Record<string, ServiceConfig>;
+};
+export type MachineInfo = {
+    machineId: string;
+
+    // Used to tell the apply tool to update it's configs now
+    applyNodeId: string;
+
+    heartbeat: number;
+    /* 
+        // TODO: ShowMore on each of the infos, so large ones are fine.
+        hostnamectl (fallback to hostname)
+        getExternalIP()
+        lscpu
+        id (fallback to whoami)
+    */
+    info: Record<string, string>;
+
+    services: Record<string, {
+        lastLaunchedTime: number;
+        errorFromLastRun: string;
+        // Only times launched for the current applyNodeId, but... still very useful.
+        totalTimesLaunched: number;
+    }>;
+};
+
+const machineConfigs = archiveJSONT<MachineConfig>(() => nestArchives("deploy/machine-configs/", getArchivesBackblaze(getDomain())));
+const machineInfos = archiveJSONT<MachineInfo>(() => nestArchives("deploy/machine-heartbeats/", getArchivesBackblaze(getDomain())));
+
+
+
+// Only works if our self id has been registered as applyNodeId (with setMachineInfo).
+let serviceConfigChangeWatchers = new Set<() => Promise<void>>();
+
+// NOTE: The callback should block until the changes are applied (possibly throwing). This way the change results can be deployed and when the changer function finishes it will automatically reload them.
+export function onServiceConfigChange(callback: () => Promise<void>): () => void {
+    serviceConfigChangeWatchers.add(callback);
+    return () => {
+        serviceConfigChangeWatchers.delete(callback);
+    };
+}
+class OnServiceChangeBase {
+    public async onServiceConfigChange() {
+        for (let callback of serviceConfigChangeWatchers) {
+            await callback();
+        }
+    }
+}
+const OnServiceChange = SocketFunction.register(
+    "on-service-change-aa6b4aaa-c325-4112-b2a8-f81c180016a0",
+    () => new OnServiceChangeBase(),
+    () => ({
+        onServiceConfigChange: {},
+    }),
+    () => ({
+        hooks: [requiresNetworkTrustHook],
+    }),
+);
+
+
+export class DeployControllerBase {
+
+    public async getAllInfo(): Promise<{ configs: MachineConfig[], infos: MachineInfo[] }> {
+        let configs = await machineConfigs.values();
+        let infos = await machineInfos.values();
+        return { configs, infos };
+    }
+    public async getMachineInfo(machineId: string): Promise<MachineInfo> {
+        let info = await machineInfos.get(machineId);
+        if (!info) throw new Error(`MachineInfo not found for ${machineId}`);
+        return info;
+    }
+    public async getMachineConfig(machineId: string): Promise<MachineConfig> {
+        let config = await machineConfigs.get(machineId);
+        if (!config) throw new Error(`MachineConfig not found for ${machineId}`);
+        return config;
+    }
+    public async setMachineInfo(machineId: string, info: MachineInfo) {
+        await machineInfos.set(machineId, info);
+    }
+    public async setServiceConfig(machineId: string, serviceId: string, config: ServiceConfig | "remove") {
+        let machineConfig = await machineConfigs.get(machineId);
+        if (!machineConfig) throw new Error(`Machine not found for ${machineId}`);
+        if (config === "remove") {
+            delete machineConfig.services[serviceId];
+        } else {
+            machineConfig.services[serviceId] = config;
+        }
+        await machineConfigs.set(machineId, machineConfig);
+        let machineInfo = await machineInfos.get(machineId);
+        if (!machineInfo) throw new Error(`MachineInfo not found for ${machineId}`);
+        await OnServiceChange.nodes[machineInfo.applyNodeId].onServiceConfigChange();
+    }
+}
+
+
+export const DeployController = getSyncedController(
+    SocketFunction.register(
+        "deploy-eda94f05-5e4d-4f5a-b1c1-98613fba60b8",
+        () => new DeployControllerBase(),
+        () => ({
+            getAllInfo: {},
+            getMachineInfo: {},
+            getMachineConfig: {},
+            setMachineInfo: {},
+            setServiceConfig: {},
+        }),
+        () => ({
+            hooks: [assertIsManagementUser],
+        }),
+    ),
+    {
+        writes: {
+            setMachineInfo: ["MachineInfo"],
+            // NOTE: Also changes the MachineInfo, but telling the machine to redeploy, which should cause the machine info (the service launched times or errors) to update!
+            setServiceConfig: ["MachineConfig", "MachineInfo"],
+        },
+        reads: {
+            getAllInfo: ["MachineConfig", "MachineInfo"],
+            getMachineInfo: ["MachineInfo"],
+            getMachineConfig: ["MachineConfig"],
+        }
+    }
+);

package/src/deployManager/machineApplyMain.ts

@@ -0,0 +1,57 @@
+import { measureWrap } from "socket-function/src/profiling/measure";
+import { getOwnMachineId } from "../-a-auth/certs";
+import { getOurNodeId, getOurNodeIdAssert } from "../-f-node-discovery/NodeDiscovery";
+import { Querysub } from "../4-querysub/QuerysubController";
+import { DeployControllerBase, MachineInfo } from "./deploySchema";
+import { runPromise } from "../functional/runCommand";
+import { getExternalIP } from "socket-function/src/networking";
+import { errorToUndefined, errorToUndefinedSilent } from "../errors";
+import { getDomain } from "../config";
+import { formatTime } from "socket-function/src/formatting/format";
+import { sort } from "socket-function/src/misc";
+import { isDefined } from "../misc";
+import { logLoadTime } from "../logModuleLoadTimes";
+
+
+
+const getLiveMachineInfo = measureWrap(async function getLiveMachineInfo() {
+    let machineInfo: MachineInfo = {
+        machineId: getOwnMachineId(),
+        applyNodeId: getOurNodeIdAssert(),
+        heartbeat: Date.now(),
+        info: {},
+        services: {},
+    };
+
+    machineInfo.info.hostnamectl = await errorToUndefinedSilent(runPromise("hostnamectl")) || "";
+    machineInfo.info.getExternalIP = await errorToUndefinedSilent(getExternalIP()) || "";
+    machineInfo.info.lscpu = await errorToUndefinedSilent(runPromise("lscpu")) || "";
+    machineInfo.info.id = await errorToUndefinedSilent(runPromise("id")) || await errorToUndefinedSilent(runPromise("whoami")) || "";
+
+    // TODO: Populate services via checking tmux for special keywords ("-dply", probably...)
+
+    return machineInfo;
+});
+const updateMachineInfo = measureWrap(async function updateMachineInfo() {
+    let machineInfo = await getLiveMachineInfo();
+    console.log("updateMachineInfo", machineInfo);
+    await new DeployControllerBase().setMachineInfo(machineInfo.machineId, machineInfo);
+});
+
+//todonext
+//  The child process will be run with shell, and then we'll watch it (don't use runPromise)
+
+function getTotalTime(module: NodeJS.Module, now: number) {
+    return (module.evalEndTime || now) - (module.evalStartTime || now);
+}
+
+
+export async function machineApplyMain() {
+    console.log(getDomain());
+    process.exit();
+    // await Querysub.hostService("machine-apply");
+
+    // await updateMachineInfo();
+}
+
+machineApplyMain().catch(console.error);

package/src/deployManager/spec.txt

@@ -0,0 +1,79 @@
+5) synced controller loading indicator
+    - Probably which wataches all controllers (on all nodes), for any pending, shown globally
+6) Validate that our loading indicator work, as well as our notifications on changed configuration, etc
+    - Test with errors, repeated crashes, etc.
+
+2) Web interface to setup configuration
+    - Needs to be able to run with NO PathValueServer, etc, just running locally
+        - I think we have a flag to recovery which allows anyone on localhost to be a superuser. Test this, as we might need it...
+    - A regular management page
+    - Shows servers
+    - Shows services
+    - Allows changing services (configuration, adding, removing, etc)
+        - Both writes to the backblaze file AND explicitly tells each service
+        - Use an editor which allows directly editting it as text?
+            - Monaco?
+            - BUT, which applies a type to it (I think monaco supports this, like with tsconfig.json).
+        - Copy the repo from the last changed service config
+    - Gives command for our setup script, in case we forget it (basically just instructions)
+3) Configuration apply utility
+    - registers with actual machineId, from getOwnMachineId (which doesn't require any permissions, however writing it to backblaze does...)
+    - tmux
+    - Work on windows as well? For testing?
+    - heartbeat machine every once in a while
+        - Updating the info as well
+        - Every 15 minutes?
+    - listener so we can be told to update our configs now
+        - otherwise poll every 5 minutes
+
+4) Run apply utility on windows, and test it in the UI, to get the UI working
+
+
+4) Always up wrapper
+    - Reruns apply utility when it crashes
+    - And error log forwarding support, so crashes get logged by the next apply script to run, by writing apply utility crashes to a special file the apply utility reads on startup
+4.1) Requisition script
+    - I guess a nodejs script, with runPromise it should be easy...
+    - Given remote IP, SSHes in, copies backblaze config from current machine, clones repo, setups crontab for always up wrapper, and then runs it
+        - OH, also installs nodejs, yarn, etc
+        - Maybe ansible, although... probably not. Especially because we can just get the AI to write this script anyway. We'll probably just have it copy over a bash script and then run that. 
+            git
+            nodejs
+            yarn
+            add git permissions
+                - Ugh... we need to use github API for this
+            clones
+            copies backblaze.json
+            copy startup.sh
+            setup crontab to startup.sh
+            run startup.sh
+    - We might need to have it use to github API to give the remote machine access to the repo?
+    - Test on a new server, that we run temporarily with some test scripts
+
+4.1) Requisition script in "bin", as a .js bootstrapper
+    - Verify it works, so we can do 
+
+5) Setup on our regular digital ocean server
+    - Remove previous startup script and kill existing tmux services
+6) Verify crash logging works with error notifications (it work if we just apply our console logs shims)
+7) Quick node removal on process crash or removal
+    Detect the nodeId of services (if they have one), and when the service dies, immediately remove "edgenodes/" file, and trigger an update of "edge-nodes-index.json"
+8) Fix deploy user notification issue, where the refresh button doesn't work?
+9) Rolling updates
+    - Written to service definition
+    - Keep previous service alive while we update
+        - If we update within that window... keep the oldest one alive, not the newest
+    - Notify server, so it can shutdown nicely
+        - We have a shutdown process, all servers should run that, for PathValueServers properly flush to disk
+        - In HTTP server, notify useers, in the same way we notify for hash updates, that they will need to switch servers
+            - Also, of course, take ourself out of the HTTP pool
+            - Maybe add this to the shutdown process?
+        - Disable infinite pollers, and wait for any outstanding to finish
+            - Maybe add this to the shutdown process?
+
+10) Reduce the heartbeat interval in NodeDiscovery to every 15 minutes... because our quick node removal should get rid of any latency with removing nodes
+    - Test it with adding / removing nodes (on the server)
+    - Also, make a huge amount of dead nodes locally, and make sure nothing breaks. It really shouldn't, because we should be checking the nodes all at once, and if a node never responds that's fine... right? (It's just a refused connection, and we don't try again for awhile, and we give up after some time).
+    - This should massively reduce our backblaze costs, as our current interval has probably cost us over 100 USD since we wrote it, for no benefit...
+
+

package/src/diagnostics/managementPages.tsx

@@ -127,6 +127,11 @@ export async function registerManagementPages2(config: {
         componentName: "RequireAuditPage",
         getModule: () => import("./misc-pages/RequireAuditPage"),
     });
+    inputPages.push({
+        title: "Deploy",
+        componentName: "DeployPage",
+        getModule: () => import("../deployManager/DeployPage"),
+    });
     inputPages.push(...config.pages);
 
     // NOTE: We don't store the UI in the database (here, or anywhere else, at least

package/src/functional/runCommand.ts

@@ -1,6 +1,61 @@
 import child_process from "child_process";
 import path from "path";
+import { red } from "socket-function/src/formatting/logColors";
 
+export const runAsync = runPromise;
+export async function runPromise(command: string, config?: { cwd?: string; quiet?: boolean; }) {
+    return new Promise<string>((resolve, reject) => {
+        const childProc = child_process.spawn(command, {
+            shell: true,
+            cwd: config?.cwd,
+            stdio: ["inherit", "pipe", "pipe"], // stdin: inherit, stdout: pipe, stderr: pipe
+        });
+
+        let stdout = "";
+        let stderr = "";
+
+        // Always collect output
+        childProc.stdout?.on("data", (data) => {
+            const chunk = data.toString();
+            stdout += chunk;
+
+            // Stream to console unless quiet mode
+            if (!config?.quiet) {
+                process.stdout.write(chunk);
+            }
+        });
+
+        childProc.stderr?.on("data", (data) => {
+            const chunk = data.toString();
+            stderr += chunk;
+            stdout += chunk;
+
+            // Stream to console unless quiet mode
+            if (!config?.quiet) {
+                process.stderr.write(red(chunk));
+            }
+        });
+
+        childProc.on("error", (err) => {
+            reject(err);
+        });
+
+        childProc.on("close", (code) => {
+            if (code === 0) {
+                resolve(stdout);
+            } else {
+                let errorMessage = `Process exited with code ${code} for command: ${command}`;
+                if (stderr) {
+                    errorMessage += `\n${stderr}`;
+                }
+                const error = new Error(errorMessage);
+                reject(error);
+            }
+        });
+    });
+}
+
+/** @deprecated, use runPromise */
 export async function runCommand(config: {
     exe: string;
     args: string[];
@@ -27,6 +82,7 @@ export async function runCommand(config: {
     });
 }
 
+/** @deprecated, use runPromise */
 export function runCommandShell(command: string, config?: { cwd?: string; maxBuffer?: number }) {
     return new Promise<string>((resolve, reject) => {
         child_process.exec(command, {

package/src/library-components/ATag.tsx

@@ -29,7 +29,11 @@ export type ATagProps = (
         rawLink?: boolean;
         lightMode?: boolean;
         noStyles?: boolean;
+
         onRef?: (element: HTMLAnchorElement | null) => void;
+
+        // On click, do this. Independent of URL behavior (which will still run if they middle click, or copy the link, etc)
+        clickOverride?: () => void;
     }
 );
 
@@ -74,6 +78,12 @@ export class ATag extends qreact.Component<ATagProps> {
                 onClick={e => {
                     if (this.props.rawLink) return;
                     if (e.button !== 0) return;
+                    if (this.props.clickOverride) {
+                        e.preventDefault();
+                        e.stopPropagation();
+                        this.props.clickOverride();
+                        return;
+                    }
                     if (this.props.target !== "_blank") {
                         e.preventDefault();
                         let resolvedValues = typeof values === "function" ? values() : values;

package/src/library-components/Button.tsx

@@ -3,6 +3,7 @@ import { isNode, list, nextId } from "socket-function/src/misc";
 import { css } from "typesafecss";
 import { Querysub } from "../4-querysub/QuerysubController";
 import { qreact } from "../4-dom/qreact";
+import { isShowingModal } from "../5-diagnostics/Modal";
 
 export type ButtonProps = (
     preact.JSX.HTMLAttributes<HTMLButtonElement>
@@ -81,6 +82,7 @@ if (!isNode()) {
 let insideAnims = new Set<Watcher>();
 let keyUpListener = new Set<() => void>();
 export function triggerKeyDown(e: KeyboardEvent, forceAmbient = true) {
+    if (Querysub.localRead(() => isShowingModal())) return;
     let isAmbientEvent = (
         e.target === document.body
         // Some elements are commonly selected, but shouldn't handling key events

package/src/library-components/SyncedController.ts

@@ -1,10 +1,11 @@
 import { atomic, atomicObjectWrite, atomicObjectWriteNoFreeze, doAtomicWrites, proxyWatcher } from "../../src/2-proxy/PathValueProxyWatcher";
 import { Querysub } from "../../src/4-querysub/Querysub";
 import { SocketFunction } from "socket-function/SocketFunction";
-import { SocketRegistered } from "socket-function/SocketFunctionTypes";
+import { FullCallType, SocketRegistered } from "socket-function/SocketFunctionTypes";
 import { onHotReload } from "socket-function/hot/HotReloadController";
 import { cache } from "socket-function/src/caching";
 import { nextId } from "socket-function/src/misc";
+import { getCallObj } from "socket-function/src/nodeProxy";
 import { MaybePromise } from "socket-function/src/types";
 
 // IMPORTANT! See cacheAsyncSynced if you just want to run promise functions
@@ -37,6 +38,7 @@ let syncedData = Querysub.createLocalSchema<{
                 [argsHash: string]: {
                     promise: Promise<unknown> | undefined;
                     result?: { result: unknown } | { error: Error };
+                    invalidated?: boolean;
                 }
             }
         }
@@ -46,20 +48,58 @@ let syncedData = Querysub.createLocalSchema<{
 type RemapFunction<T> = T extends (...args: infer Args) => Promise<infer Return>
     ? {
         (...args: Args): Return | undefined;
+        promise(...args: Args): Promise<Return>;
         reset(...args: Args): void;
         resetAll(): void;
+
+        refresh(...args: Args): void;
+        refreshAll(): void;
+        isAnyLoading(): boolean;
+        setCache(config: { args: Args, result: Return }): void;
     }
     : T;
-export function getSyncedController<T extends SocketRegistered>(controller: T): {
+
+// key =>
+const writeWatchers = new Map<string, {
+    controllerId: string;
+    fncName: string;
+}[]>();
+
+export function getSyncedController<T extends SocketRegistered>(
+    controller: T,
+    config?: {
+        /** When a controller call for a write finishes, we refresh all readers.
+         *      - Invalidation is global, across all controllers.
+         */
+        reads?: { [key in keyof T["nodes"][""]]?: string[]; };
+        writes?: { [key in keyof T["nodes"][""]]?: string[]; };
+    }
+): {
     (nodeId: string): {
         [fnc in keyof T["nodes"][""]]: RemapFunction<T["nodes"][""][fnc]>;
     } & {
         resetAll(): void;
+        refreshAll(): void;
+        isAnyLoading(): boolean;
     };
     resetAll(): void;
+    refreshAll(): void;
+    isAnyLoading(): boolean;
 } {
     let id = nextId();
     controllerIds.add(id);
+
+    for (let [fncName, keys] of Object.entries(config?.reads ?? {})) {
+        for (let key of keys || []) {
+            let watcherList = writeWatchers.get(key);
+            if (!watcherList) {
+                watcherList = [];
+                writeWatchers.set(key, watcherList);
+            }
+            watcherList.push({ controllerId: id, fncName });
+        }
+    }
+
     let result = cache((nodeId: string) => {
         SocketFunction.onNextDisconnect(nodeId, () => {
             Querysub.commitLocal(() => {
@@ -91,36 +131,76 @@ export function getSyncedController<T extends SocketRegistered>(controller: T):
                         });
                     };
                 }
+                if (fncNameUntyped === "refreshAll") {
+                    return () => {
+                        return Querysub.commitLocal(() => {
+                            for (let fnc in syncedData()[id][nodeId]) {
+                                for (let argsHash in syncedData()[id][nodeId][fnc]) {
+                                    syncedData()[id][nodeId][fnc][argsHash].invalidated = true;
+                                }
+                            }
+                        });
+                    };
+                }
+                if (fncNameUntyped === "isAnyLoading") {
+                    return () => {
+                        return Querysub.commitLocal(() => {
+                            for (let fnc in syncedData()[id][nodeId]) {
+                                for (let argsHash in syncedData()[id][nodeId][fnc]) {
+                                    if (atomic(syncedData()[id][nodeId][fnc][argsHash].promise)) {
+                                        return true;
+                                    }
+                                }
+                            }
+                        });
+                    };
+                }
                 let fncName = fncNameUntyped;
                 function call(...args: any[]) {
                     return Querysub.commitLocal(() => {
                         let argsHash = JSON.stringify(args);
                         let obj = syncedData()[id][nodeId][fncName][argsHash];
 
+                        let result = atomic(obj.result);
                         let promise = atomic(obj.promise);
-                        if (!promise) {
-                            //console.log("calling", fncName, args);
-                            promise = controller.nodes[nodeId][fncName](...args) as Promise<unknown>;
+
+                        // NOTE: If we are invalidated when the promise is running, nothing happens (as we don't watch invalidated if we are running with a promise). BUT, if the promise isn't running, we will run again, and start running again. In this way we don't queue up a lot if we invalidate a lot, but we do always run again after the invalidation to get the latest result!
+                        if (!promise && (!result || atomic(obj.invalidated))) {
+                            let promise = controller.nodes[nodeId][fncName](...args) as Promise<unknown>;
                             doAtomicWrites(() => {
                                 obj.promise = promise;
                             });
                             promise.then(
                                 result => {
-                                    //console.log("result", result);
                                     Querysub.commitLocal(() => {
                                         obj.result = atomicObjectWriteNoFreeze({ result });
+                                        obj.promise = undefined;
+
+                                        let root = syncedData();
+                                        for (let writesTo of config?.writes?.[fncName] || []) {
+                                            for (let watcher of writeWatchers.get(writesTo) || []) {
+                                                for (let nodeId in root[watcher.controllerId]) {
+                                                    for (let fnc in root[watcher.controllerId][nodeId]) {
+                                                        for (let argsHash in root[watcher.controllerId][nodeId][fnc]) {
+                                                            let obj = root[watcher.controllerId][nodeId][fnc][argsHash];
+                                                            obj.invalidated = true;
+                                                        }
+                                                    }
+                                                }
+                                            }
+                                        }
                                     });
                                 },
                                 error => {
-                                    //console.log("error", error);
                                     Querysub.commitLocal(() => {
                                         obj.result = atomicObjectWriteNoFreeze({ error });
+                                        obj.promise = undefined;
                                     });
                                 }
                             );
                         }
 
-                        let result = atomic(obj.result);
+
                         if (result) {
                             if ("error" in result) {
                                 throw result.error;
@@ -131,6 +211,16 @@ export function getSyncedController<T extends SocketRegistered>(controller: T):
                         return undefined;
                     });
                 }
+                call.promise = (...args: any[]) => {
+                    call(...args);
+                    let argsHash = JSON.stringify(args);
+                    let promise = atomic(syncedData()[id][nodeId][fncName][argsHash].promise);
+                    if (!promise) {
+                        debugger;
+                        throw new Error(`Impossible, called function, but promise is not found for ${fncName}`);
+                    }
+                    return promise;
+                };
                 call.reset = (...args: any[]) => {
                     return Querysub.commitLocal(() => {
                         let argsHash = JSON.stringify(args);
@@ -148,7 +238,36 @@ export function getSyncedController<T extends SocketRegistered>(controller: T):
                         delete syncedData()[id][nodeId][fncName];
                     });
                 };
-
+                call.refresh = (...args: any[]) => {
+                    return Querysub.commitLocal(() => {
+                        let argsHash = JSON.stringify(args);
+                        let obj = syncedData()[id][nodeId][fncName][argsHash];
+                        obj.invalidated = true;
+                    });
+                };
+                call.refreshAll = () => {
+                    return Querysub.commitLocal(() => {
+                        for (let argsHash in syncedData()[id][nodeId][fncName]) {
+                            syncedData()[id][nodeId][fncName][argsHash].invalidated = true;
+                        }
+                    });
+                };
+                call.isAnyLoading = () => {
+                    return Querysub.commitLocal(() => {
+                        for (let argsHash in syncedData()[id][nodeId][fncName]) {
+                            if (atomic(syncedData()[id][nodeId][fncName][argsHash].promise)) {
+                                return true;
+                            }
+                        }
+                    });
+                };
+                call.setCache = (config: { args: any[], result: any }) => {
+                    return Querysub.commitLocal(() => {
+                        let argsHash = JSON.stringify(config.args);
+                        syncedData()[id][nodeId][fncName][argsHash].promise = undefined;
+                        syncedData()[id][nodeId][fncName][argsHash].result = atomicObjectWriteNoFreeze({ result: config.result });
+                    });
+                };
                 return call;
             },
         });
@@ -167,5 +286,29 @@ export function getSyncedController<T extends SocketRegistered>(controller: T):
             }
         });
     };
+    result.refreshAll = () => {
+        return Querysub.commitLocal(() => {
+            for (let nodeId in syncedData()[id]) {
+                for (let fnc in syncedData()[id][nodeId]) {
+                    for (let argsHash in syncedData()[id][nodeId][fnc]) {
+                        syncedData()[id][nodeId][fnc][argsHash].invalidated = true;
+                    }
+                }
+            }
+        });
+    };
+    result.isAnyLoading = () => {
+        return Querysub.commitLocal(() => {
+            for (let nodeId in syncedData()[id]) {
+                for (let fnc in syncedData()[id][nodeId]) {
+                    for (let argsHash in syncedData()[id][nodeId][fnc]) {
+                        if (atomic(syncedData()[id][nodeId][fnc][argsHash].promise)) {
+                            return true;
+                        }
+                    }
+                }
+            }
+        });
+    };
     return result;
 }

package/src/logModuleLoadTimes.ts

@@ -0,0 +1,45 @@
+import { sort } from "socket-function/src/misc";
+import { isDefined } from "./misc";
+import { formatTime } from "socket-function/src/formatting/format";
+
+export function logLoadTime() {
+    let allModules = Object.values(require.cache).filter(isDefined);
+    let now = Date.now();
+    sort(allModules, x => x?.evalStartTime || now);
+    let timePerModule = new Map<NodeJS.Module, number>();
+    let moduleStack: NodeJS.Module[] = [];
+    let lastTime = allModules[0].evalStartTime || now;
+    function emitAtTime(time: number) {
+        for (let i = moduleStack.length - 1; i >= 0; i--) {
+            let module = moduleStack[i];
+            let endTime = module.evalEndTime || now;
+            let prevTime = timePerModule.get(module) || 0;
+            if (endTime < time) {
+                // If it ends, pop it, and account for the partial time
+                moduleStack.pop();
+                let duration = endTime - lastTime;
+                prevTime += duration;
+                lastTime = endTime;
+                timePerModule.set(module, prevTime);
+            } else {
+                // Account for the full time
+                let duration = time - lastTime;
+                prevTime += duration;
+                timePerModule.set(module, prevTime);
+                break;
+            }
+        }
+        lastTime = time;
+    }
+    for (let module of allModules) {
+        emitAtTime(module.evalStartTime || now);
+        moduleStack.push(module);
+    }
+    emitAtTime(now);
+
+    let flat = Array.from(timePerModule);
+    sort(flat, x => -x[1]);
+    for (let [module, time] of flat.slice(0, 100).reverse()) {
+        console.log(`${formatTime(time)} ${module.filename}`);
+    }
+}