querysub 0.130.0 → 0.131.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.130.0",
+  "version": "0.131.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-0-hooks/hooks.ts

@@ -1,3 +1,6 @@
+import type { EdgeNodeConfig } from "../4-deploy/edgeNodes";
+import type { ExtraMetadata } from "../5-diagnostics/nodeMetadata";
+
 // Hooks, to allow function implementations to be declared after their first call
 //  (so static calls work).
 
@@ -89,7 +92,6 @@ export const onTimeProfile = createHookFunction<
     ) => void
 >("onTimeProfile");
 
-import type { ExtraMetadata } from "../5-diagnostics/nodeMetadata";
 export const registerNodeMetadata = createHookFunction<
     (
         metadata: ExtraMetadata
@@ -116,3 +118,16 @@ export const logNodeStateStats = createHookFunction<
 export const isManagementUser = createHookFunctionReturn<
     () => Promise<boolean>
 >("isManagementUser");
+
+
+
+declare global {
+    var BOOTED_EDGE_NODE: EdgeNodeConfig | undefined;
+}
+
+
+export function getBootedEdgeNode(): EdgeNodeConfig {
+    let edgeNode = globalThis.BOOTED_EDGE_NODE;
+    if (!edgeNode) throw new Error(`No edge node booted? This should be impossible.`);
+    return edgeNode;
+}

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -23,7 +23,7 @@ import dns from "dns/promises";
 import { isDefined } from "../misc";
 import { diskLog, noDiskLogPrefix } from "../diagnostics/logs/diskLogger";
 import { getDebuggerUrl } from "../diagnostics/listenOnDebugger";
-import { getBootedEdgeNode } from "../4-deploy/edgeBootstrap";
+import { getBootedEdgeNode } from "../-0-hooks/hooks";
 
 let HEARTBEAT_INTERVAL = timeInMinute * 2;
 // Interval which we check other heartbeats

package/src/3-path-functions/PathFunctionRunner.ts

@@ -665,9 +665,9 @@ export async function preloadFunctions(specs: FunctionSpec[]) {
     let nodeIds = await getControllerNodeIdList(FunctionPreloadController);
     await Promise.allSettled(nodeIds.map(async nodeId => {
         let controller = FunctionPreloadController.nodes[nodeId.nodeId];
-        console.log(blue(`Preloading functions on ${nodeId}`));
+        console.log(blue(`Preloading functions on ${String(nodeId)}`));
         await errorToUndefined(controller.preloadFunctions(specs));
-        console.log(blue(`Finished preloading functions on ${nodeId}`));
+        console.log(blue(`Finished preloading functions on ${String(nodeId)}`));
     }));
 }
 

package/src/4-deploy/deploySchema.ts

@@ -38,36 +38,4 @@ export function setLiveDeployedHash(config: {
             deploySchema()[getDomain()].deploy.history[nextId()] = hashObj;
         },
     });
-}
-
-//todonext
-//  - Function to dynamically deploy edge service based on hash (try on old hashes)
-//  - Watch live hash, and when it changes, deploy new edge service
-//      - Add flag to deploy script which sets it (with all the flags, so we can just deploy it)
-//  - Update edge node code to pass live hash, and try to connect to a service with that
-
-//todonext
-/*
-Force client refresh mechanism / update notification ON `yarn deploy`
-    - deployonlycode / deployonlyui / deploynotifyforced
-    - OH! Actually...
-    - This HAS to use a synced value!
-    - Add a new synced value which specifies the UI hash!
-        - THEN! Have front-end servers dynamically launch this services with this hash when it's set (loading the code and then serving the specific path, with the hash, etc)
-        - Also, serve this to clients!
-    - Clients will watch this value (normally, like any other value), and notify the user when it is out of date, letting them ignore it, but eventually forcing a refresh
-    - AND, support preloading it
-    - So our order will be
-        (git push)
-        preload functions
-        preload UI
-        atomically set function hashes + ui hashes
-        clients are notified they should refresh
-        new clients use services with the latest hash (which servers are already hosting)
-        functions will run this hash
-    - Ah, also... add a flag which tells us NOT to use the service HTML cache
-        - Remove it when used
-        - Required on refresh, otherwise we might refresh and use an outdated service
-        
-ALSO, have FunctionRunner watch the hash using the get/watch mechanism (for preloading, as it will dynamically load the code depending on what is really deployed anyways...)
-*/
+}

package/src/4-deploy/edgeBootstrap.ts

@@ -3,23 +3,17 @@ import { isServer } from "../config2";
 import { EdgeNodeConfig, EdgeNodesIndex } from "./edgeNodes";
 import { timeInMinute, timeInSecond } from "socket-function/src/misc";
 import { measureBlock } from "socket-function/src/profiling/measure";
+import { URLParam } from "../library-components/URLParam";
 
 module.hotreload = true;
 module.noserverhotreload = false;
 
+const liveHashOverrideParam = "liveHashOverride";
+export const liveHashOverrideURL = new URLParam(liveHashOverrideParam, undefined as undefined | {
+    liveHash: string;
+    time: number;
+});
 
-declare global {
-    var BOOTED_EDGE_NODE: EdgeNodeConfig | undefined;
-}
-
-
-export function getBootedEdgeNode(): EdgeNodeConfig {
-    if (isServer()) throw new Error(`getBootedEdgeNode is not available on the server`);
-
-    let edgeNode = globalThis.BOOTED_EDGE_NODE;
-    if (!edgeNode) throw new Error(`No edge node booted? This should be impossible.`);
-    return edgeNode;
-}
 
 let getCachedConfig = cache(async (url: string): Promise<EdgeNodesIndex | undefined> => {
     setTimeout(() => {
@@ -217,6 +211,26 @@ async function edgeNodeFunction(config: {
 
     let cachedConfig = config.cachedConfig;
 
+    let liveHashOverride = "";
+    let liveHashOverrideExpiryTime = 0;
+
+    const liveHashOverrideParam2: typeof liveHashOverrideParam = "liveHashOverride";
+    let searchParams = new URLSearchParams(document.location.search);
+    let liveHashOverrideObj = searchParams.get(liveHashOverrideParam2);
+    // JUST use it on the first loop, so if we can't find any matches, we ignore it.
+    if (liveHashOverrideObj) {
+        try {
+            let liveHashOverrideObject = JSON.parse(liveHashOverrideObj);
+            liveHashOverrideExpiryTime = liveHashOverrideObject.time + 1000 * 30;
+            liveHashOverride = liveHashOverrideObject.liveHash;
+        } catch (e: any) {
+            console.error(`Error parsing liveHashOverride ${liveHashOverride}: ${e.stack}`);
+        }
+        // Remove it from the search params
+        searchParams.delete(liveHashOverrideParam2);
+        document.location.search = searchParams.toString();
+    }
+
     let progressUI = createProgressUI();
     progressUI.setMessage("Finding available server...");
     while (true) {
@@ -226,9 +240,12 @@ async function edgeNodeFunction(config: {
         } catch (e) {
             console.error(e);
         }
-        progressUI.setMessage("No available servers, retrying in 15s");
-        console.log(`Failing to boot, trying in 15 seconds`);
-        await new Promise(resolve => setTimeout(resolve, 1000 * 15));
+        progressUI.setMessage("No available servers, retrying soon");
+        let time = 1000 * 15;
+        if (Date.now() < liveHashOverrideExpiryTime) {
+            time = 1000 * 3;
+        }
+        await new Promise(resolve => setTimeout(resolve, time));
     }
     progressUI.stop();
 
@@ -260,6 +277,16 @@ async function edgeNodeFunction(config: {
     async function getEdgeNodeConfig(): Promise<EdgeNodeConfig> {
         let edgeIndex = cachedConfig || await (await fetch(config.edgeNodeConfigURL)).json() as EdgeNodesIndex;
         cachedConfig = undefined;
+        let liveHashForced = false;
+        if (Date.now() < liveHashOverrideExpiryTime) {
+            liveHashForced = true;
+            edgeIndex.liveHash = liveHashOverride;
+        }
+
+        // export const liveHashOverrideURL = new URLParam(liveHashOverrideParam, undefined as undefined | {
+        //     liveHash: string;
+        //     time: number;
+        // });
 
         console.group(`Found edge nodes`);
 
@@ -309,6 +336,9 @@ async function edgeNodeFunction(config: {
 
         let liveNodes = edgeNodes.filter(x => x.gitHash === edgeIndex.liveHash);
         if (liveNodes.length === 0) {
+            if (liveHashForced) {
+                throw new Error(`Could not find any live nodes (${edgeIndex.liveHash}), and the live hash is forced.`);
+            }
             let latestHash = edgeNodes[0].gitHash;
             console.warn(`Could not find any live nodes (${edgeIndex.liveHash}), falling back to latest hash: ${latestHash}`);
             liveNodes = edgeNodes.filter(x => x.gitHash === latestHash);

package/src/4-deploy/edgeClientWatcher.tsx

@@ -0,0 +1,187 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { Querysub } from "../4-querysub/QuerysubController";
+import { deploySchema } from "./deploySchema";
+import { getDomain } from "../config";
+import { throttleFunction, timeInMinute } from "socket-function/src/misc";
+import { isNode } from "typesafecss";
+import { logErrors, timeoutToError } from "../errors";
+import { blue, red } from "socket-function/src/formatting/logColors";
+import { showModal } from "../5-diagnostics/Modal";
+import { qreact } from "../4-dom/qreact";
+import { liveHashOverrideURL } from "./edgeBootstrap";
+import { css } from "typesafecss";
+import { formatTime } from "socket-function/src/formatting/format";
+import { delay } from "socket-function/src/batching";
+
+export function startEdgeNotifier() {
+    SocketFunction.expose(EdgeNotifierController);
+    let lastHash = "";
+    Querysub.createWatcher(() => {
+        let liveHash = deploySchema()[getDomain()].deploy.live.hash;
+        if (!Querysub.isAllSynced()) return;
+        if (liveHash === lastHash) return;
+        let refreshThresholdTime = deploySchema()[getDomain()].deploy.live.refreshThresholdTime;
+        lastHash = liveHash;
+        void notifyClients(liveHash, refreshThresholdTime);
+    });
+}
+
+let watchingClientNodes = new Set<string>();
+const notifyClients = throttleFunction(100, async function notifyClients(liveHash: string, refreshThresholdTime: number) {
+    await Promise.allSettled(Array.from(watchingClientNodes).map(async clientNodeId => {
+        try {
+            await timeoutToError(5000, EdgeNotifierClientController.nodes[clientNodeId].onLiveHashChange(liveHash, refreshThresholdTime), () => new Error(`Timeout for watch updates on node ${clientNodeId}`));
+        } catch (e) {
+            console.log(blue(`Client unwatching liveHash ${clientNodeId}`), e);
+            watchingClientNodes.delete(clientNodeId);
+        }
+    }));
+});
+
+// Track current notification state
+let currentNotification: { close: () => void } | null = null;
+let curHash = "";
+function onLiveHashChange(liveHash: string, refreshThresholdTime: number) {
+    let notifyIntervals = [0, 0.1, 0.5, 1];
+    console.log(blue(`Client liveHash changed ${liveHash}`));
+    let skipFirst = false;
+    if (currentNotification) {
+        currentNotification.close();
+        currentNotification = null;
+        skipFirst = true;
+    }
+    curHash = liveHash;
+
+    // Start notification loop
+    void (async () => {
+        // Show notifications at intervals
+        for (let i = 0; i < notifyIntervals.length; i++) {
+            // Don't show if a newer notification is active
+            if (curHash !== liveHash) return;
+
+            let waitDuration = (notifyIntervals[i + 1] - notifyIntervals[i]) * refreshThresholdTime;
+            if (i >= notifyIntervals.length - 1 && waitDuration <= 30 * 1000) continue;
+
+            // Update the URL override for manual refreshes
+            Querysub.localCommit(() => {
+                liveHashOverrideURL.value = { liveHash, time: Date.now() };
+            });
+            if (!skipFirst) {
+                // Show notification modal
+                currentNotification = {
+                    close: showModal({
+                        content: (
+                            <div
+                                title={`Live Hash: ${liveHash}`}
+                                className={
+                                    css.vbox(10).pad(20)
+                                        .hsla(0, 0, 0, 0.8)
+                                        .position("fixed")
+                                        .bottom(10)
+                                        .right(10)
+                                        .zIndex(1000)
+                                }
+                                onClick={() => {
+                                    if (currentNotification) {
+                                        currentNotification.close();
+                                        currentNotification = null;
+                                    }
+                                }}
+                            >
+                                <div className={css.vbox(10).maxWidth(250)}>
+                                    <h3 className={css.margin(0)}>Server Update Available</h3>
+                                    <div>The server has been updated. Please refresh the page to ensure you don't experience incompatibility issues.</div>
+                                </div>
+                                <div className={css.hbox(10).justifyContent("flex-end")}>
+                                    <button
+                                        className={css.pad(8, 16).hsl(200, 50, 50).color("white").pointer}
+                                        onClick={() => {
+                                            Querysub.localCommit(() => {
+                                                liveHashOverrideURL.value = { liveHash, time: Date.now() };
+                                            });
+                                            window.location.reload();
+                                        }}
+                                    >
+                                        Refresh Now
+                                    </button>
+                                    <button
+                                        className={css.pad(8, 16).pointer}
+                                        onClick={() => {
+                                            if (currentNotification) {
+                                                currentNotification.close();
+                                                currentNotification = null;
+                                            }
+                                        }}
+                                    >
+                                        Dismiss
+                                    </button>
+                                </div>
+                            </div>
+                        )
+                    }).close
+                };
+            }
+            skipFirst = false;
+
+            console.log(red(`Notify again in ${formatTime(waitDuration)}`));
+            await delay(waitDuration);
+        }
+
+        console.log(red(`Force refresh after 30 seconds`));
+        await delay(30 * 1000);
+
+        // Only force refresh if this is still the current notification
+        if (curHash === liveHash) {
+            window.location.reload();
+        }
+    })();
+}
+
+const EdgeNotifierClientController = SocketFunction.register(
+    "EdgeNotifierClientController-ed122e41-6ad2-4161-9492-a3f8414bd4f0",
+    new class EdgeNotifierClient {
+        public async onLiveHashChange(liveHash: string, refreshThresholdTime: number) {
+            onLiveHashChange(liveHash, refreshThresholdTime);
+        }
+    },
+    () => ({
+        onLiveHashChange: {},
+    }),
+    () => ({}),
+    {
+        noAutoExpose: true,
+    }
+);
+
+if (!isNode()) {
+    setImmediate(() => {
+        void Querysub.optionalStartupWait().finally(() => {
+            logErrors(EdgeNotifierController.nodes[SocketFunction.browserNodeId()].watchUpdates());
+        });
+        SocketFunction.expose(EdgeNotifierClientController);
+    });
+}
+
+class EdgeNotifierControllerBase {
+    public async watchUpdates() {
+        let nodeId = SocketFunction.getCaller().nodeId;
+        console.log(blue(`Client watching liveHash ${nodeId}`));
+        watchingClientNodes.add(nodeId);
+        Querysub.onNextDisconnect(nodeId, () => {
+            watchingClientNodes.delete(nodeId);
+        });
+    }
+}
+
+const EdgeNotifierController = SocketFunction.register(
+    "EdgeNotifierController-b11fbbf0-ef5e-4ba3-80ae-d5b1b8124f2d",
+    new EdgeNotifierControllerBase(),
+    () => ({
+        watchUpdates: {},
+    }),
+    () => ({}),
+    {
+        noAutoExpose: true,
+    }
+);
+

package/src/4-deploy/edgeNodes.ts

@@ -24,6 +24,7 @@ import { deploySchema } from "./deploySchema";
 import { proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
 import { Querysub } from "../4-querysub/QuerysubController";
 import { onEdgeNodesChanged } from "./edgeBootstrap";
+import { startEdgeNotifier } from "./edgeClientWatcher";
 
 const UPDATE_POLL_INTERVAL = timeInMinute;
 const DEAD_NODE_COUNT_THRESHOLD = 15;
@@ -177,8 +178,6 @@ const loadEntryPointsByHash = runInSerial(async function loadEntryPointsByHash(c
     await edgeNodeStorage.set(getNextNodePath(), Buffer.from(JSON.stringify(edgeNodeConfig)));
     console.log(magenta(`Deployed edge node`), edgeNodeConfig);
 
-    SocketFunction.expose(EdgeNodeController);
-
     await updateEdgeNodesFile();
 
     onEdgeNodesChanged();
@@ -196,6 +195,8 @@ export const getEdgeNodeConfigURL = lazy(async () => {
 const startUpdateLoop = lazy(async () => {
     await getEdgeNodeConfigURL();
     await runInfinitePollCallAtStart(UPDATE_POLL_INTERVAL, updateLoop);
+    SocketFunction.expose(EdgeNodeController);
+    startEdgeNotifier();
 });
 
 async function updateLoop() {
@@ -272,9 +273,9 @@ export async function preloadUI(hash: string) {
     let nodeIds = await getControllerNodeIdList(EdgeNodeController);
     await Promise.allSettled(nodeIds.map(async nodeId => {
         let controller = EdgeNodeController.nodes[nodeId.nodeId];
-        console.log(blue(`Preloading UI on ${nodeId}, hash: ${hash}`));
+        console.log(blue(`Preloading UI on ${String(nodeId)}, hash: ${hash}`));
         await errorToUndefined(controller.preloadUI({ hash }));
-        console.log(blue(`Finished preloading UI on ${nodeId}`));
+        console.log(blue(`Finished preloading UI on ${String(nodeId)}`));
     }));
 }
 
@@ -309,23 +310,3 @@ const EdgeNodeController = SocketFunction.register(
     }
 );
 
-// async function main() {
-//     await loadEntryPointsByHash({
-//         host: "127-0-0-1.querysub.com:1111",
-//         entryPaths: ["./src/browser.ts"],
-//         hash: "7b1eb4934d5bdfe6d0f7076049c4b7decad4e645",
-//     });
-// }
-// main().catch(console.error).finally(() => process.exit());
-
-
-
-//todonext
-//  3) Verify the deploy code works on the live site
-//      - I think it isn't? But maybe we just need to do-update again
-
-
-//  7) Client live hash watch code + notify + refresh + flag to ignore the baked in live hash + edge node
-//  - Test by pushing, deploying, and then we should immediately see a notify to refresh
-
-//  8) VERIFY writing and permissions still works

package/src/4-querysub/Querysub.ts

@@ -290,6 +290,9 @@ export class Querysub {
     public static isAllSynced = Querysub.allSynced;
     public static isAnyUnsynced = Querysub.anyUnsynced;
 
+    public static onNextDisconnect = SocketFunction.onNextDisconnect;
+    public static isNodeConnected = SocketFunction.isNodeConnected;
+
     public static pathHasAnyWatchers(get: () => unknown) {
         return clientWatcher.pathHasAnyWatchers(getProxyPath(get));
     }

package/src/library-components/URLParam.ts

@@ -63,11 +63,13 @@ export function createURLSync<T>(urlKey: string, defaultValue: T, config?: {
     if (config?.storage === "localStorage") {
         localStorageKeys.add(urlKey);
     }
-    Querysub.localCommit(() => {
-        data().defaults[urlKey] = defaultValue;
-        if (!(urlKey in loadSearchCache)) {
-            data().params[urlKey] = deepCloneJSON(defaultValue);
-        }
+    setImmediate(() => {
+        Querysub.localCommit(() => {
+            data().defaults[urlKey] = defaultValue;
+            if (!(urlKey in loadSearchCache)) {
+                data().params[urlKey] = deepCloneJSON(defaultValue);
+            }
+        });
     });
     function deleteKeys(obj: any) {
         if (!canHaveChildren(obj)) return;
@@ -222,9 +224,11 @@ function watchURLForUpdates() {
 }
 
 if (!isNode()) {
-    loadParamsFromURL();
-    watchURLForUpdates();
-    syncStateToURL();
+    setImmediate(() => {
+        loadParamsFromURL();
+        watchURLForUpdates();
+        syncStateToURL();
+    });
 }
 
 export function parseSearchString(search: string): { [key: string]: unknown } {