querysub 0.193.0 → 0.195.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.193.0",
+  "version": "0.195.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -20,7 +20,7 @@ import { isDefined } from "../misc";
 import { diskLog } from "../diagnostics/logs/diskLogger";
 import { getBootedEdgeNode } from "../-0-hooks/hooks";
 
-let HEARTBEAT_INTERVAL = timeInMinute * 2;
+let HEARTBEAT_INTERVAL = timeInMinute * 15;
 // Interval which we check other heartbeats
 let CHECK_INTERVAL = HEARTBEAT_INTERVAL;
 // If the heartbeat is older than thing, it fails the dead check
@@ -30,19 +30,19 @@ let DEAD_THRESHOLD = HEARTBEAT_INTERVAL * 2;
 let DEAD_CHECK_COUNT = 4;
 // If we find we are unable to write our heartbeat, we have to kill our own process. Otherwise it may
 //  be too out of sync, and might commit unverified data to the disk.
-let SUICIDE_HEARTBEAT_THRESHOLD = timeInMinute * 15;
+let SUICIDE_HEARTBEAT_THRESHOLD = HEARTBEAT_INTERVAL * 4;
 
-let CLIENTSIDE_POLL_RATE = timeInMinute * 5;
+let CLIENTSIDE_POLL_RATE = timeInMinute * 30;
 
 // We can't poll backblaze too often. One a minute starts to cost around 10 cents per month.
 //  So once a second would cost 6 USD per month, per service... which starts to get expensive.
-let DISK_AUDIT_RATE = timeInMinute * 5;
+let DISK_AUDIT_RATE = timeInMinute * 15;
 // We CAN poll our API frequently. The overhead for calls should be far less than 1ms, and
 //  the bandwidth should also be effectively nothing. Maybe 2.5GB per month if we send a
 //  request per second, and each request is 1000 bytes (which as we use websockets, it
-//  probably is less than that). Which is around 2.5 cents on backblaze IF we go over
+//  probably is less than that). Which is around 2.5 cents on digital ocean IF we go over
 //  our 1TB/month allowance.
-let API_AUDIT_RATE = timeInSecond * 15;
+let API_AUDIT_RATE = timeInSecond * 30;
 let API_AUDIT_COUNT = 12;
 
 
@@ -392,7 +392,7 @@ async function runMainSyncLoops(discoveryReady: PromiseObj<void>) {
 
     console.log(magenta(`Node discovery is loaded`));
 
-    await runInfinitePoll(HEARTBEAT_INTERVAL, async function nodeDiscoverHeartbeat() {
+    await runInfinitePollCallAtStart(HEARTBEAT_INTERVAL, async function nodeDiscoverHeartbeat() {
         // If we waited too long, other nodes might think we are dead. In which case, we SHOULD terminate.
         if (!isNoNetwork()) {
             // FIRST, verify we didn't delay too long (to make sure we kill any nodes that were disconnected

package/src/2-proxy/PathValueProxyWatcher.ts

@@ -759,9 +759,11 @@ export class PathValueProxyWatcher {
         ) {
             if (!watcher.pendingWrites.has(pathStr)) {
                 let parentPathStr = pathStr;
+                let isLocal = pathStr.startsWith(LOCAL_DOMAIN_PATH);
+                let depthToData = isLocal ? 1 : DEPTH_TO_DATA;
                 while (true) {
                     parentPathStr = getParentPathStr(parentPathStr);
-                    if (getPathDepth(parentPathStr) < DEPTH_TO_DATA) break;
+                    if (getPathDepth(parentPathStr) < depthToData) break;
                     // We don't need to check all parents, as if any value is set, all parents will
                     //  likely have their specialObjectWriteValue set, due to a previous run of this function!
                     if (watcher.pendingWrites.has(parentPathStr)) break;

package/src/4-deploy/edgeNodes.ts

@@ -26,7 +26,7 @@ import { onEdgeNodesChanged } from "./edgeBootstrap";
 import { startEdgeNotifier } from "./edgeClientWatcher";
 import { getGitRefLive, getGitURLLive } from "./git";
 
-const UPDATE_POLL_INTERVAL = timeInMinute;
+const UPDATE_POLL_INTERVAL = timeInMinute * 15;
 const DEAD_NODE_COUNT_THRESHOLD = 15;
 
 const edgeNodeStorage = isNodeTrue() && nestArchives("edgenodes/", getArchivesBackblazePublic(getDomain()));

package/src/deployManager/DeployPage.tsx

@@ -2,14 +2,16 @@ import { SocketFunction } from "socket-function/SocketFunction";
 import { qreact } from "../../src/4-dom/qreact";
 import { DeployController } from "./deploySchema";
 import { css } from "typesafecss";
+import { syncedIsAnyLoading } from "../library-components/SyncedController";
 
 export class DeployPage extends qreact.Component {
     render() {
         let controller = DeployController(SocketFunction.browserNodeId());
+        let allInfo = controller.getAllInfo();
         return <div className={css.vbox(10)}>
             <h1>DeployPage</h1>
             <div className={css.whiteSpace("pre-wrap")}>
-                {JSON.stringify(controller.getAllInfo(), null, 4)}
+                {JSON.stringify(allInfo, null, 4)}
             </div>
         </div>;
     }
@@ -18,20 +20,6 @@ export class DeployPage extends qreact.Component {
 
 todonext
 
-# Domain config file
-- Have the domain configured via a file in the repo, so we don't have to set it on every single script
-    - This also makes it possible to use binary scripts directly in querysub without having to add a script in our package.json to set the domain parameter
-        - FORWARD the main scripts
-            function
-            function-public
-            server
-            server-public
-            deploy
-            ... anything that uses node_modules, which is a lot...
-        - And remove all --domain arguments (which is a lot as well)
-    - ALSO, support setting arbitrary config parameter in this config file?
-        - Maybe on a per script basis, giving scripts names.
-
 5) synced controller loading indicator
     - Probably which wataches all controllers (on all nodes), for any pending, shown globally
 6) Validate that our loading indicator work, as well as our notifications on changed configuration, etc

package/src/deployManager/machineApplyMain.ts

@@ -28,7 +28,7 @@ const getLiveMachineInfo = measureWrap(async function getLiveMachineInfo() {
     machineInfo.info.lscpu = await errorToUndefinedSilent(runPromise("lscpu")) || "";
     machineInfo.info.id = await errorToUndefinedSilent(runPromise("id")) || await errorToUndefinedSilent(runPromise("whoami")) || "";
 
-    // TODO: Populate services via checking tmux for special keywords ("-dply", probably...)
+    // TODO: Populate services via checking tmux for special keywords ("-apply", probably...)
 
     return machineInfo;
 });
@@ -38,20 +38,11 @@ const updateMachineInfo = measureWrap(async function updateMachineInfo() {
     await new DeployControllerBase().setMachineInfo(machineInfo.machineId, machineInfo);
 });
 
-//todonext
-//  The child process will be run with shell, and then we'll watch it (don't use runPromise)
-
-function getTotalTime(module: NodeJS.Module, now: number) {
-    return (module.evalEndTime || now) - (module.evalStartTime || now);
-}
 
 
 export async function machineApplyMain() {
-    console.log(getDomain());
-    process.exit();
-    // await Querysub.hostService("machine-apply");
-
-    // await updateMachineInfo();
+    await Querysub.hostService("machine-apply");
+    await updateMachineInfo();
 }
 
 machineApplyMain().catch(console.error);

package/src/diagnostics/managementPages.tsx

@@ -341,7 +341,6 @@ class ManagementRoot extends qreact.Component {
                         </div>
                     }
                 </div>
-
             </>
         );
     }

package/src/library-components/SyncedController.ts

@@ -1,9 +1,10 @@
-import { atomic, atomicObjectWrite, atomicObjectWriteNoFreeze, doAtomicWrites, proxyWatcher } from "../../src/2-proxy/PathValueProxyWatcher";
+import { atomic, atomicObjectWrite, atomicObjectWriteNoFreeze, doAtomicWrites, isTransparentValue, proxyWatcher, specialObjectWriteValue } from "../../src/2-proxy/PathValueProxyWatcher";
 import { Querysub } from "../../src/4-querysub/Querysub";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { FullCallType, SocketRegistered } from "socket-function/SocketFunctionTypes";
 import { onHotReload } from "socket-function/hot/HotReloadController";
 import { cache } from "socket-function/src/caching";
+import { formatTime } from "socket-function/src/formatting/format";
 import { nextId } from "socket-function/src/misc";
 import { getCallObj } from "socket-function/src/nodeProxy";
 import { MaybePromise } from "socket-function/src/types";
@@ -31,6 +32,22 @@ onHotReload(() => {
     });
 });
 
+export function syncedIsAnyLoading() {
+    return Querysub.fastRead(() => {
+        for (let controllerId in syncedData()) {
+            for (let nodeId in syncedData()[controllerId]) {
+                for (let fnc in syncedData()[controllerId][nodeId]) {
+                    for (let argsHash in syncedData()[controllerId][nodeId][fnc]) {
+                        if (atomic(syncedData()[controllerId][nodeId][fnc][argsHash].promise)) {
+                            return true;
+                        }
+                    }
+                }
+            }
+        }
+    });
+}
+
 let syncedData = Querysub.createLocalSchema<{
     [controllerId: string]: {
         [nodeId: string]: {
@@ -166,12 +183,20 @@ export function getSyncedController<T extends SocketRegistered>(
 
                         // NOTE: If we are invalidated when the promise is running, nothing happens (as we don't watch invalidated if we are running with a promise). BUT, if the promise isn't running, we will run again, and start running again. In this way we don't queue up a lot if we invalidate a lot, but we do always run again after the invalidation to get the latest result!
                         if (!promise && (!result || atomic(obj.invalidated))) {
+                            let timeStart = Date.now();
+                            function logFinished() {
+                                let duration = Date.now() - timeStart;
+                                if (duration > 500) {
+                                    console.warn(`Slow call ${fncName} took ${formatTime(duration)}`);
+                                }
+                            }
                             let promise = controller.nodes[nodeId][fncName](...args) as Promise<unknown>;
                             doAtomicWrites(() => {
                                 obj.promise = promise;
                             });
                             promise.then(
                                 result => {
+                                    logFinished();
                                     Querysub.commitLocal(() => {
                                         obj.result = atomicObjectWriteNoFreeze({ result });
                                         obj.promise = undefined;
@@ -192,6 +217,7 @@ export function getSyncedController<T extends SocketRegistered>(
                                     });
                                 },
                                 error => {
+                                    logFinished();
                                     Querysub.commitLocal(() => {
                                         obj.result = atomicObjectWriteNoFreeze({ error });
                                         obj.promise = undefined;

package/src/library-components/SyncedControllerLoadingIndicator.tsx

@@ -0,0 +1,50 @@
+import { css } from "typesafecss";
+import { qreact } from "../4-dom/qreact";
+import { syncedIsAnyLoading } from "./SyncedController";
+import { formatTime } from "socket-function/src/formatting/format";
+
+export class SyncedControllerLoadingIndicator extends qreact.Component {
+    timeLastLoaded = Date.now();
+    lastWasLoaded = false;
+    render() {
+        if (!syncedIsAnyLoading()) {
+            if (!this.lastWasLoaded) {
+                console.log(`Loaded all SyncedController calls in ${formatTime(Date.now() - this.timeLastLoaded)}`);
+            }
+            this.timeLastLoaded = Date.now();
+            this.lastWasLoaded = true;
+            return undefined;
+        }
+        this.lastWasLoaded = false;
+        const spinAnimationClass = `SyncedControllerLoadingIndicator-spin`;
+
+        return <div className={
+            css.position("fixed").bottom(0).left(0).zIndex(1000)
+                .padding("12px 16px")
+                .background("rgba(0, 0, 0, 0.8)")
+                .color("white")
+                .borderTopRightRadius(8)
+                .hbox(8)
+        }>
+            <div className={
+                css.width("16px").height("16px")
+                    .border("2px solid rgba(255, 255, 255, 0.3)")
+                    .borderTop("2px solid white")
+                    .borderRadius("50%")
+                + " " + spinAnimationClass
+            }></div>
+            <span>Syncing data...</span>
+            <style>
+                {`
+                    @keyframes ${spinAnimationClass} {
+                        0% { transform: rotate(0deg); }
+                        100% { transform: rotate(360deg); }
+                    }
+                    .${spinAnimationClass} {
+                        animation: ${spinAnimationClass} 1s linear infinite;
+                    }
+                `}
+            </style>
+        </div>;
+    }
+}