querysub 0.260.0 → 0.262.0

package/src/deployManager/machineSchema.ts

@@ -1,4 +1,4 @@
-import { isNodeTrue, timeInMinute } from "socket-function/src/misc";
+import { isNodeTrue, list, timeInMinute, timeInSecond } from "socket-function/src/misc";
 import { nestArchives } from "../-a-archives/archives";
 import { getArchivesBackblaze } from "../-a-archives/archivesBackBlaze";
 import { getDomain } from "../config";
@@ -10,7 +10,7 @@ import { getSyncedController } from "../library-components/SyncedController";
 import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
 import { extractType } from "../misc/extractType";
 import { getOwnMachineId } from "../-a-auth/certs";
-import { runInSerial, runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { delay, runInSerial, runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { lazy } from "socket-function/src/caching";
 import { errorToUndefinedSilent } from "../errors";
 import { commitAndPush, getGitRefInfo, getGitRefLive, getGitURLLive, getGitUncommitted, getLatestRefOnUpstreamBranch, setGitRef } from "../4-deploy/git";
@@ -19,6 +19,9 @@ import path from "path";
 import fs from "fs";
 import { runPromise } from "../functional/runCommand";
 import { isNode } from "typesafecss";
+import { DeployProgress, deployFunctions, deployGetFunctions } from "../4-deploy/deployFunctions";
+import { FunctionSpec, functionSchema } from "../3-path-functions/PathFunctionRunner";
+import { Querysub } from "../4-querysub/QuerysubController";
 
 const SERVICE_FOLDER_NAME = "machine-services";
 export const SERVICE_FOLDER = `${SERVICE_FOLDER_NAME}/`;
@@ -288,8 +291,73 @@ export class MachineServiceControllerBase {
             message: commitMessage,
         });
     }
-}
 
+    public async getPendingFunctions() {
+        return await deployGetFunctions();
+    }
+    public async getLiveFunctions() {
+        return await Querysub.commitAsync(() => {
+            let base = functionSchema()[getDomain()].PathFunctionRunner;
+            let functions: FunctionSpec[] = [];
+            for (let moduleObj of Object.values(base)) {
+                for (let functionObj of Object.values(moduleObj.Sources)) {
+                    if (!functionObj) continue;
+                    functions.push(functionObj);
+                }
+            }
+            return functions;
+        });
+    }
+
+    public async deployFunctions(config: {
+        functionSpecs: FunctionSpec[];
+        notifyRefreshDelay: number;
+        deployOnlyCode?: boolean;
+        deployOnlyUI?: boolean;
+    }) {
+        let caller = SocketFunction.getCaller();
+        const onProgress: DeployProgress = (config) => {
+            void DeployProgressController.nodes[caller.nodeId].onDeployProgress(config);
+        };
+        await deployFunctions({
+            functionSpecs: config.functionSpecs,
+            notifyRefreshDelay: config.notifyRefreshDelay,
+            deployOnlyCode: config.deployOnlyCode,
+            deployOnlyUI: config.deployOnlyUI,
+            progress: onProgress,
+        });
+    }
+}
+let deployWatchers = new Set<DeployProgress>();
+class DeployProgressControllerBase {
+    async onDeployProgress(config: { section: string; progress: number; }) {
+        for (let deployWatcher of deployWatchers) {
+            deployWatcher(config);
+        }
+    }
+}
+const DeployProgressController = SocketFunction.register(
+    " DeployProgressController-9827a7e2-6ace-4156-b623-b4d1a7bab066",
+    () => new DeployProgressControllerBase(),
+    () => ({
+        onDeployProgress: {},
+    }),
+);
+export async function deployFunctionsWithProgress(config: {
+    functionSpecs: FunctionSpec[];
+    notifyRefreshDelay: number;
+    deployOnlyCode?: boolean;
+    deployOnlyUI?: boolean;
+    onProgress: DeployProgress;
+}) {
+    const { onProgress, ...rest } = config;
+    deployWatchers.add(onProgress);
+    try {
+        await MachineServiceController(SocketFunction.browserNodeId()).deployFunctions.promise(rest);
+    } finally {
+        deployWatchers.delete(onProgress);
+    }
+}
 
 export const MachineServiceController = getSyncedController(
     SocketFunction.register(
@@ -311,6 +379,9 @@ export const MachineServiceController = getSyncedController(
             getGitRefInfo: {},
             commitPushService: {},
             commitPushAndPublishQuerysub: {},
+            getPendingFunctions: {},
+            deployFunctions: {},
+            getLiveFunctions: {},
         }),
         () => ({
             hooks: [assertIsManagementUser],
@@ -326,6 +397,7 @@ export const MachineServiceController = getSyncedController(
             deleteServiceConfig: ["ServiceConfig", "ServiceConfigList"],
             commitPushService: ["gitInfo", "ServiceConfig"],
             commitPushAndPublishQuerysub: ["gitInfo", "ServiceConfig"],
+            deployFunctions: ["gitInfo"],
         },
         reads: {
             getMachineList: ["MachineInfoList"],
@@ -333,6 +405,8 @@ export const MachineServiceController = getSyncedController(
             getServiceList: ["ServiceConfigList"],
             getServiceConfig: ["ServiceConfig"],
             getGitInfo: ["gitInfo"],
+            getPendingFunctions: ["gitInfo"],
+            getLiveFunctions: ["gitInfo"],
         }
     }
 );

package/src/deployManager/spec.txt

@@ -1,29 +1,32 @@
-Our test node isn't appearing. What? Ugh...
-    - AND I think nodes are never being removed long term? Ugh...
 
-7.1) Verify quick node removal work by having a repeatedly crashing service, verifying the node exists, and then goes away on crash.
+8) Fix deploy user notification issue, where the refresh button doesn't work?
+    - For application level updates
+    - Just an application commit, then deploy the service, then the refresh button should show up, and work
 
 
-8) Fix deploy user notification issue, where the refresh button doesn't work?
+What about cloudflare registration?
+8) Use a special service for the HTTP bootstrapper, and then have 2 others that are on other ports
+    - Find a way to prevent the bootstrapper from being picked as the backend as well, so we don't need to update it!
+
+    --bootstraponly is added, and should work?
 
 9) Rolling service updates
     - Add rollingWindow to the definition
-    - Show it in the services list as well, 
-    - Keep the oldest service alive while we update
+    - Set for public facing services (but not for scripts)
+        - PathValueServer - 10 minutes
+        - FunctionRunner - 10 minutes
+        - HTTP - 4 hours
+    - Show the rolling time in the update buttons (both list and the save button on the details page), so we know it will be a rolling update
+    - Keep the oldest service alive when we update
         - Tracked per serviceId
+        - Update the rolling time, so if we remove the rolling window we kill the old service immediately
     - Notify when servers are outdated
         - In HTTP server, notify users, in the same way we notify for hash updates, that they will need to switch servers
-            - Verify this update works with a relatively low rolling update window, ensuring it force refreshes before the server actually restarts.
-
-10) Add RAM and cpu count to the machine info, as well as to the machine list page
-    - ALSO, disk size, and disk utilization?
-        - Yeah... I mean, we have similar metrics for nodes, but... I think nodes and machines will always have some overlap, and both are useful, and have their own distinct uses (top down vs bottom up, where top down is easy to identify that a problem exists, but not where it is, where as bottom up has too much detail and the problem might only exist due to many nodes, but if you see a problem it's easier to find where it is... and some problems might not exist at a machine level too, such as a single core being maxed out, or high latency of requests, or a specific resource being slow, etc, etc)
+        - Use the node registration to know the nodeId to talk to, and allow servers to register to get told when they are outdated (and how long before they are shutdown).
+        - Verify this update works with a relatively low rolling update window, ensuring it force refreshes before the server actually restarts.
 
-9) Replace yarn deploy with a page as well
-    - Only when running locally (and a super user, obviously)
-    - Can commit and push (just the main repo, not querysub, of course)
-        - Warn if querysub has changes, and link to the services page
-    - Another tab on the machines page? (it's annoying, but... we could rename the title from machines to deploy again?)
-    - Show a nice list of all the changed files and functions
-    - Give better progress, so we can see what we're waiting on, and how long we've been waiting
-    - ALSO, add a button which we can add to our main header, which will show if we have unpushed changes, and which links to the deploy page
+10) Add RAM total, ram % used, cpu count, CPU %, disk size, disk % used to machine info
+    - Show this all in the list page, with nice bars?
+        - Filled, but the total size will also depend on the maximum (to a degree), so it's very nice.
+        - A bar, with a label, or something. The AI can give us a few options
+    (Yes, it overlaps with node metrics, but that's okay, sometimes we want to look at machines, other times services, and the types of errors that show up in either changes).

package/src/deployManager/urlParams.ts

@@ -1,6 +1,6 @@
 import { URLParam } from "../library-components/URLParam";
 
-export type ViewType = "services" | "machines" | "service-detail" | "machine-detail";
+export type ViewType = "services" | "machines" | "service-detail" | "machine-detail" | "deploy";
 
 export const currentViewParam = new URLParam<ViewType>("machineview", "machines");
 export const selectedServiceIdParam = new URLParam("serviceId", "");

package/src/diagnostics/NodeViewer.tsx

@@ -85,7 +85,13 @@ export class NodeViewer extends qreact.Component {
                     let time = Date.now();
                     let data: NodeData = { nodeId };
                     try {
+                        if (nodeId.includes("b3a757")) {
+                            debugger;
+                        }
                         data.table = await controller.getMiscInfo(nodeId);
+                        if (nodeId.includes("b3a757")) {
+                            debugger;
+                        }
                         data.live_isReadReady = await controller.live_isReadReady(nodeId);
                         data.live_exposedControllers = await controller.getExposedControllers(nodeId);
                         data.live_entryPoint = await controller.live_getEntryPoint(nodeId);
@@ -99,8 +105,14 @@ export class NodeViewer extends qreact.Component {
                         } else {
                             data.devToolsURL = nodeId;
                         }
+                        if (nodeId.includes("b3a757")) {
+                            debugger;
+                        }
                     } catch (e: any) {
                         data.apiError = "Error: " + e.stack;
+                        if (nodeId.includes("b3a757")) {
+                            debugger;
+                        }
                     }
                     time = Date.now() - time;
                     data.loadTime = time;
@@ -144,8 +156,11 @@ export class NodeViewer extends qreact.Component {
             return <div>Verifying access...</div>;
         }
         let nodeDatas = Object.values(this.state.nodes);
+        let deadCount = 0;
         if (!showNotReadyNodes.value) {
-            nodeDatas = nodeDatas.filter(x => x.live_isReadReady);
+            let ready = nodeDatas.filter(x => x.live_isReadReady);
+            deadCount = nodeDatas.length - ready.length;
+            nodeDatas = ready;
         }
 
         function mergeTables(tables: TableType[]): TableType {
@@ -240,7 +255,7 @@ export class NodeViewer extends qreact.Component {
                         this.populateNodeInfos.reset();
                         this.populateNodeInfos();
                     }}>Refresh</Button>
-                    <InputLabelURL label="Show Not Ready" checkbox url={showNotReadyNodes} />
+                    <InputLabelURL label={`Show Not Ready (${deadCount})`} checkbox url={showNotReadyNodes} />
                     {this.state.error && <div class={errorMessage}>{this.state.error}</div>}
                 </div>
                 <div class={css.vbox(10)}>

package/src/library-components/ATag.tsx

@@ -1,5 +1,5 @@
 import preact from "preact";
-import { css } from "typesafecss";
+import { css, isNode } from "typesafecss";
 import { URLParam, parseSearchString, encodeSearchString } from "./URLParam";
 import { qreact } from "../4-dom/qreact";
 
@@ -97,14 +97,8 @@ export class ATag extends qreact.Component<ATagProps> {
                     // NOTE: Calculate the href dynamically, to prevent pages with a lot of links that
                     //  reference frequently changing values from resulting in a huge number of DOM changes
                     //  every time a value changes.
-                    let urlObj = new URL(document.location.href);
-                    let params = parseSearchString(urlObj.search);
                     let resolvedValues = typeof values === "function" ? values() : values;
-                    for (let value of resolvedValues || []) {
-                        params[value.param.urlKey] = value.value;
-                    }
-                    urlObj.search = encodeSearchString(params);
-                    e.currentTarget.href = urlObj.toString();
+                    e.currentTarget.href = createLink(resolvedValues || []);
                     props.onMouseOver?.(e);
                 }}
                 onKeyDown={e => {
@@ -120,4 +114,15 @@ export class ATag extends qreact.Component<ATagProps> {
     }
 }
 export const Anchor = ATag;
-export const Link = ATag;
+export const Link = ATag;
+
+export function createLink(values: URLOverride[]) {
+    if (isNode()) throw new Error(`Create link is not implemented yet on node.`);
+    let urlObj = new URL(document.location.href);
+    let params = parseSearchString(urlObj.search);
+    for (let value of values) {
+        params[value.param.urlKey] = value.value;
+    }
+    urlObj.search = encodeSearchString(params);
+    return urlObj.toString();
+}

package/src/misc/formatJSX.tsx

@@ -1,6 +1,6 @@
 import { qreact } from "../4-dom/qreact";
 import { formatTime, formatVeryNiceDateTime } from "socket-function/src/formatting/format";
 
-export function formatTimeJSX(time: number) {
+export function formatDateJSX(time: number) {
     return <span title={formatVeryNiceDateTime(time)}>{formatTime(Date.now() - time)}</span>;
 }

package/src/misc.ts

@@ -53,4 +53,12 @@ export function clamp(value: number, min: number, max: number) {
 
 export function unique<T>(array: T[]): T[] {
     return Array.from(new Set(array));
+}
+
+export function mostCommon<T>(array: T[]): T | undefined {
+    let counts = new Map<T, number>();
+    for (let item of array) {
+        counts.set(item, (counts.get(item) || 0) + 1);
+    }
+    return Array.from(counts.entries()).sort((a, b) => b[1] - a[1])[0]?.[0];
 }

package/src/server.ts

@@ -5,7 +5,6 @@ import { isNode, isNodeTrue } from "socket-function/src/misc";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { getThreadKeyCert } from "./-a-auth/certs";
 import { logErrors } from "./errors";
-import { publishMachineARecords } from "./-e-certs/EdgeCertController";
 import { pathValueAuthority2 } from "./0-path-value-core/NodePathAuthorities";
 import { green } from "socket-function/src/formatting/logColors";
 import { formatTime } from "socket-function/src/formatting/format";
@@ -14,9 +13,11 @@ import { formatTime } from "socket-function/src/formatting/format";
 //  we DO want to register some static diagnostics from Querysub, so... import it explicitly.
 import "./4-querysub/Querysub";
 import { getOurAuthorities } from "./config2";
-import { isPublic } from "./config";
+import { getDomain, isPublic } from "./config";
 
 import yargs from "yargs";
+import { Querysub } from "./4-querysub/Querysub";
+import { createSourceCheck } from "./user-implementation/canSeeSource";
 let yargObj = isNodeTrue() && yargs(process.argv)
     .option("authority", { type: "string", desc: `Defines the base paths we are an authority on (the domain is prepended to them). Either a file path to a JSON(AuthorityPath[]), or a base64 representation of the JSON(AuthorityPath[]).` })
     .option("verbose", { type: "boolean", desc: "Log all writes and reads" })
@@ -50,13 +51,12 @@ async function main() {
         pathValueAuthority2.becomePathAuthority(authority);
     }
 
-    let keyPair = await getThreadKeyCert();
-    await SocketFunction.mount({
-        port: 0, ...keyPair, public: isPublic(),
-        // SHOULD be safe, because trust is still required. Otherwise we run into issues when we have some
-        //  public services, and some private, but still want to run things locally.
-        autoForwardPort: true,
+    await Querysub.hostServer({
+        trustedDomains: [getDomain(), "noproxy." + getDomain()],
+        port: 443,
+        rootPath: __dirname + "/../",
+        clientsideEntryPoint: "./src/browser.ts",
+        hotReload: true,
+        sourceCheck: createSourceCheck(),
     });
-
-    await publishMachineARecords();
 }