querysub 0.250.0 → 0.252.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.250.0",
+  "version": "0.252.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -64,4 +64,4 @@
   "resolutions": {
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6"
   }
-}
+}

package/src/0-path-value-core/NodePathAuthorities.ts

@@ -366,6 +366,7 @@ class NodePathAuthorities {
         //  as being out of date.
         let now = Date.now();
         let isReadReady = await errorToUndefinedSilent(PathController.nodes[nodeId].isReadReady());
+
         // If the node is gone, delete it from authorities
         if (isReadReady === undefined) {
             console.error(yellow(`Node errored out, removing from authorities ${nodeId}`));

package/src/4-deploy/edgeBootstrap.ts

@@ -290,6 +290,16 @@ async function edgeNodeFunction(config: {
     }
     async function getEdgeNodeConfig(): Promise<EdgeNodeConfig> {
         let edgeIndex = cachedConfig || await (await fetch(config.edgeNodeConfigURL)).json() as EdgeNodesIndex;
+
+        edgeIndex.edgeNodes.sort((a, b) => -(a.bootTime - b.bootTime));
+        // Filter out duplicate hostnames (taking the max bootTime). We can't have multiple services using the same port, so duplicates are DEFINITELY dead. This fixes some issues with accessing old services, and generally makes reading the logs easier.
+        let hostNames = new Set<string>();
+        edgeIndex.edgeNodes = edgeIndex.edgeNodes.filter(x => {
+            if (hostNames.has(x.host)) return false;
+            hostNames.add(x.host);
+            return true;
+        });
+
         cachedConfig = undefined;
         let liveHashForced = false;
         if (Date.now() < liveHashOverrideExpiryTime) {

package/src/4-deploy/edgeNodes.ts

@@ -7,7 +7,7 @@ import { getArchivesBackblazePublic } from "../-a-archives/archivesBackBlaze";
 import { nestArchives } from "../-a-archives/archives";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { runInSerial, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
-import { compare, compareArray, isNodeTrue, timeInMinute } from "socket-function/src/misc";
+import { compare, compareArray, isNodeTrue, sort, timeInMinute } from "socket-function/src/misc";
 import { cacheLimited, lazy } from "socket-function/src/caching";
 import { canHaveChildren } from "socket-function/src/types";
 import { shutdown } from "../diagnostics/periodic";

package/src/deployManager/components/MachinesListPage.tsx

@@ -48,7 +48,7 @@ export class MachinesListPage extends qreact.Component {
                                 this.state.selectedForDeletion[machineId] = true;
                             }
                         }}>
-                        🗑️ Delete Machines
+                        🗑️ Delete Dead Machines
                     </button>
                 )}
 

package/src/deployManager/components/ServiceDetailPage.tsx

@@ -7,7 +7,7 @@ import { Querysub } from "../../4-querysub/QuerysubController";
 import { currentViewParam, selectedServiceIdParam, selectedMachineIdParam } from "../urlParams";
 import { formatTime, formatVeryNiceDateTime } from "socket-function/src/formatting/format";
 import { InputPicker } from "../../library-components/InputPicker";
-import { nextId, sort } from "socket-function/src/misc";
+import { deepCloneJSON, nextId, sort } from "socket-function/src/misc";
 import { InputLabel } from "../../library-components/InputLabel";
 import { Button } from "../../library-components/Button";
 import { isDefined } from "../../misc";
@@ -89,6 +89,8 @@ type ServiceConfig = ${serviceConfigType}
     }
 
     private updateUnsavedChanges(updatedConfig: ServiceConfig) {
+        // Do not let them update the serviceId, as that would break things
+        updatedConfig.serviceId = selectedServiceIdParam.value;
         const newConfigStr = JSON.stringify(updatedConfig, null, 4);
         this.state.unsavedChanges = updatedConfig;
 
@@ -159,6 +161,8 @@ type ServiceConfig = ${serviceConfigType}
         if (!selectedServiceId) return;
 
         Querysub.commit(() => {
+            // Do not let them update the serviceId, as that would break things
+            updatedConfig.serviceId = selectedServiceIdParam.value;
             this.state.isSaving = true;
             this.state.saveError = "";
         });
@@ -333,8 +337,10 @@ type ServiceConfig = ${serviceConfigType}
                             <div
                                 className={css.hbox(12)}
                             >
-                                <div className={css.flexGrow(1)}>
-                                    {machineId} ({machineInfo.info["getExternalIP"]})
+                                <div className={css.vbox(5)}>
+                                    <div>
+                                        {machineId} ({machineInfo.info["getExternalIP"]})
+                                    </div>
                                 </div>
                                 {isMachineDead && (
                                     <div className={css.colorhsl(0, 80, 50)}>
@@ -513,6 +519,33 @@ type ServiceConfig = ${serviceConfigType}
 
                 <div className={css.marginAuto}></div>
 
+                <button className={css.pad2(12, 8).button.bord2(0, 0, 20).hsl(110, 70, 90)}
+                    onClick={() => {
+                        if (!selectedServiceId) return;
+
+                        const newServiceId = `service-${Math.round(Date.now())}`;
+                        const clonedConfig = deepCloneJSON(config);
+                        clonedConfig.serviceId = newServiceId;
+                        clonedConfig.parameters.deploy = false;
+                        clonedConfig.info.title += " (Copy)";
+
+                        Querysub.onCommitFinished(async () => {
+                            try {
+                                await controller.addServiceConfig.promise(newServiceId, clonedConfig);
+                                Querysub.commit(() => {
+                                    selectedServiceIdParam.value = newServiceId;
+                                    this.state.unsavedChanges = undefined;
+                                });
+                            } catch (error) {
+                                Querysub.localCommit(() => {
+                                    this.state.saveError = error instanceof Error ? error.message : String(error);
+                                });
+                            }
+                        });
+                    }}>
+                    ✚ Duplicate Service
+                </button>
+
                 <button className={css.pad2(12, 8).button.bord2(0, 80, 50).hsl(0, 80, 90)}
                     onClick={() => {
                         if (!selectedServiceId) return;

package/src/deployManager/components/ServicesListPage.tsx

@@ -18,7 +18,7 @@ export class ServicesListPage extends qreact.Component {
         if (!serviceList) return <div>Loading services...</div>;
 
         let services = serviceList.map(serviceId => [serviceId, controller.getServiceConfig(serviceId)] as const);
-        sort(services, x => -(x[1]?.info.lastUpdatedTime || Date.now()));
+        sort(services, x => -(x[1]?.info.lastUpdatedTime || Date.now()) - (x[1]?.parameters.deploy && (Number.MAX_SAFE_INTEGER / 2) || 0));
 
         let getMachineInfo = cache((machineId: string) => controller.getMachineInfo(machineId));
 
@@ -59,7 +59,7 @@ export class ServicesListPage extends qreact.Component {
             </div>
             <div className={css.vbox(8)}>
                 {services.map(([serviceId, config]) => {
-                    if (!config) return <div key={serviceId}>Loading {serviceId}...</div>;
+                    if (!config) return <div key={serviceId}>Config is broken?</div>;
                     let failingMachines = config.machineIds.filter(machineId => {
                         let machineInfo = getMachineInfo(machineId);
                         return machineInfo?.services[serviceId]?.errorFromLastRun;

package/src/deployManager/machineApplyMainCode.ts

@@ -25,7 +25,7 @@ import { onServiceConfigChange } from "./machineController";
 import { PromiseObj } from "../promise";
 import path from "path";
 
-const PIPE_FILE_LINE_LIMIT = 10;
+const PIPE_FILE_LINE_LIMIT = 100_000;
 
 const getLiveMachineInfo = measureWrap(async function getLiveMachineInfo() {
     let machineInfo: MachineInfo = {

package/src/deployManager/machineSchema.ts

@@ -36,8 +36,6 @@ export type ServiceConfig = {
         command: string;
         /** Allows forcing an update */
         poke?: number;
-        /** If once we don't restart it on exit, only running it once. However we will run it again on boot, so it's not once globally, it's just running it once for now. */
-        once?: boolean;
 
         /** Not set by default, so we can setup the configuration before deploying it (or so we can undeploy easily without deleting it) */
         deploy?: boolean;
@@ -144,6 +142,7 @@ export class DeployControllerBase {
     }
 
     public async addServiceConfig(serviceId: string, config: ServiceConfig) {
+        config.info.lastUpdatedTime = Date.now();
         let existingConfig = await serviceConfigs.get(serviceId);
         if (existingConfig) {
             throw new Error(`Service ${serviceId} already exists`);
@@ -153,6 +152,7 @@ export class DeployControllerBase {
     }
 
     public async setServiceConfig(serviceId: string, config: ServiceConfig) {
+        config.info.lastUpdatedTime = Date.now();
         let serviceConfig = await serviceConfigs.get(serviceId);
 
         if (!serviceConfig) {

package/src/deployManager/spec.txt

@@ -1,36 +1,46 @@
-2) Verify when the file is truncated it still works
+Ugh...
+    1) Our local http server can't talk to the remote PathValueServer server?
+    2) The remote http server isn't loading the modules?
+        - Which... come from permissions, and so the PathValueServer?
+        - Ah, and, it can't find the PathValueServer either?
 
-2) Apply ansi coloring / whatever type we use
+OH! It is... that we keep picking the wrong server, thinking it is working, but it has an old config?
+
+bd6cf9eb4accffba8
+
+4) Add 2X redundancy to PathValueServer, and... FunctionRunner?
 
-4) Destroy our testing digital ocean server
 
 5) Setup on our regular digital ocean server
-    - Remove previous startup.sh, and crontab and kill existing tmux services
     - Setup all the services in the new UI
         - Copy from the previous startup.sh, running the same services
         - Changing the UI if anything is extremely annoying, but... I don't see how it would be...
     tmux send-keys -t server1 "cd ~/cyoa && yarn server-public" Enter
     tmux send-keys -t server2 "cd ~/cyoa && yarn server-public" Enter
     tmux send-keys -t fnc "cd ~/cyoa && yarn function-public --verbosecalls" Enter
+
     tmux send-keys -t http "cd ~/cyoa && yarn cyoa-public --verbosecalls" Enter
     tmux send-keys -t watch "cd ~/cyoa && yarn gc-watch-public" Enter
     tmux send-keys -t join "cd ~/cyoa && yarn join-public" Enter
 5) Verify the editor works
 
-6) Verify PathValueServer gracefully shutdowns, not losing any values (because it delays and flushes writes before shutting down, detecting the ctrl+c).
-
-
-7) Quick node removal on process crash OR removal
-    - In the service, check our parent folder to see if we are in a screen (/machine-services/git/), and then write our nodeId to /machine-services/nodeId
-    - If we see a nodeId when we are removing a screen, or killing the service, then delete that nodeId from the nodeId directory (and call tellEveryoneNodesChanges)
-7.1) Verify this by killing a lot of services (the function runner?), by just poking it over and over, verifying the nodes are quickly deleted
 
 
 
 8) REIMPLEMENT yarn do-update functionality, with UI on the configuration page
-    OH, maybe... buttons for querysub, button for the main site, and then buttons on each service to update them!
-        YES, and then, a button to do all of that on each service, and a button to do that and on each service.
-        - So a lot of buttons. But only 2 components, one for all, and one for a specific service.
+    - Components
+        <UpdateButtons />
+            - Commit all (add, commit, push)
+                - Only shows up if there are unsaved changes
+            - Commit querysub (publish, add, commit, push querysub, update package.json of app, add, commit, push)
+                - Only shows up if there are unsaved changes
+            - Deploy all (set hash to latest and save)
+                - Only shows up if (any) deployed hashes differ from latest for app
+        <UpdateServiceButtons service={serviceConfig} />
+            - Commit (add, commit, push)
+                - Only shows up if there are unsaved changes
+            - Deploy (set hash to latest and save)
+                - Only shows up if deployed hash differ from latest for app
     - Endpoints
         - anyQuerysubUnsaved
         - anyAppUnsaved
@@ -48,6 +58,27 @@
     - Commit, push, update hash
         - Also with an option to also update querysub (it'll tell us if querysub has changes), updating package.json to
 
+
+7) Quick node removal on process crash OR removal
+    - In the service, check our parent folder to see if we are in a screen (/machine-services/git/), and then write our nodeId to /machine-services/nodeId
+    - If we see a nodeId when we are removing a screen, or killing the service, then delete that nodeId from the nodeId directory (and call tellEveryoneNodesChanges)
+7.1) Verify this by killing a lot of services (the function runner?), by just poking it over and over, verifying the nodes are quickly deleted
+
+
+
+In general, hit our servers hard, to make sure we can launch them, they can talk to each other, etc. Even with entirely new folders (to somewhat emulate another machine), and they can all startup smoothly.
+6) BUG! On startup, new nodes might take a poll interval to be fully ready? Hmm... we should restart our servers a lot again to make sure they come back up QUICKLY
+    - OH! It might be that the identity takes some time to propagate (the DNS), which causes us to require a second poll to be alive? Maybe... our local machine deleted it, because it couldn't see the DNS? No... none of these are reasonable...
+    - Try changing the key, so it syncs again, so we start fresh with the server
+        - Reduce it to just 1 instance as well?
+6.1) HTTP required a reboot when it was running without PathValueServer, as it didn't clone the right hashes, and then never did, even with PathValueServer was up?
+    - But maybe it was a node polling issue? But we are supposed to send notifications when nodes are launched? Hmm...
+FORTUNATELY, looking at logs is now very simple, as well as updating nodes, so... this shouldn't be too hard to debug
+
+
+6) Verify PathValueServer gracefully shutdowns, not losing any values (because it delays and flushes writes before shutting down, detecting the ctrl+c).
+
+
 8) Fix deploy user notification issue, where the refresh button doesn't work?
 
 9) Rolling service updates

package/src/deployManager/urlParams.ts

@@ -3,6 +3,6 @@ import { URLParam } from "../library-components/URLParam";
 export type ViewType = "services" | "machines" | "service-detail" | "machine-detail";
 
 // URL Parameters for navigation state
-export const currentViewParam = new URLParam<ViewType>("view", "machines");
+export const currentViewParam = new URLParam<ViewType>("machineview", "machines");
 export const selectedServiceIdParam = new URLParam("serviceId", "");
 export const selectedMachineIdParam = new URLParam("machineId", ""); 

package/src/diagnostics/managementPages.tsx

@@ -70,6 +70,12 @@ export async function registerManagementPages2(config: {
     registeredModule = config.module;
     let inputPages: typeof config.pages = [];
 
+
+    inputPages.push({
+        title: "Machines",
+        componentName: "MachinesPage",
+        getModule: () => import("../deployManager/MachinesPage"),
+    });
     inputPages.push({
         title: "Nodes",
         componentName: "NodeViewer",
@@ -127,11 +133,6 @@ export async function registerManagementPages2(config: {
         componentName: "RequireAuditPage",
         getModule: () => import("./misc-pages/RequireAuditPage"),
     });
-    inputPages.push({
-        title: "Machines",
-        componentName: "MachinesPage",
-        getModule: () => import("../deployManager/MachinesPage"),
-    });
     inputPages.push(...config.pages);
 
     // NOTE: We don't store the UI in the database (here, or anywhere else, at least