querysub 0.357.0 → 0.358.0

package/.cursorrules

@@ -26,6 +26,7 @@ NEVER EVER pass state to qreact.Component as a template parameter. It should ALS
 
     t.lookup objects cannot be set with `this.state.someLookup = {}`. You can add keys or remove keys, and keys are added implicitly (`this.state.someLookup["someKey"] = { x: 1, y: 1 }`, always works, the key is automatically added if it doesn't exist). Removal uses delete `delete this.state.someLookup["someKey"]`.
 
+Follow the rule of minimum scoping. If something can be a local variable, it should be a local variable, it shouldn't be a class variable. If it can be a class variable, it should be a class variable, not a state variable. Etc, etc. Keep it simple, reference things closer, and write less code.
 
 Try not to use "null", and instead always use "undefined".
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.357.0",
+  "version": "0.358.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -56,6 +56,7 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
+    "querysub": "^0.357.0",
     "socket-function": "^1.1.2",
     "terser": "^5.31.0",
     "typesafecss": "^0.28.0",

package/src/-a-archives/archivesDisk.ts

@@ -75,7 +75,7 @@ class ArchivesDisk {
         }
     }
 
-    @measureFnc
+    // We don't time the file system writes as it's almost never blocking, it's not stopping our processing. And if it is stopping our processing, then we actually want to know the parent function which is calling it, and then we want it to just void it instead of awaiting it (And the only way to know the parent caller is to not time these functions).
     public async set(fileName: string, data: Buffer): Promise<void> {
         await this.init();
 
@@ -88,7 +88,6 @@ class ArchivesDisk {
 
         await fs.promises.writeFile(this.LOCAL_ARCHIVE_FOLDER + fileName, data);
     }
-    @measureFnc
     public async append(fileName: string, data: Buffer): Promise<void> {
         await this.init();
 
@@ -101,7 +100,6 @@ class ArchivesDisk {
 
         await fs.promises.appendFile(this.LOCAL_ARCHIVE_FOLDER + fileName, data);
     }
-    @measureFnc
     public async del(fileName: string): Promise<void> {
         await this.init();
         this.log(blue(`Deleting file ${fileName}`));
@@ -112,8 +110,16 @@ class ArchivesDisk {
         try {
             await fs.promises.rm(this.LOCAL_ARCHIVE_FOLDER + fileName, { recursive: true });
         } catch { }
-        let dir = fileName.replaceAll("\\", "/").split("/").slice(0, -1).join("/");
-        await this.gcDir(dir);
+        try {
+            let dir = fileName;
+            while (dir.length > 0) {
+                dir = dir.replaceAll("\\", "/").split("/").slice(0, -1).join("/");
+                if (dir.endsWith(":")) break;
+                let files = await fs.promises.readdir(this.LOCAL_ARCHIVE_FOLDER + dir);
+                if (files.length > 0) break;
+                await fs.promises.rmdir(this.LOCAL_ARCHIVE_FOLDER + dir);
+            }
+        } catch { }
     }
 
     public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined>; }): Promise<void> {
@@ -180,7 +186,8 @@ class ArchivesDisk {
      *      it all at once).
      * @fsErrorRetryCount Retries count for file system error (not user errors). Ex, too many open files, etc.
      */
-    @measureFnc
+    // NOTE: I've commented out the gate measuring so we can determine what is taking the time. We might want to remove all the timing from this file, as if we find any timing this file is slow, it doesn't help us optimize at all, because we then need to figure out what is calling it. 
+    //@measureFnc
     public async get(fileNameInput: string, config?: { range?: { start: number; end: number; }; retryCount?: number }): Promise<Buffer | undefined> {
         await this.init();
         this.log(blue(`Start read file ${fileNameInput}`));

package/src/-a-archives/archivesMemoryCache.ts

@@ -36,13 +36,14 @@ export function createArchivesMemoryCache(
         //  - Also, it means if there's a range read that happens, if that range read doesn't return enough bytes, for example, we try to read from 0 to 100, but we only get 50 bytes back, we can only cache the 0 to 50, and then if they read from 0 to 100 again, we can use the cache, but we have to also read 50 to 100 from the source, because the file might have gotten larger. 
         fullyImmutable?: boolean;
         stats?: ArchivesMemoryCacheStats;
+        sizeCache?: Map<string, number>;
     }
 ): Archives {
     let {
         maxSize = 1024 * 1024 * 1024 * 4,
         maxCount = 1000 * 1000,
         fullyImmutable = false,
-        extraReadSize = 1024
+        extraReadSize = 1024 * 1024
     } = config ?? {};
 
     // Cache structure: Map from path to sorted array of ranges
@@ -199,13 +200,34 @@ export function createArchivesMemoryCache(
         }
     });
 
+    function updateStats(cached: boolean, size: number | undefined) {
+        if (!config?.stats) {
+            return;
+        }
+        if (cached) {
+            config.stats.cachedReads++;
+            if (size !== undefined) {
+                config.stats.cachedReadSize += size;
+            }
+        } else {
+            config.stats.uncachedReads++;
+            if (size !== undefined) {
+                config.stats.uncachedReadSize += size;
+            }
+        }
+        config.stats.totalCacheSize = totalSize;
+        config.stats.totalCacheCount = lruArray.length;
+    }
+
     async function cachedGet(path: string, getConfig?: {
         range?: { start: number; end: number; };
         retryCount?: number;
         fastRead?: boolean;
     }): Promise<Buffer | undefined> {
         if (cacheDisabled) {
-            return await archives.get(path, getConfig);
+            let result = await archives.get(path, getConfig);
+            updateStats(false, result?.length);
+            return result;
         }
 
         let range = getConfig?.range;
@@ -214,12 +236,15 @@ export function createArchivesMemoryCache(
         if (!range) {
             // If not fully immutable, don't cache entire file reads (might have appends)
             if (!fullyImmutable) {
-                return await archives.get(path, getConfig);
+                let result = await archives.get(path, getConfig);
+                updateStats(false, result?.length);
+                return result;
             }
 
             // Get file info to determine size, then treat as range read
             let info = await archives.getInfo(path);
             if (!info) {
+                updateStats(false, undefined);
                 return undefined;
             }
 
@@ -234,12 +259,20 @@ export function createArchivesMemoryCache(
         {
             for (let entry of cacheByPath.get(path) || []) {
                 if (entry.start <= start && entry.end >= end) {
-                    return entry.data.slice(start - entry.start, end - entry.start);
+                    let result = entry.data.slice(start - entry.start, end - entry.start);
+                    updateStats(true, result.length);
+                    return result;
                 }
             }
         }
 
-        let size = await archives.getInfo(path);
+        let cachedSize = config?.sizeCache?.get(path);
+        if (cachedSize === undefined) {
+            cachedSize = ((await archives.getInfo(path))?.size || 0);
+            config?.sizeCache?.set(path, cachedSize);
+            updateStats(false, undefined);
+        }
+        let size = cachedSize;
         if (!size) {
             return undefined;
         }
@@ -250,7 +283,7 @@ export function createArchivesMemoryCache(
             let aligned = alignRange(start, end);
 
             let readStart = aligned.start;
-            let readEnd = Math.min(aligned.end, size.size);
+            let readEnd = Math.min(aligned.end, size);
 
             let overlapping = findOverlappingEntries(path, start, end);
 
@@ -283,6 +316,7 @@ export function createArchivesMemoryCache(
                 });
 
                 if (!data) {
+                    updateStats(false, undefined);
                     return undefined;
                 }
 
@@ -329,17 +363,7 @@ export function createArchivesMemoryCache(
         let result = coveringEntry.data.slice(offsetStart, offsetEnd);
 
         // Update statistics at the end when we know the final result
-        if (config?.stats) {
-            if (didRead) {
-                config.stats.uncachedReads++;
-                config.stats.uncachedReadSize += result.length;
-            } else {
-                config.stats.cachedReads++;
-                config.stats.cachedReadSize += result.length;
-            }
-            config.stats.totalCacheSize = totalSize;
-            config.stats.totalCacheCount = lruArray.length;
-        }
+        updateStats(!didRead, result.length);
 
         return result;
     }

package/src/deployManager/components/MachineDetailPage.tsx

@@ -9,6 +9,8 @@ import { ATag, Anchor } from "../../library-components/ATag";
 import { ShowMore } from "../../library-components/ShowMore";
 import { filterParam } from "../../diagnostics/logs/FastArchiveViewer";
 import { managementPageURL } from "../../diagnostics/managementPages";
+import { t } from "../../2-proxy/schema2";
+import { Querysub } from "../../4-querysub/QuerysubController";
 
 export class MachineDetailPage extends qreact.Component {
     render() {
@@ -18,12 +20,14 @@ export class MachineDetailPage extends qreact.Component {
         let controller = MachineServiceController(SocketFunction.browserNodeId());
         let machineInfo = controller.getMachineInfo(selectedMachineId);
         let serviceList = controller.getServiceList();
+        let machineConfig = controller.getMachineConfig(selectedMachineId);
 
         if (controller.isAnyLoading()) return <div>Loading machine info...</div>;
         if (!machineInfo) return <div>Machine not found</div>;
         if (!serviceList) return <div>Service list not found</div>;
 
-        const machine = machineInfo; // Create const reference for type safety
+        const machine = machineInfo;
+        const isDisabled = machineConfig?.disabled || false;
 
         // Get all service configs that target this machine
         let relevantServiceConfigs = new Map<string, ServiceConfig>();
@@ -47,12 +51,49 @@ export class MachineDetailPage extends qreact.Component {
 
         const isMachineDead = Date.now() - machine.heartbeat > (MACHINE_RESYNC_INTERVAL * 4);
 
+        let failingServices = Object.keys(machine.services).filter(serviceId => {
+            return machine.services[serviceId].errorFromLastRun;
+        });
+
+        let backgroundColor = (
+            isDisabled && css.hsl(0, 0, 50)
+            || failingServices.length > 0 && css.hsl(0, 50, 60)
+            || isMachineDead && css.hsl(45, 80, 80)
+            || css.hsl(0, 0, 100)
+        );
+
         return <div className={css.vbox(16)}>
-            <div className={css.hbox(12)}>
+            <div className={css.hbox(12).pad2(16).bord2(0, 0, 20) + backgroundColor}>
                 <h2 className={css.flexGrow(1)}>{selectedMachineId}</h2>
                 {isMachineDead && <div className={css.colorhsl(0, 80, 60)}>
                     ⚠️ Machine is likely dead
                 </div>}
+                {isDisabled && <div className={css.colorhsl(30, 80, 60).pad2(8, 4).bord2(30, 80, 60).hsl(30, 80, 95)}>
+                    ⚠️ Machine is disabled
+                </div>}
+            </div>
+
+            <div className={css.hbox(12).pad2(16).bord2(0, 0, 20).hsl(0, 0, 95)}>
+                <div className={css.flexGrow(1).vbox(4)}>
+                    <div className={css.fontSize(16).colorhsl(0, 0, 20)}>
+                        <b>Machine Status</b>
+                    </div>
+                    <div>
+                        {isDisabled && "When disabled, services will not be deployed to this machine. Re-enabling will restore all service assignments." || "This machine is enabled and available for service deployments."}
+                    </div>
+                </div>
+                <button
+                    className={css.pad2(12, 8).button.bord2(0, 0, 20) + (isDisabled ? css.hsl(120, 60, 90) : css.hsl(30, 80, 90))}
+                    onClick={() => {
+                        Querysub.onCommitFinished(async () => {
+                            await controller.setMachineConfig.promise(selectedMachineId, {
+                                machineId: selectedMachineId,
+                                disabled: !isDisabled
+                            });
+                        });
+                    }}>
+                    {isDisabled && "Enable Machine" || "Disable Machine"}
+                </button>
             </div>
 
             <div className={css.vbox(12)}>

package/src/deployManager/components/MachinesListPage.tsx

@@ -28,6 +28,8 @@ export class MachinesListPage extends qreact.Component {
         let machines = machineList.map(machineId => [machineId, controller.getMachineInfo(machineId)] as const).filter(x => x[1]);
         sort(machines, x => -(x[1]?.heartbeat || 0));
 
+        let machineConfigs = controller.getMachineConfigList();
+
         const selectedMachineIds = Object.keys(this.state.selectedForDeletion);
         const hasSelectedMachines = selectedMachineIds.length > 0;
         const DEAD_MACHINE_THRESHOLD = MACHINE_RESYNC_INTERVAL * 4;
@@ -121,6 +123,8 @@ export class MachinesListPage extends qreact.Component {
                     const serviceCount = Object.keys(machineInfo.services).length;
                     const isMachineDead = Date.now() - machineInfo.heartbeat > (MACHINE_RESYNC_INTERVAL * 4);
                     const isSelected = this.state.selectedForDeletion[machineId];
+                    const machineConfig = machineConfigs?.find(x => x.machineId === machineId);
+                    const isDisabled = machineConfig?.disabled || false;
 
                     let failingServices = Object.keys(machineInfo.services).filter(serviceId => {
                         return machineInfo.services[serviceId].errorFromLastRun;
@@ -136,8 +140,9 @@ export class MachinesListPage extends qreact.Component {
                             className={
                                 css.pad2(12).bord2(0, 0, 20).button
                                 + (
-                                    failingServices.length > 0 && css.hsl(0, 50, 60)
-                                    || isMachineDead && css.hsl(0, 0, 50)
+                                    isDisabled && css.hsl(0, 0, 50)
+                                    || failingServices.length > 0 && css.hsl(0, 50, 60)
+                                    || isMachineDead && css.hsl(45, 80, 80)
                                     || css.hsl(0, 0, 100)
                                 )
                                 + (this.state.isDeleteMode && isSelected && css.bord2(200, 80, 60, 2))
@@ -165,6 +170,9 @@ export class MachinesListPage extends qreact.Component {
                                         />
                                     </div>
                                 )}
+                                {isDisabled && <div className={css.colorhsl(30, 80, 60).pad2(4, 2).bord2(30, 80, 60).hsl(30, 80, 95)}>
+                                    🚫 Disabled
+                                </div>}
                                 {isMachineDead && <div className={css.colorhsl(0, 80, 50)}>
                                     ⚠️ Machine is likely dead
                                 </div>}

package/src/deployManager/machineApplyMainCode.ts

@@ -4,7 +4,7 @@ import { measureWrap } from "socket-function/src/profiling/measure";
 import { getOwnMachineId } from "../-a-auth/certs";
 import { forceRemoveNode, getOurNodeId, getOurNodeIdAssert } from "../-f-node-discovery/NodeDiscovery";
 import { Querysub } from "../4-querysub/QuerysubController";
-import { MACHINE_RESYNC_INTERVAL, MachineServiceControllerBase, MachineInfo, ServiceConfig, serviceConfigs, SERVICE_FOLDER, machineInfos, SERVICE_NODE_FILE_NAME } from "./machineSchema";
+import { MACHINE_RESYNC_INTERVAL, MachineServiceControllerBase, MachineInfo, ServiceConfig, serviceConfigs, SERVICE_FOLDER, machineInfos, SERVICE_NODE_FILE_NAME, getEffectiveServiceConfigs } from "./machineSchema";
 import { runPromise } from "../functional/runCommand";
 import { getExternalIP } from "socket-function/src/networking";
 import { errorToUndefined, errorToUndefinedSilent } from "../errors";
@@ -579,7 +579,7 @@ async function quickIsOutdated() {
             return true;
         }
     }
-    let configs = await serviceConfigs.values();
+    let configs = await getEffectiveServiceConfigs();
     let relevantConfigs = configs.filter(config => config.machineIds.includes(machineId)).filter(x => x.parameters.deploy);
     let screens = await getScreenState();
     let root = os.homedir() + "/" + SERVICE_FOLDER;
@@ -608,7 +608,7 @@ async function quickIsOutdated() {
 const resyncServicesBase = runInSerial(measureWrap(async function resyncServices() {
     console.log(magenta("Resyncing services"));
     let machineId = getOwnMachineId();
-    let configs = await serviceConfigs.values();
+    let configs = await getEffectiveServiceConfigs();
     let relevantConfigs = configs.filter(config => config.machineIds.includes(machineId)).filter(x => x.parameters.deploy);
 
     let machineInfo = await getLiveMachineInfo();

package/src/deployManager/machineSchema.ts

@@ -93,8 +93,13 @@ export type ServiceConfig = {
         lastUpdatedTime: number;
     };
 };
+export type MachineConfig = {
+    machineId: string;
+    disabled: boolean;
+};
 export const machineInfos = archiveJSONT<MachineInfo>(() => nestArchives("machines/machine-heartbeats/", getArchivesBackblaze(getDomain())));
 export const serviceConfigs = archiveJSONT<ServiceConfig>(() => nestArchives("machines/service-configs/", getArchivesBackblaze(getDomain())));
+export const machineConfigs = archiveJSONT<MachineConfig>(() => nestArchives("machines/machine-configs/", getArchivesBackblaze(getDomain())));
 
 export function doRegisterNodeForMachineCleanup() {
     if (isNode()) {
@@ -153,6 +158,16 @@ export class MachineServiceControllerBase {
         return await serviceConfigs.get(serviceId);
     }
 
+    public async getMachineConfigList() {
+        return await machineConfigs.values();
+    }
+    public async getMachineConfig(machineId: string): Promise<MachineConfig | undefined> {
+        return await machineConfigs.get(machineId);
+    }
+    public async setMachineConfig(machineId: string, config: MachineConfig) {
+        await machineConfigs.set(machineId, config);
+    }
+
     private async notifyMachines(newMachineIds: string[], oldMachineIds: string[]) {
         let addedNodeIds = new Set<string>();
         let removedNodeIds = new Set<string>();
@@ -358,6 +373,24 @@ export class MachineServiceControllerBase {
         });
     }
 }
+
+export async function getEffectiveServiceConfigs(): Promise<ServiceConfig[]> {
+    let configs = await serviceConfigs.values();
+    let allMachineConfigs = await machineConfigs.values();
+
+    let disabledMachineIds = new Set<string>();
+    for (let machineConfig of allMachineConfigs) {
+        if (machineConfig.disabled) {
+            disabledMachineIds.add(machineConfig.machineId);
+        }
+    }
+
+    return configs.map(config => ({
+        ...config,
+        machineIds: config.machineIds.filter(machineId => !disabledMachineIds.has(machineId)),
+    }));
+}
+
 let deployWatchers = new Set<DeployProgress>();
 class DeployProgressControllerBase {
     async onDeployProgress(config: { section: string; progress: number; }) {
@@ -401,6 +434,9 @@ export const MachineServiceController = getSyncedController(
             setMachineInfo: {},
             getServiceList: {},
             getServiceConfig: {},
+            getMachineConfigList: {},
+            getMachineConfig: {},
+            setMachineConfig: {},
             addServiceConfig: {},
             setServiceConfigs: {},
             getServiceConfigType: {},
@@ -422,6 +458,7 @@ export const MachineServiceController = getSyncedController(
             deleteMachineIds: ["MachineInfo", "MachineInfoList"],
             addMachineInfo: ["MachineInfo", "MachineInfoList"],
             setMachineInfo: ["MachineInfo"],
+            setMachineConfig: ["MachineConfig", "MachineConfigList"],
             addServiceConfig: ["ServiceConfig", "ServiceConfigList"],
             setServiceConfigs: ["ServiceConfig"],
             deleteServiceConfig: ["ServiceConfig", "ServiceConfigList"],
@@ -432,6 +469,8 @@ export const MachineServiceController = getSyncedController(
         reads: {
             getMachineList: ["MachineInfoList"],
             getMachineInfo: ["MachineInfo"],
+            getMachineConfigList: ["MachineConfigList"],
+            getMachineConfig: ["MachineConfig"],
             getServiceList: ["ServiceConfigList"],
             getServiceConfig: ["ServiceConfig"],
             getGitInfo: ["gitInfo"],

package/src/diagnostics/NodeViewer.tsx

@@ -80,6 +80,7 @@ export class NodeViewer extends qreact.Component {
                 Querysub.commit(() => this.state.nodeIds = nodeIds);
 
                 let ourIP = await controller.getCallerIP();
+                let ourExternalIP = await getExternalIP();
 
                 await Promise.allSettled(nodeIds.map(async nodeId => {
                     let time = Date.now();
@@ -97,7 +98,7 @@ export class NodeViewer extends qreact.Component {
                         data.live_entryPoint = await controller.live_getEntryPoint(nodeId);
                         data.live_authorityPaths = await controller.live_getAuthorityPaths(nodeId);
                         data.ip = await controller.getNodeIP(nodeId);
-                        if (data.ip === ourIP || data.ip === "127.0.0.1") {
+                        if (data.ip === ourExternalIP || data.ip === ourIP) {
                             data.devToolsURL = await controller.getInspectURL(nodeId);
                             if (data.devToolsURL) {
                                 data.devToolsURL = data.devToolsURL + "<inspect>";