querysub 0.358.0 → 0.359.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.358.0",
+  "version": "0.359.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-a-archives/archivesDisk.ts

@@ -1,4 +1,5 @@
 import { isNode } from "socket-function/src/misc";
+import os from "os";
 import { fsExistsAsync, getSubFolder } from "../fs";
 import fs from "fs";
 import { blue, red, yellow } from "socket-function/src/formatting/logColors";
@@ -440,5 +441,15 @@ export const getArchivesLocal = cache((domain: string): Archives => {
         archivesLocal.LOCAL_ARCHIVE_FOLDER = getSubFolder(domain).replaceAll("\\", "/");
     }
 
+    return archivesLocal;
+});
+
+export const getArchivesHome = cache((domain: string): Archives => {
+    const archivesLocal = new ArchivesDisk();
+
+    if (isNode()) {
+        archivesLocal.LOCAL_ARCHIVE_FOLDER = os.homedir() + "/querysub/" + domain + "/";
+    }
+
     return archivesLocal;
 });

package/src/deployManager/components/MachineDetailPage.tsx

@@ -118,8 +118,8 @@ export class MachineDetailPage extends qreact.Component {
                     <div className={css.vbox(10)}>
                         {Object.entries(machine.info).map(([key, value]) => (
                             <div key={key} className={css.hbox(10).hsla(0, 0, 0, 0.1).pad2(10, 2)}>
-                                <b>{key}:</b>
-                                <ShowMore className={css.whiteSpace("pre-wrap")} maxHeight={40}>
+                                <b className={css.flexShrink0}>{key}:</b>
+                                <ShowMore className={css.whiteSpace("pre-wrap")} maxHeight={80}>
                                     {(() => {
                                         if (typeof value === "object") {
                                             return <div className={css.pad2(6, 2).hsla(0, 0, 0, 0.1).relative}>

package/src/deployManager/components/ServiceDetailPage.tsx

@@ -157,9 +157,11 @@ export class ServiceDetailPage extends qreact.Component {
             let index = nextIndexes.get(machineId) || 0;
             nextIndexes.set(machineId, index + 1);
             let machineInfo = controller.getMachineInfo(machineId);
+            let machineConfig = controller.getMachineConfig(machineId);
             const serviceInfo = machineInfo?.services[selectedServiceId || ""];
             const isMachineDead = machineInfo ? Date.now() - machineInfo.heartbeat > (MACHINE_RESYNC_INTERVAL * 4) : true;
             const hasError = serviceInfo?.errorFromLastRun;
+            const isDisabled = machineConfig?.disabled || false;
 
             return {
                 machineId,
@@ -167,11 +169,12 @@ export class ServiceDetailPage extends qreact.Component {
                 serviceInfo,
                 isMachineDead,
                 hasError,
+                isDisabled,
                 heartbeat: machineInfo?.heartbeat || 0,
                 index
             };
         });
-        sort(machineStatuses, x => (x.hasError ? 0 : x.isMachineDead ? 1 : 2) * 1000000 + x.heartbeat);
+        sort(machineStatuses, x => (x.isDisabled ? 3 : x.hasError ? 0 : x.isMachineDead ? 1 : 2) * 1000000 + x.heartbeat);
         let now = Date.now();
 
         let machines = (controller.getMachineList() || []).map(x => controller.getMachineInfo(x)).filter(isDefined);
@@ -253,11 +256,13 @@ export class ServiceDetailPage extends qreact.Component {
             {configT.parameters.deploy && <div className={css.vbox(8).fillWidth}>
                 <h3>Deployed Machines ({config.machineIds.length})</h3>
                 <div className={css.vbox(4).fillWidth}>
-                    {machineStatuses.map(({ machineId, machineInfo, serviceInfo, isMachineDead, hasError, index }) => {
+                    {machineStatuses.map(({ machineId, machineInfo, serviceInfo, isMachineDead, hasError, isDisabled, index }) => {
                         if (!machineInfo) return <div key={machineId}>Loading {machineId}...</div>;
 
                         let backgroundColor = css.hsl(0, 0, 100); // Default: white
-                        if (hasError) {
+                        if (isDisabled) {
+                            backgroundColor = css.hsl(0, 0, 50); // Gray for disabled machines
+                        } else if (hasError) {
                             backgroundColor = css.hsl(0, 70, 90); // Red-ish for errors
                         } else if (isMachineDead) {
                             backgroundColor = css.hsl(0, 0, 50); // Gray for dead machines
@@ -281,6 +286,11 @@ export class ServiceDetailPage extends qreact.Component {
                                         {machineId} ({machineInfo.info["getExternalIP"]})
                                     </div>
                                 </div>
+                                {isDisabled && (
+                                    <div className={css.colorhsl(0, 0, 30)}>
+                                        🚫 Machine Disabled
+                                    </div>
+                                )}
                                 {isMachineDead && (
                                     <div className={css.colorhsl(0, 80, 50)}>
                                         ⚠️ Machine Dead

package/src/deployManager/components/ServicesListPage.tsx

@@ -21,6 +21,7 @@ export class ServicesListPage extends qreact.Component {
     render() {
         let controller = MachineServiceController(SocketFunction.browserNodeId());
         let serviceList = controller.getServiceList();
+        let machineConfigList = controller.getMachineConfigList();
 
         if (!serviceList) return <div>Loading services...</div>;
 
@@ -28,6 +29,7 @@ export class ServicesListPage extends qreact.Component {
         sort(services, x => -(x[1]?.info.lastUpdatedTime || Date.now()) - (x[1]?.parameters.deploy && (Number.MAX_SAFE_INTEGER / 2) || 0));
 
         let getMachineInfo = cache((machineId: string) => controller.getMachineInfo(machineId));
+        let getMachineConfig = cache((machineId: string) => controller.getMachineConfig(machineId));
 
         let keyCounts = new Map<string, number>();
         for (let [serviceId, config] of services) {
@@ -75,24 +77,34 @@ export class ServicesListPage extends qreact.Component {
             <div className={css.vbox(8)}>
                 {services.map(([serviceId, config]) => {
                     if (!config) return <div key={serviceId}>Config is broken? Missing value for service? Is the file corrupted?</div>;
-                    let failingMachines = config.machineIds.filter(machineId => {
+
+                    let disabledMachines = config.machineIds.filter(machineId => {
+                        let machineConfig = getMachineConfig(machineId);
+                        return machineConfig?.disabled;
+                    });
+                    let enabledMachineIds = config.machineIds.filter(machineId => {
+                        let machineConfig = getMachineConfig(machineId);
+                        return !machineConfig?.disabled;
+                    });
+
+                    let failingMachines = enabledMachineIds.filter(machineId => {
                         let machineInfo = getMachineInfo(machineId);
                         return machineInfo?.services[serviceId]?.errorFromLastRun;
                     });
-                    let runningMachines = config.machineIds.filter(machineId => {
+                    let runningMachines = enabledMachineIds.filter(machineId => {
                         let machineInfo = getMachineInfo(machineId);
                         let serviceInfo = machineInfo?.services[serviceId];
                         return serviceInfo && !serviceInfo.errorFromLastRun;
                     });
-                    let missingMachines = config.machineIds.filter(machineId => {
+                    let missingMachines = enabledMachineIds.filter(machineId => {
                         let machineInfo = getMachineInfo(machineId);
                         return !machineInfo?.services[serviceId];
                     });
-                    let totalLaunches = config.machineIds.reduce((acc, machineId) => {
+                    let totalLaunches = enabledMachineIds.reduce((acc, machineId) => {
                         let machineInfo = getMachineInfo(machineId);
                         return acc + (machineInfo?.services[serviceId]?.totalTimesLaunched || 0);
                     }, 0);
-                    let unknown = config.machineIds.length - runningMachines.length - failingMachines.length - missingMachines.length;
+                    let unknown = enabledMachineIds.length - runningMachines.length - failingMachines.length - missingMachines.length;
                     let duplicateKey = (keyCounts.get(config.parameters.key || "") || 0) > 1;
                     return <div className={css.hbox(10)}>
                         <Anchor noStyles key={serviceId}
@@ -118,7 +130,7 @@ export class ServicesListPage extends qreact.Component {
                                         {config.parameters.key}
                                     </div>
                                     <div>
-                                        {config.machineIds.length} configured {failingMachines.length > 0 && `(${failingMachines.length} failing)`} {missingMachines.length > 0 && `(${missingMachines.length} machine hasn't run service yet)`} {unknown > 0 && `(${unknown} unknown)`} • {totalLaunches} launches •
+                                        {enabledMachineIds.length} configured {disabledMachines.length > 0 && `(+${disabledMachines.length} on disabled machines)`} {failingMachines.length > 0 && `(${failingMachines.length} failing)`} {missingMachines.length > 0 && `(${missingMachines.length} machine hasn't run service yet)`} {unknown > 0 && `(${unknown} unknown)`} • {totalLaunches} launches •
                                         Deploy: {config.parameters.deploy ? "enabled" : "disabled"}
                                     </div>
                                 </div>

package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts

@@ -5,7 +5,7 @@ import { BufferIndex } from "./BufferIndex";
 import { delay, runInParallel, runInSerial, runInfinitePoll } from "socket-function/src/batching";
 import { IndexedLogResults, Reader, SearchParams, addReadToResults, createEmptyIndexedLogResults, INDEX_EXTENSION } from "./BufferIndexHelpers";
 import { getDomain, isPublic } from "../../../config";
-import { getArchivesLocal } from "../../../-a-archives/archivesDisk";
+import { getArchivesHome, getArchivesLocal } from "../../../-a-archives/archivesDisk";
 import { getArchivesBackblaze, getArchivesBackblazePrivateImmutable } from "../../../-a-archives/archivesBackBlaze";
 import { getOwnThreadId } from "../../../-a-auth/certs";
 import { ArchivesMemoryCacheStats, createArchivesMemoryCache } from "../../../-a-archives/archivesMemoryCache";
@@ -53,78 +53,6 @@ export class IndexedLogs<T> {
         }
     }
 
-    private findCallbacks = new Map<string, (match: T) => void>();
-    private resultsCallbacks = new Map<string, (results: IndexedLogResults) => void>();
-    public async clientFind(config: {
-        params: SearchParams;
-        onResult: (match: T) => void;
-        onResults?: (results: IndexedLogResults) => void;
-    }): Promise<IndexedLogResults> {
-        let controller = IndexedLogShimController.nodes[SocketFunction.getBrowserNodeId()];
-        let findId = nextId();
-        let callback = (match: T) => {
-            config.onResult(match);
-        };
-        this.findCallbacks.set(findId, callback);
-        if (config.onResults) {
-            this.resultsCallbacks.set(findId, config.onResults);
-        }
-        try {
-            return await controller.find({
-                findId,
-                indexedLogsName: this.config.name,
-                params: config.params,
-            });
-        } finally {
-            // There's some trailing time after the controller call finishes when the results will be trickling back to us. 
-            setTimeout(() => {
-                this.findCallbacks.delete(findId);
-                this.resultsCallbacks.delete(findId);
-            }, timeInMinute * 30);
-        }
-    }
-    public clientCancelAllCallbacks() {
-        this.findCallbacks.clear();
-        this.resultsCallbacks.clear();
-    }
-    public async clientGetPaths(config: {
-        startTime: number;
-        endTime: number;
-        only?: "local" | "public";
-    }): Promise<TimeFilePathWithSize[]> {
-        let controller = IndexedLogShimController.nodes[SocketFunction.getBrowserNodeId()];
-        return await controller.getPaths({
-            indexedLogsName: this.config.name,
-            startTime: config.startTime,
-            endTime: config.endTime,
-            only: config.only,
-        });
-    }
-    public onFindResult(config: {
-        findId: string;
-        result: unknown;
-    }) {
-        let callback = this.findCallbacks.get(config.findId);
-        if (!callback) throw new Error(`Find callback ${config.findId} not found`);
-        callback(config.result as T);
-    }
-    public async onResults(config: {
-        findId: string;
-        results: IndexedLogResults;
-    }): Promise<boolean> {
-        let callback = this.resultsCallbacks.get(config.findId);
-        if (!callback) throw new Error(`Results callback ${config.findId} not found`);
-        callback(config.results);
-        return true;
-    }
-
-    public async clientForceMoveLogsToPublic() {
-        let controller = IndexedLogShimController.nodes[SocketFunction.getBrowserNodeId()];
-        await controller.forceMoveLogsToPublic({
-            indexedLogsName: this.config.name,
-        });
-    }
-
     private localLogsStats = {
         cachedReads: 0,
         uncachedReads: 0,
@@ -145,7 +73,7 @@ export class IndexedLogs<T> {
     private fileSizeCache = new Map<string, number>();
 
     private getLocalLogs = lazy((): Archives => {
-        let baseDisk = getArchivesLocal(getDomain());
+        let baseDisk = getArchivesHome(getDomain());
         let archives = nestArchives("indexed-logs/" + this.config.name, baseDisk);
         archives = createArchivesMemoryCache(archives, {
             maxSize: 1024 * 1024 * 512,
@@ -158,7 +86,7 @@ export class IndexedLogs<T> {
         return archives;
     });
     private getPublicLogs = lazy((): Archives => {
-        let basePublic: Archives = getArchivesLocal(getDomain());
+        let basePublic: Archives = getArchivesHome(getDomain());
         // NOTE: The local disk is so fast that reading in 10 megabytes is nothing, And if we read in too small of a value, the overhead per read ends up making this take forever. 
         let extraReadSize = 1024 * 1024 * 10;
         if (this.config.forceUsePublicLogs || isPublic()) {
@@ -547,6 +475,80 @@ export class IndexedLogs<T> {
     }
 
 
+    // #region Client Calls
+    private findCallbacks = new Map<string, (match: T) => void>();
+    private resultsCallbacks = new Map<string, (results: IndexedLogResults) => void>();
+    public async clientFind(config: {
+        params: SearchParams;
+        onResult: (match: T) => void;
+        onResults?: (results: IndexedLogResults) => void;
+    }): Promise<IndexedLogResults> {
+        let controller = IndexedLogShimController.nodes[SocketFunction.getBrowserNodeId()];
+        let findId = nextId();
+        let callback = (match: T) => {
+            config.onResult(match);
+        };
+        this.findCallbacks.set(findId, callback);
+        if (config.onResults) {
+            this.resultsCallbacks.set(findId, config.onResults);
+        }
+        try {
+            return await controller.find({
+                findId,
+                indexedLogsName: this.config.name,
+                params: config.params,
+            });
+        } finally {
+            // There's some trailing time after the controller call finishes when the results will be trickling back to us. 
+            setTimeout(() => {
+                this.findCallbacks.delete(findId);
+                this.resultsCallbacks.delete(findId);
+            }, timeInMinute * 30);
+        }
+    }
+    public clientCancelAllCallbacks() {
+        this.findCallbacks.clear();
+        this.resultsCallbacks.clear();
+    }
+    public async clientGetPaths(config: {
+        startTime: number;
+        endTime: number;
+        only?: "local" | "public";
+    }): Promise<TimeFilePathWithSize[]> {
+        let controller = IndexedLogShimController.nodes[SocketFunction.getBrowserNodeId()];
+        return await controller.getPaths({
+            indexedLogsName: this.config.name,
+            startTime: config.startTime,
+            endTime: config.endTime,
+            only: config.only,
+        });
+    }
+    public onFindResult(config: {
+        findId: string;
+        result: unknown;
+    }) {
+        let callback = this.findCallbacks.get(config.findId);
+        if (!callback) throw new Error(`Find callback ${config.findId} not found`);
+        callback(config.result as T);
+    }
+    public async onResults(config: {
+        findId: string;
+        results: IndexedLogResults;
+    }): Promise<boolean> {
+        let callback = this.resultsCallbacks.get(config.findId);
+        if (!callback) throw new Error(`Results callback ${config.findId} not found`);
+        callback(config.results);
+        return true;
+    }
+
+    public async clientForceMoveLogsToPublic() {
+        let controller = IndexedLogShimController.nodes[SocketFunction.getBrowserNodeId()];
+        await controller.forceMoveLogsToPublic({
+            indexedLogsName: this.config.name,
+        });
+    }
+    // #endregion
+
 
     // TEST functions
     public async TEST_flushNow() {

package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx

@@ -854,6 +854,11 @@ export class LogViewer3 extends qreact.Component {
                     >
                         {savedPathsURL.value ? `Clear Frozen Files (${this.getPaths().length})` : `Freeze Files (${this.state.paths.length})`}
                     </Button>
+                    {!savedPathsURL.value && (
+                        <div className={css.pad2(8, 4).hsl(40, 60, 50).colorhsl(40, 0, 100)}>
+                            Files frozen in memory. Click Preview Files to refresh.
+                        </div>
+                    )}
                     {(() => {
                         if (!savedPathsURL.value) return undefined;
                         const paths = this.getPaths();

package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx

@@ -19,12 +19,11 @@ IMPORTANT! Now I am properly calling shutdown, so none of the streamed logs shou
 
 
 
-4) Make it easy to enable or disable an entire server, regardless of what services are on it.
-- This is annoying, but it would be very useful. I think the apply loop can probably figure it out. We should probably ask the AI to do it. I'm sure it'll fuck it up, but it'll give us a start at least. And we can also just tell it, okay, find the actual code that we're going to need to change, but not change it, and just keep and maybe even have it put a comment there. And then we just keep doing that until we're absolutely certain that we found every place that we need to change to make this work. And then the AI might be able to help with the refactor. 
-3) Start the servers again, and deploy all of our code
+OH! The archives have to be put in the home folder. They aren't?
 
-2) Create lot of remote server logs
-    - Via our refresh loop
+
+Hmm... why are not enough logs appearing?
+    - We only have logs for server.ts? Wtf?
 
 2.0) SUPPORT reading pending from multiple servers
     - The main controller has to find a node on each other machine, and call it. Only one node per machine though, so it shouldn't be too difficult.
@@ -35,6 +34,14 @@ IMPORTANT! Now I am properly calling shutdown, so none of the streamed logs shou
 2) Add a UI toggle to read public logs (only shows up on a non-public server though, as otherwise it wouldn't make sense)
     - Basically, just changes the code we're reading from multiple servers to select public servers instead, and then, of course, skip ourselves. 
 
+BUG: UGH... live logs for the remote server isn't working...
+    "new non-local WATCH"
+    - UGH... it being pending logs is annoying, as that's hard to debug locally...
+    AH! Why do we have so few logs?
+
+2) Create lot of remote server logs
+    - Via our refresh loop
+
 3) Verify true remote reads are reasonable fast
 
 3) Deploy service for movelogs
@@ -63,10 +70,12 @@ Rewrite error notification code
     AND, the only watcher will be the watcher service. You can't get recent errors, or any errors, without going through one of those
     NO dev errors. They are usually red-herrings anyways... and we should just be using public servers for regular usage
         - And we still have dev logs we can check if to see if an error happened locally
+    ALSO owns the discord messaging code
 
 
 
 Remove all old LogViewer/FastArchiveAppendable code
+    - Make sure to find all links and update them as well
 
 
 0) Add LZ4 compression to socket-function by default