querysub 0.371.0 → 0.373.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.371.0",
+  "version": "0.373.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/deployManager/components/ServicesListPage.tsx

@@ -130,7 +130,7 @@ export class ServicesListPage extends qreact.Component {
                                         {config.parameters.key}
                                     </div>
                                     <div>
-                                        {enabledMachineIds.length} configured {disabledMachines.length > 0 && `(+${disabledMachines.length} on disabled machines)`} {failingMachines.length > 0 && `(${failingMachines.length} failing)`} {missingMachines.length > 0 && `(${missingMachines.length} machine hasn't run service yet)`} {unknown > 0 && `(${unknown} unknown)`} • {totalLaunches} launches •
+                                        <span className={css.boldStyle.fontSize(14)}>{enabledMachineIds.length}</span> configured {disabledMachines.length > 0 && `(+${disabledMachines.length} on disabled machines)`} {failingMachines.length > 0 && `(${failingMachines.length} failing)`} {missingMachines.length > 0 && `(${missingMachines.length} machine hasn't run service yet)`} {unknown > 0 && `(${unknown} unknown)`} • {totalLaunches} launches •
                                         Deploy: {config.parameters.deploy ? "enabled" : "disabled"}
                                     </div>
                                 </div>

package/src/diagnostics/logs/IndexedLogs/FilePathSelector.tsx

@@ -25,6 +25,15 @@ export type FilePathsByMachine = {
     errorCount: number;
 };
 
+function getSourceColor(sourceName: string | undefined): string | undefined {
+    if (!sourceName) return undefined;
+    const lastPart = sourceName.split("/").pop()?.toLowerCase();
+    if (lastPart === "error") return css.colorhsl(0, 80, 40);
+    if (lastPart === "warn") return css.colorhsl(40, 80, 40);
+    if (lastPart === "info") return css.colorhsl(280, 80, 40);
+    if (lastPart === "log") return css.colorhsl(200, 80, 40);
+    return undefined;
+}
 
 export class FilePathSelector extends qreact.Component<{
     paths: TimeFilePathWithSize[];
@@ -136,8 +145,6 @@ export class FilePathSelector extends qreact.Component<{
                     <div>|</div>
                     <div>Machines: {formatNumber(grouped.length)}</div>
                     <div>|</div>
-                    <div>Threads: {formatNumber(totals.uniqueThreads)}</div>
-                    <div>|</div>
                     <div>Files: {formatNumber(this.props.paths.length)}</div>
                     <div>|</div>
                     <div>Pending: {formatNumber(totals.totalPending)}</div>
@@ -379,7 +386,7 @@ export class FilePathSelectorModal extends qreact.Component<{
                 </td>
                 <td className={css.pad2(2)}>{this.props.formatBytes(file.size)}</td>
                 <td className={css.pad2(2)}>{file.logCount !== undefined ? formatNumber(file.logCount) : <span className={css.colorhsl(40, 60, 40)}>pending</span>}</td>
-                <td className={css.pad2(2)}>{file.sourceName}</td>
+                <td className={css.pad2(2) + (getSourceColor(file.sourceName) || "")}>{file.sourceName}</td>
                 <td className={css.pad2(2)}>{file.dedupe}</td>
                 <td className={css.pad2(2)}>{formatDateTime(file.startTime)}</td>
                 <td className={css.pad2(2)}>
@@ -418,8 +425,7 @@ export class FilePathSelectorModal extends qreact.Component<{
                     <div className={css.vbox(5)}>
                         <div className={css.hbox(10)}>
                             <div className={css.minWidth(120)}>Machines: {formatNumber(summary.machineCount)}</div>
-                            {/* NOTE: If the files have been merged, the thread count doesn't matter anymore, so it's deceiving to put the thread count here. */}
-                            {/* <div className={css.minWidth(110)}>Threads: {formatNumber(summary.threadCount)}</div> */}
+                            <div className={css.minWidth(110)}>Threads: {formatNumber(summary.threadCount)}</div>
                             <div className={css.minWidth(140)}>Files: {formatNumber(summary.fileCount)} ({formatNumber(summary.pendingCount)} pending)</div>
                             <div className={css.minWidth(120)}>Size: {this.props.formatBytes(summary.totalSize)}</div>
                             <div className={css.minWidth(100)}>Logs: {formatNumber(summary.totalLogCount)}</div>
@@ -432,7 +438,7 @@ export class FilePathSelectorModal extends qreact.Component<{
                             <div className={css.vbox(5)}>
                                 {Array.from(bySource.entries()).map(([sourceName, stats]) => (
                                     <div key={sourceName} className={css.hbox(10)}>
-                                        <div className={css.minWidth(240)}>{sourceName}:</div>
+                                        <div className={css.minWidth(240) + (getSourceColor(sourceName) || "")}>{sourceName}:</div>
                                         <div className={css.minWidth(140)}>Files: {formatNumber(stats.fileCount)} ({formatNumber(stats.pendingCount)} pending)</div>
                                         <div className={css.minWidth(120)}>Size: {this.props.formatBytes(stats.size)}</div>
                                         <div className={css.minWidth(100)}>Logs: {formatNumber(stats.logCount)}</div>

package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts

@@ -25,6 +25,7 @@ import { blue } from "socket-function/src/formatting/logColors";
 import { LimitGroup } from "../../../functional/limitProcessing";
 import { getAllNodeIds } from "../../../-f-node-discovery/NodeDiscovery";
 import { NodeCapabilitiesController } from "../../../-g-core-values/NodeCapabilities";
+import { getLoggers2Async } from "../diskLogger";
 
 export type TimeFilePathWithSize = TimeFilePath & {
     size: number;
@@ -40,7 +41,6 @@ export class IndexedLogs<T> {
         name: string,
         maxSingleFileData?: number;
         maxCountPerFile?: number;
-        forceUsePublicLogs?: boolean;
         getTime: (result: T) => number | undefined;
     }) {
         loggerByName.set(this.config.name, this as any);
@@ -48,9 +48,10 @@ export class IndexedLogs<T> {
 
 
     private static shouldRunLoop = false;
-    public static runLogMoveLoop() {
+    public static async runLogMoveLoop() {
         IndexedLogs.shouldRunLoop = true;
-        for (let indexedLogs of loggerByName.values()) {
+        let allLoggers = await getLoggers2Async();
+        for (let indexedLogs of Object.values(allLoggers)) {
             indexedLogs.runLogMoverLoop();
         }
     }
@@ -91,7 +92,7 @@ export class IndexedLogs<T> {
         let basePublic: Archives = getArchivesHome(getDomain());
         // NOTE: The local disk is so fast that reading in 10 megabytes is nothing, And if we read in too small of a value, the overhead per read ends up making this take forever. 
         let extraReadSize = 1024 * 1024 * 10;
-        if (this.config.forceUsePublicLogs || isPublic()) {
+        if (isPublic()) {
             basePublic = getArchivesBackblaze(getDomain());
             // NOTE: While the latency to back plays is high, now we're reading in parallel, so it shouldn't be as big of an issue. 
             extraReadSize = 1024 * 1024 * 1;
@@ -274,8 +275,16 @@ export class IndexedLogs<T> {
         forceReadPublic?: boolean;
     }): Promise<TimeFilePathWithSize[]> {
         let finalPaths: TimeFilePathWithSize[] = [];
+        if (config.forceReadPublic && !isPublic()) {
+            let machineNodes = await this.getMachineNodes();
+            if (machineNodes.length === 0) throw new Error(`Cannot find any public nodes to read from`);
+            return await IndexedLogShimController.nodes[machineNodes[0]].getPaths({
+                ...config,
+                indexedLogsName: this.config.name,
+            });
+        }
 
-        if (!config.only && (config.forceReadPublic || isPublic())) {
+        if (config.only !== "local" && isPublic()) {
             let machineNodes = await this.getMachineNodes();
             await Promise.all(machineNodes.map(async (machineNode) => {
                 try {
@@ -291,11 +300,6 @@ export class IndexedLogs<T> {
             }));
         }
 
-        // If we're forcefully reading from the public server, but we're not public, the code above will be the only code which adds to results
-        if (!isPublic() && config.forceReadPublic) {
-            return finalPaths;
-        }
-
         let localLogs = this.getLocalLogs();
         let backblazeLogs = this.getPublicLogs();
         let paths: TimeFilePath[] = [];
@@ -347,6 +351,16 @@ export class IndexedLogs<T> {
         onResult: (match: T) => void;
         onResults?: (results: IndexedLogResults) => Promise<boolean>;
     }): Promise<IndexedLogResults> {
+
+        if (config.params.forceReadPublic && !isPublic()) {
+            let machineNodes = await this.getMachineNodes();
+            if (machineNodes.length === 0) throw new Error(`Cannot find any public nodes to read from`);
+            return await this.clientFind({
+                ...config,
+                nodeId: machineNodes[0],
+            });
+        }
+
         let startTime = Date.now();
         let interval: NodeJS.Timeout | undefined;
 
@@ -360,7 +374,8 @@ export class IndexedLogs<T> {
         let allResults = new Map<string, IndexedLogResults>();
         let allDone: Promise<void>[] = [];
 
-        if (config.params.only !== "local" && (config.params.forceReadPublic || isPublic())) {
+        // Read other nodes if on a public server
+        if (config.params.only !== "local" && isPublic()) {
             let machineNodes = await this.getMachineNodes();
             allDone.push(...machineNodes.map(async (machineNode) => {
                 try {
@@ -385,12 +400,6 @@ export class IndexedLogs<T> {
             }));
         }
 
-        // If we're forcefully reading from the public server, but we're not public, the code above will be the only code which adds to results
-        if (!isPublic() && config.params.forceReadPublic) {
-            await Promise.all(allDone);
-            return getFinalResults();
-        }
-
 
         let results: IndexedLogResults = createEmptyIndexedLogResults();
         allResults.set("", results);
@@ -588,6 +597,7 @@ export class IndexedLogs<T> {
         }
 
         await moveLogsToPublic({
+            loggerName: this.config.name,
             forceAll,
             localLogs: this.getLocalLogs(),
             publicLogs: this.getPublicLogs(),

package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx

@@ -686,11 +686,23 @@ export class LogViewer3 extends qreact.Component {
                         checkbox
                         label="Exclude Pending Results"
                         url={excludePendingResults}
+                        onChangeValue={(newValue) => {
+                            if (newValue) {
+                                savedPathsURL.value = "";
+                                void this.loadPaths();
+                            }
+                        }}
                     />
                     {!isPublic() && <InputLabelURL
                         checkbox
                         label="Read Public Logs"
                         url={readPublicLogs}
+                        onChangeValue={(newValue) => {
+                            if (newValue) {
+                                savedPathsURL.value = "";
+                                void this.loadPaths();
+                            }
+                        }}
                     />}
                     <InputLabelURL
                         label="Limit"

package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts

@@ -5,12 +5,13 @@ import { getOwnThreadId } from "../../../-f-node-discovery/NodeDiscovery";
 import { BufferIndex } from "./BufferIndex";
 import { LogStreamer } from "./LogStreamer";
 import { TimeFileTree } from "./TimeFileTree";
-import { green, magenta } from "socket-function/src/formatting/logColors";
+import { blue, green, magenta } from "socket-function/src/formatting/logColors";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 
 
 
 export async function moveLogsToPublic(config: {
+    loggerName: string;
     publicMoveThreshold: number;
     maxSingleFileData: number;
     movingTimeout: number;
@@ -128,7 +129,7 @@ export async function moveLogsToPublic(config: {
 
     if (!await tryToGetMoveLock()) return;
 
-    console.log(magenta(`Moving ${localPaths.length} log files to public`));
+    console.log(magenta(`Moving ${localPaths.length} log files to public (${blue(config.loggerName)})`), { loggerName: config.loggerName, localPaths });
 
     let byStartTime = keyByArray(localPaths, x => x.startTime);
 

package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx

@@ -20,21 +20,22 @@ IMPORTANT! Now I am properly calling shutdown, so none of the streamed logs shou
 
 
 
-AH, make sure force move works with force public flag!
-
-2) Create lot of remote server logs
-    - Via our refresh loop
-
-3) Ensure our moveloops are working correctly
-
-
+3) Ensure our moveloops are working correctly, on both dev and public
 
 
 1) Fix missing __NAME__
-     "Received PathValue for path" misses name?
-     - Maybe the missing name only happens when we rate limit?
+    "Received PathValue for path" misses name?
+    - Maybe the missing name only happens when we rate limit?
 
+2) Create lot of remote server logs
+    - Via our refresh loop
 
+3) Verify search is still fast, and works correctly!
+    - Try to get a rare value in that is in the logs, but is hard to find
+    Good rare search. Checks for sending emails, which we do very rarely. If we pack in 10GB of logs after this, it will really stress our indexing system.
+        - At the moment our pending index has a good hit rate (0 false positives out of 1720). We'll see if this continues for our remote index.
+    https://127-0-0-1.querysubtest.com:7007/?hot&readPublicLogs&showingmanagement&machineview=services&managementpage=LogViewer3&searchText=Sending%20email&selectedFields=%7B%22path%22%3Atrue%7D
+    
 
 
 Rewrite error notification code