querysub 0.367.0 → 0.369.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.367.0",
+  "version": "0.369.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -109,6 +109,10 @@ export function isOwnNodeId(nodeId: string): boolean {
     return false;
 }
 
+export function isNodeIdOnOwnMachineId(nodeId: string): boolean {
+    return certs.getMachineId(nodeId) === getOwnMachineId() || nodeId.startsWith("127-0-0-1.");
+}
+
 let nodeOverrides: string[] | undefined;
 let beforeGetNodeAllId = async () => { };
 export async function getAllNodeIds() {

package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts

@@ -12,7 +12,7 @@ import { ArchivesMemoryCacheStats, createArchivesMemoryCache } from "../../../-a
 import { registerShutdownHandler } from "../../periodic";
 import { measureBlock, measureFnc } from "socket-function/src/profiling/measure";
 import { isNode } from "typesafecss";
-import { getOwnMachineId } from "../../../-f-node-discovery/NodeDiscovery";
+import { getOwnMachineId, isNodeIdOnOwnMachineId, isOwnNodeId } from "../../../-f-node-discovery/NodeDiscovery";
 import { TimeFilePath, TimeFileTree } from "./TimeFileTree";
 import { LogStreamer } from "./LogStreamer";
 import { moveLogsToPublic } from "./moveIndexLogsToPublic";
@@ -226,10 +226,11 @@ export class IndexedLogs<T> {
 
     private machineNodeCache = new Map<string, string>();
     private async getMachineNodes(): Promise<string[]> {
-        let ownMachineId = getOwnMachineId();
         let nodeIds = await getAllNodeIds();
+        nodeIds = nodeIds.filter(x => !isNodeIdOnOwnMachineId(x));
         let byMachineId = keyByArray(nodeIds, x => getMachineId(x));
-        byMachineId.delete(ownMachineId);
+
+        let nextCache = new Map<string, string>();
 
         await Promise.all(Array.from(byMachineId.entries()).map(async ([machineId, nodeIds]) => {
             let added = false;
@@ -239,12 +240,12 @@ export class IndexedLogs<T> {
                 if (!hasLogger) return false;
                 // NOTE: Prefer to do the searching on the move logs service. However, if it's not available, any service can do searching. It just might lag that server... 
                 if (preferredOnly) {
-                    let entryPoint = await NodeCapabilitiesController.nodes[nodeId].getEntryPoint();
-                    if (!entryPoint.includes("movelogs")) return false;
+                    let entryPoint = await timeoutToUndefinedSilent(2500, NodeCapabilitiesController.nodes[nodeId].getEntryPoint());
+                    if (!entryPoint?.includes("movelogs")) return false;
                 }
                 added = true;
 
-                this.machineNodeCache.set(machineId, nodeId);
+                nextCache.set(machineId, nodeId);
                 return true;
             };
             let cached = this.machineNodeCache.get(machineId);
@@ -261,6 +262,7 @@ export class IndexedLogs<T> {
             }
         }));
 
+        this.machineNodeCache = nextCache;
         return Array.from(this.machineNodeCache.values());
     }
 
@@ -298,21 +300,28 @@ export class IndexedLogs<T> {
         let backblazeLogs = this.getPublicLogs();
         let paths: TimeFilePath[] = [];
 
+        let promises: Promise<void>[] = [];
+
         if (config.only !== "public") {
-            let localPaths = await new TimeFileTree(localLogs).findAllPaths({
-                startTime: config.startTime,
-                endTime: config.endTime,
-            });
-            paths.push(...localPaths);
+            promises.push((async () => {
+                let localPaths = await new TimeFileTree(localLogs).findAllPaths({
+                    startTime: config.startTime,
+                    endTime: config.endTime,
+                });
+                paths.push(...localPaths);
+            })());
         }
 
         if (config.only !== "local") {
-            let backblazePaths = await new TimeFileTree(backblazeLogs).findAllPaths({
-                startTime: config.startTime,
-                endTime: config.endTime,
-            });
-            paths.push(...backblazePaths);
+            promises.push((async () => {
+                let backblazePaths = await new TimeFileTree(backblazeLogs).findAllPaths({
+                    startTime: config.startTime,
+                    endTime: config.endTime,
+                });
+                paths.push(...backblazePaths);
+            })());
         }
+        await Promise.all(promises);
 
         // Bake in the size to ensure more cache hits (without this, every single access will always result in a file/remote call).
         let pathsWithSize = await Promise.all(paths.map(async (path): Promise<TimeFilePathWithSize> => {
@@ -351,12 +360,16 @@ export class IndexedLogs<T> {
         let allResults = new Map<string, IndexedLogResults>();
         let allDone: Promise<void>[] = [];
 
-        if (!config.params.only && (config.params.forceReadPublic || isPublic())) {
+        if (config.params.only !== "local" && (config.params.forceReadPublic || isPublic())) {
             let machineNodes = await this.getMachineNodes();
             allDone.push(...machineNodes.map(async (machineNode) => {
                 try {
                     let resultObj = await this.clientFind({
                         ...config,
+                        params: {
+                            ...config.params,
+                            only: "local",
+                        },
                         nodeId: machineNode,
                         onResult(match) {
                             config.onResult(match);
@@ -374,6 +387,7 @@ export class IndexedLogs<T> {
 
         // If we're forcefully reading from the public server, but we're not public, the code above will be the only code which adds to results
         if (!isPublic() && config.params.forceReadPublic) {
+            await Promise.all(allDone);
             return getFinalResults();
         }
 
@@ -696,7 +710,6 @@ class IndexedLogClient {
         indexedLogsName: string;
         result: unknown;
     }) {
-        if (isNode()) return;
         let indexedLogs = loggerByName.get(config.indexedLogsName);
         if (!indexedLogs) throw new Error(`Indexed logs ${config.indexedLogsName} not found`);
         indexedLogs.onFindResult({
@@ -709,7 +722,6 @@ class IndexedLogClient {
         indexedLogsName: string;
         results: IndexedLogResults;
     }): Promise<boolean> {
-        if (isNode()) return true;
         let indexedLogs = loggerByName.get(config.indexedLogsName);
         if (!indexedLogs) throw new Error(`Indexed logs ${config.indexedLogsName} not found`);
         return await indexedLogs.onResults({

package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx

@@ -25,6 +25,7 @@ import { errorToUndefined } from "../../../errors";
 import { IndexedLogResults, createEmptyIndexedLogResults, mergeIndexedLogResults } from "./BufferIndexHelpers";
 import { TimeFilePath } from "./TimeFileTree";
 import { isPublic } from "../../../config";
+import { Zip } from "../../../zip";
 
 let searchText = new URLParam("searchText", "");
 let excludePendingResults = new URLParam("excludePendingResults", false);
@@ -76,7 +77,7 @@ export class LogViewer3 extends qreact.Component {
         let paths: TimeFilePathWithSize[] = [];
         if (savedPathsURL.value) {
             const compressed = Buffer.from(savedPathsURL.value, "base64");
-            const decompressed = LZ4.decompress(compressed);
+            const decompressed = Zip.gunzipSync(compressed);
             paths = JSON.parse(decompressed.toString("utf8"));
         } else {
             paths = this.state.paths;
@@ -95,7 +96,7 @@ export class LogViewer3 extends qreact.Component {
     freezePaths() {
         const json = JSON.stringify(this.state.paths);
         const buffer = Buffer.from(json, "utf8");
-        const compressed = LZ4.compress(buffer);
+        const compressed = Zip.gzipSync(buffer, 9);
         savedPathsURL.value = compressed.toString("base64");
     }
 
@@ -342,6 +343,8 @@ export class LogViewer3 extends qreact.Component {
 
         Querysub.commitLocal(() => {
             this.state.loadingPaths = true;
+            this.state.results = [];
+            this.state.stats = undefined;
         });
 
         let loggers = await getLoggers2Async();
@@ -364,15 +367,15 @@ export class LogViewer3 extends qreact.Component {
         let allPaths: TimeFilePathWithSize[] = [];
         let range = Querysub.localRead(() => getTimeRange());
 
-        for (let logger of selectedLoggers) {
+        await Promise.all(selectedLoggers.map(async (logger) => {
             let paths = await logger.clientGetPaths({
                 startTime: range.startTime,
                 endTime: range.endTime,
-                only: excludePendingResults.value ? "local" : undefined,
+                only: excludePendingResults.value ? "public" : undefined,
                 forceReadPublic: readPublicLogs.value,
             });
             allPaths.push(...paths);
-        }
+        }));
         sort(allPaths, x => -x.startTime);
 
         Querysub.commitLocal(() => {
@@ -679,7 +682,6 @@ export class LogViewer3 extends qreact.Component {
         return (
             <div className={css.vbox(20).pad2(20).fillBoth}>
                 <div className={css.hbox(20)}>
-                    <div>Log Viewer 3</div>
                     <InputLabelURL
                         checkbox
                         label="Exclude Pending Results"
@@ -740,7 +742,9 @@ export class LogViewer3 extends qreact.Component {
                     </Button>
                     <TimeRangeSelector />
                 </div>
-                {!isPublic() && readPublicLogs.value && <h1>Reading public logs</h1>}
+                {!isPublic() && readPublicLogs.value && <div className={css.fontSize(40).pad2(12, 6).hsl(280, 40, 50).colorhsl(0, 0, 100).boldStyle}>
+                    Reading public logs
+                </div>}
 
                 <div className={css.hbox(10).fillWidth}>
                     <InputLabelURL

package/src/diagnostics/logs/IndexedLogs/TimeFileTree.ts

@@ -156,12 +156,12 @@ export class TimeFileTree {
                 // Find all days in this month
                 const days = await this.archives.find(monthFolder + "/", { shallow: true, type: "folders" });
 
-                for (const dayFolder of days) {
+                await Promise.all(days.map(async (dayFolder) => {
                     // dayFolder is a full path like "2024/01/15"
                     const day = parseInt(dayFolder.split("/").at(-1) || "", 10);
 
                     if (isNaN(day) || day < 1 || day > 31) {
-                        continue;
+                        return;
                     }
 
                     // Check if this day is in range
@@ -169,7 +169,7 @@ export class TimeFileTree {
                     const dayEndDate = new Date(Date.UTC(year, month - 1, day, 23, 59, 59, 999));
 
                     if (dayEndDate.getTime() < config.startTime || dayDate.getTime() > config.endTime) {
-                        continue;
+                        return;
                     }
 
                     // Find all log files in this day
@@ -187,7 +187,7 @@ export class TimeFileTree {
                             results.push(decoded);
                         }
                     }
-                }
+                }));
             }
         }
 

package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx

@@ -23,6 +23,8 @@ IMPORTANT! Now I am properly calling shutdown, so none of the streamed logs shou
 1) Verify isPublic searching works (from our local machine)
 
 
+1) Deploy our changes
+
 
 2) Create lot of remote server logs
     - Via our refresh loop