querysub 0.368.0 → 0.370.0

package/bin/movelogs.js

@@ -1,4 +1,7 @@
 #!/usr/bin/env node
 
+process.argv.push("--nobreak");
+process.argv.push("--public");
+
 require("typenode");
 require("../src/diagnostics/logs/IndexedLogs/moveLogsEntry");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.368.0",
+  "version": "0.370.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -109,6 +109,10 @@ export function isOwnNodeId(nodeId: string): boolean {
     return false;
 }
 
+export function isNodeIdOnOwnMachineId(nodeId: string): boolean {
+    return certs.getMachineId(nodeId) === getOwnMachineId() || nodeId.startsWith("127-0-0-1.");
+}
+
 let nodeOverrides: string[] | undefined;
 let beforeGetNodeAllId = async () => { };
 export async function getAllNodeIds() {

package/src/3-path-functions/PathFunctionRunnerMain.ts

@@ -23,6 +23,7 @@ import { green, magenta } from "socket-function/src/formatting/logColors";
 import { parseFilterSelector } from "../misc/filterable";
 import path from "path";
 import { Querysub } from "../4-querysub/QuerysubController";
+import { IndexedLogs } from "../diagnostics/logs/IndexedLogs/IndexedLogs";
 
 async function main() {
     Error.stackTraceLimit = 20;
@@ -39,6 +40,10 @@ async function main() {
 
     await Querysub.hostService("PathFunctionRunnerMain");
 
+    if (!isPublic()) {
+        IndexedLogs.runLogMoveLoop();
+    }
+
     // Use a fairly high stick time (the default is 10s), because having wait to sync data is very slow,
     //  and the function runner SHOULD have more memory than the clients, and much faster network speeds
     //  (it should be on the same local network as the path value authorities).

package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts

@@ -12,7 +12,7 @@ import { ArchivesMemoryCacheStats, createArchivesMemoryCache } from "../../../-a
 import { registerShutdownHandler } from "../../periodic";
 import { measureBlock, measureFnc } from "socket-function/src/profiling/measure";
 import { isNode } from "typesafecss";
-import { getOwnMachineId } from "../../../-f-node-discovery/NodeDiscovery";
+import { getOwnMachineId, isNodeIdOnOwnMachineId, isOwnNodeId } from "../../../-f-node-discovery/NodeDiscovery";
 import { TimeFilePath, TimeFileTree } from "./TimeFileTree";
 import { LogStreamer } from "./LogStreamer";
 import { moveLogsToPublic } from "./moveIndexLogsToPublic";
@@ -226,10 +226,11 @@ export class IndexedLogs<T> {
 
     private machineNodeCache = new Map<string, string>();
     private async getMachineNodes(): Promise<string[]> {
-        let ownMachineId = getOwnMachineId();
         let nodeIds = await getAllNodeIds();
+        nodeIds = nodeIds.filter(x => !isNodeIdOnOwnMachineId(x));
         let byMachineId = keyByArray(nodeIds, x => getMachineId(x));
-        byMachineId.delete(ownMachineId);
+
+        let nextCache = new Map<string, string>();
 
         await Promise.all(Array.from(byMachineId.entries()).map(async ([machineId, nodeIds]) => {
             let added = false;
@@ -244,7 +245,7 @@ export class IndexedLogs<T> {
                 }
                 added = true;
 
-                this.machineNodeCache.set(machineId, nodeId);
+                nextCache.set(machineId, nodeId);
                 return true;
             };
             let cached = this.machineNodeCache.get(machineId);
@@ -261,6 +262,7 @@ export class IndexedLogs<T> {
             }
         }));
 
+        this.machineNodeCache = nextCache;
         return Array.from(this.machineNodeCache.values());
     }
 
@@ -358,7 +360,7 @@ export class IndexedLogs<T> {
         let allResults = new Map<string, IndexedLogResults>();
         let allDone: Promise<void>[] = [];
 
-        if (!config.params.only && (config.params.forceReadPublic || isPublic())) {
+        if (config.params.only !== "local" && (config.params.forceReadPublic || isPublic())) {
             let machineNodes = await this.getMachineNodes();
             allDone.push(...machineNodes.map(async (machineNode) => {
                 try {
@@ -385,6 +387,7 @@ export class IndexedLogs<T> {
 
         // If we're forcefully reading from the public server, but we're not public, the code above will be the only code which adds to results
         if (!isPublic() && config.params.forceReadPublic) {
+            await Promise.all(allDone);
             return getFinalResults();
         }
 
@@ -707,7 +710,6 @@ class IndexedLogClient {
         indexedLogsName: string;
         result: unknown;
     }) {
-        if (isNode()) return;
         let indexedLogs = loggerByName.get(config.indexedLogsName);
         if (!indexedLogs) throw new Error(`Indexed logs ${config.indexedLogsName} not found`);
         indexedLogs.onFindResult({
@@ -720,7 +722,6 @@ class IndexedLogClient {
         indexedLogsName: string;
         results: IndexedLogResults;
     }): Promise<boolean> {
-        if (isNode()) return true;
         let indexedLogs = loggerByName.get(config.indexedLogsName);
         if (!indexedLogs) throw new Error(`Indexed logs ${config.indexedLogsName} not found`);
         return await indexedLogs.onResults({

package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx

@@ -371,7 +371,7 @@ export class LogViewer3 extends qreact.Component {
             let paths = await logger.clientGetPaths({
                 startTime: range.startTime,
                 endTime: range.endTime,
-                only: excludePendingResults.value ? "local" : undefined,
+                only: excludePendingResults.value ? "public" : undefined,
                 forceReadPublic: readPublicLogs.value,
             });
             allPaths.push(...paths);
@@ -682,7 +682,6 @@ export class LogViewer3 extends qreact.Component {
         return (
             <div className={css.vbox(20).pad2(20).fillBoth}>
                 <div className={css.hbox(20)}>
-                    <div>Log Viewer 3</div>
                     <InputLabelURL
                         checkbox
                         label="Exclude Pending Results"

package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx

@@ -19,17 +19,12 @@ IMPORTANT! Now I am properly calling shutdown, so none of the streamed logs shou
 
 
 
-0) Deploy again
-1) Verify isPublic searching works (from our local machine)
-
-
 
 2) Create lot of remote server logs
     - Via our refresh loop
 
-
-3) Create and deploy service for movelogs
-3.0) Update preferred entrypoint to be the movelogs entrypoint
+3) Deploy service for movelogs
+    "yarn movelogs"
 
 0) Run move logs in function runner, in development, just so we don't get too far behind