querysub 0.373.0 → 0.375.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.373.0",
+  "version": "0.375.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -34,9 +34,7 @@
     "join": "./bin/join.js",
     "join-public": "./bin/join-public.js",
     "movelogs": "./bin/movelogs.js",
-    "addsuperuser": "./bin/addsuperuser.js",
-    "error-email": "./bin/error-email.js",
-    "error-im": "./bin/error-im.js"
+    "addsuperuser": "./bin/addsuperuser.js"
   },
   "dependencies": {
     "@types/fs-ext": "^2.0.3",

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -185,7 +185,7 @@ function addNodeIdBase(nodeId: string) {
 function setNodeIds(nodeIds: string[]) {
     nodeIds = nodeIds.filter(x => x !== SPECIAL_NODE_ID_FOR_UNMOUNTED_NODE);
 
-    logDisk("log", "setNodeIds", { nodeIds });
+    console.info("setNodeIds", { nodeIds });
     // Also try all localhost ports, if we are developing and not in public mode
     if (isNode() && !isPublic() && isDevDebugbreak()) {
         let ports = new Set(nodeIds.map(nodeId => decodeNodeId(nodeId)?.port).filter(isDefined));
@@ -350,7 +350,7 @@ async function runHeartbeatAuditLoop() {
                 }
             } else {
                 deadCount.delete(nodeId);
-                logDisk("log", "Read node heartbeat", { nodeId, lastTime });
+                console.info("Read node heartbeat", { nodeId, lastTime });
             }
         }
         if (pendingDeadCount) {

package/src/0-path-value-core/PathValueCommitter.ts

@@ -177,7 +177,7 @@ class PathValueCommitter {
                 markArrayAsSplitable(values);
                 const { Querysub } = await import("../4-querysub/Querysub");
                 let serializedValues = await pathValueSerializer.serialize(values, { compress: Querysub.COMPRESS_NETWORK });
-                logDisk("log", "Send PathValues to server", { valueCount: values.length, targetId: otherAuthority, });
+                console.info("Send PathValues to server", { valueCount: values.length, targetId: otherAuthority, });
                 let forwardPromise = PathValueController.nodes[otherAuthority].forwardWrites(
                     serializedValues,
                     undefined,

package/src/0-path-value-core/PathValueController.ts

@@ -86,9 +86,9 @@ class PathValueControllerBase {
                 auditLog("RECEIVE VALUE", { path: value.path, time: value.time.time, sourceNodeId });
             }
         }
-        logDisk("log", "Received PathValues via forwardWrites", { valueCount: values.length, callerId, });
+        console.info("Received PathValues via forwardWrites", { valueCount: values.length, callerId, });
         for (let value of values) {
-            logDisk("log", "Received PathValue for path", { path: value.path, time: value.time.time, callerId });
+            console.info("Received PathValue for path", { path: value.path, time: value.time.time, callerId });
         }
 
         if (isCoreQuiet) {

package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts

@@ -309,7 +309,7 @@ class TransactionLocker {
     }
     public async createConfirm(key: string) {
         let path = this.getConfirmKey(key);
-        logDisk("log", "Creating confirmation for ${key}");
+        console.info("Creating confirmation for ${key}");
         await this.storage.setValue(path, Buffer.from(""));
         return path;
     }
@@ -362,7 +362,7 @@ class TransactionLocker {
             delete: ellipsize(deletes.map(a => debugFileInfo(a.key)).join(","), 50),
         });
 
-        logDisk("log", "Writing transaction", {
+        console.info("Writing transaction", {
             name,
             ops: transaction.ops.length,
         });
@@ -488,7 +488,7 @@ class TransactionLocker {
                 }
             }
 
-            logDisk("log", "Read archive state", {
+            console.info("Read archive state", {
                 rawFilesCount: files.length,
                 confirmedCount: currentDataFiles.size,
                 rawFiles: files.map(a => a.file),
@@ -507,7 +507,7 @@ class TransactionLocker {
             let result = await tryToRead();
             if (result) {
                 let timeToRead = Date.now() - startTime;
-                logDisk("log", `Read data state in ${formatTime(timeToRead)}`);
+                console.info(`Read data state in ${formatTime(timeToRead)}`);
                 return result;
             }
         }
@@ -546,7 +546,7 @@ class TransactionLocker {
         let rawLookup = new Set(Array.from(rawDataFiles).map(a => a.file));
         // If any creates are not confirmed, it must not have been applied
         if (transaction.ops.some(a => a.type === "create" && rawLookup.has(a.key) && !confirmedKeys.has(a.key))) {
-            logDisk("log", `Transaction not applied (has pending confirmations of creates)`, {
+            console.info(`Transaction not applied (has pending confirmations of creates)`, {
                 keys: transaction.ops
                     .filter(a => a.type === "create" && rawLookup.has(a.key) && !confirmedKeys.has(a.key))
                     .map(a => a.key)
@@ -555,7 +555,7 @@ class TransactionLocker {
         }
         // If any deletes still exist, it must not have been applied
         if (transaction.ops.some(a => a.type === "delete" && confirmedKeys.has(a.key))) {
-            logDisk("log", `Transaction not applied (has pending deletes)`, {
+            console.info(`Transaction not applied (has pending deletes)`, {
                 keys: transaction.ops
                     .filter(a => a.type === "delete" && confirmedKeys.has(a.key))
                     .map(a => a.key)
@@ -568,7 +568,7 @@ class TransactionLocker {
         let createCount = transaction.ops.filter(a => a.type === "create").length;
         let deleteCount = transaction.ops.filter(a => a.type === "delete").length;
         let lockedFiles = transaction.lockedFilesMustEqual?.length;
-        logDisk("log", `Applying transaction with ${createCount} file creates and ${deleteCount} file deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`, {
+        console.info(`Applying transaction with ${createCount} file creates and ${deleteCount} file deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`, {
             transactions: transaction.ops.map(x => JSON.stringify(x)),
         });
         logNodeStats(`archives|TΔ Apply`, formatNumber, 1);
@@ -594,7 +594,7 @@ class TransactionLocker {
         };
         await Promise.all(list(CONCURRENT_WRITE_COUNT).map(runThread));
 
-        logDisk("log", `Applied transaction with ${createCount} file creates and file ${deleteCount} deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`, {
+        console.info(`Applied transaction with ${createCount} file creates and file ${deleteCount} deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`, {
             transactions: transaction.ops.map(x => JSON.stringify(x)),
         });
     }
@@ -660,7 +660,7 @@ class TransactionLocker {
         let threshold = activeT.createTime + this.storage.propagationTime;
         if (Date.now() < threshold) {
             let waitTime = threshold - Date.now();
-            logDisk("log", `Waiting ${formatTime(waitTime)} for transaction ${activeT.seqNum} to settle.`);
+            console.info(`Waiting ${formatTime(waitTime)} for transaction ${activeT.seqNum} to settle.`);
             await new Promise(resolve => setTimeout(resolve, waitTime));
             return this.getFilesBase();
         }
@@ -768,7 +768,7 @@ class TransactionLocker {
         let dels = transaction.ops.filter(a => a.type === "delete").length;
         let creates = transaction.ops.filter(a => a.type === "create").length;
         let createBytes = transaction.ops.map(a => a.type === "create" && a.value?.length || 0).reduce((a, b) => a + b, 0);
-        logDisk("log", `Starting transaction with ${creates} file creates and ${dels} file deletes, ${formatNumber(createBytes)}B`, {
+        console.info(`Starting transaction with ${creates} file creates and ${dels} file deletes, ${formatNumber(createBytes)}B`, {
             createFilesNames: transaction.ops.filter(a => a.type === "create").map(a => a.key),
             deleteFilesNames: transaction.ops.filter(a => a.type === "delete").map(a => a.key),
         });
@@ -797,7 +797,7 @@ class TransactionLocker {
             let beforeData = await this.getFilesBase();
             if (!this.isTransactionValid(transaction, beforeData.dataFiles, beforeData.rawDataFiles)) {
                 logNodeStats(`archives|TΔ Rejected`, formatNumber, 1);
-                logDisk("log", `Finished transaction with rejection, ${transaction.ops.length} ops`);
+                console.info(`Finished transaction with rejection, ${transaction.ops.length} ops`);
                 return "rejected";
             }
 
@@ -806,7 +806,7 @@ class TransactionLocker {
             let afterData = await this.getFilesBase();
             if (this.wasTransactionApplied(transaction, afterData.dataFiles, afterData.rawDataFiles)) {
                 logNodeStats(`archives|TΔ Accepted`, formatNumber, 1);
-                logDisk("log", `Finished transaction with ${transaction.ops.length} ops`);
+                console.info(`Finished transaction with ${transaction.ops.length} ops`);
                 return "accepted";
             }
         }

package/src/0-path-value-core/auditLogs.ts

@@ -58,7 +58,7 @@ function debugLogBase(type: string, values: { [key: string]: unknown }) {
         return;
     }
     if (isDiskAudit()) {
-        logDisk("log", type, values, { "diskAudit": true });
+        console.info(type, values, { "diskAudit": true });
     }
     let newEntry: DebugLog = { type, time: Date.now(), values };
     logHistory.push(newEntry);

package/src/0-path-value-core/pathValueCore.ts

@@ -1261,7 +1261,7 @@ class PathWatcher {
                     auditLog("new non-local WATCH PARENT", { path, watcher: config.callback });
                 }
             }
-            logDisk("log", `New PathValue watches`, {
+            console.info(`New PathValue watches`, {
                 newPathsWatched: newPathsWatched.size,
                 newParentsWatched: newParentsWatched.size,
             });
@@ -1394,7 +1394,7 @@ class PathWatcher {
         }
 
         if (fullyUnwatched.paths.length > 0 || fullyUnwatched.parentPaths.length > 0) {
-            logDisk("log", `Unwatched PathValue watches`, {
+            console.info(`Unwatched PathValue watches`, {
                 unwatchedPaths: fullyUnwatched.paths.length,
                 unwatchedParents: fullyUnwatched.parentPaths.length,
             });

package/src/3-path-functions/PathFunctionRunner.ts

@@ -689,7 +689,7 @@ export class PathFunctionRunner {
         let syncTime = wallTime - evalTime;
 
 
-        logDisk("log", "Finished FunctionRunner function", {
+        console.info("Finished FunctionRunner function", {
             ...callPath, argsEncoded: "", functionSpec,
             wallTime, syncTime, evalTime,
             loops: runCount,

package/src/3-path-functions/PathFunctionRunnerMain.ts

@@ -41,7 +41,7 @@ async function main() {
     await Querysub.hostService("PathFunctionRunnerMain");
 
     if (!isPublic()) {
-        IndexedLogs.runLogMoveLoop();
+        void IndexedLogs.runLogMoveLoop();
     }
 
     // Use a fairly high stick time (the default is 10s), because having wait to sync data is very slow,

package/src/4-dom/qreact.tsx

@@ -2119,7 +2119,7 @@ function updateDOMNodeFields(domNode: DOMNode, vNode: VirtualDOM, prevVNode: Vir
                                     if (name === "blur") {
                                         let target = args[0].currentTarget as HTMLElement;
                                         if (!target.getAttribute("data-blur-on-unmount") && !target.isConnected) {
-                                            logDisk("log", "Ignoring blur for disconnected element. You can use data-blur-on-unmount to re-enable blurs on this element.", target);
+                                            console.info("Ignoring blur for disconnected element. You can use data-blur-on-unmount to re-enable blurs on this element.", target);
                                             return;
                                         }
                                     }
@@ -2428,7 +2428,7 @@ function blurFixOnMouseDownHack(event: MouseEvent) {
 
     // Looks like we are going to blur, so blur now
     if (selected instanceof HTMLElement && !selected.hasAttribute("data-no-early-blur")) {
-        logDisk("log", `Simulating early blur to prevent blur from firing after mousedown. This solves a problem where mousedown changes the UI, and then the blur fires on the wrong element. You can use data-no-early-blur to opt-out of this feature`, selected);
+        console.info(`Simulating early blur to prevent blur from firing after mousedown. This solves a problem where mousedown changes the UI, and then the blur fires on the wrong element. You can use data-no-early-blur to opt-out of this feature`, selected);
         selected.blur();
     }
 }

package/src/4-querysub/QuerysubController.ts

@@ -480,7 +480,7 @@ export class QuerysubControllerBase {
                 //      make the UI look cleaner (instead of showing stale values, it shows nothing)?
                 let undefinedValues: PathValue[] = newPathsNotAllowed.map(path => ({ path, value: undefined, canGCValue: true, isTransparent: true, time: epochTime, locks: [], lockCount: 0, valid: true, event: false }));
 
-                logDisk("log", "Disallowing PathValue watches due to disallowed permissions", { count: newPathsNotAllowed.length, callerId });
+                console.info("Disallowing PathValue watches due to disallowed permissions", { count: newPathsNotAllowed.length, callerId });
 
                 ignoreErrors(pathValueSerializer.serialize(undefinedValues, { compress: Querysub.COMPRESS_NETWORK }).then(buffers =>
                     PathValueController.nodes[callerId].forwardWrites(

package/src/5-diagnostics/diskValueAudit.ts

@@ -188,7 +188,7 @@ async function checkAuthority(authority: AuthorityPath, threshold: number) {
         console.error(`Value mismatch between disk and memory for ${formatNumber(changedValues.size)} values. Ex: ${firstValue.path}`);
     }
     if (removedValues.size > 0) {
-        logDisk("log", green(`Removing ${formatNumber(removedValues.size)} paths from memory which have been GCed on the disk.`));
+        console.info(green(`Removing ${formatNumber(removedValues.size)} paths from memory which have been GCed on the disk.`));
     }
     if (changedValues.size > 0 || removedValues.size > 0) {
         let allValues = new Set([...changedValues, ...removedValues]);

package/src/deployManager/components/MachineDetailPage.tsx

@@ -7,10 +7,10 @@ import { formatNumber, formatVeryNiceDateTime } from "socket-function/src/format
 import { sort } from "socket-function/src/misc";
 import { ATag, Anchor } from "../../library-components/ATag";
 import { ShowMore } from "../../library-components/ShowMore";
-import { filterParam } from "../../diagnostics/logs/FastArchiveViewer";
 import { managementPageURL } from "../../diagnostics/managementPages";
 import { t } from "../../2-proxy/schema2";
 import { Querysub } from "../../4-querysub/QuerysubController";
+import { getLogViewerParams } from "../../diagnostics/logs/IndexedLogs/LogViewerParams";
 
 export class MachineDetailPage extends qreact.Component {
     render() {
@@ -105,10 +105,7 @@ export class MachineDetailPage extends qreact.Component {
                     <div>
                         Apply Node ID: {machine.applyNodeId}
                     </div>
-                    <ATag values={[
-                        managementPageURL.getOverride("LogViewer2"),
-                        filterParam.getOverride(`__machineId = ${machineInfo.machineId}`),
-                    ]}>
+                    <ATag values={getLogViewerParams({ __machineId: machineInfo.machineId })}>
                         Machine Logs
                     </ATag>
                 </div>

package/src/deployManager/components/ServiceDetailPage.tsx

@@ -20,8 +20,8 @@ import { PrimitiveDisplay } from "../../diagnostics/logs/ObjectDisplay";
 import { parseAnsiColors, rgbToHsl } from "../../diagnostics/logs/ansiFormat";
 import { RenderGitRefInfo, UpdateServiceButtons, bigEmoji, buttonStyle } from "./deployButtons";
 import { TypedConfigEditor } from "../../library-components/TypedConfigEditor";
-import { filterParam } from "../../diagnostics/logs/FastArchiveViewer";
 import { managementPageURL } from "../../diagnostics/managementPages";
+import { getLogViewerParams } from "../../diagnostics/logs/IndexedLogs/LogViewerParams";
 
 
 
@@ -401,10 +401,7 @@ export class ServiceDetailPage extends qreact.Component {
                                     {isWatching ? "Stop Watching Output" : "Watch Screen Output"}
                                 </div>
 
-                                <ATag values={[
-                                    managementPageURL.getOverride("LogViewer2"),
-                                    filterParam.getOverride(`__machineId = ${machineId}`),
-                                ]}>
+                                <ATag values={getLogViewerParams({ __machineId: machineId })}>
                                     Machine Logs
                                 </ATag>
                             </div>

package/src/deployManager/machineApplyMainCode.ts

@@ -866,6 +866,13 @@ export async function machineApplyMain() {
         if (await quickIsOutdated()) {
             console.log(magenta("Likely outdated, resyncing now"));
             await resyncServices();
+        } else {
+            let config = await machineInfos.get(getOwnMachineId());
+            if (config) {
+                config.heartbeat = Date.now();
+                console.log(magenta(`Updating heartbeat for ${getOwnMachineId()} to ${config.heartbeat}`));
+                await machineInfos.set(getOwnMachineId(), config);
+            }
         }
     });
 

package/src/diagnostics/NodeViewer.tsx

@@ -36,8 +36,8 @@ import { SocketRegistered } from "socket-function/SocketFunctionTypes";
 import { ATag } from "../library-components/ATag";
 import { getSyncedController } from "../library-components/SyncedController";
 import child_process from "child_process";
-import { filterParam } from "./logs/FastArchiveViewer";
 import { getHTTPSKeyCert } from "../-e-certs/certAuthority";
+import { getLogViewerParams } from "./logs/IndexedLogs/LogViewerParams";
 
 
 type NodeData = {
@@ -296,10 +296,9 @@ export class NodeViewer extends qreact.Component {
                                         if (str.startsWith("http")) return formatValue(obj);
 
                                         return (
-                                            <ATag values={[
-                                                managementPageURL.getOverride("LogViewer2"),
-                                                filterParam.getOverride(`__nodeId = ${str}`),
-                                            ]}>
+                                            <ATag values={getLogViewerParams({
+                                                __nodeId: str,
+                                            })}>
                                                 Logs
                                             </ATag>
                                         );

package/src/diagnostics/logs/IndexedLogs/BufferIndexHelpers.ts

@@ -16,7 +16,7 @@ export type SearchParams = {
     findBuffer: Buffer;
     pathOverrides?: TimeFilePathWithSize[];
     only?: "local" | "public";
-    forceReadPublic?: boolean;
+    forceReadProduction?: boolean;
 };
 
 export type Unit = number;

package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts

@@ -272,10 +272,10 @@ export class IndexedLogs<T> {
         startTime: number;
         endTime: number;
         only?: "local" | "public";
-        forceReadPublic?: boolean;
+        forceReadProduction?: boolean;
     }): Promise<TimeFilePathWithSize[]> {
         let finalPaths: TimeFilePathWithSize[] = [];
-        if (config.forceReadPublic && !isPublic()) {
+        if (config.forceReadProduction && !isPublic()) {
             let machineNodes = await this.getMachineNodes();
             if (machineNodes.length === 0) throw new Error(`Cannot find any public nodes to read from`);
             return await IndexedLogShimController.nodes[machineNodes[0]].getPaths({
@@ -352,7 +352,7 @@ export class IndexedLogs<T> {
         onResults?: (results: IndexedLogResults) => Promise<boolean>;
     }): Promise<IndexedLogResults> {
 
-        if (config.params.forceReadPublic && !isPublic()) {
+        if (config.params.forceReadProduction && !isPublic()) {
             let machineNodes = await this.getMachineNodes();
             if (machineNodes.length === 0) throw new Error(`Cannot find any public nodes to read from`);
             return await this.clientFind({
@@ -649,7 +649,7 @@ export class IndexedLogs<T> {
         startTime: number;
         endTime: number;
         only?: "local" | "public";
-        forceReadPublic?: boolean;
+        forceReadProduction?: boolean;
 
     }): Promise<TimeFilePathWithSize[]> {
         let controller = IndexedLogShimController.nodes[SocketFunction.getBrowserNodeId()];
@@ -658,7 +658,7 @@ export class IndexedLogs<T> {
             startTime: config.startTime,
             endTime: config.endTime,
             only: config.only,
-            forceReadPublic: config.forceReadPublic,
+            forceReadProduction: config.forceReadProduction,
         });
     }
     public onFindResult(config: {
@@ -778,7 +778,7 @@ class IndexedLogShim {
         startTime: number;
         endTime: number;
         only?: "local" | "public";
-        forceReadPublic?: boolean;
+        forceReadProduction?: boolean;
     }): Promise<TimeFilePathWithSize[]> {
         let indexedLogs = loggerByName.get(config.indexedLogsName);
         if (!indexedLogs) throw new Error(`Indexed logs ${config.indexedLogsName} not found`);
@@ -786,7 +786,7 @@ class IndexedLogShim {
             startTime: config.startTime,
             endTime: config.endTime,
             only: config.only,
-            forceReadPublic: config.forceReadPublic,
+            forceReadProduction: config.forceReadProduction,
         });
     }