querysub 0.372.0 → 0.374.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.372.0",
+  "version": "0.374.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -185,7 +185,7 @@ function addNodeIdBase(nodeId: string) {
 function setNodeIds(nodeIds: string[]) {
     nodeIds = nodeIds.filter(x => x !== SPECIAL_NODE_ID_FOR_UNMOUNTED_NODE);
 
-    logDisk("log", "setNodeIds", { nodeIds });
+    console.info("setNodeIds", { nodeIds });
     // Also try all localhost ports, if we are developing and not in public mode
     if (isNode() && !isPublic() && isDevDebugbreak()) {
         let ports = new Set(nodeIds.map(nodeId => decodeNodeId(nodeId)?.port).filter(isDefined));
@@ -350,7 +350,7 @@ async function runHeartbeatAuditLoop() {
                 }
             } else {
                 deadCount.delete(nodeId);
-                logDisk("log", "Read node heartbeat", { nodeId, lastTime });
+                console.info("Read node heartbeat", { nodeId, lastTime });
             }
         }
         if (pendingDeadCount) {

package/src/0-path-value-core/PathValueCommitter.ts

@@ -177,7 +177,7 @@ class PathValueCommitter {
                 markArrayAsSplitable(values);
                 const { Querysub } = await import("../4-querysub/Querysub");
                 let serializedValues = await pathValueSerializer.serialize(values, { compress: Querysub.COMPRESS_NETWORK });
-                logDisk("log", "Send PathValues to server", { valueCount: values.length, targetId: otherAuthority, });
+                console.info("Send PathValues to server", { valueCount: values.length, targetId: otherAuthority, });
                 let forwardPromise = PathValueController.nodes[otherAuthority].forwardWrites(
                     serializedValues,
                     undefined,

package/src/0-path-value-core/PathValueController.ts

@@ -86,9 +86,9 @@ class PathValueControllerBase {
                 auditLog("RECEIVE VALUE", { path: value.path, time: value.time.time, sourceNodeId });
             }
         }
-        logDisk("log", "Received PathValues via forwardWrites", { valueCount: values.length, callerId, });
+        console.info("Received PathValues via forwardWrites", { valueCount: values.length, callerId, });
         for (let value of values) {
-            logDisk("log", "Received PathValue for path", { path: value.path, time: value.time.time, callerId });
+            console.info("Received PathValue for path", { path: value.path, time: value.time.time, callerId });
         }
 
         if (isCoreQuiet) {

package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts

@@ -309,7 +309,7 @@ class TransactionLocker {
     }
     public async createConfirm(key: string) {
         let path = this.getConfirmKey(key);
-        logDisk("log", "Creating confirmation for ${key}");
+        console.info("Creating confirmation for ${key}");
         await this.storage.setValue(path, Buffer.from(""));
         return path;
     }
@@ -362,7 +362,7 @@ class TransactionLocker {
             delete: ellipsize(deletes.map(a => debugFileInfo(a.key)).join(","), 50),
         });
 
-        logDisk("log", "Writing transaction", {
+        console.info("Writing transaction", {
             name,
             ops: transaction.ops.length,
         });
@@ -488,7 +488,7 @@ class TransactionLocker {
                 }
             }
 
-            logDisk("log", "Read archive state", {
+            console.info("Read archive state", {
                 rawFilesCount: files.length,
                 confirmedCount: currentDataFiles.size,
                 rawFiles: files.map(a => a.file),
@@ -507,7 +507,7 @@ class TransactionLocker {
             let result = await tryToRead();
             if (result) {
                 let timeToRead = Date.now() - startTime;
-                logDisk("log", `Read data state in ${formatTime(timeToRead)}`);
+                console.info(`Read data state in ${formatTime(timeToRead)}`);
                 return result;
             }
         }
@@ -546,7 +546,7 @@ class TransactionLocker {
         let rawLookup = new Set(Array.from(rawDataFiles).map(a => a.file));
         // If any creates are not confirmed, it must not have been applied
         if (transaction.ops.some(a => a.type === "create" && rawLookup.has(a.key) && !confirmedKeys.has(a.key))) {
-            logDisk("log", `Transaction not applied (has pending confirmations of creates)`, {
+            console.info(`Transaction not applied (has pending confirmations of creates)`, {
                 keys: transaction.ops
                     .filter(a => a.type === "create" && rawLookup.has(a.key) && !confirmedKeys.has(a.key))
                     .map(a => a.key)
@@ -555,7 +555,7 @@ class TransactionLocker {
         }
         // If any deletes still exist, it must not have been applied
         if (transaction.ops.some(a => a.type === "delete" && confirmedKeys.has(a.key))) {
-            logDisk("log", `Transaction not applied (has pending deletes)`, {
+            console.info(`Transaction not applied (has pending deletes)`, {
                 keys: transaction.ops
                     .filter(a => a.type === "delete" && confirmedKeys.has(a.key))
                     .map(a => a.key)
@@ -568,7 +568,7 @@ class TransactionLocker {
         let createCount = transaction.ops.filter(a => a.type === "create").length;
         let deleteCount = transaction.ops.filter(a => a.type === "delete").length;
         let lockedFiles = transaction.lockedFilesMustEqual?.length;
-        logDisk("log", `Applying transaction with ${createCount} file creates and ${deleteCount} file deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`, {
+        console.info(`Applying transaction with ${createCount} file creates and ${deleteCount} file deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`, {
             transactions: transaction.ops.map(x => JSON.stringify(x)),
         });
         logNodeStats(`archives|TΔ Apply`, formatNumber, 1);
@@ -594,7 +594,7 @@ class TransactionLocker {
         };
         await Promise.all(list(CONCURRENT_WRITE_COUNT).map(runThread));
 
-        logDisk("log", `Applied transaction with ${createCount} file creates and file ${deleteCount} deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`, {
+        console.info(`Applied transaction with ${createCount} file creates and file ${deleteCount} deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`, {
             transactions: transaction.ops.map(x => JSON.stringify(x)),
         });
     }
@@ -660,7 +660,7 @@ class TransactionLocker {
         let threshold = activeT.createTime + this.storage.propagationTime;
         if (Date.now() < threshold) {
             let waitTime = threshold - Date.now();
-            logDisk("log", `Waiting ${formatTime(waitTime)} for transaction ${activeT.seqNum} to settle.`);
+            console.info(`Waiting ${formatTime(waitTime)} for transaction ${activeT.seqNum} to settle.`);
             await new Promise(resolve => setTimeout(resolve, waitTime));
             return this.getFilesBase();
         }
@@ -768,7 +768,7 @@ class TransactionLocker {
         let dels = transaction.ops.filter(a => a.type === "delete").length;
         let creates = transaction.ops.filter(a => a.type === "create").length;
         let createBytes = transaction.ops.map(a => a.type === "create" && a.value?.length || 0).reduce((a, b) => a + b, 0);
-        logDisk("log", `Starting transaction with ${creates} file creates and ${dels} file deletes, ${formatNumber(createBytes)}B`, {
+        console.info(`Starting transaction with ${creates} file creates and ${dels} file deletes, ${formatNumber(createBytes)}B`, {
             createFilesNames: transaction.ops.filter(a => a.type === "create").map(a => a.key),
             deleteFilesNames: transaction.ops.filter(a => a.type === "delete").map(a => a.key),
         });
@@ -797,7 +797,7 @@ class TransactionLocker {
             let beforeData = await this.getFilesBase();
             if (!this.isTransactionValid(transaction, beforeData.dataFiles, beforeData.rawDataFiles)) {
                 logNodeStats(`archives|TΔ Rejected`, formatNumber, 1);
-                logDisk("log", `Finished transaction with rejection, ${transaction.ops.length} ops`);
+                console.info(`Finished transaction with rejection, ${transaction.ops.length} ops`);
                 return "rejected";
             }
 
@@ -806,7 +806,7 @@ class TransactionLocker {
             let afterData = await this.getFilesBase();
             if (this.wasTransactionApplied(transaction, afterData.dataFiles, afterData.rawDataFiles)) {
                 logNodeStats(`archives|TΔ Accepted`, formatNumber, 1);
-                logDisk("log", `Finished transaction with ${transaction.ops.length} ops`);
+                console.info(`Finished transaction with ${transaction.ops.length} ops`);
                 return "accepted";
             }
         }

package/src/0-path-value-core/auditLogs.ts

@@ -58,7 +58,7 @@ function debugLogBase(type: string, values: { [key: string]: unknown }) {
         return;
     }
     if (isDiskAudit()) {
-        logDisk("log", type, values, { "diskAudit": true });
+        console.info(type, values, { "diskAudit": true });
     }
     let newEntry: DebugLog = { type, time: Date.now(), values };
     logHistory.push(newEntry);

package/src/0-path-value-core/pathValueCore.ts

@@ -1261,7 +1261,7 @@ class PathWatcher {
                     auditLog("new non-local WATCH PARENT", { path, watcher: config.callback });
                 }
             }
-            logDisk("log", `New PathValue watches`, {
+            console.info(`New PathValue watches`, {
                 newPathsWatched: newPathsWatched.size,
                 newParentsWatched: newParentsWatched.size,
             });
@@ -1394,7 +1394,7 @@ class PathWatcher {
         }
 
         if (fullyUnwatched.paths.length > 0 || fullyUnwatched.parentPaths.length > 0) {
-            logDisk("log", `Unwatched PathValue watches`, {
+            console.info(`Unwatched PathValue watches`, {
                 unwatchedPaths: fullyUnwatched.paths.length,
                 unwatchedParents: fullyUnwatched.parentPaths.length,
             });

package/src/3-path-functions/PathFunctionRunner.ts

@@ -689,7 +689,7 @@ export class PathFunctionRunner {
         let syncTime = wallTime - evalTime;
 
 
-        logDisk("log", "Finished FunctionRunner function", {
+        console.info("Finished FunctionRunner function", {
             ...callPath, argsEncoded: "", functionSpec,
             wallTime, syncTime, evalTime,
             loops: runCount,

package/src/3-path-functions/PathFunctionRunnerMain.ts

@@ -41,7 +41,7 @@ async function main() {
     await Querysub.hostService("PathFunctionRunnerMain");
 
     if (!isPublic()) {
-        IndexedLogs.runLogMoveLoop();
+        void IndexedLogs.runLogMoveLoop();
     }
 
     // Use a fairly high stick time (the default is 10s), because having wait to sync data is very slow,

package/src/4-dom/qreact.tsx

@@ -2119,7 +2119,7 @@ function updateDOMNodeFields(domNode: DOMNode, vNode: VirtualDOM, prevVNode: Vir
                                     if (name === "blur") {
                                         let target = args[0].currentTarget as HTMLElement;
                                         if (!target.getAttribute("data-blur-on-unmount") && !target.isConnected) {
-                                            logDisk("log", "Ignoring blur for disconnected element. You can use data-blur-on-unmount to re-enable blurs on this element.", target);
+                                            console.info("Ignoring blur for disconnected element. You can use data-blur-on-unmount to re-enable blurs on this element.", target);
                                             return;
                                         }
                                     }
@@ -2428,7 +2428,7 @@ function blurFixOnMouseDownHack(event: MouseEvent) {
 
     // Looks like we are going to blur, so blur now
     if (selected instanceof HTMLElement && !selected.hasAttribute("data-no-early-blur")) {
-        logDisk("log", `Simulating early blur to prevent blur from firing after mousedown. This solves a problem where mousedown changes the UI, and then the blur fires on the wrong element. You can use data-no-early-blur to opt-out of this feature`, selected);
+        console.info(`Simulating early blur to prevent blur from firing after mousedown. This solves a problem where mousedown changes the UI, and then the blur fires on the wrong element. You can use data-no-early-blur to opt-out of this feature`, selected);
         selected.blur();
     }
 }

package/src/4-querysub/QuerysubController.ts

@@ -480,7 +480,7 @@ export class QuerysubControllerBase {
                 //      make the UI look cleaner (instead of showing stale values, it shows nothing)?
                 let undefinedValues: PathValue[] = newPathsNotAllowed.map(path => ({ path, value: undefined, canGCValue: true, isTransparent: true, time: epochTime, locks: [], lockCount: 0, valid: true, event: false }));
 
-                logDisk("log", "Disallowing PathValue watches due to disallowed permissions", { count: newPathsNotAllowed.length, callerId });
+                console.info("Disallowing PathValue watches due to disallowed permissions", { count: newPathsNotAllowed.length, callerId });
 
                 ignoreErrors(pathValueSerializer.serialize(undefinedValues, { compress: Querysub.COMPRESS_NETWORK }).then(buffers =>
                     PathValueController.nodes[callerId].forwardWrites(

package/src/5-diagnostics/diskValueAudit.ts

@@ -188,7 +188,7 @@ async function checkAuthority(authority: AuthorityPath, threshold: number) {
         console.error(`Value mismatch between disk and memory for ${formatNumber(changedValues.size)} values. Ex: ${firstValue.path}`);
     }
     if (removedValues.size > 0) {
-        logDisk("log", green(`Removing ${formatNumber(removedValues.size)} paths from memory which have been GCed on the disk.`));
+        console.info(green(`Removing ${formatNumber(removedValues.size)} paths from memory which have been GCed on the disk.`));
     }
     if (changedValues.size > 0 || removedValues.size > 0) {
         let allValues = new Set([...changedValues, ...removedValues]);

package/src/diagnostics/logs/IndexedLogs/FilePathSelector.tsx

@@ -25,6 +25,15 @@ export type FilePathsByMachine = {
     errorCount: number;
 };
 
+function getSourceColor(sourceName: string | undefined): string | undefined {
+    if (!sourceName) return undefined;
+    const lastPart = sourceName.split("/").pop()?.toLowerCase();
+    if (lastPart === "error") return css.colorhsl(0, 80, 40);
+    if (lastPart === "warn") return css.colorhsl(40, 80, 40);
+    if (lastPart === "info") return css.colorhsl(280, 80, 40);
+    if (lastPart === "log") return css.colorhsl(200, 80, 40);
+    return undefined;
+}
 
 export class FilePathSelector extends qreact.Component<{
     paths: TimeFilePathWithSize[];
@@ -377,7 +386,7 @@ export class FilePathSelectorModal extends qreact.Component<{
                 </td>
                 <td className={css.pad2(2)}>{this.props.formatBytes(file.size)}</td>
                 <td className={css.pad2(2)}>{file.logCount !== undefined ? formatNumber(file.logCount) : <span className={css.colorhsl(40, 60, 40)}>pending</span>}</td>
-                <td className={css.pad2(2)}>{file.sourceName}</td>
+                <td className={css.pad2(2) + (getSourceColor(file.sourceName) || "")}>{file.sourceName}</td>
                 <td className={css.pad2(2)}>{file.dedupe}</td>
                 <td className={css.pad2(2)}>{formatDateTime(file.startTime)}</td>
                 <td className={css.pad2(2)}>
@@ -429,7 +438,7 @@ export class FilePathSelectorModal extends qreact.Component<{
                             <div className={css.vbox(5)}>
                                 {Array.from(bySource.entries()).map(([sourceName, stats]) => (
                                     <div key={sourceName} className={css.hbox(10)}>
-                                        <div className={css.minWidth(240)}>{sourceName}:</div>
+                                        <div className={css.minWidth(240) + (getSourceColor(sourceName) || "")}>{sourceName}:</div>
                                         <div className={css.minWidth(140)}>Files: {formatNumber(stats.fileCount)} ({formatNumber(stats.pendingCount)} pending)</div>
                                         <div className={css.minWidth(120)}>Size: {this.props.formatBytes(stats.size)}</div>
                                         <div className={css.minWidth(100)}>Logs: {formatNumber(stats.logCount)}</div>

package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts

@@ -25,6 +25,7 @@ import { blue } from "socket-function/src/formatting/logColors";
 import { LimitGroup } from "../../../functional/limitProcessing";
 import { getAllNodeIds } from "../../../-f-node-discovery/NodeDiscovery";
 import { NodeCapabilitiesController } from "../../../-g-core-values/NodeCapabilities";
+import { getLoggers2Async } from "../diskLogger";
 
 export type TimeFilePathWithSize = TimeFilePath & {
     size: number;
@@ -40,7 +41,6 @@ export class IndexedLogs<T> {
         name: string,
         maxSingleFileData?: number;
         maxCountPerFile?: number;
-        forceUsePublicLogs?: boolean;
         getTime: (result: T) => number | undefined;
     }) {
         loggerByName.set(this.config.name, this as any);
@@ -48,9 +48,10 @@ export class IndexedLogs<T> {
 
 
     private static shouldRunLoop = false;
-    public static runLogMoveLoop() {
+    public static async runLogMoveLoop() {
         IndexedLogs.shouldRunLoop = true;
-        for (let indexedLogs of loggerByName.values()) {
+        let allLoggers = await getLoggers2Async();
+        for (let indexedLogs of Object.values(allLoggers)) {
             indexedLogs.runLogMoverLoop();
         }
     }
@@ -91,7 +92,7 @@ export class IndexedLogs<T> {
         let basePublic: Archives = getArchivesHome(getDomain());
         // NOTE: The local disk is so fast that reading in 10 megabytes is nothing, And if we read in too small of a value, the overhead per read ends up making this take forever. 
         let extraReadSize = 1024 * 1024 * 10;
-        if (this.config.forceUsePublicLogs || isPublic()) {
+        if (isPublic()) {
             basePublic = getArchivesBackblaze(getDomain());
             // NOTE: While the latency to back plays is high, now we're reading in parallel, so it shouldn't be as big of an issue. 
             extraReadSize = 1024 * 1024 * 1;
@@ -274,8 +275,16 @@ export class IndexedLogs<T> {
         forceReadPublic?: boolean;
     }): Promise<TimeFilePathWithSize[]> {
         let finalPaths: TimeFilePathWithSize[] = [];
+        if (config.forceReadPublic && !isPublic()) {
+            let machineNodes = await this.getMachineNodes();
+            if (machineNodes.length === 0) throw new Error(`Cannot find any public nodes to read from`);
+            return await IndexedLogShimController.nodes[machineNodes[0]].getPaths({
+                ...config,
+                indexedLogsName: this.config.name,
+            });
+        }
 
-        if (!config.only && (config.forceReadPublic || isPublic())) {
+        if (config.only !== "local" && isPublic()) {
             let machineNodes = await this.getMachineNodes();
             await Promise.all(machineNodes.map(async (machineNode) => {
                 try {
@@ -291,11 +300,6 @@ export class IndexedLogs<T> {
             }));
         }
 
-        // If we're forcefully reading from the public server, but we're not public, the code above will be the only code which adds to results
-        if (!isPublic() && config.forceReadPublic) {
-            return finalPaths;
-        }
-
         let localLogs = this.getLocalLogs();
         let backblazeLogs = this.getPublicLogs();
         let paths: TimeFilePath[] = [];
@@ -347,6 +351,16 @@ export class IndexedLogs<T> {
         onResult: (match: T) => void;
         onResults?: (results: IndexedLogResults) => Promise<boolean>;
     }): Promise<IndexedLogResults> {
+
+        if (config.params.forceReadPublic && !isPublic()) {
+            let machineNodes = await this.getMachineNodes();
+            if (machineNodes.length === 0) throw new Error(`Cannot find any public nodes to read from`);
+            return await this.clientFind({
+                ...config,
+                nodeId: machineNodes[0],
+            });
+        }
+
         let startTime = Date.now();
         let interval: NodeJS.Timeout | undefined;
 
@@ -360,7 +374,8 @@ export class IndexedLogs<T> {
         let allResults = new Map<string, IndexedLogResults>();
         let allDone: Promise<void>[] = [];
 
-        if (config.params.only !== "local" && (config.params.forceReadPublic || isPublic())) {
+        // Read other nodes if on a public server
+        if (config.params.only !== "local" && isPublic()) {
             let machineNodes = await this.getMachineNodes();
             allDone.push(...machineNodes.map(async (machineNode) => {
                 try {
@@ -385,12 +400,6 @@ export class IndexedLogs<T> {
             }));
         }
 
-        // If we're forcefully reading from the public server, but we're not public, the code above will be the only code which adds to results
-        if (!isPublic() && config.params.forceReadPublic) {
-            await Promise.all(allDone);
-            return getFinalResults();
-        }
-
 
         let results: IndexedLogResults = createEmptyIndexedLogResults();
         allResults.set("", results);

package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx

@@ -686,6 +686,12 @@ export class LogViewer3 extends qreact.Component {
                         checkbox
                         label="Exclude Pending Results"
                         url={excludePendingResults}
+                        onChangeValue={(newValue) => {
+                            if (newValue) {
+                                savedPathsURL.value = "";
+                                void this.loadPaths();
+                            }
+                        }}
                     />
                     {!isPublic() && <InputLabelURL
                         checkbox

package/src/diagnostics/logs/diskLogger.ts

@@ -146,7 +146,7 @@ void Promise.resolve().then(() => {
 
 
 const logDiskDontShim = logDisk;
-/** NOTE: Calling this directly means we lose __FILE__ tracking. But... that's probably fine... */
+/** @deprecated, Don't call this directly, call console info instead, which our shim will prevent from logging to the console, but it will still call logDisk. */
 export function logDisk(type: "log" | "warn" | "info" | "error", ...args: unknown[]) {
     if (!isNode()) return;
     try {

package/src/diagnostics/logs/diskShimConsoleLogs.ts

@@ -55,6 +55,8 @@ export function shimConsoleLogs() {
                 // Some arguments might throw if accessed (as they might be proxies), so
                 //  catch and ignore errors
             }
+            // NOTE: Info really has absolutely no purpose. There's no reason to use info instead of log, so we're going to give it a purpose. Infos are not going to be shown in the console and are only going to be logged to disk. This helps fix our shimming issue, where when we call log disk directly, we lose out on the source file information.
+            if (fncName === "info") return;
             return originalFnc(...args);
         };
     }

package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx

@@ -20,8 +20,6 @@ IMPORTANT! Now I am properly calling shutdown, so none of the streamed logs shou
 
 
 
-UGH... I'm seeing the remote move logs saying that it moved a lot of files. 
-
 3) Ensure our moveloops are working correctly, on both dev and public
 
 

package/src/diagnostics/watchdog.ts

@@ -72,7 +72,7 @@ function logProfileMeasuresTimingsNow() {
     measureObj = startMeasure();
     function diskLogMeasureObj(table: FormattedMeasureTable | undefined) {
         if (!table) return;
-        logDisk("log", table.title, { entries: table.entries });
+        console.info(table.title, { entries: table.entries });
     }
     diskLogMeasureObj(logMeasureTable(profile, {
         name: `watchdog at ${new Date().toLocaleString()}`,

package/src/user-implementation/userData.ts

@@ -626,7 +626,7 @@ function sendLoginEmail(config: {
             );
             if (alreadyAllowed) {
                 data().machineSecure[machineId].userId = userId;
-                logDisk("log", `User ${userId} already allowed for current ip and machine id, no need to send email.`);
+                console.info(`User ${userId} already allowed for current ip and machine id, no need to send email.`);
                 return;
             }
         }
@@ -818,14 +818,14 @@ function inviteUser(config: { email: string }) {
     Querysub.ignorePermissionsChecks(() => {
         let curUserObj = getUserObjAssert();
         if (config.email in curUserObj.invitedUsers2) {
-            logDisk("log", `User ${config.email} already invited`);
+            console.info(`User ${config.email} already invited`);
             return;
         }
         // If the user already exists, don't invite
         const { email } = config;
         let userId = atomic(data().secure.emailToUserId[email]) || createNewUserId();
         if (userId in data().users) {
-            logDisk("log", `User ${userId} already exists, no need to invite`);
+            console.info(`User ${userId} already exists, no need to invite`);
             return;
         }