querysub 0.311.0 → 0.313.0

package/src/-d-trust/NetworkTrust2.ts

@@ -1,5 +1,5 @@
 import { measureWrap } from "socket-function/src/profiling/measure";
-import { getCommonName, getIdentityCA, getMachineId, getOwnMachineId } from "../-a-auth/certs";
+import { getIdentityCA, getMachineId, getOwnMachineId } from "../-a-auth/certs";
 import { getArchives } from "../-a-archives/archives";
 import { isNode, throttleFunction, timeInSecond } from "socket-function/src/misc";
 import { SocketFunctionHook } from "socket-function/SocketFunctionTypes";
@@ -14,12 +14,13 @@ import { devDebugbreak, getDomain, isDevDebugbreak, isPublic, isRecovery } from
 import { formatTime } from "socket-function/src/formatting/format";
 import { runInSerial } from "socket-function/src/batching";
 import { Querysub } from "../4-querysub/QuerysubController";
+import { magenta } from "socket-function/src/formatting/logColors";
 
 // Cache the untrust list, to prevent bugs from causing too many backend reads (while also allowing
 //  bad servers which make request before their trust is verified from staying broken).
 const UNTRUST_CACHE_TIME = 30 * timeInSecond;
 
-const archives = lazy(() => getArchives("trust/"));
+const archives = lazy(() => getArchives("trust2/"));
 
 export const requiresNetworkTrustHook: SocketFunctionHook = async config => {
     // HACK: On the clientside we strip the domain process and machine id, so we can no longer determine
@@ -39,10 +40,11 @@ export const requiresNetworkTrustHook: SocketFunctionHook = async config => {
     if (getNodeIdIP(caller.nodeId) === "127.0.0.1" && isRecovery()) {
         return;
     }
-    let machineId = IdentityController_getMachineId(SocketFunction.getCaller());
+    let machineId = IdentityController_getMachineId(caller);
     let trusted = await isTrusted(machineId);
     if (!trusted) {
         devDebugbreak();
+        let machineId = IdentityController_getMachineId(caller);
         throw new Error(`Calling machine is not trusted. Caller ${machineId} is not trusted by ${SocketFunction.mountedNodeId} to make call ${config.call.classGuid}.${config.call.functionName}. To gain trust add backblaze permissions (see hasBackblazePermissions) or set --nonetwork.`);
     }
 };
@@ -51,7 +53,7 @@ export const assertIsNetworkTrusted = requiresNetworkTrustHook;
 let lastArchivesTrusted: string[] | undefined;
 let trustedCache = new Set<string>();
 let untrustedCache = new Map<string, number>();
-export const isTrusted = runInSerial(async function isTrusted(machineId: string) {
+export const isTrusted = measureWrap(async function isTrusted(machineId: string) {
     // See the comment in requiresNetworkTrustHook for why clients have to trust all callers.
     if (isClient()) return true;
 
@@ -68,25 +70,31 @@ export const isTrusted = runInSerial(async function isTrusted(machineId: string)
         return false;
     }
 
-    // Find is faster than get, and usually we don't need the full certificate
-    let trustedMachineIds = await archives().find("");
-    lastArchivesTrusted = trustedMachineIds.slice();
-    // Always trust ourself
-    trustedMachineIds.push(getOwnMachineId());
-    // IF developing, trust localhost. This allows us to develop without port forwards,
-    //  on our services, which INCREASES security, and prevents dev machines from being
-    //  connected to by attackers (as dev machines might reveal unfinished content, or even
-    //  have security vulnerabilities).
-    //  - Don't trust this on public, as in theory an attacker MIGHT be able to connect
-    //      from localhost (but not have disk read/write access)? Maybe...
-    if (!isPublic()) {
-        trustedMachineIds.push("127-0-0-1." + getDomain());
+    return await isTrustedBase(machineId);
+});
+let trustedCachePopulated = false;
+const isTrustedBase = runInSerial(measureWrap(async function isTrustedBase(machineId: string) {
+    if (!trustedCachePopulated) {
+        trustedCachePopulated = true;
+        let trustedMachineIds = await archives().find("");
+        lastArchivesTrusted = trustedMachineIds.slice();
+        for (let trustedMachineId of trustedMachineIds) {
+            trustedCache.add(trustedMachineId);
+            // NOTE: We don't load trust certs here, as we need to load them on demand in case the trust changes after our initial startup.
+        }
+    } else {
+        // Checking a single entry is a lot faster (as find is slow)
+        let trusted = await archives().get(machineId);
+        if (trusted) {
+            trustedCache.add(machineId);
+        }
     }
+    // Always trust ourself
+    trustedCache.add(getOwnMachineId());
 
-    // NOTE: This should be safe from collisions with existing files, because... while there might be a metadata
-    //  file which exists, it will never also be a valid hash/public key, which machineId will always be.
-    for (let trustedMachineId of trustedMachineIds) {
-        trustedCache.add(trustedMachineId);
+    // NOTE: The only happens in the case WE connected to it (ex, "127-0-0-1.querysubtest.com:15358"). It can't look like this if it connected to us, in which case the nodeId will be "client:...", being mostly random, and created by us (UNTIL they prove they have another id). So... I'm not even sure the isPublic check is required? We only connect to nodes we discover through node discovery, which requires backblaze write permissions. But I guess it's fine to be extra careful about it...
+    if (!isPublic()) {
+        trustedCache.add("127-0-0-1");
     }
 
     if (!trustedCache.has(machineId)) {
@@ -95,7 +103,7 @@ export const isTrusted = runInSerial(async function isTrusted(machineId: string)
     } else {
         return true;
     }
-});
+}));
 
 export async function isNodeTrusted(nodeId: string) {
     let domainName = getNodeIdDomainMaybeUndefined(nodeId);
@@ -112,15 +120,14 @@ const loadServerCert = cache(async (machineId: string) => {
         console.warn(`Could not find certificate in archives for ${machineId}`);
         return;
     }
+    console.log(magenta(`Loading certificate for ${machineId}`));
     trustCertificate(certFile);
 });
 
-const ensureWeAreTrusted = lazy(measureWrap(async () => {
+export const ensureWeAreTrusted = lazy(measureWrap(async () => {
     let machineKeyCert = getIdentityCA();
-    let machineId = getCommonName(machineKeyCert.cert);
-
-    await isTrusted(machineId);
-    if (!lastArchivesTrusted?.includes(machineId)) {
+    let machineId = getOwnMachineId();
+    if (!await archives().get(machineId)) {
         await archives().set(machineId, machineKeyCert.cert);
     }
 }));
@@ -159,6 +166,7 @@ const TrustedController = SocketFunction.register(
 if (isNode()) {
     // We have to be trusted if we make calls to a trusted endpoint, OR our mounting
     //  (really only if we are mounting a trusted endpoint, but we don't actually know that)
+    // ONLY done on received calls, not on calls we made. If we make a call we assume that the server we called is known to us through a trusted route, and therefore, trusted.
     requiresNetworkTrustHook.clientHook = async config => {
         await ensureWeAreTrusted();
     };
@@ -166,10 +174,9 @@ if (isNode()) {
     // Load the remote certificate, in the almost certain case it isn't a real certificate, and is just internal
     SocketFunction.addGlobalClientHook(async config => {
         await measureWrap(async function checkTrust() {
-            await Promise.all([
-                ensureWeAreTrusted(),
-                loadTrustCerts(config.call.nodeId)
-            ]);
+            // IMPORTANT! We SHOULDN'T need to add our machineId before we connect, as we only check machineId on received calls, so we only need to set it before we make a call (so by the time anyone receives a call, they trust us!)
+            await ensureWeAreTrusted();
+            await loadTrustCerts(config.call.nodeId);
         })();
     });
 }

package/src/-e-certs/EdgeCertController.ts

@@ -37,7 +37,7 @@ export const getHostedIP = lazy(async () => {
     }
     return await getExternalIP();
 });
-const getIPDomain = lazy(async () => {
+export const getIPDomain = lazy(async () => {
     let ip = await getHostedIP();
     return ip.replaceAll(".", "-") + "." + getDomain();
 });
@@ -56,7 +56,7 @@ export async function getSNICerts(config: {
         [getDomain()]: async (callback) => {
             await EdgeCertController_watchHTTPSKeyCert(callback);
         },
-        [getOwnMachineId()]: async (callback) => {
+        [getOwnMachineId() + "." + getDomain()]: async (callback) => {
             let threadCert = await getThreadKeyCert();
             callback({
                 key: threadCert.key,
@@ -273,7 +273,7 @@ async function getHTTPSKeyCertInner(callerIP: string) {
                         console.warn(`Tried to serve an edge node on the local domain, but SocketFunction.mount did NOT specify a public ip (ex, { ip: "0.0.0.0" }) AND there are already existing public servers. You can't load balance between real ips and 127.0.0.1! ${existingIPs.join(", ")}. You will need to use 127-0-0-1.${edgeDomain} to access the local server (instead of just ${edgeDomain}).`);
                     }
                     */
-                    console.log(yellow(`Current process is not marked as public, but machine previous had public services. NOT setting ${edgeDomain} to 127.0.0.1, as this would make the public services inaccessible. Assuming the current process is for development, I recommend using noproxy.${edgeDomain} or 127-0-0-1.${edgeDomain} to access the server.`));
+                    console.log(yellow(`Current process is not marked as public, but machine previous had public services. NOT setting ${edgeDomain} to 127.0.0.1, as this would make the public services inaccessible. Assuming the current process is for development, I recommend using 127-0-0-1.${edgeDomain} or 127-0-0-1.${edgeDomain} to access the server.`));
                 }
             } else {
                 if (existingIPs.includes("127.0.0.1")) {
@@ -289,7 +289,6 @@ async function getHTTPSKeyCertInner(callerIP: string) {
                     }
                 });
             }
-            promises.push(addRecord("A", "noproxy." + edgeDomain, "127.0.0.1"));
             promises.push(addRecord("A", "127-0-0-1." + edgeDomain, "127.0.0.1"));
             // Add records in parallel, so we can wait for DNS propagation in parallel
             await Promise.all(promises);

package/src/-e-certs/certAuthority.ts

@@ -10,7 +10,7 @@ import { formatDateTime, formatTime } from "socket-function/src/formatting/forma
 import { delay } from "socket-function/src/batching";
 import { timeInMinute } from "socket-function/src/misc";
 
-const archives = lazy(() => getArchives(`https_certs_2/`));
+const archives = lazy(() => getArchives(`https_certs_3/`));
 // Expire EXPIRATION_THRESHOLD% of the way through the certificate's lifetime
 const EXPIRATION_THRESHOLD = 0.4;
 
@@ -50,7 +50,6 @@ export const getHTTPSKeyCert = cache(async (domain: string): Promise<{ key: stri
     const accountKey = getAccountKey();
     let altDomains: string[] = [];
 
-    // altDomains.push("noproxy." + domain);
     // // NOTE: Allowing local access is just an optimization, not to avoid having to forward ports
     // //  (unless you type 127-0-0-1.domain into the browser... then I guess you don't have to forward ports?)
     // altDomains.push("127-0-0-1." + domain);

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -4,7 +4,7 @@ import { getDomain, isDevDebugbreak, isNoNetwork, isPublic } from "../config";
 import { measureBlock } from "socket-function/src/profiling/measure";
 import { isNode, sha256Hash, throttleFunction, timeInMinute, timeInSecond } from "socket-function/src/misc";
 import { errorToUndefinedSilent, ignoreErrors, logErrors, timeoutToUndefinedSilent } from "../errors";
-import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
+import { ensureWeAreTrusted, requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
 import { delay, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { getNodeId, getNodeIdFromLocation } from "socket-function/src/nodeCache";
 import { lazy } from "socket-function/src/caching";
@@ -17,9 +17,10 @@ import { waitForFirstTimeSync } from "socket-function/time/trueTimeShim";
 import { decodeNodeId, decodeNodeIdAssert } from "../-a-auth/certs";
 
 import { isDefined } from "../misc";
-import { diskLog } from "../diagnostics/logs/diskLogger";
 import { getBootedEdgeNode } from "../-0-hooks/hooks";
 import { EdgeNodeConfig } from "../4-deploy/edgeNodes";
+import * as certs from "../-a-auth/certs";
+import { logDisk } from "../diagnostics/logs/diskLogger";
 
 let HEARTBEAT_INTERVAL = timeInMinute * 15;
 // Interval which we check other heartbeats
@@ -44,6 +45,8 @@ let DISK_AUDIT_RATE = timeInMinute * 15;
 //  probably is less than that). Which is around 2.5 cents on digital ocean IF we go over
 //  our 1TB/month allowance.
 let API_AUDIT_RATE = timeInSecond * 30;
+// BUT, for now, poll less often... because I think it is lagging our 2 core potato digital ocean server.
+API_AUDIT_RATE = timeInMinute * 5;
 let API_AUDIT_COUNT = 12;
 
 
@@ -84,10 +87,10 @@ export function getOwnNodeIdAssert(): string {
 }
 
 export const getOwnThreadId = lazy(() => {
-    return decodeNodeIdAssert(getOwnNodeIdAssert()).threadId;
+    return certs.getOwnThreadId();
 });
 export const getOwnMachineId = lazy(() => {
-    return decodeNodeIdAssert(getOwnNodeIdAssert()).machineId;
+    return certs.getOwnMachineId();
 });
 
 export function isOwnNodeId(nodeId: string): boolean {
@@ -178,7 +181,7 @@ function addNodeIdBase(nodeId: string) {
 function setNodeIds(nodeIds: string[]) {
     nodeIds = nodeIds.filter(x => x !== SPECIAL_NODE_ID_FOR_UNMOUNTED_NODE);
 
-    diskLog("setNodeIds", { nodeIds });
+    logDisk("log", "setNodeIds", { nodeIds });
     // Also try all localhost ports, if we are developing and not in public mode
     if (isNode() && !isPublic() && isDevDebugbreak()) {
         let ports = new Set(nodeIds.map(nodeId => decodeNodeId(nodeId)?.port).filter(isDefined));
@@ -343,7 +346,7 @@ async function runHeartbeatAuditLoop() {
                 }
             } else {
                 deadCount.delete(nodeId);
-                diskLog("Read node heartbeat", { nodeId, lastTime });
+                logDisk("log", "Read node heartbeat", { nodeId, lastTime });
             }
         }
         if (pendingDeadCount) {
@@ -353,7 +356,7 @@ async function runHeartbeatAuditLoop() {
         if (removedNodeIds.length > 0) {
             console.log(blue(`Removed ${removedNodeIds.length}/${nodeIds.length} nodes from node list`), { removedNodeIds });
             await syncArchives();
-            await tellEveryoneNodesChanges();
+            await tellEveryoneNodesChanges(`removedNodeIds ${removedNodeIds.join("|")}`);
         }
     });
 }
@@ -471,7 +474,8 @@ export async function onNodeDiscoveryReady() {
 }
 
 if (isServer()) {
-    setImmediate(() => {
+    setImmediate(async () => {
+
         logErrors(runHeartbeatAuditLoop());
         logErrors(runMemoryAuditLoop());
         // NOTE: We used to wait until we mounted, but... we should be able to find nodes
@@ -517,7 +521,7 @@ if (isServer()) {
 
 export async function forceRemoveNode(nodeId: string) {
     await archives().del(nodeId);
-    void tellEveryoneNodesChanges();
+    void tellEveryoneNodesChanges(`forceRemoveNode ${nodeId}`);
 }
 
 
@@ -528,13 +532,14 @@ export async function nodeDiscoveryShutdown() {
     if (isServer()) {
         await archives().del(getOwnNodeId());
     }
-    void tellEveryoneNodesChanges();
+    void tellEveryoneNodesChanges("nodeDiscoveryShutdown");
 }
-const tellEveryoneNodesChanges = throttleFunction(1000, function tellEveryoneNodesChanges() {
+const tellEveryoneNodesChanges = throttleFunction(1000, function tellEveryoneNodesChanges(reason: string) {
     if (isClient()) return;
+    console.log(red(`Telling everyone nodes changed`));
     for (let nodeId of allNodeIds2) {
         if (isOwnNodeId(nodeId)) continue;
-        ignoreErrors(NodeDiscoveryController.nodes[nodeId].resyncNodes());
+        ignoreErrors(NodeDiscoveryController.nodes[nodeId].resyncNodes(reason));
     }
 });
 
@@ -544,7 +549,9 @@ class NodeDiscoveryControllerBase {
         console.log(magenta(`Received addNode`), { nodeId });
         addNodeId(nodeId);
     }
-    public async resyncNodes() {
+    public async resyncNodes(reason: string) {
+        let caller = SocketFunction.getCaller();
+        console.log(magenta(`Received resyncNodes from ${caller.nodeId}, reason = ${reason}`));
         await syncArchives();
     }
     public async getAllNodesHash(): Promise<string> {

package/src/-g-core-values/NodeCapabilities.ts

@@ -78,7 +78,12 @@ export async function getControllerNodeIdList(
             passedNodeIds.set(nodeId, entryPoint);
         }
     }));
-    return Array.from(passedNodeIds.entries()).map(([nodeId, entryPoint]) => ({ nodeId, entryPoint }));
+
+    let results = Array.from(passedNodeIds.entries()).map(([nodeId, entryPoint]) => ({ nodeId, entryPoint }));
+    // Ignore the special local IDs, otherwise we'll be returning duplicates. And our caller probably doesn't want the fastest path, they probably just want every path. 
+    // TODO: We should detect if the local ID and the remote ID is the same and then always pick the local ID instead. However, I have no idea how to do this as it doesn't include the machine ID, just the port. So how do we know if they're the same? I think we actually have to talk to them and identify their machine IDs, which we need to cache, which then takes longer, et cetera, et cetera. 
+    results = results.filter(x => !x.nodeId.startsWith("127-0-0-1."));
+    return results;
 }
 
 

package/src/0-path-value-core/NodePathAuthorities.ts

@@ -21,7 +21,6 @@ import { cache, cacheLimited } from "socket-function/src/caching";
 import { IdentityController_getCurrentReconnectNodeIdAssert, IdentityController_getReconnectNodeIdAssert } from "../-c-identity/IdentityController";
 import { getBufferFraction, getBufferInt, getShortNumber } from "../bits";
 import { devDebugbreak, getDomain, isDevDebugbreak } from "../config";
-import { diskLog } from "../diagnostics/logs/diskLogger";
 import { waitForFirstTimeSync } from "socket-function/time/trueTimeShim";
 
 export const LOCAL_DOMAIN = "LOCAL";
@@ -262,6 +261,7 @@ class NodePathAuthorities {
                     this.previouslyNotAvailableNodes.add(nodeId);
                     return;
                 }
+
                 this.previouslyNotAvailableNodes.delete(nodeId);
 
                 time = Date.now() - time;

package/src/0-path-value-core/PathValueCommitter.ts

@@ -21,7 +21,7 @@ import { formatNumber, formatTime } from "socket-function/src/formatting/format"
 import { isClient } from "../config2";
 import { remoteWatcher } from "../1-path-client/RemoteWatcher";
 import { auditLog, isDebugLogEnabled } from "./auditLogs";
-import { diskLog } from "../diagnostics/logs/diskLogger";
+import { logDisk } from "../diagnostics/logs/diskLogger";
 
 /*
 - commitWrites <= creating writes
@@ -177,7 +177,7 @@ class PathValueCommitter {
                 markArrayAsSplitable(values);
                 const { Querysub } = await import("../4-querysub/Querysub");
                 let serializedValues = await pathValueSerializer.serialize(values, { compress: Querysub.COMPRESS_NETWORK });
-                diskLog(`Send PathValues to server`, { valueCount: values.length, targetId: otherAuthority, });
+                logDisk("log", "Send PathValues to server", { valueCount: values.length, targetId: otherAuthority, });
                 let forwardPromise = PathValueController.nodes[otherAuthority].forwardWrites(
                     serializedValues,
                     undefined,
@@ -662,4 +662,4 @@ function trackLag(now: number, lag: number) {
 }
 
 
-export const pathValueCommitter = new PathValueCommitter();
+export const pathValueCommitter = new PathValueCommitter();

package/src/0-path-value-core/PathValueController.ts

@@ -16,7 +16,7 @@ import debugbreak from "debugbreak";
 import { ClientWatcher } from "../1-path-client/pathValueClientWatcher";
 import { auditLog, isDebugLogEnabled } from "./auditLogs";
 import { debugNodeId } from "../-c-identity/IdentityController";
-import { diskLog } from "../diagnostics/logs/diskLogger";
+import { logDisk } from "../diagnostics/logs/diskLogger";
 export { pathValueCommitter };
 
 class PathValueControllerBase {
@@ -86,9 +86,9 @@ class PathValueControllerBase {
                 auditLog("RECEIVE VALUE", { path: value.path, time: value.time.time, sourceNodeId });
             }
         }
-        diskLog(`Received PathValues via forwardWrites`, { valueCount: values.length, callerId, });
+        logDisk("log", "Received PathValues via forwardWrites", { valueCount: values.length, callerId, });
         for (let value of values) {
-            diskLog("Received PathValue for path", { path: value.path, time: value.time.time, callerId });
+            logDisk("log", "Received PathValue for path", { path: value.path, time: value.time.time, callerId });
         }
 
         if (isCoreQuiet) {

package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts

@@ -12,8 +12,9 @@ import { devDebugbreak } from "../../config";
 import { logErrors } from "../../errors";
 import { saveSnapshot } from "./archiveSnapshots";
 import { getNodeId } from "socket-function/src/nodeCache";
-import { diskLog } from "../../diagnostics/logs/diskLogger";
 import { logNodeStateStats, logNodeStats } from "../../-0-hooks/hooks";
+import { parsePath, toFileNameKVP, parseFileNameKVP } from "../../misc";
+import { logDisk } from "../../diagnostics/logs/diskLogger";
 
 /** Clean up old files after a while */
 const DEAD_CREATE_THRESHOLD = timeInHour * 12;
@@ -301,14 +302,14 @@ class TransactionLocker {
         private isTransactionValidOverride?: (transaction: Transaction, dataFiles: FileInfo[], rawDataFiles: FileInfo[]) => boolean
     ) { }
 
-    // #region Base File Operations
+    // #region Base File Ops
     public getConfirmKey(key: string): string {
         let { dir, name } = parsePath(key);
         return `${dir}confirm_${name}.confirm`;
     }
     public async createConfirm(key: string) {
         let path = this.getConfirmKey(key);
-        diskLog(`Creating confirmation for ${key}`);
+        logDisk("log", "Creating confirmation for ${key}");
         await this.storage.setValue(path, Buffer.from(""));
         return path;
     }
@@ -361,7 +362,7 @@ class TransactionLocker {
             delete: ellipsize(deletes.map(a => debugFileInfo(a.key)).join(","), 50),
         });
 
-        diskLog(`Writing transaction`, {
+        logDisk("log", "Writing transaction", {
             name,
             ops: transaction.ops.length,
         });
@@ -483,7 +484,7 @@ class TransactionLocker {
                 }
             }
 
-            diskLog(`Read archive state`, {
+            logDisk("log", "Read archive state", {
                 rawFilesCount: files.length,
                 confirmedCount: currentDataFiles.size,
                 rawFiles: files.map(a => a.file),
@@ -502,7 +503,7 @@ class TransactionLocker {
             let result = await tryToRead();
             if (result) {
                 let timeToRead = Date.now() - startTime;
-                diskLog(`Read data state in ${formatTime(timeToRead)}`);
+                logDisk("log", `Read data state in ${formatTime(timeToRead)}`);
                 return result;
             }
         }
@@ -541,7 +542,7 @@ class TransactionLocker {
         let rawLookup = new Set(Array.from(rawDataFiles).map(a => a.file));
         // If any creates are not confirmed, it must not have been applied
         if (transaction.ops.some(a => a.type === "create" && rawLookup.has(a.key) && !confirmedKeys.has(a.key))) {
-            diskLog(`Transaction not applied (has pending confirmations of creates)`, {
+            logDisk("log", `Transaction not applied (has pending confirmations of creates)`, {
                 keys: transaction.ops
                     .filter(a => a.type === "create" && rawLookup.has(a.key) && !confirmedKeys.has(a.key))
                     .map(a => a.key)
@@ -550,7 +551,7 @@ class TransactionLocker {
         }
         // If any deletes still exist, it must not have been applied
         if (transaction.ops.some(a => a.type === "delete" && confirmedKeys.has(a.key))) {
-            diskLog(`Transaction not applied (has pending deletes)`, {
+            logDisk("log", `Transaction not applied (has pending deletes)`, {
                 keys: transaction.ops
                     .filter(a => a.type === "delete" && confirmedKeys.has(a.key))
                     .map(a => a.key)
@@ -563,7 +564,7 @@ class TransactionLocker {
         let createCount = transaction.ops.filter(a => a.type === "create").length;
         let deleteCount = transaction.ops.filter(a => a.type === "delete").length;
         let lockedFiles = transaction.lockedFilesMustEqual?.length;
-        diskLog(`Applying transaction with ${createCount} file creates and ${deleteCount} file deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`, {
+        logDisk("log", `Applying transaction with ${createCount} file creates and ${deleteCount} file deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`, {
             transactions: transaction.ops.map(x => JSON.stringify(x)),
         });
         logNodeStats(`archives|TΔ Apply`, formatNumber, 1);
@@ -589,7 +590,7 @@ class TransactionLocker {
         };
         await Promise.all(list(CONCURRENT_WRITE_COUNT).map(runThread));
 
-        diskLog(`Applied transaction with ${createCount} file creates and file ${deleteCount} deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`, {
+        logDisk("log", `Applied transaction with ${createCount} file creates and file ${deleteCount} deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`, {
             transactions: transaction.ops.map(x => JSON.stringify(x)),
         });
     }
@@ -655,7 +656,7 @@ class TransactionLocker {
         let threshold = activeT.createTime + this.storage.propagationTime;
         if (Date.now() < threshold) {
             let waitTime = threshold - Date.now();
-            diskLog(`Waiting ${formatTime(waitTime)} for transaction ${activeT.seqNum} to settle.`);
+            logDisk("log", `Waiting ${formatTime(waitTime)} for transaction ${activeT.seqNum} to settle.`);
             await new Promise(resolve => setTimeout(resolve, waitTime));
             return this.getFilesBase();
         }
@@ -759,7 +760,7 @@ class TransactionLocker {
         let dels = transaction.ops.filter(a => a.type === "delete").length;
         let creates = transaction.ops.filter(a => a.type === "create").length;
         let createBytes = transaction.ops.map(a => a.type === "create" && a.value?.length || 0).reduce((a, b) => a + b, 0);
-        diskLog(`Starting transaction with ${creates} file creates and ${dels} file deletes, ${formatNumber(createBytes)}B`, {
+        logDisk("log", `Starting transaction with ${creates} file creates and ${dels} file deletes, ${formatNumber(createBytes)}B`, {
             createFilesNames: transaction.ops.filter(a => a.type === "create").map(a => a.key),
             deleteFilesNames: transaction.ops.filter(a => a.type === "delete").map(a => a.key),
         });
@@ -788,7 +789,7 @@ class TransactionLocker {
             let beforeData = await this.getFilesBase();
             if (!this.isTransactionValid(transaction, beforeData.dataFiles, beforeData.rawDataFiles)) {
                 logNodeStats(`archives|TΔ Rejected`, formatNumber, 1);
-                diskLog(`Finished transaction with rejection, ${transaction.ops.length} ops`);
+                logDisk("log", `Finished transaction with rejection, ${transaction.ops.length} ops`);
                 return "rejected";
             }
 
@@ -797,33 +798,10 @@ class TransactionLocker {
             let afterData = await this.getFilesBase();
             if (this.wasTransactionApplied(transaction, afterData.dataFiles, afterData.rawDataFiles)) {
                 logNodeStats(`archives|TΔ Accepted`, formatNumber, 1);
-                diskLog(`Finished transaction with ${transaction.ops.length} ops`);
+                logDisk("log", `Finished transaction with ${transaction.ops.length} ops`);
                 return "accepted";
             }
         }
     }
 }
 
-function parsePath(path: string): { dir: string; name: string } {
-    path = path.replaceAll("\\", "/");
-    let lastSlash = path.lastIndexOf("/");
-    if (lastSlash === -1) return { dir: "", name: path };
-    return { dir: path.slice(0, lastSlash + 1), name: path.slice(lastSlash + 1) };
-}
-
-// Loses spaces in keys and values
-function toFileNameKVP(kvp: { [key: string]: string }): string {
-    function s(v: string) {
-        return v.replaceAll(" ", "_");
-    }
-    return "   " + Object.entries(kvp).map(([key, value]) => `${s(key)}=${s(value)}`).join("   ") + "   ";
-}
-function parseFileNameKVP(fileName: string): { [key: string]: string } {
-    let parts = fileName.trim().split("   ");
-    let obj: { [key: string]: string } = {};
-    for (let part of parts) {
-        let [key, value] = part.split("=");
-        obj[key] = value || key;
-    }
-    return obj;
-}

package/src/0-path-value-core/pathValueCore.ts

@@ -29,7 +29,8 @@ import { sha256 } from "js-sha256";
 import { PromiseObj } from "../promise";
 import { ClientWatcher } from "../1-path-client/pathValueClientWatcher";
 import { auditLog, isDebugLogEnabled } from "./auditLogs";
-import { diskLog } from "../diagnostics/logs/diskLogger";
+import { logDisk } from "../diagnostics/logs/diskLogger";
+
 
 let yargObj = isNodeTrue() && yargs(process.argv)
     .option("noarchive", { type: "boolean", alias: ["noarchives"], desc: "Don't save writes to disk. Still reads from disk." })
@@ -1260,7 +1261,7 @@ class PathWatcher {
                     auditLog("new local WATCH PARENT", { path });
                 }
             }
-            diskLog(`New PathValue watches`, {
+            logDisk("log", `New PathValue watches`, {
                 newPathsWatched: newPathsWatched.size,
                 newParentsWatched: newParentsWatched.size,
             });
@@ -1393,7 +1394,7 @@ class PathWatcher {
         }
 
         if (fullyUnwatched.paths.length > 0 || fullyUnwatched.parentPaths.length > 0) {
-            diskLog(`Unwatched PathValue watches`, {
+            logDisk("log", `Unwatched PathValue watches`, {
                 unwatchedPaths: fullyUnwatched.paths.length,
                 unwatchedParents: fullyUnwatched.parentPaths.length,
             });

package/src/3-path-functions/PathFunctionRunner.ts

@@ -18,13 +18,13 @@ import { parseArgs } from "./PathFunctionHelpers";
 import { PERMISSIONS_FUNCTION_ID, getAllDevelopmentModulesIds, getDevelopmentModule, getExportPath, getModuleRelativePath, getSchemaObject } from "./syncSchema";
 import { formatTime } from "socket-function/src/formatting/format";
 import { getControllerNodeIdList, set_debug_getFunctionRunnerShards } from "../-g-core-values/NodeCapabilities";
-import { diskLog } from "../diagnostics/logs/diskLogger";
 import { FilterSelector, Filterable, doesMatch } from "../misc/filterable";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
 import { getDomain, isLocal } from "../config";
 import { getGitRefSync, getGitURLSync } from "../4-deploy/git";
 import { DeployProgress } from "../4-deploy/deployFunctions";
+import { logDisk } from "../diagnostics/logs/diskLogger";
 
 export const functionSchema = rawSchema<{
     [domainName: string]: {
@@ -679,7 +679,7 @@ export class PathFunctionRunner {
         let syncTime = wallTime - evalTime;
 
 
-        diskLog(`Finished FunctionRunner function`, {
+        logDisk("log", "Finished FunctionRunner function", {
             ...callPath, argsEncoded: "", functionSpec,
             wallTime, syncTime, evalTime,
             loops: runCount,

package/src/4-dom/qreact.tsx

@@ -2067,7 +2067,7 @@ function updateDOMNodeFields(domNode: DOMNode, vNode: VirtualDOM, prevVNode: Vir
                                     if (name === "blur") {
                                         let target = args[0].currentTarget as HTMLElement;
                                         if (!target.getAttribute("data-blur-on-unmount") && !target.isConnected) {
-                                            console.info("Ignoring blur for disconnected element. You can use data-blur-on-unmount to re-enable blurs on this element.", target);
+                                            logDisk("log", "Ignoring blur for disconnected element. You can use data-blur-on-unmount to re-enable blurs on this element.", target);
                                             return;
                                         }
                                     }
@@ -2376,7 +2376,7 @@ function blurFixOnMouseDownHack(event: MouseEvent) {
 
     // Looks like we are going to blur, so blur now
     if (selected instanceof HTMLElement && !selected.hasAttribute("data-no-early-blur")) {
-        console.info(`Simulating early blur to prevent blur from firing after mousedown. This solves a problem where mousedown changes the UI, and then the blur fires on the wrong element. You can use data-no-early-blur to opt-out of this feature`, selected);
+        logDisk("log", `Simulating early blur to prevent blur from firing after mousedown. This solves a problem where mousedown changes the UI, and then the blur fires on the wrong element. You can use data-no-early-blur to opt-out of this feature`, selected);
         selected.blur();
     }
 }
@@ -2745,4 +2745,5 @@ function triggerGlobalOnMountWatch(component: QRenderClass) {
 
 // NOTE: Import Querysub at the end, so we can export qreact before we require it. That way Querysub
 //      can statically access qreact.
-import { Querysub } from "../4-querysub/Querysub";
+import { Querysub } from "../4-querysub/Querysub";
+import { logDisk } from "../diagnostics/logs/diskLogger";

package/src/4-querysub/Querysub.ts

@@ -849,11 +849,11 @@ export class Querysub {
             globalThis.remapImportRequestsClientside = globalThis.remapImportRequestsClientside || [];
             globalThis.remapImportRequestsClientside.push(async (args) => {
                 try {
-                    let key: typeof identityStorageKey = "machineCA_6";
+                    let key: typeof identityStorageKey = "machineCA_9";
                     let storageValueJSON = localStorage.getItem(key);
                     if (!storageValueJSON) return args;
                     let storageValue = JSON.parse(storageValueJSON) as IdentityStorageType;
-                    let machineId = storageValue.domain;
+                    let machineId = storageValue.domain.split(".").at(-3) || "";
 
                     let pem = Buffer.from(storageValue.keyB64, "base64");
                     let privateKey = exports.extractRawED25519PrivateKey(pem);