querysub 0.136.0 → 0.138.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.136.0",
+  "version": "0.138.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/0-path-value-core/PathValueCommitter.ts

@@ -11,7 +11,7 @@ import { errorToUndefined, logErrors, timeoutToUndefined } from "../errors";
 import { getPathFromStr } from "../path";
 import { nodePathAuthority, pathValueAuthority2 } from "./NodePathAuthorities";
 import { PathValueController } from "./PathValueController";
-import { PathValue, MAX_ACCEPTED_CHANGE_AGE, lockWatcher, authorityStorage, writeValidStorage, WriteState, lockToCallback, pathWatcher, MAX_CHANGE_AGE, debugPathValue, debugPathValuePath } from "./pathValueCore";
+import { PathValue, MAX_ACCEPTED_CHANGE_AGE, lockWatcher, authorityStorage, writeValidStorage, WriteState, lockToCallback, pathWatcher, MAX_CHANGE_AGE, debugPathValue, debugPathValuePath, compareTime, epochTime } from "./pathValueCore";
 import debugbreak from "debugbreak";
 import { red } from "socket-function/src/formatting/logColors";
 import { registerDynamicResource } from "../diagnostics/trackResources";
@@ -20,7 +20,7 @@ import { formatStats } from "socket-function/src/profiling/statsFormat";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import { isClient } from "../config2";
 import { remoteWatcher } from "../1-path-client/RemoteWatcher";
-import { debugLog, isDebugLogEnabled } from "./debugLogs";
+import { auditLog, isDebugLogEnabled } from "./auditLogs";
 import { diskLog } from "../diagnostics/logs/diskLogger";
 
 /*
@@ -130,7 +130,7 @@ class PathValueCommitter {
             let valuesPerOtherAuthority = new Map<string, PathValue[]>();
             for (let pathValue of values) {
                 if (isDebugLogEnabled()) {
-                    debugLog("CREATE VALUE", { path: pathValue.path, time: pathValue.time.time });
+                    auditLog("CREATE VALUE", { path: pathValue.path, time: pathValue.time.time });
                 }
 
                 let otherAuthorities = await pathValueAuthority2.getWriteNodes(pathValue.path);
@@ -233,13 +233,13 @@ class PathValueCommitter {
             sourceNodeId: string;
             initialTrigger?: "initialTrigger";
         }[]) => {
-            let initialTriggers = { values: new Set<PathValue>(), parentPaths: new Set<string>() };
+            let initialTriggers = { values: new Set<string>(), parentPaths: new Set<string>() };
             // Check received values to make sure the authority for them is the same as the sourceNodeId
             //  (OR that WE are the authority). If not... it means we received values after changing the watcher,
             //      which requires us to ignore the other values
             // (Clients get most of their values from the same server, so they shouldn't require this check)
+            const self = this;
             if (!isClient()) {
-                const self = this;
                 measureBlock(function ignoreUnrequestedValues() {
                     for (let batch of batched) {
                         batch.pathValues = batch.pathValues.filter(value => {
@@ -255,8 +255,16 @@ class PathValueCommitter {
                             //  When we start watching, those values will get clobbered.
                             if (watchingAuthorityId === undefined) return true;
                             if (watchingAuthorityId === batch.sourceNodeId) return true;
+                            // epochTimes are just indicators that the value has no value, and so are safe to sync
+                            //  from any source. They won't cause future conflicts, because any other value overrides them.
+                            //  - This might not be required?
+                            if (compareTime(value.time, epochTime) === 0) {
+                                return true;
+                            }
+
                             // Also warn, because... if we get a lot of these, there might be a bug.
                             //  A few when we change watches is possible, but it should be rare.
+                            auditLog("IGNORING VALUE FROM DIFFERENT AUTHORITY", { path: value.path, watchingAuthorityId, receivedFromAuthority: batch.sourceNodeId });
                             console.warn(`Ignoring a value that was received from a different authority than the one we are watching. Path: ${value.path}, Watching Authority (should be source): ${watchingAuthorityId}, Actual Source Authority: ${batch.sourceNodeId}`);
                             return false;
                         });
@@ -264,29 +272,84 @@ class PathValueCommitter {
                 });
             }
 
-            // The first time we receive value for a watch from a specific node, we need to clobber our history
-            //  (unless we are the authority). Otherwise bad rejected values can stick around,
+            let specialInitialParentCausedRejections: WriteState[] = [];
+
+            // The first time we receive value for a watch from a specific node, we need to reject future
+            //  missing values (unless we are the authority). Otherwise bad rejected values can stick around,
             //  because they are newer, and our new source doesn't know we have them.
             measureBlock(function resetOnInitialTrigger() {
                 for (let batch of batched) {
                     if (!batch.initialTrigger) continue;
                     for (let pathValue of batch.pathValues) {
-                        initialTriggers.values.add(pathValue);
-                        authorityStorage.resetForInitialTrigger(pathValue.path);
+                        initialTriggers.values.add(pathValue.path);
+                        auditLog("INITIAL SYNC", { path: pathValue.path, time: pathValue.time.time, sourceNodeId: batch.sourceNodeId });
+                        // We shouldn't receive the initialTrigger flag if we are the authority anyways
+                        if (nodePathAuthority.isSelfAuthority(pathValue.path)) continue;
+                        let history = authorityStorage.getValuePlusHistory(pathValue.path);
+                        for (let value of history) {
+                            if (compareTime(value.time, pathValue.time) > 0) {
+                                specialInitialParentCausedRejections.push({
+                                    path: value.path,
+                                    time: value.time,
+                                    isValid: false,
+                                    isTransparent: !!value.isTransparent,
+                                    reason: "future missing (initial trigger)",
+                                });
+                            }
+                        }
                     }
-                    // Reset child paths as well, otherwise extra keys that have been deleted
-                    //  don't get properly removed.
+                    // IF the parent is synced, but we don't receive a path value, then invalidate the
+                    //  old PathValue (as this means it is invalid, but the remote doesn't know to
+                    //  tell us it is invalid, as we only watched the parent, not the specific path).
+                    //  - If the values become unrejected in the future, we will receive them again, which
+                    //      will cause their valid state to be changed to accepted.
+                    //  - Also, if another batch sets the value, it will override our valid state write,
+                    //      as we are writing this in a special array will is always handled first.
                     if (batch.parentsSynced) {
                         for (let parentPath of batch.parentsSynced) {
-                            initialTriggers.parentPaths.add(parentPath);
+                            auditLog("INITIAL PARENT SYNC", { path: parentPath, sourceNodeId: batch.sourceNodeId });
                             let allChildPaths = authorityStorage.getPathsFromParent(parentPath) || [];
                             let childChecker = remoteWatcher.getChildPatchWatchChecker({
                                 parentPath,
                                 nodeId: batch.sourceNodeId
                             });
                             for (let childPath of allChildPaths) {
-                                if (childChecker.isWatchedByNodeId(childPath)) {
-                                    authorityStorage.resetForInitialTrigger(childPath);
+                                if (!childChecker.isWatchedByNodeId(childPath)) continue;
+                                // We shouldn't receive the initialTrigger flag if we are the authority anyways.
+                                if (nodePathAuthority.isSelfAuthority(childPath)) continue;
+                                initialTriggers.parentPaths.add(childPath);
+
+                                // NOTE: On initial trigger, we send all child paths, so they should be
+                                //  in the values list, if they exist. AND if we receive them after,
+                                //  they will clobber this rejection
+                                //  - See PathWatcher.watchPath, where when !noInitialTrigger (aka, initialTrigger),
+                                //      we get all paths via authorityStorage.getPathsFromParent, and send their values.
+                                if (initialTriggers.values.has(childPath)) continue;
+
+                                let values = authorityStorage.getValuePlusHistory(childPath);
+                                if (values.length === 0) continue;
+
+                                values = values.filter(value => {
+                                    // DO NOT reject epoch values. For missing values, we still need a signal, and the parent
+                                    //  sync won't send this, so we need to NOT reject them.
+                                    if (compareTime(value.time, epochTime) === 0) {
+                                        return false;
+                                    }
+                                    return true;
+                                });
+                                if (values.length === 0) continue;
+
+                                debugbreak(2);
+                                debugger;
+                                console.log(self.getExistingWatchRemoteNodeId(childPath));
+                                for (let value of values) {
+                                    specialInitialParentCausedRejections.push({
+                                        path: value.path,
+                                        time: value.time,
+                                        isValid: false,
+                                        isTransparent: !!value.isTransparent,
+                                        reason: "child missing (initial trigger)",
+                                    });
                                 }
                             }
                         }
@@ -326,10 +389,12 @@ class PathValueCommitter {
                 pathValues = pathValues.filter(value => {
                     if (!nodePathAuthority.isSelfAuthority(value.path)) return true;
                     if (value.time.time < maxAge) {
+                        auditLog("DISCARD INVALID GENESIS VALUE", { path: value.path, time: value.time.time });
                         console.error(red(`Ignoring a value that is ${formatTime(Date.now() - value.time.time)} pass change time, which would break assumptions made by archives / locks about change stability. ${debugPathValuePath(value)}`));
                         return false;
                     }
                     if (value.time.time > maxFutureAge) {
+                        auditLog("DISCARD INVALID FUTURE VALUE", { path: value.path, time: value.time.time });
                         console.error(red(`Ignoring a value that is ${formatTime(value.time.time - Date.now())} in the future, which would break writes to that path until that time passes. ${debugPathValuePath(value)}`));
                         return false;
                     }
@@ -338,7 +403,7 @@ class PathValueCommitter {
             });
 
             // NOTE: Values MUST be synchronously ingested, otherwise our "initialTrigger" resetting will break things.
-            this.ingestValues(pathValues, parentsSynced, parentSyncedSources, initialTriggers);
+            this.ingestValues(pathValues, parentsSynced, parentSyncedSources, initialTriggers, specialInitialParentCausedRejections);
             // Wait a microtick to allow triggered watchers to finish.
             await Promise.resolve();
             if (!isNode()) {
@@ -355,7 +420,8 @@ class PathValueCommitter {
         pathValues: PathValue[],
         parentsSynced: string[] | undefined,
         parentSyncedSources: Map<string, string[]> | undefined,
-        initialTriggers?: { values: Set<PathValue>; parentPaths: Set<string> }
+        initialTriggers?: { values: Set<string>; parentPaths: Set<string> },
+        specialInitialParentCausedRejections?: WriteState[],
     ): void {
         if (pathValues.length === 0 && !parentsSynced?.length) return;
 
@@ -370,7 +436,7 @@ class PathValueCommitter {
         }
 
         // NOTE: Also triggers rejections of watched paths
-        this.ingestValidStates([], parentsSynced, undefined, pathValues, initialTriggers);
+        this.ingestValidStates(specialInitialParentCausedRejections ?? [], parentsSynced, undefined, pathValues, initialTriggers);
     }
 
     // NOTE: Technically, because we trigger all remotes any time a value changes, even
@@ -385,14 +451,10 @@ class PathValueCommitter {
         parentsSynced?: string[],
         recomputeValidState?: "recomputeValidState",
         alsoPathValues?: PathValue[],
-        initialTriggers?: { values: Set<PathValue>; parentPaths: Set<string> }
+        initialTriggers?: { values: Set<string>; parentPaths: Set<string> }
     ) {
         if (remoteLocksChanged.length === 0 && !parentsSynced?.length && !alsoPathValues?.length) return;
 
-        if (alsoPathValues?.some(x => x.path.includes("#BREAK"))) {
-            debugger;
-        }
-
         let now = Date.now();
 
         let valuesChanged = new Set<PathValue>();

package/src/0-path-value-core/PathValueController.ts

@@ -14,7 +14,7 @@ import { formatNumber, formatTime } from "socket-function/src/formatting/format"
 import { sha256 } from "js-sha256";
 import debugbreak from "debugbreak";
 import { ClientWatcher } from "../1-path-client/pathValueClientWatcher";
-import { debugLog, isDebugLogEnabled } from "./debugLogs";
+import { auditLog, isDebugLogEnabled } from "./auditLogs";
 import { debugNodeId } from "../-c-identity/IdentityController";
 import { diskLog } from "../diagnostics/logs/diskLogger";
 export { pathValueCommitter };
@@ -83,7 +83,7 @@ class PathValueControllerBase {
         if (isDebugLogEnabled()) {
             let sourceNodeId = debugNodeId(callerId);
             for (let value of values) {
-                debugLog("RECEIVE VALUE", { path: value.path, time: value.time.time, sourceNodeId });
+                auditLog("RECEIVE VALUE", { path: value.path, time: value.time.time, sourceNodeId });
             }
         }
         diskLog(`Received PathValues via forwardWrites`, { valueCount: values.length, callerId, });
@@ -178,10 +178,10 @@ class PathValueControllerBase {
         if (isDebugLogEnabled()) {
             let sourceNodeId = debugNodeId(callerId);
             for (let value of config.paths) {
-                debugLog("WATCH PATH", { path: value, sourceNodeId });
+                auditLog("WATCH PATH", { path: value, sourceNodeId });
             }
             for (let value of config.parentPaths) {
-                debugLog("WATCH PARENT PATH", { path: value, sourceNodeId });
+                auditLog("WATCH PARENT PATH", { path: value, sourceNodeId });
             }
         }
         pathWatcher.watchPath({ paths: config.paths, parentPaths: config.parentPaths, callback: callerId, initialTrigger: true });
@@ -191,10 +191,10 @@ class PathValueControllerBase {
         if (isDebugLogEnabled()) {
             let sourceNodeId = debugNodeId(SocketFunction.getCaller().nodeId);
             for (let value of config.paths) {
-                debugLog("UNWATCH PATH", { path: value, sourceNodeId });
+                auditLog("UNWATCH PATH", { path: value, sourceNodeId });
             }
             for (let value of config.parentPaths) {
-                debugLog("UNWATCH PARENT PATH", { path: value, sourceNodeId });
+                auditLog("UNWATCH PARENT PATH", { path: value, sourceNodeId });
             }
         }
         pathWatcher.unwatchPath({ paths: config.paths, parentPaths: config.parentPaths, callback: callerId });

package/src/0-path-value-core/{debugLogs.ts → auditLogs.ts}

@@ -4,18 +4,20 @@ import { isDevDebugbreak } from "../config";
 import { measureWrap } from "socket-function/src/profiling/measure";
 import { QueueLimited } from "socket-function/src/misc";
 
-export interface DebugLog {
+export type DebugLog = {
     type: string;
     time: number;
     values: { [key: string]: unknown };
-}
+};
 
-let ENABLED_LOGGING = isDevDebugbreak();
-export function enableDebugLogging() {
+// NOTE: For now, this is actually fairly light, so we'll just turn it on by default. It is very useful
+//  in debugging synchronization issues.
+let ENABLED_LOGGING = true;
+export function enableAuditLogging() {
     ENABLED_LOGGING = true;
     debugLogFnc = debugLogBase;
 }
-export function disableDebugLogging() {
+export function disableAuditLogging() {
     ENABLED_LOGGING = false;
     debugLogFnc = NOOP;
 }
@@ -32,20 +34,22 @@ export function getLogHistoryIncludes(includes: string) {
         return Object.values(x.values).some(y => String(y).includes(includes));
     });
 }
+(globalThis as any).getLogHistoryEquals = getLogHistoryEquals;
+(globalThis as any).getPathAuditLogs = getLogHistoryEquals;
 export function getLogHistoryEquals(value: string) {
     return logHistory.getAllUnordered().filter(x => {
         return Object.values(x.values).some(y => y === value);
     });
 }
 
-export function debugLog(type: string, values: { [key: string]: unknown }) {
+export function auditLog(type: string, values: { [key: string]: unknown }) {
     debugLogFnc(type, values);
 }
 let debugLogFnc = debugLogBase;
 let NOOP = () => { };
 function debugLogBase(type: string, values: { [key: string]: unknown }) {
     if (!ENABLED_LOGGING) {
-        disableDebugLogging();
+        disableAuditLogging();
         return;
     }
     let newEntry: DebugLog = { type, time: Date.now(), values };

package/src/0-path-value-core/pathValueCore.ts

@@ -28,7 +28,7 @@ import yargs from "yargs";
 import { sha256 } from "js-sha256";
 import { PromiseObj } from "../promise";
 import { ClientWatcher } from "../1-path-client/pathValueClientWatcher";
-import { debugLog, isDebugLogEnabled } from "./debugLogs";
+import { auditLog, isDebugLogEnabled } from "./auditLogs";
 import { diskLog } from "../diagnostics/logs/diskLogger";
 
 let yargObj = isNodeTrue() && yargs(process.argv)
@@ -207,6 +207,7 @@ export type WriteState = {
     //      - HOWEVER, writers should just be smart, as it causes a lot of thrashing and work
     //          on all the clients per write, even if nothing changes!
     isTransparent: boolean;
+    reason?: string;
 };
 
 // NOTE: Object values ARE allowed here. They are effectively immutable though, so it is recommended
@@ -679,7 +680,7 @@ class AuthorityPathValueStorage {
 
     public resetForInitialTrigger(path: string) {
         if (isDebugLogEnabled()) {
-            debugLog("REMOVE FOR INITIAL SYNC", { path });
+            auditLog("REMOVE FOR INITIAL SYNC", { path });
         }
         this.removePathFromStorage(path, "reset for initial trigger");
     }
@@ -720,12 +721,20 @@ class AuthorityPathValueStorage {
         this.getMultiNodesForParent = fnc;
     }
     @measureFnc
-    public ingestValues(newValues: PathValue[], parentsSynced: string[] | undefined, parentSyncedSources: Map<string, string[]> | undefined) {
+    public ingestValues(
+        newValues: PathValue[],
+        parentsSynced: string[] | undefined,
+        parentSyncedSources: Map<string, string[]> | undefined,
+        // Skips archiving (used if we received the values from another node, in which case
+        //  not only should the values already be archives, they are likely too old to be archives
+        //  again!)
+        skipArchive?: "skipArchive"
+    ) {
         let now = Date.now();
 
         if (isDebugLogEnabled()) {
             for (let value of newValues) {
-                debugLog("INGEST VALUE", { path: value.path, time: value.time.time });
+                auditLog("INGEST VALUE", { path: value.path, time: value.time.time });
             }
         }
 
@@ -762,7 +771,7 @@ class AuthorityPathValueStorage {
                 this.possiblyUndefinedPaths().value.add(value.path);
             }
 
-            if (!value.event && !value.path.startsWith(LOCAL_DOMAIN_PATH)) {
+            if (!skipArchive && !value.event && !value.path.startsWith(LOCAL_DOMAIN_PATH)) {
                 let authorityPath = pathValueAuthority2.getSelfArchiveAuthority(value.path);
 
                 // Store in pending archive lookup
@@ -983,7 +992,7 @@ class AuthorityPathValueStorage {
         this.values.delete(path);
 
         if (isDebugLogEnabled()) {
-            debugLog("REMOVE PATH", { path, reason });
+            auditLog("REMOVE PATH", { path, reason });
         }
 
         // NOTE: parentPathLookup is important because clients often use Object.keys() to get paths,
@@ -1348,7 +1357,7 @@ class PathWatcher {
         }
     }
 
-    public triggerValuesChanged(valuesChanged: Set<PathValue>, parentsSynced?: string[], initialTriggers?: { values: Set<PathValue>; parentPaths: Set<string> }) {
+    public triggerValuesChanged(valuesChanged: Set<PathValue>, parentsSynced?: string[], initialTriggers?: { values: Set<string>; parentPaths: Set<string> }) {
         let changedPerCallbacks = this.getWatchers(valuesChanged, { parentsSynced, pathsAreSynced: true });
         for (let [watch, changes] of changedPerCallbacks) {
             if (initialTriggers) {
@@ -1357,7 +1366,7 @@ class PathWatcher {
                 let valuesOther = new Set<PathValue>();
                 let parentsOther = new Set<string>();
                 for (let value of changes) {
-                    if (initialTriggers.values.has(value)) {
+                    if (initialTriggers.values.has(value.path)) {
                         valuesFromInitialTrigger.add(value);
                     } else {
                         valuesOther.add(value);
@@ -1486,7 +1495,7 @@ class PathWatcher {
                 if (isDebugLogEnabled()) {
                     for (let pathValue of changes) {
 
-                        debugLog("SEND VALUE", { path: pathValue.path, time: pathValue.time.time, nodeId: debugNodeId(watcher), transparent: pathValue.isTransparent, canGC: pathValue.canGCValue });
+                        auditLog("SEND VALUE", { path: pathValue.path, time: pathValue.time.time, nodeId: debugNodeId(watcher), transparent: pathValue.isTransparent, canGC: pathValue.canGCValue });
                     }
                 }
                 await PathValueController.nodes[watcher].forwardWrites(
@@ -1677,6 +1686,11 @@ class WriteValidStorage {
         });
     }
     public setWriteValidState(write: WriteState, lockless?: boolean): boolean {
+        if (write.isValid) {
+            auditLog(write.reason || "ACCEPTING VALUE", { path: write.path, time: write.time.time });
+        } else {
+            auditLog(write.reason || "REJECTING VALUE", { path: write.path, time: write.time.time });
+        }
         this.ensureGarbageCollectOldState();
 
         // If lockless then it starts valid, and always stays valid.

package/src/1-path-client/RemoteWatcher.ts

@@ -17,7 +17,66 @@ import { appendToPathStr, getParentPathStr, getPathDepth, getPathIndexAssert, ge
 import { isEmpty } from "../misc";
 import debugbreak from "debugbreak";
 import { isDevDebugbreak } from "../config";
-import { debugLog } from "../0-path-value-core/debugLogs";
+import { auditLog } from "../0-path-value-core/auditLogs";
+
+//todonext
+//  BOTH PathValueServers crashed due to trying to archive a value that is too far in the past.
+//  - Ah, that would happen if we did an audit, so... let's not archive auditted values.
+
+//todonext
+//  Ugh... now all paths are failing to sync?
+//      ".,querysub._com.,PathFunctionRunner.,book3_0.,Data.,data.,sliftist@gmail._com.,books.,bb1c01e5fb3be5828._querysub._com_1712516071167._0005.,pages.,1728369494296._706_0._30634508051801834.,items.,"
+
+//todonext
+//  OH! Right, if the value doesn't exist... 
+
+
+// This path was received, but ignored, because apparently t
+
+//todonext
+//  Alright, failing to sync paths is easy
+//  - Let's first see if it's on the client
+//  - We do receive the value, so... it's a clientside issue
+// ".,querysub._com.,PathFunctionRunner.,book3_0.,Data.,data.,sliftist@gmail._com.,books.,bb1c01e5fb3be5828._querysub._com_1712516071167._0005.,pages.,1728369494296._706_0._30634508051801834.,items.,hasOwnProperty.,"
+
+
+//todonext
+//  BUG! Every time on load (after 15 seconds), we fail to sync UndoWatch. This shouldn't be happening...
+//  - It looks like we are trying to load a lot of nested values, which should be atomic? So... why is our schema
+//      not working, and even if it isn't... why don't we receive the values eventually?
+//  - https://127-0-0-1.querysub.com:1111/?managementpage=AuditLogPage&selected=%221728769700117.415_0.13356892448729063%22&book=bb1c01e5fb3be5828.querysub.com_1712516071167.0005&path=.%2Cquerysub._com.%2CPathFunctionRunner.%2Cbook3_0.%2CData.%2Cdata.%2Csliftist%40gmail._com.%2Cbooks.%2Cbb1c01e5fb3be5828._querysub._com_1712516073485._0002.%2Cpages.%2Cb5da74489986c2f47._querysub._com_1708134597635._0002.%2Citems.%2C0.%2C
+
+
+//todonext
+//  BUG! Our predictions seem to be rejected for a frame before we get the new value. Our predictions
+//      can't break like this for such a simple thing...
+
+
+//todonext
+//  Yep, found the bug. We were missing child values in the sync due to having them split between two authorities,
+//      but the parent sent both, so it's got lost, and then it deleted the other paths because it figured
+//      they are erroneous (as in, bad predictions).
+//  - The fix was basically just updating the code that gets the remote node id to use the parent syncer instead
+
+//todonext
+//  Ugh... we failed to load a value, after starting on the main book page, and then drilling down. And then
+//      we loaded it on server restart. UGH... we might be ignoring values we shouldn't...
+
+
+
+//todonext
+// Enable this flag, and verify it stops double sending of values
+//  Ex, of this path:
+//  https://127-0-0-1.querysub.com:1111/?managementpage=AuditLogPage&selected=%221728769700117.415_0.13356892448729063%22&path=.%2Cquerysub._com.%2CPathFunctionRunner.%2Cbook3_0.%2CData.%2Cdata.%2Csliftist%40gmail._com.%2Cbooks.%2Cbb1c01e5fb3be5828._querysub._com_1712516073485._0002.%2Cpages.%2Cb5da74489986c2f47._querysub._com_1708134597635._0002.%2Citems.%2C0.%2C
+//todonext
+//  Add a breakpoint at "IGNORING VALUE..."  when we turn stop double sends on. Presently, this happens
+//      a lot, due to double syncing. BUT, once we stop double syncing... this should mostly go away.
+const STOP_KEYS_DOUBLE_SENDS = false;
+
+//todonext;
+// DEPLOY (do-update AND yarn deploy), and then test some more!
+//  - Also get archive.org to recache our site, as we changed our bootstrapper
+
 
 export class RemoteWatcher {
     public static DEBUG = false;
@@ -33,6 +92,9 @@ export class RemoteWatcher {
             start: number;
             end: number;
         }>;
+        // Watches which we suppressed, due to this parent sync. If this parent sync stops,
+        //  we will need to re-send the watches.
+        suppressedWatches: Set<string>;
         fullNodeCount: number;
     }>());
 
@@ -154,6 +216,8 @@ export class RemoteWatcher {
     private internalWatchLatest(config: WatchConfig & {
         debugName?: string;
         tryReconnect?: boolean;
+        // Watch, even if we were already watching
+        forceWatch?: boolean;
     }) {
         let watchesPerAuthority = new Map<string, { paths: string[]; parentPaths: string[] }>();
 
@@ -173,6 +237,14 @@ export class RemoteWatcher {
                 //  to come back out of order, causing watching functions to trigger out of order!
                 let remoteAuthorityId = this.remoteWatchPaths.get(path);
                 if (remoteAuthorityId !== undefined) {
+                    if (config.forceWatch) {
+                        let watchesPer = watchesPerAuthority.get(remoteAuthorityId);
+                        if (!watchesPer) {
+                            watchesPer = { paths: [], parentPaths: [] };
+                            watchesPerAuthority.set(remoteAuthorityId, watchesPer);
+                        }
+                        watchesPer.paths.push(path);
+                    }
                     this.disconnectedPaths.delete(path);
                     continue;
                 }
@@ -187,9 +259,11 @@ export class RemoteWatcher {
                     if (!this.disconnectedPaths.has(path)) {
                         newDisconnectPaths++;
                         this.disconnectedPaths.add(path);
+                        auditLog("NO AUTHORITY FOR PATH", { path });
                     }
                     continue;
                 }
+                auditLog("FOUND AUTHORITY FOR PATH", { path, authorityId });
                 this.disconnectedPaths.delete(path);
                 foundPaths++;
 
@@ -281,7 +355,7 @@ export class RemoteWatcher {
                 }
 
                 if (!watchObj) {
-                    watchObj = { nodesId: new Map(), fullNodeCount: 0 };
+                    watchObj = { nodesId: new Map(), fullNodeCount: 0, suppressedWatches: new Set() };
                     this.remoteWatchParents.set(path, watchObj);
                 }
                 watchObj.fullNodeCount = nodes.length;
@@ -335,13 +409,23 @@ export class RemoteWatcher {
 
             if (isDevDebugbreak()) {
                 for (let path of paths) {
-                    debugLog("remoteWatcher outer WATCH", { path, remoteNodeId: authorityId });
+                    auditLog("remoteWatcher outer WATCH", { path, remoteNodeId: authorityId });
                 }
                 for (let path of parentPaths) {
-                    debugLog("remoteWatcher outer WATCH PARENT", { path, remoteNodeId: authorityId });
+                    auditLog("remoteWatcher outer WATCH PARENT", { path, remoteNodeId: authorityId });
                 }
             }
 
+            if (STOP_KEYS_DOUBLE_SENDS) {
+                paths = paths.filter(path => {
+                    let parentPath = getParentPathStr(path);
+                    let parentObj = this.remoteWatchParents.get(parentPath);
+                    if (!parentObj) return true;
+                    parentObj.suppressedWatches.add(path);
+                    return false;
+                });
+            }
+
             this.reconnectCache(authorityId);
             // Don't throw, as the reconnectCache will eventually resolve errors and watch successfully
             logErrors(this.watchLatestRemote({
@@ -407,10 +491,10 @@ export class RemoteWatcher {
                 for (let [authorityId, { paths, parentPaths }] of byAuthority) {
                     if (isDevDebugbreak()) {
                         for (let path of paths) {
-                            debugLog("remoteWatcher inner WATCH", { path, remoteNodeId: authorityId });
+                            auditLog("remoteWatcher inner WATCH", { path, remoteNodeId: authorityId });
                         }
                         for (let path of parentPaths) {
-                            debugLog("remoteWatcher inner WATCH PARENT", { path, remoteNodeId: authorityId });
+                            auditLog("remoteWatcher inner WATCH PARENT", { path, remoteNodeId: authorityId });
                         }
                     }
                     // NOTE: We log watches here because we want to log batched watches
@@ -460,7 +544,6 @@ export class RemoteWatcher {
         };
         const innerParentPathRemove = (path: string, remoteAuthorityId: string | undefined) => {
             if (remoteAuthorityId === undefined) return;
-            this.remoteWatchParents.delete(path);
             this.disconnectedParents.delete(path);
             let paths = unwatchesPerAuthority.get(remoteAuthorityId);
             if (!paths) {
@@ -474,21 +557,36 @@ export class RemoteWatcher {
             let remoteAuthorityId = this.remoteWatchPaths.get(path);
             innerPathRemove(path, remoteAuthorityId);
         }
+        let suppressedRewatches = new Set<string>();
         for (let path of config.parentPaths) {
             let watchObj = this.remoteWatchParents.get(path);
             if (!watchObj) continue;
+            // Rewatch anything that we suppressed watching (which happens when STOP_KEYS_DOUBLE_SENDS is set)
+            for (let suppressedWatch of watchObj.suppressedWatches) {
+                // Only if we are still watching it (which most of the time, we won't be)
+                if (!this.remoteWatchPaths.has(suppressedWatch)) continue;
+                suppressedRewatches.add(suppressedWatch);
+            }
             for (let authorityId of watchObj.nodesId.keys()) {
                 innerParentPathRemove(path, authorityId);
             }
             this.remoteWatchParents.delete(path);
         }
+        if (suppressedRewatches.size > 0) {
+            this.internalWatchLatest({
+                paths: Array.from(suppressedRewatches),
+                parentPaths: [],
+                debugName: "watchSuppressedWatches",
+                forceWatch: true,
+            });
+        }
         for (let [authorityIdBase, { paths, parentPaths }] of unwatchesPerAuthority.entries()) {
             if (isDevDebugbreak()) {
                 for (let path of paths) {
-                    debugLog("remoteWatcher inner UNWATCH", { path, remoteNodeId: authorityIdBase });
+                    auditLog("remoteWatcher inner UNWATCH", { path, remoteNodeId: authorityIdBase });
                 }
                 for (let path of parentPaths) {
-                    debugLog("remoteWatcher inner UNWATCH PARENT", { path, remoteNodeId: authorityIdBase });
+                    auditLog("remoteWatcher inner UNWATCH PARENT", { path, remoteNodeId: authorityIdBase });
                 }
             }
             if (!isOwnNodeId(authorityIdBase)) {
@@ -505,7 +603,24 @@ export class RemoteWatcher {
         this.unwatchLatest(config);
     }
 
-    public getExistingWatchRemoteNodeId(path: string) {
+    @measureFnc
+    public getExistingWatchRemoteNodeId(path: string): string | undefined {
+        let parentPath = getParentPathStr(path);
+        let parentWatchObj = this.remoteWatchParents.get(parentPath);
+        if (parentWatchObj) {
+            for (let [nodeId, obj] of parentWatchObj.nodesId) {
+                if (matchesParentRangeFilter({
+                    parentPath,
+                    fullPath: path,
+                    start: obj.start,
+                    end: obj.end,
+                })) {
+                    return nodeId;
+                }
+            }
+            // NOTE: I think it is a bug if we hit here. There shouldn't be a parent, but have us not match it?
+            //  But... I also don't see the harm in checking remoteWatchPaths?
+        }
         return this.remoteWatchPaths.get(path);
     }
     public getAllRemoteWatchedNodeIds() {

package/src/1-path-client/pathValueClientWatcher.ts

@@ -36,7 +36,7 @@ import { remoteWatcher } from "./RemoteWatcher";
 import { heapTagObj } from "../diagnostics/heapTag";
 import { Querysub } from "../4-querysub/QuerysubController";
 import { isDevDebugbreak } from "../config";
-import { debugLog } from "../0-path-value-core/debugLogs";
+import { auditLog } from "../0-path-value-core/auditLogs";
 
 const pathValueCtor = heapTagObj("PathValue");
 
@@ -433,10 +433,10 @@ export class ClientWatcher {
         let expiryTime = Date.now() + ClientWatcher.WATCH_STICK_TIME;
         if (isDevDebugbreak()) {
             for (let path of fullyUnwatchedPaths) {
-                debugLog("clientWatcher UNWATCH", { path });
+                auditLog("clientWatcher UNWATCH", { path });
             }
             for (let path of fullyUnwatchedParents) {
-                debugLog("clientWatcher UNWATCH PARENT", { path });
+                auditLog("clientWatcher UNWATCH PARENT", { path });
             }
         }
         for (let path of fullyUnwatchedPaths) {
@@ -533,10 +533,10 @@ export class ClientWatcher {
 
         if (isDevDebugbreak()) {
             for (let path of pathsArray) {
-                debugLog("clientWatcher WATCH", { path });
+                auditLog("clientWatcher WATCH", { path });
             }
             for (let path of parentPathsArray) {
-                debugLog("clientWatcher WATCH PARENT", { path });
+                auditLog("clientWatcher WATCH PARENT", { path });
             }
         }
 

package/src/2-proxy/PathValueProxyWatcher.ts

@@ -422,9 +422,10 @@ export class PathValueProxyWatcher {
     /** NOTE: This results in VERY slow performance, and will log A LOT of information. */
     public static TRACE = false;
     public static TRACE_WRITES = false;
+    public static TRACE_WATCHERS = new Set<string>();
 
     private SHOULD_TRACE(watcher: SyncWatcher) {
-        return PathValueProxyWatcher.DEBUG || PathValueProxyWatcher.TRACE || PathValueProxyWatcher.TRACE_WRITES && watcher.options.canWrite || ClientWatcher.DEBUG_TRIGGERS === "heavy";
+        return PathValueProxyWatcher.DEBUG || PathValueProxyWatcher.TRACE || PathValueProxyWatcher.TRACE_WRITES && watcher.options.canWrite || ClientWatcher.DEBUG_TRIGGERS === "heavy" || PathValueProxyWatcher.TRACE_WATCHERS.size > 0 && Array.from(PathValueProxyWatcher.TRACE_WATCHERS).some(x => watcher.debugName.includes(x));
     }
 
     private isUnsyncCheckers = new Set<{ value: number }>();
@@ -917,6 +918,9 @@ export class PathValueProxyWatcher {
                     let result = authorityStorage.temporaryOverride(options.overrides, () =>
                         options.watchFunction()
                     );
+                    // Clone, otherwise proxies get out of the watcher, which can result in accesses outside
+                    //  of a synced watcher.
+                    result = deepCloneCborx(result);
                     options.onResultUpdated?.({ result }, undefined, outerWatcher);
                 } catch (e: any) {
                     options.onResultUpdated?.({ error: e.stack }, undefined, outerWatcher);

package/src/4-deploy/deployMain.ts

@@ -37,17 +37,6 @@ let yargObj = yargs(process.argv)
 setIsDeploy();
 
 export async function deployMain() {
-    if (true as boolean) {
-        // Mount, so we can write directly to the database
-        await SocketFunction.mount({ port: 0, ...await getThreadKeyCert(), public: isPublic() });
-
-        await setLiveDeployedHash({ hash: "7b94606dc1de8d03acb8b23952ff5d49e716a45e", refreshThresholdTime: 0, });
-        //await setLiveDeployedHash({ hash: "4cba43649682967bf1ff919d941820bea6719ef7", refreshThresholdTime: 0, });
-        console.log(magenta(`SHUTTING DOWN`));
-        await shutdown();
-        return;
-    }
-
     //SocketFunction.logMessages = true;
     //quietCoreMode();
     //ClientWatcher.DEBUG_READS = true;

package/src/4-deploy/edgeBootstrap.ts

@@ -236,6 +236,7 @@ async function edgeNodeFunction(config: {
 
     let progressUI = createProgressUI();
     progressUI.setMessage("Finding available server...");
+    let retryCount = 0;
     while (true) {
         try {
             let booted = await tryToBoot();
@@ -243,7 +244,8 @@ async function edgeNodeFunction(config: {
         } catch (e) {
             console.error(e);
         }
-        progressUI.setMessage("No available servers, retrying soon");
+        retryCount++;
+        progressUI.setMessage(`No available servers, retried ${retryCount++} times`);
         let time = 1000 * 15;
         if (Date.now() < liveHashOverrideExpiryTime) {
             time = 1000 * 3;

package/src/5-diagnostics/diskValueAudit.ts

@@ -8,7 +8,7 @@ import { formatNumber, formatTime } from "socket-function/src/formatting/format"
 import { green } from "socket-function/src/formatting/logColors";
 import debugbreak from "debugbreak";
 import { isDevDebugbreak } from "../config";
-import { DebugLog, getLogHistoryEquals } from "../0-path-value-core/debugLogs";
+import { DebugLog, getLogHistoryEquals } from "../0-path-value-core/auditLogs";
 import { getOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
 import { sort } from "socket-function/src/misc";
 
@@ -200,7 +200,7 @@ async function checkAuthority(authority: AuthorityPath, threshold: number) {
                 authorityStorage.resetForInitialTrigger(valuePath.path);
             }
         }
-        authorityStorage.ingestValues(Array.from(allValues), undefined, undefined);
+        authorityStorage.ingestValues(Array.from(allValues), undefined, undefined, "skipArchive");
 
         if (allValues.size > 0) {
             // NOTE: By triggering all of these as initial syncs... it SHOULD result in us clobbering the old
@@ -209,7 +209,7 @@ async function checkAuthority(authority: AuthorityPath, threshold: number) {
             //  - Except predictions might be clobbered... but that's fine, they don't really matter.
             pathWatcher.triggerValuesChanged(allValues, undefined, {
                 parentPaths: new Set(),
-                values: allValues,
+                values: new Set(Array.from(allValues).map(x => x.path)),
             });
         }
     }

package/src/5-diagnostics/memoryValueAudit.ts

@@ -14,7 +14,7 @@ import { remoteWatcher } from "../1-path-client/RemoteWatcher";
 import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
 import { getOwnNodeId, isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
 import debugbreak from "debugbreak";
-import { DebugLog, DebugLogController, getLogHistoryEquals, getLogHistoryIncludes } from "../0-path-value-core/debugLogs";
+import { DebugLog, DebugLogController, getLogHistoryEquals, getLogHistoryIncludes } from "../0-path-value-core/auditLogs";
 import { getParentPathStr } from "../path";
 import { diskAuditNow } from "./diskValueAudit";
 import { yellow, green } from "socket-function/src/formatting/logColors";
@@ -247,7 +247,7 @@ async function auditSpecificPathsBase(config: {
             continue;
         }
 
-        if (!pathValueSerializer.compareValuePaths(value, ourValue)) continue;
+        if (!pathValueSerializer.compareValuePaths(value, ourValue) || !!value?.valid !== !!ourValue?.valid) continue;
 
         let ourTime = ourValue ? ourValue.time.time : 0;
         let remoteTime = value ? value.time.time : 0;
@@ -262,7 +262,7 @@ async function auditSpecificPathsBase(config: {
         ourValue = authorityStorage.getValueAtTime(path);
         value = (await ValueAuditController.nodes[authorityNodeId].getPathValue(path)).value;
 
-        if (!pathValueSerializer.compareValuePaths(value, ourValue)) continue;
+        if (!pathValueSerializer.compareValuePaths(value, ourValue) || !!value?.valid !== !!ourValue?.valid) continue;
         let THRESHOLD2 = Date.now() - MAX_ASSUMED_PROPAGATION_DELAY;
         let ourTime2 = authorityStorage.getLastChangedTime(path)?.time || 0;
         let oursIsNew = ourTime2 > THRESHOLD2;
@@ -297,8 +297,8 @@ async function auditSpecificPathsBase(config: {
             }
             let atTime = Date.now();
             console.error(`Audit mismatch for ${path} from ${authorityNodeId} detected at ${atTime}. Watching because: ${reason}`);
-            console.log(`Value Remote:`, pathValueSerializer.getPathValue(value));
-            console.log(`Value Local:`, pathValueSerializer.getPathValue(ourValue));
+            console.log(`Value Remote (${value?.valid ? "valid" : "invalid"}):`, pathValueSerializer.getPathValue(value));
+            console.log(`Value Local (${ourValue?.valid ? "valid" : "invalid"}):`, pathValueSerializer.getPathValue(ourValue));
             console.log(`Time Remote:`, value?.time.time);
             console.log(`Time Local:`, ourValue?.time.time);
             console.log(`Our node id:`, ourNodeId);

package/src/diagnostics/AuditLogPage.tsx

@@ -0,0 +1,148 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { qreact } from "../4-dom/qreact";
+import { assertIsManagementUser } from "./managementPages";
+import { css } from "typesafecss";
+import { getSyncedController } from "../library-components/SyncedController";
+import { DebugLog, DebugLogController, getLogHistoryEquals, getLogHistoryIncludes } from "../0-path-value-core/auditLogs";
+import { remoteWatcher } from "../1-path-client/RemoteWatcher";
+import { pathValueAuthority2 } from "../0-path-value-core/NodePathAuthorities";
+import { getParentPathStr } from "../path";
+import { getBrowserUrlNode, getOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
+import { nextId, sort } from "socket-function/src/misc";
+import { t } from "../2-proxy/schema2";
+import { InputLabelURL } from "../library-components/InputLabel";
+import { URLParam } from "../library-components/URLParam";
+import { Table } from "../5-diagnostics/Table";
+import { ObjectDisplay } from "./logs/ObjectDisplay";
+
+/**
+TODO:
+    1) Support watching multiple paths at once
+    2) Show extra metadata for the paths
+        - Current value (raw, as in, don't sync it)
+        - If it is synced
+        - The authority of it
+        - An expander which shows a table of the full history of values, with valid state, etc
+*/
+
+type DebugLogWithSource = DebugLog & {
+    sourceType: "local" | "remote";
+    sourceNodeId: string;
+}
+
+let path = new URLParam("path", "");
+let includeDescendants = new URLParam("includeDescendants", false);
+export class AuditLogPage extends qreact.Component {
+    loadId = nextId();
+    state = t.state({
+        loadedId: t.string
+    });
+    render() {
+        let logs: DebugLogWithSource[] = [];
+        let loaded = this.loadId === this.state.loadedId;
+        if (loaded) {
+            let obj = auditLogController(getBrowserUrlNode()).getPathHistory({
+                path: path.value,
+                includeDescendants: includeDescendants.value,
+            });
+            if (obj) {
+                logs = obj.fullLogs;
+            }
+        }
+        return (
+            <div class={css.pad2(10).vbox(10).fillWidth}>
+                <InputLabelURL
+                    label="Path"
+                    url={path}
+                    fillWidth
+                    onChange={() => this.loadId = nextId()}
+                />
+                <InputLabelURL
+                    label="Include Descendants"
+                    url={includeDescendants}
+                    checkbox
+                    onChange={() => this.loadId = nextId()}
+                />
+                {!loaded && <button onClick={() => this.state.loadedId = this.loadId}>Load</button>}
+                <Table
+                    rows={logs}
+                    columns={{
+                        type: {},
+                        time: {},
+                        sourceType: {},
+                        sourceNodeId: {},
+                        values: {
+                            formatter(value) {
+                                return <ObjectDisplay value={value} />;
+                            },
+                        },
+                    }}
+                />
+            </div>
+        );
+    }
+}
+
+
+class AuditLogControllerBase {
+    public async getPathHistory(config: {
+        path: string;
+        includeDescendants?: boolean;
+    }) {
+        let { path, includeDescendants } = config;
+        let authorityNodeId = remoteWatcher.getExistingWatchRemoteNodeId(path);
+        if (!authorityNodeId) {
+            authorityNodeId = await pathValueAuthority2.getSingleReadNodeSync(path);
+        }
+        let logs: DebugLog[] = [];
+        let remoteLogs: DebugLog[] = [];
+        if (includeDescendants) {
+            // Also look for the parent, so we can capture keys() syncs as well
+            //  (this should give us child paths too, because that's how includes works)
+            logs = getLogHistoryIncludes(path);
+            if (authorityNodeId) {
+                remoteLogs = await DebugLogController.nodes[authorityNodeId].getLogHistoryEquals(path);
+            }
+        } else {
+            logs = getLogHistoryEquals(path);
+            if (authorityNodeId) {
+                remoteLogs = await DebugLogController.nodes[authorityNodeId].getLogHistoryEquals(path);
+            }
+        }
+
+        let fullLogs: DebugLogWithSource[] = [];
+        let ownNodeId = getOwnNodeId();
+        for (let log of logs) {
+            fullLogs.push({
+                ...log,
+                sourceType: "local",
+                sourceNodeId: ownNodeId,
+            });
+        }
+        if (authorityNodeId) {
+            for (let log of remoteLogs) {
+                fullLogs.push({
+                    ...log,
+                    sourceType: "remote",
+                    sourceNodeId: authorityNodeId,
+                });
+            }
+        }
+        sort(fullLogs, x => x.time);
+        return {
+            fullLogs,
+        };
+    }
+}
+
+export const AuditLogController = SocketFunction.register(
+    "AuditLogController-bf798faf-c803-4add-ac2c-99d0f44e753a",
+    new AuditLogControllerBase(),
+    () => ({
+        getPathHistory: {},
+    }),
+    () => ({
+        hooks: [assertIsManagementUser],
+    })
+);
+export const auditLogController = getSyncedController(AuditLogController);

package/src/diagnostics/managementPages.tsx

@@ -64,7 +64,7 @@ export async function registerManagementPages2(config: {
         throw new Error(`registerManagementPages2 called with a different module than previously registered. This will break isManagementUser, and so is not allowed.`);
     }
     registeredModule = config.module;
-    let inputPages = config.pages.slice();
+    let inputPages: typeof config.pages = [];
 
     inputPages.push({
         title: "Nodes",
@@ -89,12 +89,19 @@ export async function registerManagementPages2(config: {
         componentName: "LogClassifiers",
         getModule: () => import("./errorLogs/LogClassifiers"),
     });
+    inputPages.push({
+        title: "Audit Paths",
+        componentName: "AuditLogPage",
+        controllerName: "AuditLogController",
+        getModule: () => import("./AuditLogPage"),
+    });
     inputPages.push({
         title: "Time",
         componentName: "TimeDebug",
         controllerName: "",
         getModule: () => import("./TimeDebug"),
     });
+    inputPages.push(...config.pages);
 
     // NOTE: We don't store the UI in the database (here, or anywhere else, at least
     //  not yet), but we do want to store the permission function call, as we may