querysub 0.406.0 → 0.408.0

package/src/diagnostics/PathDistributionInfo.tsx

@@ -0,0 +1,102 @@
+import { qreact } from "../4-dom/qreact";
+import { css } from "../4-dom/css";
+import { sort } from "socket-function/src/misc";
+import { proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
+import { pathWatcher } from "../0-path-value-core/PathWatcher";
+import { PathRouter } from "../0-path-value-core/PathRouter";
+import { QuerysubController, querysubNodeId } from "../4-querysub/QuerysubController";
+import { getSyncedController } from "../library-components/SyncedController";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { isCurrentUserSuperUser } from "../user-implementation/userData";
+import { delay } from "socket-function/src/batching";
+import { remoteWatcher } from "../1-path-client/RemoteWatcher";
+
+module.hotreload = true;
+
+let querysubController = getSyncedController(QuerysubController);
+
+let pathNodeCache = new Map<string, string | "missing">();
+let pendingPaths = new Set<string>();
+let flushScheduled = false;
+
+function scheduleFlush() {
+    if (flushScheduled) return;
+    flushScheduled = true;
+    let promise = Promise.resolve().then(async () => {
+        flushScheduled = false;
+        if (pendingPaths.size === 0) return;
+        let paths = Array.from(pendingPaths);
+        pendingPaths.clear();
+        let nodeId = await querysubNodeId();
+        await remoteWatcher.flushWatchers();
+        if (!nodeId) return;
+        let result = await QuerysubController.nodes[nodeId].debugGetPathNodeIds(paths);
+        for (let path of paths) {
+            pathNodeCache.set(path, result.get(path) ?? "missing");
+        }
+    });
+    proxyWatcher.triggerOnPromiseFinish(promise, { waitReason: "pathDistribution" });
+}
+
+function getNodeForPath(path: string): string | "missing" | undefined {
+    let cached = pathNodeCache.get(path);
+    if (cached !== undefined) return cached;
+    pendingPaths.add(path);
+    scheduleFlush();
+    return undefined;
+}
+
+export class PathDistributionInfo extends qreact.Component {
+    render() {
+        if (!isCurrentUserSuperUser()) return undefined;
+
+        let nodeSpecs = querysubController(SocketFunction.browserNodeId()).debugGetNodeSpecs();
+
+        let paths = pathWatcher.getAllWatchedPaths().filter(p => !PathRouter.isLocalPath(p));
+        let distribution = new Map<string, string[]>();
+        let missingPaths: string[] = [];
+        for (let path of paths) {
+            let nodeId = getNodeForPath(path);
+            if (nodeId === undefined) continue;
+            if (nodeId === "missing") {
+                missingPaths.push(path);
+                continue;
+            }
+            let nodePaths = distribution.get(nodeId);
+            if (!nodePaths) {
+                nodePaths = [];
+                distribution.set(nodeId, nodePaths);
+            }
+            nodePaths.push(path);
+        }
+
+        let entries = Array.from(distribution.entries());
+        sort(entries, ([nodeId, p]) => nodeId);
+        sort(entries, ([nodeId, p]) => (nodeSpecs?.get(nodeId)?.routeStart ?? 0));
+
+        let titleParts = entries.map(([nodeId, p]) => {
+            let spec = nodeSpecs?.get(nodeId);
+            let range = spec ? ` (${spec.routeStart}-${spec.routeEnd})` : "";
+            return `${nodeId}${range}: ${p.length}`;
+        });
+        if (missingPaths.length > 0) titleParts.push(`missing: ${missingPaths.length}`);
+        let title = titleParts.join("\n");
+
+        let parts: preact.ComponentChild[] = [];
+        for (let i = 0; i < entries.length; i++) {
+            let [nodeId, nodePaths] = entries[i];
+            if (i > 0) parts.push(" | ");
+            let spec = nodeSpecs?.get(nodeId);
+            parts.push(<span className={css.button} onClick={() => console.log(`Paths for ${nodeId}:`, nodePaths, spec && { range: `${spec.routeStart}-${spec.routeEnd}` })}>{nodePaths.length}</span>);
+        }
+        if (missingPaths.length > 0) {
+            if (parts.length > 0) parts.push(" | ");
+            parts.push(<span className={css.button.boldStyle.hslcolor(0, 100, 60)} onClick={() => console.log("Missing paths:", missingPaths)}>{missingPaths.length}</span>);
+        }
+
+        return <div className={css.hbox(4)} title={title}>
+            <span>🌐</span>
+            {parts.length > 0 && parts || "..."}
+        </div>;
+    }
+}

package/src/diagnostics/SyncTestPage.tsx

@@ -17,11 +17,9 @@ import { PathAuditerController } from "./pathAuditer";
 import { t } from "../2-proxy/schema2";
 
 
-let { data, functions } = Querysub.createSchema<{
-    values: {
-        [key: string]: number;
-    };
-}>()({
+let { data, functions } = Querysub.createSchema({
+    values: t.lookup(t.number)
+})({
     functions: {
         setValue(key: string, value: number) {
             data().values[key] = value;
@@ -90,6 +88,14 @@ export class SyncTestPage extends qreact.Component {
         allThreads = allThreads.slice();
         sort(allThreads, x => x.entrypoint);
 
+        function refreshDelayed() {
+            Querysub.onCommitFinished(() => {
+                setTimeout(() => {
+                    syncTestController.refreshAll();
+                }, 500);
+            });
+        }
+
         return (
             <div className={css.pad2(20).vbox(40)}>
                 <h1>Sync Test Page</h1>
@@ -125,8 +131,11 @@ export class SyncTestPage extends qreact.Component {
                                 <div className={css.hbox(10)}>
                                     <span class={css.boldStyle}>{key}</span>
                                     <span>{value}</span>
-                                    <Button onClick={() => functions.incrementValue(key)}>Increment</Button>
-                                    <InputLabel value={value} number onChangeValue={value => functions.setValue(key, +value)} />
+                                    <Button onClick={() => { functions.incrementValue(key); refreshDelayed(); }}>Increment</Button>
+                                    <InputLabel value={value} number onChangeValue={value => {
+                                        functions.setValue(key, +value);
+                                        refreshDelayed();
+                                    }} />
                                 </div>
                                 <div className={css.hbox(10, 2).wrap.marginLeft(20)}>
                                     {allThreads.map(thread => {
@@ -149,6 +158,7 @@ export class SyncTestPage extends qreact.Component {
                                                         path: path,
                                                         nodeId: thread.nodeId
                                                     });
+                                                    refreshDelayed();
                                                 }}>
                                                     Delete Value
                                                 </Button>
@@ -168,6 +178,7 @@ export class SyncTestPage extends qreact.Component {
                     label="Insert new key"
                     onChangeValue={value => {
                         functions.setValue(value, 0);
+                        refreshDelayed();
                     }}
                 />
                 <Button onClick={() => {
@@ -205,7 +216,7 @@ class SyncTestControllerBase {
     }
 
     public async getValueAndTime(path: string) {
-        let pathValue = authorityStorage.getValueAtTime(path);
+        let pathValue = authorityStorage.getValueAtTime(path, undefined, false, "noAudit");
         let value = pathValueSerializer.getPathValue(pathValue);
         let time = pathValue?.time.time;
         return { value, time };

package/src/diagnostics/auditDiskValues.ts

@@ -0,0 +1,221 @@
+import { getControllerNodeIdList } from "../-g-core-values/NodeCapabilities";
+import { AuthoritySpec, PathRouter } from "../0-path-value-core/PathRouter";
+import { PathValueController, PathValueControllerBase, AuditSnapshotEntry } from "../0-path-value-core/PathValueController";
+import { MAX_CHANGE_AGE, MAX_TIME_UNTIL_DISK_FLUSH, pathValueArchives, compareTime, PathValue, Time } from "../0-path-value-core/pathValueCore";
+import { binarySearchIndex, compare } from "socket-function/src/misc";
+import { formatNumber, formatTime } from "socket-function/src/formatting/format";
+import { magenta, green, yellow, red } from "socket-function/src/formatting/logColors";
+
+function logDiscrepancy(message: string, diskEntry: { path: string; time: Time } | undefined, serverEntry: AuditSnapshotEntry | undefined, authorityNodeId: string) {
+    let details: Record<string, unknown> = { authorityNodeId };
+
+    if (diskEntry) {
+        details.path = diskEntry.path;
+        details.diskTime = diskEntry.time.time;
+        details.diskTimeVersion = diskEntry.time.version;
+        details.diskCreatorId = diskEntry.time.creatorId;
+    }
+
+    if (serverEntry) {
+        details.path = serverEntry.path;
+        details.serverTime = serverEntry.time.time;
+        details.serverTimeVersion = serverEntry.time.version;
+        details.serverCreatorId = serverEntry.time.creatorId;
+    }
+
+    console.error(`Disk audit: ${message}`, details);
+}
+
+export async function auditDiskValues(spec: AuthoritySpec) {
+    let startTime = Date.now();
+    console.log(magenta("=== Starting Disk Audit ==="));
+
+    let timeStartWait = Date.now();
+    await PathRouter.waitUntilReady();
+    let timeWaitReady = Date.now() - timeStartWait;
+
+    let checkCutOff = Date.now() - MAX_TIME_UNTIL_DISK_FLUSH;
+    let authoritiesToCheck = PathRouter.getAllOverlappingAuthorities(spec);
+
+    console.log(magenta(`Checking ${green(formatNumber(authoritiesToCheck.length))} authorities`));
+
+    let timeStartLoadDisk = Date.now();
+    let snapshot = await pathValueArchives.loadValues(spec);
+    let diskValues = Object.values(snapshot.values).flat();
+    let timeLoadDisk = Date.now() - timeStartLoadDisk;
+
+    console.log(green(`Loaded ${magenta(formatNumber(diskValues.length))} values from disk in ${formatTime(timeLoadDisk)}`));
+
+    let diskValuesByPath = new Map<string, PathValue>();
+    for (let value of diskValues) {
+        if (!value.valid) continue;
+        if (value.canGCValue) continue;
+
+        let existing = diskValuesByPath.get(value.path);
+        if (!existing || compareTime(value.time, existing.time) > 0) {
+            diskValuesByPath.set(value.path, value);
+        }
+    }
+
+    let serverSnapshotsByAuthority = new Map<string, { spec: AuthoritySpec; serverMap: Map<string, AuditSnapshotEntry> }>();
+
+    let timeStartLoadAuthorities = Date.now();
+    let authoritiesLoaded = 0;
+    await Promise.all(authoritiesToCheck.map(async authority => {
+        try {
+            let serverSnapshot = await PathValueControllerBase.getAuditSnapshot({
+                nodeId: authority.nodeId,
+                spec,
+            });
+
+            let serverMap = new Map<string, AuditSnapshotEntry>();
+            for (let entry of serverSnapshot) {
+                serverMap.set(entry.path, entry);
+            }
+            serverSnapshotsByAuthority.set(authority.nodeId, { spec: authority, serverMap });
+            authoritiesLoaded++;
+            console.log(magenta(`Loaded authority ${magenta(String(authoritiesLoaded))}/${green(formatNumber(authoritiesToCheck.length))}: ${authority.nodeId} (${magenta(formatNumber(serverSnapshot.length))} entries)`));
+        } catch (error) {
+            console.error(`Failed to get audit snapshot from authority ${authority.nodeId}`, error);
+        }
+    }));
+    let timeLoadAuthorities = Date.now() - timeStartLoadAuthorities;
+
+    let totalServerValues = 0;
+    for (let [, { serverMap }] of serverSnapshotsByAuthority) {
+        totalServerValues += serverMap.size;
+    }
+
+    console.log(yellow(`Starting comparison: ${green(formatNumber(diskValuesByPath.size))} disk values vs ${green(formatNumber(totalServerValues))} server values`));
+
+    let timeStartComparison = Date.now();
+    let issuesOnDiskMissingFromServer = 0;
+    let issuesTimeMismatch = 0;
+    let issuesOnServerMissingFromDisk = 0;
+    let valuesToSync: PathValue[] = [];
+    let serverEntriesToFetch = new Map<string, AuditSnapshotEntry[]>();
+
+    function compareValues(a: PathValue, b: PathValue): number {
+        let pathComparison = compare(a.path, b.path);
+        if (pathComparison !== 0) return pathComparison;
+        return compareTime(a.time, b.time);
+    }
+
+    function addValueToSync(value: PathValue) {
+        let index = binarySearchIndex(valuesToSync.length, i => compareValues(valuesToSync[i], value));
+        if (index >= 0) return;
+
+        index = ~index;
+        valuesToSync.splice(index, 0, value);
+    }
+
+    for (let [authorityNodeId, { spec: authoritySpec, serverMap }] of serverSnapshotsByAuthority) {
+        for (let [path, diskValue] of diskValuesByPath) {
+            if (diskValue.time.time > checkCutOff) continue;
+            if (diskValue.canGCValue) continue;
+
+            if (!PathRouter.matchesAuthoritySpec(authoritySpec, path)) continue;
+
+            let diskEntry = { path: diskValue.path, time: diskValue.time };
+            let serverEntry = serverMap.get(path);
+            if (!serverEntry) {
+                logDiscrepancy("Value on disk but missing from server", diskEntry, undefined, authorityNodeId);
+                issuesOnDiskMissingFromServer++;
+                addValueToSync(diskValue);
+            } else {
+                if (serverEntry.time.time > checkCutOff) continue;
+
+                let comparison = compareTime(diskValue.time, serverEntry.time);
+                if (comparison !== 0) {
+                    logDiscrepancy("Time mismatch between disk and server", diskEntry, serverEntry, authorityNodeId);
+                    issuesTimeMismatch++;
+
+                    if (comparison > 0) {
+                        addValueToSync(diskValue);
+                    } else {
+                        let entries = serverEntriesToFetch.get(authorityNodeId);
+                        if (!entries) {
+                            entries = [];
+                            serverEntriesToFetch.set(authorityNodeId, entries);
+                        }
+                        entries.push(serverEntry);
+                    }
+                }
+            }
+        }
+
+        for (let [path, serverEntry] of serverMap) {
+            if (serverEntry.time.time > checkCutOff) continue;
+
+            if (!PathRouter.matchesAuthoritySpec(authoritySpec, path)) continue;
+
+            let diskValue = diskValuesByPath.get(path);
+            if (!diskValue) {
+                logDiscrepancy("Value on server but missing from disk", undefined, serverEntry, authorityNodeId);
+                issuesOnServerMissingFromDisk++;
+                let entries = serverEntriesToFetch.get(authorityNodeId);
+                if (!entries) {
+                    entries = [];
+                    serverEntriesToFetch.set(authorityNodeId, entries);
+                }
+                entries.push(serverEntry);
+            }
+        }
+    }
+    let timeComparison = Date.now() - timeStartComparison;
+
+    let timeStartFetch = Date.now();
+    for (let [authorityNodeId, entries] of serverEntriesToFetch) {
+        try {
+            let values = await PathValueControllerBase.getValuesByPathAndTime({
+                nodeId: authorityNodeId,
+                entries,
+            });
+            for (let value of values) {
+                addValueToSync(value);
+            }
+        } catch (error) {
+            console.error(`Failed to fetch values from authority ${authorityNodeId}`, error);
+        }
+    }
+    let timeFetch = Date.now() - timeStartFetch;
+
+    let timeStartSync = Date.now();
+    if (valuesToSync.length > 0) {
+        let valuesPerAuthority = PathRouter.getAllAuthoritiesForValues(valuesToSync);
+        for (let [authorityNodeId, values] of valuesPerAuthority) {
+            await PathValueControllerBase.sendValues({ nodeId: authorityNodeId, pathValues: values });
+        }
+    }
+    let timeSync = Date.now() - timeStartSync;
+
+    let totalTime = Date.now() - startTime;
+
+    let totalIssues = issuesOnDiskMissingFromServer + issuesTimeMismatch + issuesOnServerMissingFromDisk;
+
+    console.log(magenta("=== Disk Audit Summary ==="));
+    if (totalIssues === 0) {
+        console.log(green(`✓ No issues found`));
+    } else {
+        console.log(yellow(`Found ${red(formatNumber(totalIssues))} total issues:`));
+        if (issuesOnDiskMissingFromServer > 0) {
+            console.log(yellow(`  - ${red(formatNumber(issuesOnDiskMissingFromServer))} values on disk but missing from server`));
+        }
+        if (issuesTimeMismatch > 0) {
+            console.log(yellow(`  - ${red(formatNumber(issuesTimeMismatch))} time mismatches between disk and server`));
+        }
+        if (issuesOnServerMissingFromDisk > 0) {
+            console.log(yellow(`  - ${red(formatNumber(issuesOnServerMissingFromDisk))} values on server but missing from disk`));
+        }
+        console.log(yellow(`Synced ${magenta(formatNumber(valuesToSync.length))} values`));
+    }
+
+    console.log(magenta("=== Timing Breakdown ==="));
+    console.log(magenta(`Wait for ready: ${formatTime(timeWaitReady)}`));
+    console.log(magenta(`Load disk: ${formatTime(timeLoadDisk)}`));
+    console.log(magenta(`Load server: ${formatTime(timeLoadAuthorities)}`));
+    console.log(magenta(`Comparison: ${formatTime(timeComparison)}`));
+    console.log(magenta(`Fetch values: ${formatTime(timeFetch)}`));
+    console.log(magenta(`Sync values: ${formatTime(timeSync)}`));
+    console.log(green(`Total time: ${formatTime(totalTime)}`));
+}

package/src/diagnostics/auditDiskValuesEntry.ts

@@ -0,0 +1,43 @@
+import { isNodeTrue, timeInHour } from "socket-function/src/misc";
+import { AuthoritySpec } from "../0-path-value-core/PathRouter";
+import { getAllAuthoritySpec } from "../0-path-value-core/PathRouterServerAuthoritySpec";
+import { auditDiskValues } from "./auditDiskValues";
+import yargs from "yargs";
+import { runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { Querysub } from "../4-querysub/Querysub";
+
+let yargObj = isNodeTrue() && yargs(process.argv)
+    .option("watch", { type: "boolean", desc: "Audit in a loop (otherwise just audit once)" })
+    .argv || {}
+    ;
+
+
+const SPLIT_COUNT = 10;
+async function auditDiskValuesNow() {
+    let baseSpec = await getAllAuthoritySpec();
+    let splitSpecs: AuthoritySpec[] = [];
+    for (let i = 0; i < SPLIT_COUNT; i++) {
+        splitSpecs.push({
+            ...baseSpec,
+            routeStart: i * baseSpec.routeEnd / SPLIT_COUNT,
+            routeEnd: (i + 1) * baseSpec.routeEnd / SPLIT_COUNT,
+        });
+    }
+    for (let spec of splitSpecs) {
+        await auditDiskValues(spec);
+    }
+}
+async function main() {
+    await Querysub.hostService("audit-disk-values");
+    if (yargObj.watch) {
+        await runInfinitePollCallAtStart(timeInHour, auditDiskValuesNow);
+    } else {
+        try {
+            // Force, as they are running this manually, and so they probably want to see something happen...
+            await auditDiskValuesNow();
+        } finally {
+            process.exit();
+        }
+    }
+}
+main().catch(console.error);

package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx

@@ -304,7 +304,11 @@ export class LogViewer3 extends qreact.Component {
                 },
                 onResult: (match: LogDatum) => {
                     results.push(match);
-                    sort(results, x => -x.time);
+                    if (range.searchFromStart) {
+                        sort(results, x => x.time);
+                    } else {
+                        sort(results, x => -x.time);
+                    }
                     void updateResults();
                 },
                 onResults: (loggerStats: IndexedLogResults) => {

package/src/diagnostics/logs/TimeRangeSelector.tsx

@@ -80,9 +80,9 @@ export class TimeRangeSelector extends qreact.Component {
                     <InputLabel
                         label="Search from Start"
                         checkbox
-                        value={searchFromStartParam.value ? "true" : ""}
-                        onChangeValue={value => {
-                            searchFromStartParam.value = !!value;
+                        checked={searchFromStartParam.value}
+                        onChange={e => {
+                            searchFromStartParam.value = e.currentTarget.checked;
                         }}
                     />
                     <Button

package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleRenderer.tsx

@@ -209,6 +209,8 @@ export class LifeCycleInstanceRenderer extends qreact.Component<{
             }
             title={statusTitle}
             onClick={(e) => {
+                // If it's already been removed from the DOM, then don't trigger the toggle
+                if (!(e.target as HTMLElement).parentElement) return;
                 if ((e.target as HTMLElement).closest(".LifeCycleRenderer-contents")) {
                     return;
                 }

package/src/diagnostics/managementPages.tsx

@@ -28,6 +28,8 @@ import { closeAllModals } from "../5-diagnostics/Modal";
 import { delay } from "socket-function/src/batching";
 import { currentViewParam, selectedServiceIdParam } from "../deployManager/urlParams";
 import { FunctionCallInfo } from "./FunctionCallInfo";
+import { PathDistributionInfo } from "./PathDistributionInfo";
+import { isCurrentUserSuperUser } from "../user-implementation/userData";
 
 export const managementPageURL = new URLParam("managementpage", "");
 export const showingManagementURL = new URLParam("showingmanagement", false);
@@ -114,6 +116,12 @@ export async function registerManagementPages2(config: {
         controllerName: "AuditLogController",
         getModule: () => import("./AuditLogPage"),
     });
+    inputPages.push({
+        title: "Node Connections",
+        componentName: "NodeConnectionsPage",
+        controllerName: "NodeConnectionsController",
+        getModule: () => import("./NodeConnectionsPage"),
+    });
     inputPages.push({
         title: "Time",
         componentName: "TimeDebug",
@@ -357,7 +365,8 @@ class ManagementRoot extends qreact.Component {
                         {pages.map(page =>
                             <ATag values={[{ param: managementPageURL, value: page.componentName }]}>{page.title}</ATag>
                         )}
-                        <FunctionCallInfo />
+                        {isCurrentUserSuperUser() && <FunctionCallInfo />}
+                        {isCurrentUserSuperUser() && <PathDistributionInfo />}
                     </div>
                     {currentPage &&
                         <div

package/src/diagnostics/misc-pages/ArchiveViewer.tsx

@@ -101,7 +101,7 @@ export class ArchiveViewer extends qreact.Component {
                 return;
             }
             try {
-                let newValuePaths = await controller.getValuePaths(getAllAuthoritySpec());
+                let newValuePaths = await controller.getValuePaths(await getAllAuthoritySpec());
                 Querysub.localCommit(() => this.state.valuePaths = newValuePaths);
             } catch (e: any) {
                 Querysub.localCommit(() => {
@@ -715,7 +715,8 @@ class ProcessProgress extends qreact.Component<{
 const EMPTY_BUFFER = Buffer.alloc(0);
 class ArchiveViewerControllerBase {
     public async getValuePaths(authority: AuthoritySpec): Promise<string[]> {
-        return pathValueArchives.getValuePaths(authority);
+        let { pickedPaths } = await pathValueArchives.getValuePaths(authority);
+        return pickedPaths;
     }
     public async getValuePathSizes(files: string[]): Promise<number[]> {
         return pathValueArchives.getValuePathSizes(files);

package/src/diagnostics/pathAuditer.ts

@@ -19,6 +19,7 @@ import { isNode } from "typesafecss";
 import { isClient } from "../config2";
 import { isLocal } from "../config";
 import { pathWatcher } from "../0-path-value-core/PathWatcher";
+import { debugNodeId } from "../-c-identity/IdentityController";
 
 if (!isClient()) {
     // Comment this line out to disable our functionality
@@ -272,10 +273,12 @@ function trackSyncAge(config: {
 
 async function auditAuthority(nodeId: string, pathsToAudit: { path: string }[], now: number) {
     let requests: PathTimeRequest[] = [];
+    let originalValues = new Map<string, PathValue | undefined>();
 
     for (let pathObj of pathsToAudit) {
         let path = pathObj.path;
         let ourLatest = authorityStorage.getValueAtTime(path);
+        originalValues.set(path, ourLatest);
 
         // Ask for their latest valid (time: undefined means latest)
         requests.push({ path, time: undefined });
@@ -293,14 +296,32 @@ async function auditAuthority(nodeId: string, pathsToAudit: { path: string }[],
         }
     }
 
+    // Wait. That way, we basically can't have race conditions, as there can't be anything which we have seen both before and after the request, which the remote server hasn't seen, as we're giving it three seconds to receive anything we haven't seen.
+    await new Promise(resolve => setTimeout(resolve, 3000));
     let responses = await PathAuditerController.nodes[nodeId].getValidStates(requests);
+    // Wait again, also for race conditions. That way, if the server saw something, if we wait three seconds, we'll probably have seen it by the time this timeout is up. 
+    await new Promise(resolve => setTimeout(resolve, 3000));
 
     let valuesToRequest: PathTimeRequest[] = [];
     let valuesToSend: PathValue[] = [];
     let pathsToForceSync = new Set<string>();
 
     for (let response of responses) {
+        let originalValue = originalValues.get(response.path);
         let ourValue = authorityStorage.getValueAtTime(response.path) || createMissingEpochValue(response.path);
+
+        let localValueChanged = false;
+        if (originalValue === undefined && ourValue.time !== epochTime) {
+            localValueChanged = true;
+        } else if (originalValue !== undefined && (originalValue.time !== ourValue.time || originalValue.valid !== ourValue.valid)) {
+            localValueChanged = true;
+        }
+
+        if (localValueChanged) {
+            onPathInteracted(response.path, 2);
+            continue;
+        }
+
         if (ourValue.isTransparent && response.isTransparent) continue;
 
         // it's latest valid is newer than ours

package/src/path.ts

@@ -171,9 +171,16 @@ export function getParentPathStr(pathStr: string) {
     return pathStr.slice(0, getStartOfLastPart(pathStr));
 }
 
-/** === getPathFromStr(pathStr).slice(-1)[0] */
+/** === getPathFromStr(pathStr).slice(-1)[0] || "" */
 export function getLastPathPart(pathStr: string) {
-    return unescapePathPart(pathStr.slice(getStartOfLastPart(pathStr) - pathDelimitEscaped.length, -pathDelimitEscaped.length));
+    let lastPartIndex = pathStr.lastIndexOf(pathDelimitEscaped);
+    if (lastPartIndex < 0) return "";
+    return unescapePathPart(
+        pathStr.slice(
+            getStartOfLastPart(pathStr),
+            lastPartIndex,
+        )
+    );
 }
 
 /** === getPathStr(getPathFromStr(pathStr).slice(0, -1)) */

package/src/rangeMath.ts

@@ -0,0 +1,41 @@
+export type Range = {
+    start: number;
+    end: number;
+};
+
+export type Ranges = Range[];
+
+export function rangesOverlap(a: Range, b: Range): boolean {
+    return a.start < b.end && a.end > b.start;
+}
+
+export function removeRange(ranges: Ranges, range: Range): {
+    removedRanges: Range[];
+} {
+    let s = range.start;
+    let e = range.end;
+    let removedRanges: Range[] = [];
+    for (let i = ranges.length - 1; i >= 0; i--) {
+        let rangeInRanges = ranges[i];
+        if (s >= rangeInRanges.end || e <= rangeInRanges.start) continue;
+        let startTaken = Math.max(rangeInRanges.start, s);
+        let endTaken = Math.min(rangeInRanges.end, e);
+        removedRanges.push({ start: startTaken, end: endTaken });
+        ranges.splice(i, 1);
+        // Add back the parts we didn't overlap
+        if (rangeInRanges.start < s) {
+            ranges.push({
+                start: rangeInRanges.start,
+                end: s,
+            });
+        }
+        if (rangeInRanges.end > e) {
+            ranges.push({
+                start: e,
+                end: rangeInRanges.end,
+            });
+        }
+
+    }
+    return { removedRanges };
+}