querysub 0.403.0 → 0.405.0

package/src/0-path-value-core/PathValueController.ts

@@ -1,239 +1,277 @@
-import { SocketFunction } from "socket-function/SocketFunction";
-import { errorToUndefined, logErrors } from "../errors";
-import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
-import { authorityStorage, ReadLock, pathWatcher, PathValue, PathValueSnapshot, WriteState, writeValidStorage, lockToCallback, isCoreQuiet, WatchConfig, MAX_ACCEPTED_CHANGE_AGE, MAX_CHANGE_AGE, debugPathValuePath } from "./pathValueCore";
-import { measureCodeSync, measureFnc } from "socket-function/src/profiling/measure";
-import { ActionsHistory } from "../diagnostics/ActionsHistory";
-import { isNode } from "socket-function/src/misc";
-import { magenta, red } from "socket-function/src/formatting/logColors";
-import { AuthorityPath, pathValueAuthority2 } from "./NodePathAuthorities";
-import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
-import { pathValueCommitter } from "./PathValueCommitter";
-import { Benchmark } from "../diagnostics/benchmark";
-import { formatNumber, formatTime } from "socket-function/src/formatting/format";
-import { sha256 } from "js-sha256";
-import debugbreak from "debugbreak";
-import { ClientWatcher } from "../1-path-client/pathValueClientWatcher";
-import { auditLog, isDebugLogEnabled } from "./auditLogs";
-import { debugNodeId, debugNodeThread } from "../-c-identity/IdentityController";
-import { logDisk } from "../diagnostics/logs/diskLogger";
-import { isDiskAudit } from "../config";
-import { decodeNodeId } from "../-a-auth/certs";
-export { pathValueCommitter };
-
-class PathValueControllerBase {
-    /**
-        Use for writers to send writes to authorities, and for authorities to send writes to watchers.
-            - Values we are an authority on will have their valid states computed (recursively), otherwise
-                we just trust the remote valid states.
-
-        Triggers all watcher of the values, as well as all watches of any changed valid states.
-
-        NOTE: We don't check for max age in forwarded values, and instead just insert them, as
-            we trust our other authorities to not forward us bad values (and rejecting it would
-            cause inconsistencies between nodes, which is worse).
-    */
-    public async forwardWrites(
-        valuesBuffers: Buffer[],
-        parentsSynced?: string[],
-        watchLocks?: "watchLocks",
-        initialTrigger?: "initialTrigger"
-    ) {
-        let machineId = SocketFunction.getCaller().nodeId;
-        let callerId = SocketFunction.getCaller().nodeId;
-
-        const values = await errorToUndefined(pathValueSerializer.deserialize(valuesBuffers) as Promise<PathValue[]>);
-        if (!values) {
-            return;
-        }
-
-        Benchmark.onReceivedValues(values);
-        ActionsHistory.OnRead(values);
-
-        // NOTE: We don't check for age here, because this only matters for new writes, and this isn't the correct place to check for this.
-        // // IMPORTANT!
-        // //  We DO NOT reject old values here. Only trusted servers can call this function, and... writes are
-        // //      sent to multiple servers, so if we reject a write here it has a high chance of resulting in
-        // //      different writes between servers. Because writes are usually accepts, and receiving an old
-        // //      write is likely due to an issue on our end (our internet lagging, our server lagging, etc),
-        // //      the write will likely be accepted (as most writes are accepted), so defaulting to accept
-        // //      it will likely result in a good state.
-        // let threshold = Date.now() - MAX_CHANGE_AGE;
-        // for (let value of values) {
-        //     let pastThreshold = threshold - value.time.time;
-        //     if (pastThreshold > 0) {
-        //         console.error(red(`Received a value that is ${formatTime(pastThreshold)} pass change time, which will cause it to be forcefully accepted (locks will be ignored): ${value.path}`));
-        //     }
-        // }
-
-        // Set up VALID watches for the caller. Because we are only adding the locks, this won't trigger
-        //  a valid state send unless the initial valid state is rejected.
-        // IMPORTANT! This needs to be done here, and cannot be done in another call, as we NEED the valid
-        //  states for created values to be watched immediately, otherwise the creator (such as the FunctionRunner),
-        //  might miss the rejection (and not rerun the rejected function).
-        if (watchLocks) {
-            for (let value of values) {
-                lockToCallback.watchLock({
-                    path: value.path,
-                    startTime: value.time,
-                    endTime: value.time,
-                    readIsTransparent: value.canGCValue || false,
-                }, callerId);
-            }
-        }
-
-        if (isDebugLogEnabled() || isDiskAudit()) {
-            let sourceNodeId = debugNodeId(callerId);
-            for (let value of values) {
-                auditLog("RECEIVE VALUE", {
-                    path: value.path,
-                    time: value.time.time,
-                    sourceNodeId,
-                    sourceNodeThreadId: decodeNodeId(sourceNodeId)?.threadId,
-                });
-            }
-        }
-        console.info("Received PathValues via forwardWrites", { valueCount: values.length, callerId, });
-
-        if (isCoreQuiet) {
-            await pathValueCommitter.ingestRemoteValues({
-                pathValues: values,
-                parentsSynced,
-                sourceNodeId: callerId,
-                initialTrigger,
-            });
-        } else {
-            let sumAge = 0;
-            let now = Date.now();
-            for (let value of values) {
-                sumAge += now - value.time.time;
-            }
-            console.log(`(${now}) Received writes: ${values.length}, ${formatTime(sumAge / values.length)} AGE, parents: ${parentsSynced?.length} from ${machineId}`);
-            await measureCodeSync(function forwardWritesMeasureOverhead() {
-                return pathValueCommitter.ingestRemoteValues({
-                    pathValues: values,
-                    parentsSynced,
-                    sourceNodeId: callerId,
-                    initialTrigger,
-                });
-            });
-        }
-    }
-
-    /** Watches that the startTime is valid, and that there are no valid values between startTime/endTime
-     *  - Puts the onus of syncing the lock, and it's readLocks, recursively, on the callee.
-     *      Of course, the callee has to be an authority for this it work, and authorities are
-     *      automatically doing this. Due to authorities being sharded between of large swathes of data,
-     *      as well as data (hopefully) being relatively localized to our shard lines, it is expected
-     *      there will be relatively few watchLockValid calls. Also, it is expected that relatively
-     *      few readLocks will become invalid (rejected).
-     *  - Notifies of invalid states by calling PathValidWatcher.onValidChange (which the client watches
-     *      statically using createValidStateWatcher).
-     *  - All the lock paths are assumed to be ours (we are the authority for them)
-     *  - Triggers invalid range readLocks by passing the valid state of the value that conflicted.
-     *      As any watchers will be watching this range locally, this will trigger them to ingest the valid
-     *      state, and trigger the value that uses it, recalculate the value that uses it (as they are the
-     *      authority on that), and then realize it is invalid and reject it.
-     */
-    public async watchLockValid(locks: ReadLock[]) {
-        let callerId = SocketFunction.getCaller().nodeId;
-        for (let lock of locks) {
-            lockToCallback.watchLock(lock, callerId);
-        }
-        let now = Date.now();
-        let validStates: WriteState[] = locks.map(lock => writeValidStorage.getWriteState(lock, now));
-        logErrors(PathValueController.nodes[callerId].onValidChange(validStates));
-    }
-
-    // TODO: Batch these calls before processing them
-    public async onValidChange(config: WriteState[]) {
-        if (!isCoreQuiet || !isNode() || ClientWatcher.DEBUG_READS) {
-            let rejected = config.filter(x => !x.isValid);
-            if (rejected.length > 0) {
-                console.group(red(`Received rejection of paths`));
-                for (let value of rejected) {
-                    console.log(debugPathValuePath(value));
-                }
-                console.groupEnd();
-            }
-        }
-
-        // Ignore any remote changes for values that WE are an authority on!
-        config = config.filter(x => !pathValueAuthority2.isSelfAuthority(x.path));
-        pathValueCommitter.ingestValidStates(config);
-    }
-
-    /** Returns serialized PathValue[] */
-    public async getSnapshot(config: { authorityPath: AuthorityPath; }): Promise<Buffer[]> {
-        let snapshot = await authorityStorage.getSnapshot(config.authorityPath);
-        let values = Object.values(snapshot.values).flat();
-        let buffers = await pathValueSerializer.serialize(values);
-        let totalSize = buffers.reduce((a, b) => a + b.length, 0);
-        console.log(`Sending snapshot: ${formatNumber(values.length)} values, ${formatNumber(buffers.length)} buffers, ${formatNumber(totalSize)}B`);
-        return buffers;
-    }
-
-
-    // Returns the initial states of the watches as well
-    // If there is no value, we call return { path, value: undefined, time: { time: 0, creatorId: 0 }, readLocks: [] }
-    //  to indicate there is no value.
-    public async watchLatest(config: WatchConfig) {
-        let callerId = SocketFunction.getCaller().nodeId;
-        if (isDebugLogEnabled()) {
-            let sourceNodeId = debugNodeId(callerId);
-            for (let value of config.paths) {
-                auditLog("WATCH PATH", { path: value, sourceNodeId, sourceNodeThreadId: debugNodeThread(callerId) });
-            }
-            for (let value of config.parentPaths) {
-                auditLog("WATCH PARENT PATH", { path: value, sourceNodeId });
-            }
-        }
-        pathWatcher.watchPath({ paths: config.paths, parentPaths: config.parentPaths, callback: callerId, initialTrigger: true });
-    }
-    public async unwatchLatest(config: WatchConfig) {
-        let callerId = SocketFunction.getCaller().nodeId;
-        if (isDebugLogEnabled()) {
-            let sourceNodeId = debugNodeId(SocketFunction.getCaller().nodeId);
-            for (let value of config.paths) {
-                auditLog("UNWATCH PATH", { path: value, sourceNodeId, sourceNodeThreadId: debugNodeThread(callerId) });
-            }
-            for (let value of config.parentPaths) {
-                auditLog("UNWATCH PARENT PATH", { path: value, sourceNodeId });
-            }
-        }
-        pathWatcher.unwatchPath({ paths: config.paths, parentPaths: config.parentPaths, callback: callerId });
-    }
-
-    public async ping() { }
-
-    public async test(obj: unknown) {
-        return obj;
-    }
-}
-
-export const PathValueController = SocketFunction.register(
-    "PathValueController-1e062e2c-81c9-497b-b414-a46d0a4c2313",
-    new PathValueControllerBase(),
-    () => ({
-        forwardWrites: {
-        },
-
-        watchLockValid: {},
-        onValidChange: {},
-
-        watchLatest: {},
-        unwatchLatest: {},
-
-        getSnapshot: {},
-
-        ping: {},
-        test: {},
-    }),
-    () => ({
-        hooks: [requiresNetworkTrustHook],
-    }),
-    {
-        noFunctionMeasure: !isNode(),
-    }
-);
-
+import { SocketFunction } from "socket-function/SocketFunction";
+import { isNodeTrusted, requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
+import { PathValue, WriteState, WatchConfig, MAX_CHANGE_AGE, getCompressNetwork, authorityStorage } from "./pathValueCore";
+import { pathWatcher } from "./PathWatcher";
+import { validStateComputer } from "./ValidStateComputer";
+import { ActionsHistory } from "../diagnostics/ActionsHistory";
+import { isNode, nextId, timeInSecond, timeoutToUndefined } from "socket-function/src/misc";
+import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
+import { pathValueCommitter } from "./PathValueCommitter";
+import { Benchmark } from "../diagnostics/benchmark";
+import { auditLog, isDebugLogEnabled } from "./auditLogs";
+import { debugNodeId, debugNodeThread } from "../-c-identity/IdentityController";
+import { isDiskAudit } from "../config";
+import { decodeNodeId } from "../-a-auth/certs";
+import { areNodeIdsEqual, isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
+import { getNodeIdIP } from "socket-function/src/nodeCache";
+import { authorityLookup } from "./AuthorityLookup";
+import { timeoutToError } from "../errors";
+import { AuthoritySpec, PathRouter } from "./PathRouter";
+export { pathValueCommitter };
+
+// The sync test page has some functions which related to PathValueController significantly, and should always be exposed if PathValueController is exposed.
+if (isNode()) {
+    setImmediate(() => {
+        import("../diagnostics/SyncTestPage");
+    });
+}
+
+
+// NOTE: initialTriggers = clobber values on these paths (or all children of a parent), And take the values that we've provided. This allows you to get a client into a nice state by just saying, I don't know what you had before from other servers, but drop all that. Here's the correct state. 
+
+export class PathValueControllerBase {
+    public static async createValues(config: {
+        nodeId: string;
+        pathValues: PathValue[];
+    }): Promise<void | "refused"> {
+        const { Querysub } = await import("../4-querysub/Querysub");
+        let { pathValues, nodeId } = config;
+
+        let serializedValues = await pathValueSerializer.serialize(pathValues, { compress: Querysub.COMPRESS_NETWORK });
+        if (isDebugLogEnabled()) {
+            for (let value of pathValues) {
+                auditLog("SEND CREATED VALUE", {
+                    path: value.path,
+                    timeId: value.time.time,
+                    targetNodeThreadId: debugNodeThread(nodeId),
+                    transparent: value.isTransparent,
+                    source: value.source,
+                    totalValues: pathValues.length,
+                });
+            }
+        }
+        return await PathValueController.nodes[nodeId].sendData({
+            valueBuffers: serializedValues,
+            initialCreation: true,
+        });
+    }
+    private static logSendValues(config: {
+        nodeId: string;
+        pathValues: PathValue[];
+        initialTriggers?: { values: Set<string>; parentPaths: Set<string> },
+    }) {
+        let { nodeId, pathValues, initialTriggers } = config;
+        if (isDebugLogEnabled()) {
+            for (let pathValue of pathValues) {
+                auditLog("SEND VALUE", {
+                    path: pathValue.path,
+                    timeId: pathValue.time.time,
+                    watcher: nodeId,
+                    isValid: pathValue.valid,
+                    nodeId: debugNodeId(nodeId),
+                    targetNodeId: debugNodeId(nodeId),
+                    targetNodeThreadId: debugNodeThread(nodeId),
+                    transparent: pathValue.isTransparent,
+                    canGC: pathValue.canGCValue,
+                    initialTrigger: initialTriggers?.values.has(pathValue.path),
+                    triggerId: nextId(),
+                    totalChanges: pathValues.length,
+                });
+            }
+        }
+    }
+    public static async sendValues(config: {
+        nodeId: string;
+        pathValues: PathValue[];
+        initialTriggers?: { values: Set<string>; parentPaths: Set<string> },
+    }) {
+        let changes = config.pathValues;
+        let { nodeId, initialTriggers } = config;
+        let allowSource = await isNodeTrusted(nodeId) || getNodeIdIP(nodeId) === "127.0.0.1";
+        let buffers = await pathValueSerializer.serialize(changes, {
+            noLocks: true,
+            compress: getCompressNetwork(),
+            stripSource: !allowSource,
+        });
+        this.logSendValues({ nodeId, pathValues: changes, initialTriggers, });
+        return await PathValueController.nodes[nodeId].sendData({
+            valueBuffers: buffers,
+            initialTriggers: initialTriggers,
+        });
+    }
+
+    public async sendData(config: {
+        // If it's the initial creation, then we can actually reject them. However, we do need to provide a rejection pathway for the client side to know that we have accepted none of the changes and that it should remove them locally. 
+        initialCreation?: boolean;
+
+        valueBuffers?: Buffer[];
+        initialTriggers?: {
+            /** This means that we will provided the full history for all these paths. If you have any other history, it should be removed.  */
+            values: Set<string>;
+            /** This means that we will have provided all of the child values for these parent paths, and so if you have any child values which were not provided in our value list, those values should be removed.  */
+            parentPaths: Set<string>;
+        },
+
+    }): Promise<void | "refused"> {
+        let callerId = SocketFunction.getCaller().nodeId;
+        let { initialCreation, valueBuffers } = config;
+
+        let values: PathValue[] = [];
+        if (valueBuffers) {
+            values = await pathValueSerializer.deserialize(valueBuffers);
+            Benchmark.onReceivedValues(values);
+            ActionsHistory.OnRead(values);
+        }
+
+        if (initialCreation) {
+            let sourceNodeId = debugNodeId(callerId);
+            let rejected = false;
+            let threshold = Date.now() - MAX_CHANGE_AGE;
+            for (let value of values) {
+                let pastThreshold = threshold - value.time.time;
+                if (pastThreshold > 0) {
+                    rejected = true;
+                    console.error(`Rejecting values as one is too old. It likely got caught in the pipeline too long and now can't be committed without trying to undone history which can already been committed to disk.`, {
+                        path: value.path,
+                        timeId: value.time.time,
+                        sourceNodeId,
+                        sourceNodeThreadId: decodeNodeId(sourceNodeId)?.threadId,
+                        totalValueCount: values.length,
+                    });
+                }
+            }
+            if (rejected) {
+                return "refused";
+            }
+        }
+
+        if (isDebugLogEnabled() || isDiskAudit()) {
+            let sourceNodeId = debugNodeId(callerId);
+            for (let value of values) {
+                auditLog("RECEIVE VALUE", {
+                    path: value.path,
+                    // NOTE: Yes, this might conflict as we're throwing away a lot of information, however in practice no, it's not going to conflict when we're debugging, and this + path will be globally unique for every value we investigate.
+                    timeId: value.time.time,
+                    sourceNodeId,
+                    sourceNodeThreadId: decodeNodeId(sourceNodeId)?.threadId,
+                });
+            }
+        }
+
+        try {
+            let parentSyncs = Array.from(config.initialTriggers?.parentPaths || [])
+                .map(x => ({ parentPath: x, sourceNodeId: callerId }));
+            let initialTriggers = config.initialTriggers || { values: new Set(), parentPaths: new Set() };
+            validStateComputer.ingestValuesAndValidStates({
+                pathValues: values,
+                parentSyncs,
+                initialTriggers,
+            });
+        } catch (error) {
+            console.error("Error ingesting values and valid states", error);
+        }
+
+
+        if (config.initialCreation) {
+            // NOTE: This will mean that all of the authorities for a value will write it to disk. So it will be written to disk multiple times. This should be fine, and it might be a good idea for redundancy, etc. And eventually, they will get compressed to one value, so it's not inefficient.
+            await PathValueControllerBase.authorityShareValues({ pathValues: values });
+        }
+    }
+
+    public static async authorityShareValues(config: {
+        pathValues: PathValue[];
+    }) {
+        let { pathValues } = config;
+        let valuesPerOtherAuthority = PathRouter.getAllAuthoritiesForValues(pathValues);
+        let promises = Array.from(valuesPerOtherAuthority.entries()).map(async ([otherAuthority, values]) => {
+            try {
+                await timeoutToError(timeInSecond * 15, PathValueControllerBase.sendValues({
+                    nodeId: otherAuthority,
+                    pathValues: values,
+                }), () => new Error(`Timeout forwarding shared values (${values.length} values) to authority ${otherAuthority}`));
+            } catch (error: any) {
+                console.error(error.message, { otherAuthority, count: values.length });
+            }
+        });
+        await Promise.all(promises);
+    }
+
+
+    public async watchLatest(config: WatchConfig) {
+        let callerId = SocketFunction.getCaller().nodeId;
+        if (isDebugLogEnabled()) {
+            let sourceNodeId = debugNodeId(callerId);
+            for (let value of config.paths) {
+                auditLog("WATCH PATH", { path: value, sourceNodeId, sourceNodeThreadId: debugNodeThread(callerId) });
+            }
+            for (let value of config.parentPaths) {
+                auditLog("WATCH PARENT PATH", { path: value, sourceNodeId });
+            }
+        }
+        pathWatcher.watchPath({
+            nodeId: callerId,
+            paths: config.paths,
+            parentPaths: config.parentPaths,
+            initialTrigger: true,
+            fullHistory: config.fullHistory,
+        });
+    }
+    public async unwatchLatest(config: WatchConfig) {
+        let callerId = SocketFunction.getCaller().nodeId;
+        if (isDebugLogEnabled()) {
+            let sourceNodeId = debugNodeId(SocketFunction.getCaller().nodeId);
+            for (let value of config.paths) {
+                auditLog("UNWATCHING PATH", { path: value, sourceNodeId, sourceNodeThreadId: debugNodeThread(callerId) });
+            }
+            for (let value of config.parentPaths) {
+                auditLog("UNWATCHING PARENT PATH", { path: value, sourceNodeId });
+            }
+        }
+        pathWatcher.unwatchPath({ paths: config.paths, parentPaths: config.parentPaths, callback: callerId });
+    }
+
+    public static async getInitialValues(config: {
+        nodeId: string;
+        spec: AuthoritySpec;
+        // Only returns values with a create time in this window
+        startTime: number;
+        endTime: number;
+    }): Promise<PathValue[]> {
+        let buffers = await PathValueController.nodes[config.nodeId].getInitialValues(config);
+        return await pathValueSerializer.deserialize(buffers);
+    }
+    // NOTE: Sends all the responses using the regular 
+    public async getInitialValues(config: {
+        spec: AuthoritySpec;
+        // Only returns values with a create time in this window
+        startTime: number;
+        endTime: number;
+    }): Promise<Buffer[]> {
+        let values = authorityStorage.getAllValues(config);
+        let buffers = await pathValueSerializer.serialize(values, {
+            noLocks: true,
+            compress: getCompressNetwork(),
+        });
+        return buffers;
+    }
+}
+
+export const PathValueController = SocketFunction.register(
+    "PathValueController-1e062e2c-81c9-497b-b414-a46d0a4c2313",
+    new PathValueControllerBase(),
+    () => ({
+        sendData: {},
+
+        getInitialValues: {},
+
+        watchLatest: {},
+        unwatchLatest: {},
+    }),
+    () => ({
+        hooks: [requiresNetworkTrustHook],
+    }),
+    {
+        noFunctionMeasure: !isNode(),
+    }
+);
+