querysub 0.2.0

package/src/0-path-value-core/PathValueController.ts

@@ -0,0 +1,231 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { errorToUndefined, logErrors } from "../errors";
+import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
+import { authorityStorage, ReadLock, pathWatcher, PathValue, PathValueSnapshot, WriteState, writeValidStorage, lockToCallback, isCoreQuiet, WatchConfig, MAX_ACCEPTED_CHANGE_AGE, MAX_CHANGE_AGE, debugPathValuePath } from "./pathValueCore";
+import { measureCodeSync, measureFnc } from "socket-function/src/profiling/measure";
+import { ActionsHistory } from "../diagnostics/ActionsHistory";
+import { isNode } from "socket-function/src/misc";
+import { magenta, red } from "socket-function/src/formatting/logColors";
+import { AuthorityPath, pathValueAuthority2 } from "./NodePathAuthorities";
+import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
+import { pathValueCommitter } from "./PathValueCommitter";
+import { Benchmark } from "../diagnostics/benchmark";
+import { formatNumber, formatTime } from "socket-function/src/formatting/format";
+import { sha256 } from "js-sha256";
+import debugbreak from "debugbreak";
+import { ClientWatcher } from "../1-path-client/pathValueClientWatcher";
+import { debugLog, isDebugLogEnabled } from "./debugLogs";
+import { debugNodeId } from "../-c-identity/IdentityController";
+import { diskLog } from "../diagnostics/logs/diskLogger";
+export { pathValueCommitter };
+
+class PathValueControllerBase {
+    /**
+        Use for writers to send writes to authorities, and for authorities to send writes to watchers.
+            - Values we are an authority on will have their valid states computed (recursively), otherwise
+                we just trust the remote valid states.
+
+        Triggers all watcher of the values, as well as all watches of any changed valid states.
+
+        NOTE: We don't check for max age in forwarded values, and instead just insert them, as
+            we trust our other authorities to not forward us bad values (and rejecting it would
+            cause inconsistencies between nodes, which is worse).
+    */
+    public async forwardWrites(
+        valuesBuffers: Buffer[],
+        parentsSynced?: string[],
+        watchLocks?: "watchLocks",
+        initialTrigger?: "initialTrigger"
+    ) {
+        let machineId = SocketFunction.getCaller().nodeId;
+        let callerId = SocketFunction.getCaller().nodeId;
+
+        const values = await errorToUndefined(pathValueSerializer.deserialize(valuesBuffers) as Promise<PathValue[]>);
+        if (!values) {
+            return;
+        }
+
+        Benchmark.onReceivedValues(values);
+        ActionsHistory.OnRead(values);
+
+        // NOTE: We don't check for age here, because this only matters for new writes, and this isn't the correct place to check for this.
+        // // IMPORTANT!
+        // //  We DO NOT reject old values here. Only trusted servers can call this function, and... writes are
+        // //      sent to multiple servers, so if we reject a write here it has a high chance of resulting in
+        // //      different writes between servers. Because writes are usually accepts, and receiving an old
+        // //      write is likely due to an issue on our end (our internet lagging, our server lagging, etc),
+        // //      the write will likely be accepted (as most writes are accepted), so defaulting to accept
+        // //      it will likely result in a good state.
+        // let threshold = Date.now() - MAX_CHANGE_AGE;
+        // for (let value of values) {
+        //     let pastThreshold = threshold - value.time.time;
+        //     if (pastThreshold > 0) {
+        //         console.error(red(`Received a value that is ${formatTime(pastThreshold)} pass change time, which will cause it to be forcefully accepted (locks will be ignored): ${value.path}`));
+        //     }
+        // }
+
+        // Set up VALID watches for the caller. Because we are only adding the locks, this won't trigger
+        //  a valid state send unless the initial valid state is rejected.
+        // IMPORTANT! This needs to be done here, and cannot be done in another call, as we NEED the valid
+        //  states for created values to be watched immediately, otherwise the creator (such as the FunctionRunner),
+        //  might miss the rejection (and not rerun the rejected function).
+        if (watchLocks) {
+            for (let value of values) {
+                lockToCallback.watchLock({
+                    path: value.path,
+                    startTime: value.time,
+                    endTime: value.time,
+                    readIsTranparent: value.canGCValue || false,
+                }, callerId);
+            }
+        }
+
+        if (isDebugLogEnabled()) {
+            let sourceNodeId = debugNodeId(callerId);
+            for (let value of values) {
+                debugLog("RECEIVE VALUE", { path: value.path, time: value.time.time, sourceNodeId });
+            }
+        }
+        diskLog(`Received PathValues via forwardWrites`, { valueCount: values.length, callerId, });
+        for (let value of values) {
+            diskLog("Received PathValue for path", { path: value.path, time: value.time.time, callerId });
+        }
+
+        if (isCoreQuiet) {
+            await pathValueCommitter.ingestRemoteValues({
+                pathValues: values,
+                parentsSynced,
+                sourceNodeId: callerId,
+                initialTrigger,
+            });
+        } else {
+            let sumAge = 0;
+            let now = Date.now();
+            for (let value of values) {
+                sumAge += now - value.time.time;
+            }
+            console.log(`(${now}) Received writes: ${values.length}, ${formatTime(sumAge / values.length)} AGE, parents: ${parentsSynced?.length} from ${machineId}`);
+            await measureCodeSync(function forwardWritesMeasureOverhead() {
+                return pathValueCommitter.ingestRemoteValues({
+                    pathValues: values,
+                    parentsSynced,
+                    sourceNodeId: callerId,
+                    initialTrigger,
+                });
+            });
+        }
+    }
+
+    /** Watches that the startTime is valid, and that there are no valid values between startTime/endTime
+     *  - Puts the onus of syncing the lock, and it's readLocks, recursively, on the callee.
+     *      Of course, the callee has to be an authority for this it work, and authorities are
+     *      automatically doing this. Due to authorities being sharded between of large swathes of data,
+     *      as well as data (hopefully) being relatively localized to our shard lines, it is expected
+     *      there will be relatively few watchLockValid calls. Also, it is expected that relatively
+     *      few readLocks will become invalid (rejected).
+     *  - Notifies of invalid states by calling PathValidWatcher.onValidChange (which the client watches
+     *      statically using createValidStateWatcher).
+     *  - All the lock paths are assumed to be ours (we are the authority for them)
+     *  - Triggers invalid range readLocks by passing the valid state of the value that conflicted.
+     *      As any watchers will be watching this range locally, this will trigger them to ingest the valid
+     *      state, and trigger the value that uses it, recalculate the value that uses it (as they are the
+     *      authority on that), and then realize it is invalid and reject it.
+     */
+    public async watchLockValid(locks: ReadLock[]) {
+        let callerId = SocketFunction.getCaller().nodeId;
+        for (let lock of locks) {
+            lockToCallback.watchLock(lock, callerId);
+        }
+        let now = Date.now();
+        let validStates: WriteState[] = locks.map(lock => writeValidStorage.getWriteState(lock, now));
+        logErrors(PathValueController.nodes[callerId].onValidChange(validStates));
+    }
+
+    // TODO: Batch these calls before processing them
+    public async onValidChange(config: WriteState[]) {
+        if (!isCoreQuiet || !isNode() || ClientWatcher.DEBUG_READS) {
+            let rejected = config.filter(x => !x.isValid);
+            if (rejected.length > 0) {
+                console.group(red(`Received rejection of paths`));
+                for (let value of rejected) {
+                    console.log(debugPathValuePath(value));
+                }
+                console.groupEnd();
+            }
+        }
+
+        // Ignore any remote changes for values that WE are an authority on!
+        config = config.filter(x => !pathValueAuthority2.isSelfAuthority(x.path));
+        pathValueCommitter.ingestValidStates(config);
+    }
+
+    /** Returns serialized PathValue[] */
+    public async getSnapshot(config: { authorityPath: AuthorityPath; }): Promise<Buffer[]> {
+        let snapshot = await authorityStorage.getSnapshot(config.authorityPath);
+        let values = Object.values(snapshot.values).flat();
+        let buffers = await pathValueSerializer.serialize(values);
+        let totalSize = buffers.reduce((a, b) => a + b.length, 0);
+        console.log(`Sending snapshot: ${formatNumber(values.length)} values, ${formatNumber(buffers.length)} buffers, ${formatNumber(totalSize)}B`);
+        return buffers;
+    }
+
+
+    // Returns the initial states of the watches as well
+    // If there is no value, we call return { path, value: undefined, time: { time: 0, creatorId: 0 }, readLocks: [] }
+    //  to indicate there is no value.
+    public async watchLatest(config: WatchConfig) {
+        let callerId = SocketFunction.getCaller().nodeId;
+        if (isDebugLogEnabled()) {
+            let sourceNodeId = debugNodeId(callerId);
+            for (let value of config.paths) {
+                debugLog("WATCH PATH", { path: value, sourceNodeId });
+            }
+            for (let value of config.parentPaths) {
+                debugLog("WATCH PARENT PATH", { path: value, sourceNodeId });
+            }
+        }
+        pathWatcher.watchPath({ paths: config.paths, parentPaths: config.parentPaths, callback: callerId, initialTrigger: true });
+    }
+    public async unwatchLatest(config: WatchConfig) {
+        let callerId = SocketFunction.getCaller().nodeId;
+        if (isDebugLogEnabled()) {
+            let sourceNodeId = debugNodeId(SocketFunction.getCaller().nodeId);
+            for (let value of config.paths) {
+                debugLog("UNWATCH PATH", { path: value, sourceNodeId });
+            }
+            for (let value of config.parentPaths) {
+                debugLog("UNWATCH PARENT PATH", { path: value, sourceNodeId });
+            }
+        }
+        pathWatcher.unwatchPath({ paths: config.paths, parentPaths: config.parentPaths, callback: callerId });
+    }
+
+    public async ping() { }
+
+    public async test(obj: unknown) {
+        return obj;
+    }
+}
+
+export const PathValueController = SocketFunction.register(
+    "PathValueController-1e062e2c-81c9-497b-b414-a46d0a4c2313",
+    new PathValueControllerBase(),
+    () => ({
+        forwardWrites: {},
+
+        watchLockValid: {},
+        onValidChange: {},
+
+        watchLatest: {},
+        unwatchLatest: {},
+
+        getSnapshot: {},
+
+        ping: {},
+        test: {},
+    }),
+    () => ({
+        hooks: [requiresNetworkTrustHook],
+    })
+);
+

package/src/0-path-value-core/archiveLocks/ArchiveLocks.ts

@@ -0,0 +1,154 @@
+//  4.2) Turn on read locking, so when we read paths we actually talk to the lock system
+//  5) Write merge script (otherwise we can't ACTUALLY test it...)
+//      - We need to use minTime/maxTime (throw if any non-genesis values are missing them),
+//          to determine if we are allowed to GC missing paths. We can only GC a block where
+//          it's maxTime < anyBlock.minTime (STRICLY less than, <= is not enough).
+//      - Our blocks will be sorted, so this isn't SO hard.
+//      - Have N blocks at each level, and when we exceed that limit, we merge. IF the size
+//          exceeds the current level limit, we add it to the next level, potentially
+//          merging on that level, etc, etc.
+//          - If a block has a minTime < blocks on the next level, it is considered on that level,
+//              even if it's size isn't enough. (This prevents blocks from dropping levels,
+//              such as if we GC a lot of deletions).
+//  5.1) Merge stats in management page
+//      - Read bytes, read count, write bytes, write count, removed bytes, removed count
+//  5.3) Run merge script on our actual data. This should result in a lot of merging,
+//      and take a few minutes to reduce our files.
+//      - The merges files should either take significantly less time to load OR allow us to
+//          do unzipping on multiple threads (because now the files will be large
+//          enough that this will be feasible).
+//      - Reload our data to verify nothing breaks
+//  5.2) Merge test data generation
+//      - Writes lot of new paths, and then delete the values, to verify we at least
+//          have deletion GCing.
+//      - We should see files at each level be created, and eventually progress to the last level,
+//          where the deletions finally disappear.
+//      - BECAUSE we can only remove values from the oldest file, we will mostly be ensuring
+//          a total file limit, merging small files to keep < 1000 files AND then ONLY merging
+//          when we READ the files and determine that the merge will reduce our size by enough.
+//          - This is because even for other gcs, such as app gc, anyone else can just
+//              read multiple files and merge them as well, just like we are doing.
+//  6) Run it. And then debug it if/when it fails
+//  7) Verify our spec.txt todo is done
+//  8) Next todo (fast app GC I believe)
+/*
+    Proof
+        Invert transactions, and think of files as all having two "sheets" on them.
+            - A transaction removes one sheet
+            - A confirmed transaction removes the other sheet
+            - A conflicting transaction (time < this.time, same data) adds a red sheet
+                - We also add sheet back by deleting the conflicting transaction, which can never be undone, and doesn't require keeping transaction around forever.
+                - And by deleting the underlying files we prevent any new transactions from
+                    removing their confirmation sheet, even after our transaction is deleted.
+        We remove one sheet, check all other sheets for conflicts, and if we find any,
+            we add a red sheet. If not, we remove the other sheet.
+        Of course, applying transactions (confirming them), requires a few steps
+            (breaking the transaction into parts, etc). The system itself has to make
+            sure these are safe, which is does by:
+            - Unique files names and every transaction requiring a delete means we can detect
+                if transactions conflict.
+            - Time stamping means we know which transaction should win.
+            - Transactions must be atomic before they are confirmed.
+            - Detaching transactions allows for better cleanup, while still blocking
+                new transactions.
+
+        OR, rather, think of it as boxes, with either letters in them OR crossed out letters
+            - We number these boxes so we know the order to apply them
+            - We are writing these boxes to paper, with pen, so we can't delete them.
+            - OR, lines on a piece of paper
+
+        ALSO, IMPORTANT! We get the time BEFORE we start reading. So that means any changes while
+            reading, are ignored (deletions are fine, we would apply those anyways).
+
+        We always transition from valid => invalid/complete, NEVER invalid => valid.
+            (or we are always invalid)
+            - This is because any transaction that causes us to be invalid will be applied,
+                removing the conflicting files, meaning we are STILL invalid.
+        
+        Using our state transition assurances, we can make sure every transaction we read is applied,
+            if it was ever valid. This makes our iteration consistent, even if our file reading is slow
+            and done in pieces.
+
+        By double reading the dir, and retrying if it changes, we ensure we get atomic dir states. Otherwise
+            reading might get some old state, and some new state, missing the intermediate state entirely.
+*/
+/*
+    Steps
+        1) Create a transaction
+        2) Wait until transaction.time > 1 ready threshold
+        3) Create confirmation of transaction / delete invalid transaction
+            - Has a time indicator, overriding the create time so it is sorted
+                just before the base transaction
+        4) Wait until confirmation.time > 1 ready threshold
+        5) Create detached create, actually delete files, delete transaction
+        6) Wait until confirmation.time > 2 ready thresholds
+        7) Delete confirmation of transaction
+*/
+/*
+    Hanging constraints
+        - ANY transaction can be created at any time (obviously)
+        - If a transaction has EVER been confirmed, the confirmation file might be created
+            multiple times, at any time.
+        - If a transaction is EVER invalid, it might be deleted.
+        - If a transaction is EVER confirmed, the create confirmation may be created at any time,
+            and the files might deleted at any time, in any order.
+        - Pending confirmed transactions might be deleted at any time.
+    The main issue would be when creating confirmations and deleting, but... the confirmation
+        means that step can be restarted without too much issue.
+    Keeping our order strict is important. The confirmed transaction can't leave any room
+        for new transactions to sneak in, as the underlying transaction may be invalidated
+        before we apply the file changes.
+*/
+
+
+export type FileInfo = {
+    file: string;
+    createTime: number;
+    size: number;
+};
+
+export type ArchiveTransaction = {
+    createFiles: {
+        file: string;
+        data: Buffer;
+    }[];
+    deleteFiles: FileInfo[];
+};
+
+export interface ArchiveLocker {
+    /** Should resolve fairly quickly, as it doesn't mutate, it just reads files. */
+    getAllValidFiles(): Promise<FileInfo[]>;
+
+    /** Swaps createFiles for deleteFiles. Guarantees correctness when observed
+     *      from getAllValidFiles.
+     *      - Will take a while to return.
+     *      - Might not change any files, you will have to call getAllValidFiles to see
+     *          if we might have changed something. If the creates files are deleted
+     *          quickly, you might not see them, and if the deleted files are gone,
+     *          it could be someone else who deleted them (for another reason).
+     *          - Basically, you shouldn't depend on this in any way, and every time
+     *              you read make the best single change, and don't rely on it for
+     *              the next set of changes.
+     *      - Might create a file and delete only some or none of the requested delete files
+     *          (if we crash while deleting multiple files).
+     */
+    atomicSwapFiles(
+        config: {
+            // Runs without locking the existence of files, so duplicates can appear. Results in less rejections,
+            //  but... then you might also have duplicate (or even redundant but slightly different) files.
+            allowDuplicates?: boolean;
+        },
+        code: (
+            validFiles: FileInfo[],
+            readFiles: (files: FileInfo[]) => Promise<(Buffer | undefined)[]>,
+        ) => Promise<ArchiveTransaction[]>
+    ): Promise<"accepted" | "rejected">;
+
+    /** Unsafely sets the current files. For loading snapshots.
+     *  - Might fail, might break all your files, etc. BUT, if you are only running a single server,
+     *      this will probably work. AND if you are reverting to an old snapshot, something that will
+     *      probably work is better than a data state which is definitely broken.
+     */
+    unsafeSetFiles(files: string[]): Promise<void>;
+    unsafeGetFileLocation(file: string): Promise<"live" | "zombie" | "recycled" | "missing">;
+}