querysub 0.2.0

package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts

@@ -0,0 +1,820 @@
+import { list, sort, timeInDay, timeInHour, timeInMinute } from "socket-function/src/misc";
+import { MaybePromise } from "socket-function/src/types";
+import { decodeNodeId } from "../../-a-auth/certs";
+import { getOwnNodeId, getOwnNodeIdAssert } from "../../-f-node-discovery/NodeDiscovery";
+import { pathValueArchives } from "../pathValueArchives";
+import { ArchiveLocker, ArchiveTransaction } from "./ArchiveLocks";
+import { Archives } from "../../-a-archives/archives";
+import debugbreak from "debugbreak";
+import { formatNumber, formatTime } from "socket-function/src/formatting/format";
+import { blue, green, magenta, red } from "socket-function/src/formatting/logColors";
+import { logNodeStateStats, logNodeStats } from "../../5-diagnostics/nodeMetadata";
+import { devDebugbreak } from "../../config";
+import { logErrors } from "../../errors";
+import { saveSnapshot } from "./archiveSnapshots";
+import { getNodeId } from "socket-function/src/nodeCache";
+import { diskLog } from "../../diagnostics/logs/diskLogger";
+
+/** Clean up old files after a while */
+const DEAD_CREATE_THRESHOLD = timeInHour * 12;
+const ARCHIVE_PROPAGATION_TIME = 5000;
+const LOG = false;
+// NOTE: There isn't a reason for transaction to not apply. If applying throws, we throw,
+//  and can throw infinite times, but if it doesn't throw, it SHOULD be applied. We retry
+//  a few times though, just in case the storage system was having some issues. After enough
+//  tries though... we HAVE to just skip it, otherwise we will be stuck forever.
+const MAX_APPLY_TRIES = 10;
+const CONCURRENT_READ_COUNT = 32;
+const CONCURRENT_WRITE_COUNT = 16;
+
+export function createArchiveLocker2(config: {
+    archiveValues: Archives;
+    /** IMPORTANT! This must be the same independent of archiveValues, otherwise we won't
+     *      lock moves between directories.
+     */
+    archiveLocks: Archives;
+    archiveRecycleBin: Archives;
+}): ArchiveLocker {
+    let { archiveValues, archiveLocks, archiveRecycleBin } = config;
+    function getArchives(key: string) {
+        return (key.endsWith(".data") || key.endsWith(".locked")) ? archiveValues : archiveLocks;
+    }
+    let debugKey = archiveLocks.getDebugName();
+    let storage: StorageType = {
+        debugKey,
+        propagationTime: ARCHIVE_PROPAGATION_TIME,
+
+        async getKeys() {
+            let filesRawValues = await archiveValues.findInfo("", { type: "files" });
+            let filesRawLocks = await archiveLocks.findInfo("", { type: "files" });
+            let files: FileInfo[] = [...filesRawValues, ...filesRawLocks].map(x => ({
+                file: x.path,
+                createTime: x.createTime,
+                size: x.size,
+            }));
+            return files;
+        },
+
+        async setValue(key, value) {
+            await getArchives(key).set(key, value);
+            logNodeStats(`archives|Created TΔ`, formatNumber)(1);
+        },
+        async getValue(key) {
+            return getArchives(key).get(key);
+        },
+        async deleteKey(key) {
+            let archives = await getArchives(key);
+            if (
+                !await archives.getInfo(key)
+                && !await archives.get(key)
+            ) {
+                return;
+            }
+            // ACTUALLY delete confirms, as they don't contain that much information, and just clutter up the recycle bin.
+            //  - DO archive transactions, as it might be useful to view the old transaction history.
+            if (archives === archiveLocks && key.endsWith(".confirm")) {
+                try {
+                    await archives.del(key);
+                } catch { }
+                return;
+            }
+            try {
+                await archives.move({
+                    path: key,
+                    targetPath: key,
+                    target: archiveRecycleBin,
+                });
+                logNodeStats(`archives|Deleted TΔ`, formatNumber)(1);
+            } catch {
+                // It was probably just moved by another process
+            }
+        },
+    };
+    async function unsafeSetFiles(files: string[]): Promise<void> {
+        let valuePaths = new Set((await archiveValues.findInfo("")).map(x => x.path));
+        let correctConfirms = new Set<string>();
+        let correctFiles = new Set<string>();
+
+        async function deleteAllOtherFiles() {
+            // Delete all old confirms / transactions / etc
+            let allConfirms = (await archiveLocks.findInfo("")).map(x => x.path);
+            await Promise.all(allConfirms.map(async confirm => {
+                if (correctConfirms.has(confirm)) return;
+                await storage.deleteKey(confirm);
+            }));
+
+            valuePaths = new Set((await archiveValues.findInfo("")).map(x => x.path));
+            // Delete all files we didn't just set (move to recycle bin)
+            await Promise.all(Array.from(valuePaths).map(async file => {
+                if (correctFiles.has(file)) return;
+                await archiveValues.move({
+                    path: file,
+                    targetPath: file,
+                    target: archiveRecycleBin,
+                });
+            }));
+        }
+
+        console.log(magenta(`Deleting all other files`));
+        // Deleting early deletes any transactions, which might be reapplied due to our non-transaction changes
+        await deleteAllOtherFiles();
+
+        // Undelete, by retrieving all files from the recycle bin
+        let promises: Promise<void>[] = [];
+        for (let file of files) {
+            if (valuePaths.has(file)) {
+                correctFiles.add(file);
+            }
+            let isInRecycleBin = !!(await archiveRecycleBin.getInfo(file));
+            if (isInRecycleBin) {
+                console.log(`Restoring ${file}`);
+                promises.push(archiveRecycleBin.move({
+                    path: file,
+                    targetPath: file,
+                    target: archiveValues,
+                }));
+                correctFiles.add(file);
+            }
+        }
+        await Promise.all(promises);
+
+        console.log(magenta(`Creating confirms`));
+        // Confirm all files
+        await Promise.all(Array.from(correctFiles).map(async file => {
+            let confirm = await locker.createConfirm(file);
+            correctConfirms.add(confirm);
+        }));
+
+        console.log(magenta(`Deleting all other files again`));
+        // Delete again, in case anything else was created while we were restoring. Not great, but... should
+        //  reduce the race conditions considerably.
+        await deleteAllOtherFiles();
+    }
+    async function unsafeGetFileLocation(file: string): Promise<"live" | "zombie" | "recycled" | "missing"> {
+        if (await archiveValues.getInfo(file)) {
+            if (!file.endsWith(".locked")) {
+                return "live";
+            }
+            let confirmKey = await locker.getConfirmKey(file);
+            if (await archiveLocks.getInfo(confirmKey)) {
+                return "live";
+            } else {
+                return "zombie";
+            }
+        }
+        if (await archiveRecycleBin.getInfo(file)) {
+            return "recycled";
+        }
+        return "missing";
+    }
+
+    let locker = new TransactionLocker(storage);
+
+    return {
+        getAllValidFiles() {
+            return locker.getFiles();
+        },
+        unsafeSetFiles,
+        unsafeGetFileLocation,
+        async atomicSwapFiles(
+            config,
+            code: (
+                validFiles: FileInfo[],
+                readFiles: (files: FileInfo[]) => Promise<(Buffer | undefined)[]>,
+            ) => Promise<ArchiveTransaction[]>
+        ): Promise<"accepted" | "rejected"> {
+            let files = await locker.getFiles();
+            let readFiles = async (files: FileInfo[]) => {
+                let pendingFiles = files.slice();
+                let readResults = new Map<string, Buffer | undefined>();
+                async function runReadThread() {
+                    while (pendingFiles.length > 0) {
+                        let file = pendingFiles.pop()!;
+                        let value = await storage.getValue(file.file);
+                        readResults.set(file.file, value);
+                    }
+                }
+                await Promise.all(list(CONCURRENT_READ_COUNT).map(runReadThread));
+                return files.map(a => {
+                    return readResults.get(a.file);
+                });
+            };
+            let transactionRaw = await code(files, readFiles);
+            if (transactionRaw.length === 0) return "accepted";
+            let newTransaction: Transaction = {
+                ops: [],
+            };
+            if (!config.allowDuplicates) {
+                newTransaction.lockedFilesMustEqual = files.map(a => a.file).filter(a => a.endsWith(".locked"));
+            }
+            for (let base of transactionRaw) {
+                for (let obj of base.createFiles) {
+                    newTransaction.ops.push({ type: "create", key: obj.file, value: obj.data });
+                }
+                for (let obj of base.deleteFiles) {
+                    newTransaction.ops.push({ type: "delete", key: obj.file });
+                }
+            }
+            let status = await locker.addTransaction(newTransaction);
+            if (status === "accepted") {
+                let newFiles = new Set<string>();
+                for (let file of files) {
+                    newFiles.add(file.file);
+                }
+                for (let op of newTransaction.ops) {
+                    const type = op.type;
+                    if (type === "create") {
+                        newFiles.add(op.key);
+                    } else if (type === "delete") {
+                        newFiles.delete(op.key);
+                    } else {
+                        let unhandled: never = type;
+                    }
+                }
+                logErrors(saveSnapshot({ files: Array.from(newFiles) }));
+            }
+            return status;
+        }
+    };
+}
+
+export type FileInfo = {
+    file: string;
+    createTime: number;
+    size: number;
+};
+type StorageType = {
+    debugKey: string;
+    propagationTime: number;
+    getKeys(): Promise<FileInfo[]>;
+    setValue(key: string, value: Buffer): Promise<void>;
+    getValue(key: string): Promise<Buffer | undefined>;
+    deleteKey(key: string): Promise<void>;
+    /** The amount of time before we have to wait until it is assumed that all reads
+     *      this old will have been received.
+     *      - On the local disk it could be 0, but we use a higher value so development is closer to production
+     *      - On backblaze writes might be routed to different servers, so a value might be written
+     *          and not be immediately readable. It might also be readable in a different order than
+     *          we wrote it (random writes might take longer). HOWEVER, if we wait propagationTime
+     *          all writes should now be readable.
+     */
+};
+type Transaction = {
+    ops: ({
+        type: "create";
+        key: string;
+        value: Buffer | undefined;
+    } | {
+        type: "delete";
+        key: string;
+    })[];
+    // Forces the locked files to strictly equal this set. This allows lockfree writes
+    //      (such as from genesis writers) to be added, but allows writers to do things
+    //      such as ensuring a file with a certain flag exists, and not have to worry
+    //      about two files being created due to a race condition.
+    lockedFilesMustEqual?: string[];
+};
+/**
+ *      Guarantees
+ *          - The output state will never include partial Transactions
+ *          - Transactions will apply as if they exclusively locked all the files OR,
+ *              they won't be applied at all.
+ *              - EXCEPT, A single identical transactions might have deletes applied multiple times,
+ *                  and might be delayed (if the original writer crashes or hangs). It is recommended
+ *                  file names are never reused (annoying, but feasible if you add some random data to
+ *                  file names), which will prevent this from ever causing issues.
+ *          - IMPORTANT! We only make the file names atomic. If you go to read a file, and it isn't there,
+ *              you should get the file names again and try again (caching the read file in memory, otherwise
+ *              you might never catch up to the new writes).
+ *          - Changes created file name so they end with ".locked"
+ *              - This is a bit annoying, but it should be easier for callers to ignore this.
+ *          - Any files created outside of TransactionLocker (that don't end with ".locked", ".confirm", or ".transaction")
+ *              will automatically confirmed.
+ *          - We assume all files are immutable
+ */
+class TransactionLocker {
+    public constructor(
+        private storage: StorageType,
+        // For example, this could be used to ensure that only 1 file with "singleSourceOfTruth"
+        //      can exist, unless the new "singleSourceOfTruth" is > a day newer than the old file.
+        //      - Completely overrides the valid check, so if you still want to verify that deletes
+        //          exist, you will have to manually check them.
+        private isTransactionValidOverride?: (transaction: Transaction, dataFiles: FileInfo[], rawDataFiles: FileInfo[]) => boolean
+    ) { }
+
+    // #region Base File Operations
+    public getConfirmKey(key: string): string {
+        let { dir, name } = parsePath(key);
+        return `${dir}confirm_${name}.confirm`;
+    }
+    public async createConfirm(key: string) {
+        let path = this.getConfirmKey(key);
+        if (LOG) {
+            console.log(`Creating confirmation for ${key}`);
+        }
+        await this.storage.setValue(path, Buffer.from(""));
+        return path;
+    }
+    private async deleteDataFile(key: string, reason: string): Promise<void> {
+        //await this.storage.setValue(key + ".reason", Buffer.from(reason));
+        // Delete file, and confirmation as well
+        await this.storage.deleteKey(key);
+        await this.storage.deleteKey(this.getConfirmKey(key));
+    }
+    // Writes the createFiles, so they can be confirmed in the future
+    private async prepareTransaction(transaction: Transaction): Promise<void> {
+        let opsRemaining = transaction.ops.slice();
+        const runThread = async () => {
+            while (true) {
+                let op = opsRemaining.pop();
+                if (!op) return;
+                if (op.type === "create") {
+                    let key = op.key;
+                    await this.storage.setValue(key, op.value || Buffer.from(""));
+                }
+            }
+        };
+        await Promise.all(list(CONCURRENT_WRITE_COUNT).map(runThread));
+    }
+    // Just writes the transaction (in a format readDataState can read)
+    private async writeTransaction(transaction: Transaction & { seqNum: number; }): Promise<void> {
+        function ellipsize(text: string, length: number) {
+            if (text.length <= length) return text;
+            return text.slice(0, length - 3) + "...";
+        }
+        let niceThreadId = decodeNodeId(getOwnNodeIdAssert())?.threadId.slice(0, 4) || "unknown";
+        function debugFileInfo(file: string) {
+            let obj = pathValueArchives.decodeDataPath(file);
+            if (!obj.seqNum) {
+                return "X";
+            }
+            return obj.seqNum + "@" + obj.threadId.slice(0, 4);
+        }
+
+        let creates = transaction.ops.filter(a => a.type === "create");
+        let deletes = transaction.ops.filter(a => a.type === "delete");
+        let name = toFileNameKVP({
+            tSeqNum: transaction.seqNum + "",
+            tWriteTime: Date.now() + "",
+            thread: niceThreadId,
+            cCount: transaction.ops.filter(a => a.type === "create").length + "",
+            dCount: transaction.ops.filter(a => a.type === "delete").length + "",
+            create: ellipsize(creates.map(a => debugFileInfo(a.key)).join(","), 50),
+            delete: ellipsize(deletes.map(a => debugFileInfo(a.key)).join(","), 50),
+        });
+
+        if (LOG) {
+            console.log(`Writing transaction ${transaction.seqNum} with ${transaction.ops.length} ops`);
+        }
+
+        let key = `transaction_${name}.transaction`;
+        let strippedTransaction: Transaction = { ops: transaction.ops.map(a => ({ ...a, value: undefined })) };
+        let value = Buffer.from(JSON.stringify(strippedTransaction));
+        await this.storage.setValue(key, value);
+    }
+    private async readDataState(): Promise<{
+        rawDataFiles: FileInfo[];
+        /** Confirmed FileInfos are === the FileInfos in rawDataFiles */
+        confirmedDataFiles: FileInfo[];
+        transactions: (Transaction & {
+            seqNum: number;
+            writeTime: number;
+            createTime: number;
+            source: FileInfo;
+        })[];
+    }> {
+        let bufferCache = new Map<string, Buffer>();
+        const tryToRead = async () => {
+            let files = await this.storage.getKeys();
+
+            let transactions: (Transaction & {
+                seqNum: number;
+                writeTime: number;
+                createTime: number;
+                source: FileInfo;
+            })[] = [];
+            for (let file of files) {
+                if (!file.file.endsWith(".transaction")) continue;
+                let tFile = file;
+                let buffer = await this.storage.getValue(tFile.file);
+                if (!buffer) return false;
+                bufferCache.set(tFile.file, buffer);
+                let transaction: Transaction | undefined;
+                try {
+                    transaction = JSON.parse(buffer.toString());
+                } catch (e: any) {
+                    // Maybe it is actively being written to? Or it is just broken. Skip it.
+                    console.error(`Failed to parse transaction. Ignoring, ${tFile.file}, ${e.stack}`);
+                }
+                if (!transaction) continue;
+
+                let { name } = parsePath(tFile.file);
+                let kvp = parseFileNameKVP(name);
+                transactions.push({
+                    ...transaction,
+                    seqNum: Number(kvp.tSeqNum) || 0,
+                    writeTime: Number(kvp.tWriteTime),
+                    createTime: tFile.createTime,
+                    source: tFile,
+                });
+            }
+
+            // Check all of our files to see if any have changed (or if there a new files).
+            //  Because we don't reuse file names, or change them, it means if a file exists
+            //      it must not have changed (if it was deleted and recreated... then we reuse
+            //      a file name, so there is no way for the state to have changed).
+            //  - Of course, if a file was created and deleted while we weren't watching we will miss that,
+            //      but validity should/can only depend on the current state, not past event transitions
+            //      (it pretty much only depends on files existing).
+            {
+                let filesVerify = await this.storage.getKeys();
+                let filesVerifySet = new Set(filesVerify.map(a => a.file));
+                // If it changes while reading, read again. Otherwise, if there were no changes while reading,
+                //  we know the state was all the state we read, at one time (maybe not now, but at an instant
+                //  in the past).
+                if (filesVerify.length !== files.length || files.some(a => !filesVerifySet.has(a.file))) {
+                    let message = "Files changed while reading. Retrying.";
+                    for (let file of files) {
+                        if (!filesVerifySet.has(file.file)) {
+                            message += `\n    Missing file: ${file.file}`;
+                        }
+                    }
+                    let fileSet = new Set(files.map(a => a.file));
+                    for (let file of filesVerify) {
+                        if (!fileSet.has(file.file)) {
+                            message += `\n    New file: ${file.file}`;
+                        }
+                    }
+                    console.warn(message);
+                    logNodeStats(`archives|TΔ Atomic Retry`, formatNumber)(1);
+                    return false;
+                }
+            }
+
+            let existingFiles = new Map(files.map(a => [a.file, a]));
+
+            let currentDataFiles = new Map<string, FileInfo>();
+            for (let file of files) {
+                if (!(
+                    file.file.endsWith(".locked")
+                    || file.file.endsWith(".confirm")
+                    || file.file.endsWith(".transaction")
+                    || file.file.endsWith(".reason")
+                )) {
+                    currentDataFiles.set(file.file, file);
+                    continue;
+                }
+                if (!file.file.endsWith(".locked")) continue;
+                let confirmKey = this.getConfirmKey(file.file);
+                let confirmFile = existingFiles.get(confirmKey);
+                if (confirmFile) {
+                    currentDataFiles.set(file.file, file);
+                }
+            }
+
+            diskLog(`Read archive state`, {
+                rawFilesCount: files.length,
+                confirmedCount: currentDataFiles.size,
+                rawFiles: files.map(a => a.file),
+                confirmedFiles: Array.from(currentDataFiles.values()).map(a => a.file),
+            });
+
+            return {
+                rawDataFiles: files,
+                confirmedDataFiles: Array.from(currentDataFiles.values()),
+                transactions,
+            };
+        };
+
+        let startTime = Date.now();
+        while (true) {
+            let result = await tryToRead();
+            if (result) {
+                if (LOG) {
+                    let timeToRead = Date.now() - startTime;
+                    console.log(`Read data state in ${formatTime(timeToRead)}`);
+                }
+                return result;
+            }
+        }
+    }
+    // #endregion
+
+    private isTransactionValid(transaction: Transaction, dataFiles: FileInfo[], rawDataFiles: FileInfo[]): boolean {
+        let override = this.isTransactionValidOverride;
+        if (override) return override(transaction, dataFiles, rawDataFiles);
+        if (transaction.lockedFilesMustEqual) {
+            let newLockedFiles = new Set(dataFiles.map(a => a.file).filter(a => a.endsWith(".locked")));
+            let mustEqualSet = new Set(transaction.lockedFilesMustEqual);
+            if (mustEqualSet.size !== newLockedFiles.size) return false;
+            if (Array.from(mustEqualSet).some(a => !newLockedFiles.has(a))) return false;
+        }
+        let existingFiles = new Set(dataFiles.map(a => a.file));
+        let existingRawFiles = new Set(rawDataFiles.map(a => a.file));
+        return transaction.ops.every(a => {
+            const type = a.type;
+            // Create files might disappear if we hang and they get cleaned up
+            if (type === "create") return existingRawFiles.has(a.key);
+            if (type === "delete") return existingFiles.has(a.key);
+            let unhandled: never = type;
+            throw new Error(`Unhandled type: ${unhandled}`);
+        });
+    }
+
+    // NOTE: If the transaction was applied long in the past, this will return false. HOWEVER, this is fine,
+    //  as if it has been applied, when we look around again, we will no longer see it
+    private wasTransactionApplied(
+        transaction: Transaction,
+        confirmedDataFiles: FileInfo[],
+        rawDataFiles: FileInfo[]
+    ) {
+        let confirmedKeys = new Set(Array.from(confirmedDataFiles).map(a => a.file));
+        let rawLookup = new Set(Array.from(rawDataFiles).map(a => a.file));
+        // If any creates are not confirmed, it must not have been applied
+        if (transaction.ops.some(a => a.type === "create" && rawLookup.has(a.key) && !confirmedKeys.has(a.key))) {
+            if (LOG) {
+                console.log(`Transaction has pending confirmations of creates.`);
+                let pending = transaction.ops.filter(a => a.type === "create" && rawLookup.has(a.key) && !confirmedKeys.has(a.key));
+                for (let p of pending) {
+                    console.log(`    ${p.key}`);
+                }
+            }
+            return false;
+        }
+        // If any deletes still exist, it must not have been applied
+        if (transaction.ops.some(a => a.type === "delete" && confirmedKeys.has(a.key))) {
+            if (LOG) {
+                console.log(`transaction has pending deletes`);
+            }
+            return false;
+        }
+        return true;
+    }
+    private async applyTransaction(transaction: Transaction): Promise<void> {
+        let createCount = transaction.ops.filter(a => a.type === "create").length;
+        let deleteCount = transaction.ops.filter(a => a.type === "delete").length;
+        let lockedFiles = transaction.lockedFilesMustEqual?.length;
+        if (LOG) {
+            console.log(`Applying transaction with ${createCount} creates and ${deleteCount} deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`);
+        }
+        logNodeStats(`archives|TΔ Apply`, formatNumber)(1);
+        let opsRemaining = transaction.ops.slice();
+        // NOTE: Order doesn't matter here. If anything is reading the values
+        //  1) If it runs after we start, it will see our transaction and apply it
+        //  2) If it runs before we start, if it can't find some Buffers, it will abort and
+        //      reread all of the files, until it can read all of the Buffers without having
+        //      a delete happen.
+        const runThread = async () => {
+            while (true) {
+                let op = opsRemaining.pop();
+                if (!op) return;
+                if (op.type === "create") {
+                    await this.createConfirm(op.key);
+                } else if (op.type === "delete") {
+                    await this.deleteDataFile(op.key, `transaction (${getOwnNodeId()})`);
+                } else {
+                    let unhandled: never = op;
+                    throw new Error(`Unhandled type: ${unhandled}`);
+                }
+            }
+        };
+        await Promise.all(list(CONCURRENT_WRITE_COUNT).map(runThread));
+
+        if (LOG) {
+            console.log(`Applied transaction with ${createCount} creates and ${deleteCount} deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`);
+        }
+    }
+
+    /** Only returns data files (no transaction files, or confirmations).
+     *      - Might run a transaction
+     */
+    public async getFiles(): Promise<FileInfo[]> {
+        let obj = await this.getFilesBase();
+        logNodeStateStats(`ArchiveLock Data File`, formatNumber)(obj.dataFiles.length);
+        return obj.dataFiles;
+    }
+    private transactionAppliedCount = new Map<number, number>();
+    private async getFilesBase(): Promise<{
+        dataFiles: FileInfo[];
+        rawDataFiles: FileInfo[];
+        nextSeqNum: number;
+    }> {
+        let readTime = Date.now();
+        let dataState = await this.readDataState();
+
+        //  - Sort transactions by [-seqNum, createTime]
+        let transactions = dataState.transactions.slice();
+        transactions.sort((a, b) => {
+            if (a.seqNum !== b.seqNum) return b.seqNum - a.seqNum;
+            return a.createTime - b.createTime;
+        });
+        {
+            //  - Also by [-seqNum, writeTime]. If we find the order is different warn,
+            //      as it might be indicative of a hanging storage writer (although more likely
+            //      it is due a hanging write of ours).
+            let transationsTest = dataState.transactions.slice();
+            transationsTest.sort((a, b) => {
+                if (a.seqNum !== b.seqNum) return b.seqNum - a.seqNum;
+                return a.writeTime - b.writeTime;
+            });
+            if (transactions.map(a => a.seqNum).join(",") !== transationsTest.map(a => a.seqNum).join(",")) {
+                console.warn("Transaction order is different when sorting by writeTime. This is likely due to a hanging writes.");
+                logNodeStats(`archives|TΔ Possible Hanging Write`, formatNumber)(1);
+            }
+        }
+
+        // IMPORTANT! DO NOT check for isTransactionValid here! This must be done BEFORE
+        //  we write, otherwise partial applications of transactions will invalidate the transaction
+        //  itself, which breaks things!
+        //  - And checking before we write is safe, because we use the next seqNum, if we write
+        //      a transaction and it is applied, it must mean no transactions snuck in between,
+        //      so it transaction is still as valid as before it was written!
+
+        // Initialization code / lock reset recovery code
+        if (transactions.length === 0) {
+            return {
+                dataFiles: dataState.confirmedDataFiles,
+                rawDataFiles: dataState.rawDataFiles,
+                nextSeqNum: 1,
+            };
+        }
+
+        let activeT = transactions[0];
+        // Wait until activeTransaction is old enough to be applied,
+        //  and then rerun the function (if transactions have appeared which are older,
+        //  which is possible, as storage might give reads out of order)
+        let threshold = activeT.createTime + this.storage.propagationTime;
+        if (Date.now() < threshold) {
+            let waitTime = threshold - Date.now();
+            if (LOG) {
+                console.log(`Waiting ${formatTime(waitTime)} for transaction ${activeT.seqNum} to settle.`);
+            }
+            await new Promise(resolve => setTimeout(resolve, waitTime));
+            return this.getFilesBase();
+        }
+
+        let applyCount = this.transactionAppliedCount.get(activeT.seqNum) || 0;
+
+        // IF the transaction is already applied, THEN we can return the data state!
+        if (
+            applyCount > MAX_APPLY_TRIES
+            || this.wasTransactionApplied(activeT, dataState.confirmedDataFiles, dataState.rawDataFiles)
+        ) {
+            // Delete any transactions with a seqNum < activeTransaction.seqNum
+            //  (otherwise our transactions stick around forever)
+            //  - We can't delete the transaction we applied, otherwise the next transaction won't
+            //      have a seqNum to compare against!
+            //  - Don't delete their created files. If they were applied, we need those, if not
+            //      the old orphaned file cleanup will clean them up.
+            for (let t of transactions) {
+                if (t.seqNum < activeT.seqNum) {
+                    await this.storage.deleteKey(t.source.file);
+                }
+            }
+
+            // Delete any create files that are WAY too old, and not confirmed
+            {
+                let oldThreshold = readTime - DEAD_CREATE_THRESHOLD;
+                let veryOldFiles = dataState.rawDataFiles.filter(x => x.file.endsWith(".locked")).filter(a => a.createTime < oldThreshold);
+                let confirmedLookup = new Set(dataState.confirmedDataFiles.map(a => a));
+                let unconfirmedOldFiles = veryOldFiles.filter(a => !confirmedLookup.has(a));
+
+                if (unconfirmedOldFiles.length > 0) {
+                    let doubleCheck = await this.readDataState();
+                    let doubleCheckLookup = new Set(doubleCheck.confirmedDataFiles.map(a => a));
+                    let doubleCheckDataFiles = new Set(doubleCheck.rawDataFiles.map(a => a.file));
+                    // IMPORTANT! Make sure they still aren't confirmed AND that the rawDataFiles exist. This helps prevent cases
+                    //  where backblaze returns no files? Which it might be doing, as multiple times this code has
+                    //  triggered (without this check), and deletes all of our files...
+                    let unconfirmedOldFiles2 = veryOldFiles.filter(a => !doubleCheckLookup.has(a) && doubleCheckDataFiles.has(a.file));
+                    console.warn(red(`Deleted ${unconfirmedOldFiles2.length} very old unconfirmed files`));
+                    if (LOG) {
+                        logNodeStats(`archives|TΔ Delete Old Rejected File`, formatNumber)(unconfirmedOldFiles2.length);
+                    }
+                    // At the point the file was very old when we started reading, not part of the active transaction.
+                    for (let file of unconfirmedOldFiles2) {
+                        await this.deleteDataFile(file.file, `old unconfirmed file (${getOwnNodeId()}, ${process.argv[1]})`);
+                    }
+                }
+            }
+
+
+            // Delete any confirmations that have no corresponding file, and are very old
+            {
+                let oldThreshold = readTime - DEAD_CREATE_THRESHOLD;
+                let usedConfirmations = new Set(dataState.confirmedDataFiles.map(a => this.getConfirmKey(a.file)));
+                let oldEnoughConfirms = dataState.rawDataFiles.filter(x => x.file.endsWith(".confirm") && x.createTime < oldThreshold);
+                let deprecatedFiles = oldEnoughConfirms.filter(a => !usedConfirmations.has(a.file));
+                if (deprecatedFiles.length > 0) {
+                    if (LOG) {
+                        console.warn(red(`Deleted ${deprecatedFiles.length} / ${oldEnoughConfirms.length} confirmations, for not having corresponding data files`));
+                        logNodeStats(`archives|TΔ Delete Deprecated Confirm`, formatNumber)(deprecatedFiles.length);
+                    }
+                    for (let file of deprecatedFiles) {
+                        await this.storage.deleteKey(file.file);
+                    }
+                }
+            }
+
+            // Delete some debug files
+            {
+                for (let file of dataState.rawDataFiles.filter(x => x.file.endsWith(".reason"))) {
+                    await this.storage.deleteKey(file.file);
+                }
+            }
+
+            return {
+                dataFiles: Array.from(dataState.confirmedDataFiles),
+                rawDataFiles: dataState.rawDataFiles,
+                nextSeqNum: activeT.seqNum + 1
+            };
+        }
+
+        {
+            console.log(`Applying transaction ${activeT.seqNum}, cur try ${applyCount} / ${MAX_APPLY_TRIES}`);
+        }
+        await this.applyTransaction(activeT);
+        this.transactionAppliedCount.set(activeT.seqNum, applyCount + 1);
+
+        // Run again, until we can be reasonable sure activeT isn't changing. We can be wrong,
+        //      though, which will just result in any transaction depending on activeT being
+        //      rejecting, and having to be inserted again.
+        return this.getFilesBase();
+    }
+
+    /** If any deleted files were deleted by other transactions, then we will be rejected. */
+    public async addTransaction(transaction: Transaction): Promise<"accepted" | "rejected"> {
+        if (LOG) {
+            let dels = transaction.ops.filter(a => a.type === "delete").length;
+            let creates = transaction.ops.filter(a => a.type === "create").length;
+            let createBytes = transaction.ops.map(a => a.type === "create" && a.value?.length || 0).reduce((a, b) => a + b, 0);
+            console.log(blue(`Starting transaction with ${creates} creates and ${dels} deletes, ${formatNumber(createBytes)}B`));
+        }
+        transaction = { ...transaction, ops: transaction.ops.slice() };
+        function normalizePath(path: string) {
+            // Replace duplicate slashes with a single slash
+            return path.replaceAll(/\/+/g, "/");
+        }
+        for (let op of transaction.ops) {
+            if (op.type === "create") {
+                op.key += ".locked";
+            }
+            // NOTE: If we fail to normalize, we might think values are deleted which exist, or that
+            //  creates are gone when they are actually there.
+            op.key = normalizePath(op.key);
+        }
+        // We check for valid AFTER applying the latest transaction. Then when we
+        //  write, if anything changes, it will be from a transaction which superceded us,
+        //  which will cause our transaction to never apply. Otherwise... we MUST still
+        //  be valid!
+
+        logNodeStats(`archives|TΔ Create File`, formatNumber)(1);
+        await this.prepareTransaction(transaction);
+
+        while (true) {
+            let beforeData = await this.getFilesBase();
+            if (!this.isTransactionValid(transaction, beforeData.dataFiles, beforeData.rawDataFiles)) {
+                logNodeStats(`archives|TΔ Rejected`, formatNumber)(1);
+                if (LOG) {
+                    console.log(red(`Finished transaction with rejection, ${transaction.ops.length} ops`));
+                }
+                return "rejected";
+            }
+
+            await this.writeTransaction({ ...transaction, seqNum: beforeData.nextSeqNum });
+
+            let afterData = await this.getFilesBase();
+            if (this.wasTransactionApplied(transaction, afterData.dataFiles, afterData.rawDataFiles)) {
+                logNodeStats(`archives|TΔ Accepted`, formatNumber)(1);
+                if (LOG) {
+                    console.log(green(`Finished transaction with ${transaction.ops.length} ops`));
+                }
+                return "accepted";
+            }
+        }
+    }
+}
+
+function parsePath(path: string): { dir: string; name: string } {
+    path = path.replaceAll("\\", "/");
+    let lastSlash = path.lastIndexOf("/");
+    if (lastSlash === -1) return { dir: "", name: path };
+    return { dir: path.slice(0, lastSlash + 1), name: path.slice(lastSlash + 1) };
+}
+
+// Loses spaces in keys and values
+function toFileNameKVP(kvp: { [key: string]: string }): string {
+    function s(v: string) {
+        return v.replaceAll(" ", "_");
+    }
+    return "   " + Object.entries(kvp).map(([key, value]) => `${s(key)}=${s(value)}`).join("   ") + "   ";
+}
+function parseFileNameKVP(fileName: string): { [key: string]: string } {
+    let parts = fileName.trim().split("   ");
+    let obj: { [key: string]: string } = {};
+    for (let part of parts) {
+        let [key, value] = part.split("=");
+        obj[key] = value || key;
+    }
+    return obj;
+}