querysub 0.592.0 → 0.594.0

package/.claude/settings.local.json

@@ -31,7 +31,11 @@
       "WebFetch(domain:developers.cloudflare.com)",
       "PowerShell(yarn type *)",
       "PowerShell(Remove-Item -Confirm:$false \"C:\\\\Users\\\\quent\\\\.claude\\\\projects\\\\D--repos-querysub\\\\memory\\\\no-naming-new-bins.md\")",
-      "Bash(python -)"
+      "Bash(python -)",
+      "PowerShell(yarn upreal *)",
+      "PowerShell($env:COMPUTERNAME; $env:USERNAME)",
+      "PowerShell($env:COMPUTERNAME)",
+      "PowerShell($env:USERNAME)"
     ]
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.592.0",
+  "version": "0.594.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -74,8 +74,8 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
-    "sliftutils": "^1.7.69",
-    "socket-function": "^1.2.27",
+    "sliftutils": "^1.7.74",
+    "socket-function": "^1.2.29",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",
     "typesafecss": "^0.32.0",

package/spec.txt

@@ -1,3 +1,15 @@
+It could be it's just taking a while to get valid. Let's check it two times to see. Although if it is, that's a bug, but it's probably an easy bug to fix. 
+
+Wait, what transaction is it applying now? If it's not a valid transaction and we don't even write it, so what? 
+
+Well, we might have to restore from a snapshot if it breaks it, but I'm pretty confident we're not going to lose all of our data. 
+
+Ugh... I mean it seems like our storage is just really out of sync. Like, we're reading files and it changes every time, etc. 
+
+
+
+
+
 Trigger values
     1) validStateComputer.ingestValuesAndValidStates
     1.5) authorityStorage.ingestValues

package/src/-a-archives/archives2.ts

@@ -4,6 +4,7 @@ import { cache, cacheJSONArgsEqual } from "socket-function/src/caching";
 import { formatDateTime } from "socket-function/src/formatting/format";
 import { getDomain } from "../config";
 import { timeInMinute } from "socket-function/src/misc";
+import { FindConfig } from "sliftutils/storage/IArchives";
 
 export const STORAGE_ACCOUNT = "root";
 
@@ -44,31 +45,34 @@ function archiveBuilder(bucket: string, overrides: Partial<RemoteConfigBase>) {
     const ONTARIO = `https://99-250-124-91.querysubtest.com:5234/file/${STORAGE_ACCOUNT}/${bucket}/storage/storagerouting.json`;
     const HETZNER = `https://65-109-93-113.querysubtest.com:5234/file/${STORAGE_ACCOUNT}/${bucket}/storage/storagerouting.json`;
     const BACKBLAZE = `https://f002.backblazeb2.com/file/${bucket}/storage/storagerouting.json`;
+
+    function stripeSources(urls: string[]) {
+        let sources: Omit<Exclude<RemoteConfigBase, string>, "validWindow">[] = [];
+        for (let j = 0; j < urls.length; j++) {
+            // Shift by j
+            let currentUrls = urls.slice(j).concat(urls.slice(0, j));
+            for (let i = 0; i < urls.length; i++) {
+                sources.push({ type: "remote", url: currentUrls[i], route: [i / urls.length, (i + 1) / urls.length], });
+            }
+        }
+        return sources;
+    }
+
+
     let sources = createSourceWindows(overrides, [
         {
-            startTime: +new Date("2026-07-20 18:00:00-04:00"),
-            sources: [
-                { type: "remote", url: ONTARIO, route: [0.5, 1], },
-                { type: "remote", url: HETZNER, route: [0, 0.5], },
-                { type: "remote", url: ONTARIO, },
-                { type: "remote", url: HETZNER, },
-                { type: "backblaze", url: BACKBLAZE, },
-            ]
-        },
-        {
-            startTime: +new Date("2026-07-21 02:59:00-04:00"),
+            startTime: +new Date("2026-07-20 00:00:00-04:00"),
             sources: [
-                { type: "remote", url: ONTARIO, route: [0, 0.5], },
-                { type: "remote", url: HETZNER, route: [0.5, 1], },
-                { type: "remote", url: ONTARIO, },
-                { type: "remote", url: HETZNER, },
+                // NOTE: Once we add new sources, we need to add something at a new start time in the future so we have time to move the files. 
+                // TODO: EVENTUALLY, Once we have enough data, instead of restriping, we need to bisect the data. That way we can slowly move over the data instead of trying to rewrite all of our data all at once. However, for the moment, moving all the data is probably fine. On the data center, we should be able to move terabytes of data every hour, so we should be able to handle quite a bit, especially if we have a few days to run the restriping. 
+                ...stripeSources([ONTARIO, HETZNER]),
                 { type: "backblaze", url: BACKBLAZE, },
             ]
         },
     ]);
 
     return createArchives({
-        version: 16,
+        version: 18,
         sources,
     });
 }
@@ -89,8 +93,8 @@ function nestBucket(folder: string, archives: ReturnType<typeof createArchives>)
         get: (path: string, config?: GetConfig) => archives.get(folder + path, config),
         set: (path: string, data: Buffer, config?: SetConfig) => archives.set(folder + path, data, config),
         del: (path: string) => archives.del(folder + path),
-        find: async (prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }) => (await archives.find(folder + prefix, config)).map(x => x.slice(folder.length)),
-        findInfo: async (prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }) => (await archives.findInfo(folder + prefix, config)).map(x => ({ ...x, path: x.path.slice(folder.length) })),
+        find: async (prefix: string, config?: FindConfig) => (await archives.find(folder + prefix, config)).map(x => x.slice(folder.length)),
+        findInfo: async (prefix: string, config?: FindConfig) => (await archives.findInfo(folder + prefix, config)).map(x => ({ ...x, path: x.path.slice(folder.length) })),
         get2: (path: string, config?: GetConfig) => archives.get2(folder + path, config),
         setLargeFile: (config: { path: string; lastModified?: number; getNextData(): Promise<Buffer | undefined>; }) => archives.setLargeFile({ ...config, path: folder + config.path }),
         getInfo: (path: string) => archives.getInfo(folder + path),

package/src/-d-trust/NetworkTrust2.ts

@@ -87,7 +87,7 @@ export const isTrusted = measureWrap(async function isTrusted(machineId: string)
     }
 });
 let populateTrustedCache = lazy(async () => {
-    let trustedMachineIds = await archives().find("");
+    let trustedMachineIds = await archives().find("", { fallbacks: true });
     lastArchivesTrusted = trustedMachineIds.slice();
     for (let trustedMachineId of trustedMachineIds) {
         trustedCache.add(trustedMachineId);

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -342,7 +342,7 @@ async function syncArchives() {
     if (isServer()) {
         // Make sure we are present
         await writeHeartbeat();
-        let nodeIds = await archives().find("");
+        let nodeIds = await archives().find("", { fallbacks: true });
         console.info(`Synced node ids from archives`, { nodeIds });
         await setNodeIds(nodeIds);
     } else {

package/src/0-path-value-core/PathRouter.ts

@@ -13,6 +13,7 @@ import { sha256 } from "js-sha256";
 import { rangesOverlap, removeRange } from "../rangeMath";
 import { decodeParentFilter } from "./hackedPackedPathParentFiltering";
 import { getBufferInt } from "socket-function/src/bits";
+import { safeLoop } from "socket-function/src/batching";
 
 import { LOCAL_DOMAIN, LOCAL_DOMAIN_PATH } from "./PathRouterConstants";
 export { LOCAL_DOMAIN, LOCAL_DOMAIN_PATH };
@@ -320,7 +321,7 @@ export class PathRouter {
     // NOTE: Encodes all the data, even if it matches our spec or not. If you don't want this, you have to filter first
     //  - If this becomes an issue we COULD filter, as we can do it quickly, but I don't think it is required, as all the present usecases prefilter anyways.
     @measureFnc
-    public static getPathIdentifierTargets(values: PathValue[], ourSpec: AuthoritySpec): Map<string, PathValue[]> {
+    public static async getPathIdentifierTargets(values: PathValue[], ourSpec: AuthoritySpec): Promise<Map<string, PathValue[]>> {
         // NOTE: The file size limit is 1024 bytes. But we also have our folder, etc, so we want to add enough buffer
         //  - Shorter hashes means we can store more, but there's a point when the collisions make it less useful.
         const MAX_PREFIXES_PER_FILE = 50;
@@ -333,17 +334,17 @@ export class PathRouter {
         let prefixes = ourSpec.prefixes.slice();
         sort(prefixes, x => x.prefixPartLength);
 
-        // NOTE: If there are few enough path values for a prefix, we don't even need to calculate the routing hash. 
+        // NOTE: If there are few enough path values for a prefix, we don't even need to calculate the routing hash.
         let byPrefix = new Map<PrefixMatcher | undefined, PathValue[]>();
-        for (let value of values) {
+        await safeLoop({ data: values, name: "PathRouter.getPathIdentifierTargets|groupByPrefix" }, value => {
             let prefix = getMatchingPrefix(ourSpec, value.path);
-            let values = byPrefix.get(prefix);
-            if (!values) {
-                values = [];
-                byPrefix.set(prefix, values);
+            let prefixValues = byPrefix.get(prefix);
+            if (!prefixValues) {
+                prefixValues = [];
+                byPrefix.set(prefix, prefixValues);
             }
-            values.push(value);
-        }
+            prefixValues.push(value);
+        });
 
         let prefixGroups = Array.from(byPrefix.entries()).map(([prefix, values]) => ({
             prefix,
@@ -391,17 +392,17 @@ export class PathRouter {
         }
 
         let finalFiles = new Map<string, PathValue[]>();
-        for (let group of groups) {
+        await safeLoop({ data: groups, name: "PathRouter.getPathIdentifierTargets|shardGroups" }, async group => {
             if (group.count < SHARD_THRESHOLD) {
                 let identifier = this.encodeIdentifier({ prefixes: group.prefixes, rangeStart: 0, rangeEnd: 1 });
                 finalFiles.set(identifier, group.values.flat());
-                continue;
+                return;
             }
             // Split by routing hash
-            let values = group.values.flat();
-            let splitCount = Math.max(MIN_SHARD_FILE_COUNT, Math.ceil(values.length / TARGET_SHARD_SIZE));
+            let groupValues = group.values.flat();
+            let splitCount = Math.max(MIN_SHARD_FILE_COUNT, Math.ceil(groupValues.length / TARGET_SHARD_SIZE));
             let byRouteGroup = new Map<number, PathValue[]>();
-            for (let value of values) {
+            await safeLoop({ data: groupValues, name: "PathRouter.getPathIdentifierTargets|routeValues" }, value => {
                 let route = this.getRouteFull({
                     path: value.path,
                     spec: {
@@ -418,14 +419,14 @@ export class PathRouter {
                     byRouteGroup.set(routeIndex, routeValues);
                 }
                 routeValues.push(value);
-            }
+            });
             for (let [routeIndex, routeValues] of byRouteGroup) {
                 let rangeStart = routeIndex / splitCount;
                 let rangeEnd = (routeIndex + 1) / splitCount;
                 let identifier = this.encodeIdentifier({ prefixes: group.prefixes, rangeStart, rangeEnd });
                 finalFiles.set(identifier, routeValues);
             }
-        }
+        });
 
         // NOTE: There could be a huge number of prefixes and we can't pack them all into one file because of the prefix limit, so this will write any remaining values. 
         if (remainingValues.length > 0) {

package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts

@@ -5,7 +5,6 @@ import { getOwnNodeId, getOwnNodeIdAssert } from "../../-f-node-discovery/NodeDi
 import { pathValueArchives } from "../pathValueArchives";
 import { ArchiveLocker, ArchiveTransaction } from "./ArchiveLocks";
 import { copyArchiveFile, IArchives } from "sliftutils/storage/IArchives";
-import debugbreak from "debugbreak";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import { blue, green, magenta, red } from "socket-function/src/formatting/logColors";
 import { devDebugbreak, getDomain } from "../../config";
@@ -15,7 +14,7 @@ import { getNodeId } from "socket-function/src/nodeCache";
 import { logNodeStateStats, logNodeStats } from "../../-0-hooks/hooks";
 import { parsePath, toFileNameKVP, parseFileNameKVP } from "../../misc";
 import { logDisk } from "../../diagnostics/logs/diskLogger";
-import { retryFunctional, runInParallel } from "socket-function/src/batching";
+import { delay, retryFunctional, runInParallel } from "socket-function/src/batching";
 
 /*
     Data starts a .data files, from servers creating files.
@@ -30,13 +29,16 @@ const ARCHIVE_PROPAGATION_TIME = timeInSecond * 5;
 //  a few times though, just in case the storage system was having some issues. After enough
 //  tries though... we HAVE to just skip it, otherwise we will be stuck forever.
 const MAX_APPLY_TRIES = 10;
-const CONCURRENT_READ_COUNT = 32;
-const CONCURRENT_WRITE_COUNT = 16;
+// Confirms used to be zero-byte files, which the storage layer now treats as deletes, wiping every historical confirmation. Locked files created before this date are assumed confirmed, as anything invalid that old would have already been cleaned up.
+const ASSUME_CONFIRMED_BEFORE = Date.parse("2026-07-23T00:00:00.000Z");
+// NOTE: These values have been increased as our storage servers can handle way more throughput than before because it's using WebSockets. 
+const CONCURRENT_READ_COUNT = 128;
+const CONCURRENT_WRITE_COUNT = 128;
 const CONCURRENT_TRANSACTION_READS = 8;
 
 /** Moves by copy-then-delete (IArchives has no move). copyArchiveFile stamps the target with the source's write time, so ordering-by-writeTime survives the move. A missing source means another process already moved it, which is fine. */
 async function moveArchiveFile(from: IArchives, to: IArchives, path: string): Promise<void> {
-    let copied = await copyArchiveFile({ from, to, path });
+    let copied = await copyArchiveFile({ from, to, path, noFallbacks: true });
     if (!copied) return;
     await from.del(path);
 }
@@ -61,7 +63,8 @@ export function createArchiveLocker2(config: {
         async getKeys() {
             let filesRawValues = await archiveValues.findInfo("", { type: "files" });
             let filesRawLocks = await archiveLocks.findInfo("", { type: "files" });
-            let files: FileInfo[] = [...filesRawValues, ...filesRawLocks].map(x => ({
+            // Zero-byte files are deletes on new storage servers. Old storage servers still list them, so we filter here to make them behave the same.
+            let files: FileInfo[] = [...filesRawValues, ...filesRawLocks].filter(x => x.size > 0).map(x => ({
                 file: x.path,
                 createTime: x.createTime,
                 size: x.size,
@@ -74,16 +77,11 @@ export function createArchiveLocker2(config: {
             logNodeStats(`archives|Created TΔ`, formatNumber, 1);
         },
         async getValue(key) {
-            return getArchives(key).get(key);
+            // NOTE: I think we wanna not use fallbacks, Because we really want this to be atomic and safe. 
+            return getArchives(key).get(key, { noFallbacks: true });
         },
         async deleteKey(key) {
             let archives = await getArchives(key);
-            if (
-                !await archives.getInfo(key)
-                && !await archives.get(key)
-            ) {
-                return;
-            }
             // ACTUALLY delete confirms, as they don't contain that much information, and just clutter up the recycle bin.
             //  - DO archive transactions, as it might be useful to view the old transaction history.
             if (archives === archiveLocks && key.endsWith(".confirm")) {
@@ -217,7 +215,8 @@ export function createArchiveLocker2(config: {
                     while (pendingFiles.length > 0) {
                         let file = pendingFiles.pop()!;
                         let value = await storage.getValue(file.file);
-                        readResults.set(file.file, value);
+                        // NOTE: The way we've updated our archives is it's going to return a result unless it throws. And it shouldn't throw very often. The only reason it would return undefined is if the file was empty. After I do the deploy, get keys won't return empty files, but I need to fix this to do the deploy. So we're just going to default any empty returns to have a value. So we stop saying that files are missing and rereading infinitely. 
+                        readResults.set(file.file, value ?? Buffer.alloc(0));
                     }
                 }
                 await Promise.all(list(CONCURRENT_READ_COUNT).map(runReadThread));
@@ -420,6 +419,19 @@ class TransactionLocker {
             }
         };
         await Promise.all(list(CONCURRENT_WRITE_COUNT).map(runThread));
+
+        // Verify the files still exist. There shouldn't be any valid case where they disappear. If they do disappear, it's our storage's fault. 
+        let keys = await this.storage.getKeys();
+        let keysSet = new Set(keys.map(a => a.file));
+        let creates = transaction.ops.filter(a => a.type === "create") as { type: "create"; key: string; value: Buffer | undefined }[];
+        let missingCreates = creates.filter(a => !keysSet.has(a.key));
+        if (missingCreates.length > 0) {
+            console.error(`We are missing files that we just wrote to storage. This shouldn't be possible because the files can't be cleaned up until they get old enough, so this means our storage is broken`, {
+                missingCount: missingCreates.length,
+                firstMissing: missingCreates[0]?.key,
+                firstMissingSize: missingCreates[0]?.value?.length,
+            });
+        }
     }
     // Just writes the transaction (in a format readDataState can read)
     private async writeTransaction(transaction: Transaction & { seqNum: number; }): Promise<void> {
@@ -592,7 +604,7 @@ class TransactionLocker {
                 if (!file.file.endsWith(".locked")) continue;
                 let confirmKey = this.getConfirmKey(file.file);
                 let confirmFile = existingFiles.get(confirmKey);
-                if (confirmFile) {
+                if (confirmFile || file.createTime < ASSUME_CONFIRMED_BEFORE) {
                     currentDataFiles.set(file.file, file);
                 } else {
                     uncomfirmedCount++;
@@ -627,25 +639,67 @@ class TransactionLocker {
     }
     // #endregion
 
-    private isTransactionValid(transaction: Transaction, dataFiles: FileInfo[], rawDataFiles: FileInfo[]): boolean {
+    private async isTransactionValid(transaction: Transaction, dataFiles: FileInfo[], rawDataFiles: FileInfo[]): Promise<boolean> {
         let override = this.isTransactionValidOverride;
         if (override) return override(transaction, dataFiles, rawDataFiles);
         if (transaction.lockedFilesMustEqual) {
             let newLockedFiles = new Set(dataFiles.map(a => a.file).filter(a => a.endsWith(".locked")));
             let mustEqualSet = new Set(transaction.lockedFilesMustEqual);
-            if (mustEqualSet.size !== newLockedFiles.size) return false;
-            if (Array.from(mustEqualSet).some(a => !newLockedFiles.has(a))) return false;
+            let missingFiles = Array.from(mustEqualSet).filter(a => !newLockedFiles.has(a));
+            let extraFiles = Array.from(newLockedFiles).filter(a => !mustEqualSet.has(a));
+            let missingFile = missingFiles[0];
+            if (missingFile) {
+                console.error(`Missing file, cannot apply transaction`, {
+                    missing: missingFile,
+                    totalMissing: missingFiles.length,
+                    totalExtra: extraFiles.length,
+                });
+                return false;
+            }
+            let extraFile = extraFiles[0];
+            if (extraFile) {
+                let extra = await this.storage.getValue(extraFile);
+                console.error(`Extra file, cannot apply transaction`, {
+                    extra: extraFile,
+                    extraLength: extra?.length,
+                    extraFirst1024: extra?.slice(0, 1024).toString("hex"),
+                    totalMissing: missingFiles.length,
+                    totalExtra: extraFiles.length,
+                });
+                return false;
+            }
+            if (mustEqualSet.size !== newLockedFiles.size) {
+                console.error(`Transaction not applied (locked files must equal but size mismatch)`, {
+                    mustEqualSetSize: mustEqualSet.size,
+                    newLockedFilesSize: newLockedFiles.size,
+                });
+                return false;
+            }
         }
         let existingFiles = new Set(dataFiles.map(a => a.file));
         let existingRawFiles = new Set(rawDataFiles.map(a => a.file));
-        return transaction.ops.every(a => {
-            const type = a.type;
-            // Create files might disappear if we hang and they get cleaned up
-            if (type === "create") return existingRawFiles.has(a.key);
-            if (type === "delete") return existingFiles.has(a.key);
-            let unhandled: never = type;
-            throw new Error(`Unhandled type: ${unhandled}`);
-        });
+        let missingCreates = transaction.ops.filter(a => a.type === "create" && !existingRawFiles.has(a.key)) as { type: "create"; key: string; value: Buffer | undefined }[];
+        let missingDeletes = transaction.ops.filter(a => a.type === "delete" && !existingFiles.has(a.key)) as { type: "delete"; key: string }[];
+        if (missingCreates.length > 0 || missingDeletes.length > 0) {
+            console.error(`Transaction not applied (missing creates or deletes)`, {
+                missingCreateFirst: missingCreates[0]?.key,
+                missingDeleteFirst: missingDeletes[0]?.key,
+                missingCreateFirstSize: missingCreates[0]?.value?.length,
+                totalMissingCreates: missingCreates.length,
+                totalMissingDeletes: missingDeletes.length,
+            });
+            return false;
+        }
+        for (let op of transaction.ops) {
+            const type = op.type;
+            if (type === "create") {
+            } else if (type === "delete") {
+            } else {
+                let unhandled: never = type;
+                throw new Error(`Unhandled op type: ${unhandled}`);
+            }
+        }
+        return true;
     }
 
     // NOTE: If the transaction was applied long in the past, this will return false. HOWEVER, this is fine,
@@ -681,7 +735,7 @@ class TransactionLocker {
         let createCount = transaction.ops.filter(a => a.type === "create").length;
         let deleteCount = transaction.ops.filter(a => a.type === "delete").length;
         let lockedFiles = transaction.lockedFilesMustEqual?.length;
-        console.info(`Applying transaction with ${createCount} file creates and ${deleteCount} file deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`, {
+        console.log(`Applying transaction with ${createCount} file creates and ${deleteCount} file deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`, {
             transactions: transaction.ops.map(x => JSON.stringify(x)),
         });
         logNodeStats(`archives|TΔ Apply`, formatNumber, 1);
@@ -707,9 +761,29 @@ class TransactionLocker {
         };
         await Promise.all(list(CONCURRENT_WRITE_COUNT).map(runThread));
 
-        console.info(`Applied transaction with ${createCount} file creates and file ${deleteCount} deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`, {
+        console.log(`Applied transaction with ${createCount} file creates and file ${deleteCount} deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`, {
             transactions: transaction.ops.map(x => JSON.stringify(x)),
         });
+
+        let newKeys = await this.storage.getKeys();
+        let newKeysSet = new Set(newKeys.map(a => a.file));
+        let createFiles = transaction.ops.filter(a => a.type === "create") as { type: "create"; key: string; value: Buffer | undefined }[];
+        let deleteFiles = transaction.ops.filter(a => a.type === "delete") as { type: "delete"; key: string }[];
+        let deleteSet = new Set(deleteFiles.map(a => a.key));
+
+        let missingCreates = createFiles.filter(a => !newKeysSet.has(a.key));
+        let extraFiles = newKeys.filter(a => deleteSet.has(a.file));
+
+        if (missingCreates.length > 0 || extraFiles.length > 0) {
+            console.error(`The storage system is broken and did not actually apply our transaction changes (transaction ${transaction.source.file})`, {
+                missingCreateFirst: missingCreates[0]?.key,
+                missingDeleteFirst: extraFiles[0]?.file,
+                missingCreateFirstSize: missingCreates[0]?.value?.length,
+                missingDeleteFirstSize: extraFiles[0]?.size,
+                totalMissingCreates: missingCreates.length,
+                totalExtraFiles: extraFiles.length,
+            });
+        }
     }
 
     /** Only returns data files (no transaction files, or confirmations).
@@ -871,6 +945,13 @@ class TransactionLocker {
         this.perf.checkpoint("apply transaction");
         this.transactionAppliedCount.set(activeT.seqNum, applyCount + 1);
 
+        {
+            let dataState = await this.readDataState();
+            if (!this.wasTransactionApplied(activeT, dataState.confirmedDataFiles, dataState.rawDataFiles)) {
+                console.error(`Transaction ${activeT.seqNum} was not applied, even though we just applied it.`);
+            }
+        }
+
         // Run again, until we can be reasonable sure activeT isn't changing. We can be wrong,
         //      though, which will just result in any transaction depending on activeT being
         //      rejecting, and having to be inserted again.
@@ -882,7 +963,7 @@ class TransactionLocker {
         let dels = transaction.ops.filter(a => a.type === "delete").length;
         let creates = transaction.ops.filter(a => a.type === "create").length;
         let createBytes = transaction.ops.map(a => a.type === "create" && a.value?.length || 0).reduce((a, b) => a + b, 0);
-        console.info(`Starting transaction with ${creates} file creates and ${dels} file deletes, ${formatNumber(createBytes)}B`, {
+        console.log(`Starting transaction with ${creates} file creates and ${dels} file deletes, ${formatNumber(createBytes)}B`, {
             createFilesNames: transaction.ops.filter(a => a.type === "create").map(a => a.key),
             deleteFilesNames: transaction.ops.filter(a => a.type === "delete").map(a => a.key),
         });
@@ -909,9 +990,12 @@ class TransactionLocker {
 
         while (true) {
             let beforeData = await this.getFilesBase();
-            if (!this.isTransactionValid(transaction, beforeData.dataFiles, beforeData.rawDataFiles)) {
+            if (!await this.isTransactionValid(transaction, beforeData.dataFiles, beforeData.rawDataFiles)) {
                 logNodeStats(`archives|TΔ Rejected`, formatNumber, 1);
-                console.info(`Finished transaction with rejection, ${transaction.ops.length} ops`);
+                console.log(`Finished transaction with rejection, ${transaction.ops.length} ops`);
+                await delay(30 * 1000);
+                let valid2 = await this.isTransactionValid(transaction, beforeData.dataFiles, beforeData.rawDataFiles);
+                console.log(`Valid2: ${valid2}`);
                 return "rejected";
             }
 
@@ -920,7 +1004,7 @@ class TransactionLocker {
             let afterData = await this.getFilesBase();
             if (this.wasTransactionApplied(transaction, afterData.dataFiles, afterData.rawDataFiles)) {
                 logNodeStats(`archives|TΔ Accepted`, formatNumber, 1);
-                console.info(`Finished transaction with ${transaction.ops.length} ops`);
+                console.log(`Finished transaction with ${transaction.ops.length} ops`);
                 return "accepted";
             }
         }

package/src/0-path-value-core/pathValueArchives.ts

@@ -181,7 +181,7 @@ export class PathValueArchives {
 
     @measureFnc
     public async archiveValues(values: PathValue[]) {
-        let parts = PathRouter.getPathIdentifierTargets(values, authorityLookup.getOurSpec());
+        let parts = await PathRouter.getPathIdentifierTargets(values, authorityLookup.getOurSpec());
 
         for (let [pathIdentifier, values] of parts) {
             let encodedObj = await this.encodeValuePaths(values);
@@ -316,8 +316,9 @@ export class PathValueArchives {
                     let dataPath = pendingDataPaths.pop()!;
                     if (readCache.has(dataPath)) continue;
                     let data = await archives().get(dataPath);
-                    if (!data) continue;
-                    readCache.set(dataPath, data);
+                    // NOTE: The way we've updated our archives is it's going to return a result unless it throws. And it shouldn't throw very often. The only reason it would return undefined is if the file was empty. After I do the deploy, get keys won't return empty files, but I need to fix this to do the deploy. So we're just going to default any empty returns to have a value. So we stop saying that files are missing and rereading infinitely. 
+                    //if (!data) continue;
+                    readCache.set(dataPath, data || Buffer.alloc(0));
                 }
             }
             await Promise.all(list(32).map(runReadThread));
@@ -359,6 +360,7 @@ export class PathValueArchives {
 
         await safeLoop({ data: dataPaths, name: blue("Load PathValues From Buffer") }, async dataFile => {
             let data = readCache.get(dataFile);
+            if (!data?.length) return;
             if (!data) throw new Error(`Data file ${dataFile} not in lookup which we just checked?`);
             totalSize += data.byteLength;
             fileCount++;
@@ -410,6 +412,7 @@ export class PathValueArchives {
         tooOldValues: number;
     }> {
         const { path, data } = config;
+        if (data.length === 0) return { values: [], rawCount: 0, tooOldValues: 0 };
         let tooOldValues = 0;
 
         let dataValues: PathValue[] = [];
@@ -423,10 +426,10 @@ export class PathValueArchives {
                 skipValues: config.skipValues,
             });
             if (dataValues.length !== decodedObj.valueCount) {
-                console.warn(`Bad archive data file at ${config.path}, Decoded count ${formatNumber(decodedObj.valueCount)} !== count in file name ${formatNumber(dataValues.length)} (${decodedObj.valueCount} !== ${dataValues.length})`);
+                console.warn(`Bad archive data file (size ${data.length}) at ${config.path}, Decoded count ${formatNumber(decodedObj.valueCount)} !== count in file name ${formatNumber(dataValues.length)} (${decodedObj.valueCount} !== ${dataValues.length})`);
             }
         } catch (e: any) {
-            console.warn(`Bad archive data file at ${config.path}, error: ${e.stack}`);
+            console.warn(`Bad archive data file (size ${data.length}) at ${config.path}, error: ${e.stack}`);
         }
         let rawCount = dataValues.length;
 

package/src/2-proxy/archiveMoveHarness.ts

@@ -250,7 +250,7 @@ export async function runArchiveMover(config: {
 
 
             for (let [key, values] of Object.entries(result.newValues)) {
-                let targets = PathRouter.getPathIdentifierTargets(values, authority);
+                let targets = await PathRouter.getPathIdentifierTargets(values, authority);
                 for (let [target, values] of targets) {
                     await addValues(target + "/", values, key);
                 }

package/src/archiveapps/archiveJoinEntry.ts

@@ -99,7 +99,7 @@ async function runGenesisJoinIteration(config?: { force?: boolean }) {
                 transaction.deleteFiles.push(file);
             }
 
-            let targets = PathRouter.getPathIdentifierTargets(allCombinedValues, authoritySpec);
+            let targets = await PathRouter.getPathIdentifierTargets(allCombinedValues, authoritySpec);
             for (let [target, values] of targets) {
 
                 let dataObj = await pathValueArchives.encodeValuePaths(values, {

package/src/deployManager/components/StoragePage.tsx

@@ -6,7 +6,7 @@ import { sort, timeInSecond } from "socket-function/src/misc";
 import preact from "preact";
 import { Querysub } from "../../4-querysub/Querysub";
 import { listServerBuckets, clearServerWriteStats, activateServerBucket, getServerActiveBucket } from "sliftutils/storage/remoteStorage/createArchives";
-import type { ServerBucketInfo, BucketDiskInfo, BucketWriteStats, ActiveBucketInfo } from "sliftutils/storage/remoteStorage/storageServerState";
+import type { ServerBucketInfo, BucketWriteStats, ActiveBucketInfo } from "sliftutils/storage/remoteStorage/storageServerState";
 import type { ArchivesConfig, SyncActivity, RemoteConfig, HostedConfig } from "sliftutils/storage/IArchives";
 import { parseHostedUrl } from "sliftutils/storage/remoteStorage/remoteConfig";
 import { UsageBar, getUsageThresholds } from "../../library-components/UsageBar";
@@ -19,6 +19,7 @@ import { Table } from "../../5-diagnostics/Table";
 import { RouteConfigView } from "./RouteConfigView";
 import { Tag, SourceTag, getSourceTags } from "./Tag";
 import { STORAGE_COMMAND_PREFIX } from "../serviceCategories";
+import { BucketDiskInfo } from "sliftutils/storage/remoteStorage/bucketDisk";
 
 module.hotreload = true;
 

package/src/diagnostics/logs/errorNotifications2/errorNotifications.ts

@@ -48,7 +48,7 @@ let pendingDiscordNotifications = new Map<string, {
     errorCount: number;
     isNew: boolean;
 }>();
-async function getSuppressionEntries(): Promise<SuppressionEntry[]> {
+export async function getSuppressionEntries(): Promise<SuppressionEntry[]> {
     return await suppression.values();
 }
 

package/src/library-components/SyncedController.ts

@@ -295,14 +295,12 @@ export function getSyncedController<T extends {
                         });
                         // We have to wait until we actually commit before making the call. Otherwise, if this commit is rejected because we need to do synchronized values, we'll call the function twice.
                         let fnc = controller.nodes[nodeId][fncName] as any;
-                        Querysub.onCommitFinished(() => {
-                            // Doesn't on commit finished also implicitly mean we're all synced? Pretty sure that isAll synced is making things break. 
-                            //if (Querysub.isAllSynced()) {
+                        // NOTE: We can't do on commit finish because we later have a trigger on promise finish for our own promise. And if we don't start the promise call until we finish the commit, And we can't finish the commit until we start the promise call, it'll never finish. So... The best we can do is just check for if everything's synced at this current point. 
+                        if (Querysub.isAllSynced()) {
                             void Promise.resolve().then(() => {
                                 doPromiseCall();
                             });
-                            //}
-                        });
+                        }
                         function doPromiseCall() {
                             let promise = fnc(...args) as Promise<unknown>;
                             promiseObjBase.resolve(promise);

package/src/storageSetup.ts

@@ -15,7 +15,7 @@ async function main() {
 
     let trustedMachineIds = await getTrustedMachineIds();
 
-    let { setTrustedMachines } = await import("sliftutils/storage/remoteStorage/storageServerState");
+    let { setTrustedMachines } = await import("sliftutils/storage/remoteStorage/serverConfig");
     await setTrustedMachines({ account: "root", machineIds: trustedMachineIds });
     console.log(`Synchronized ${trustedMachineIds.length} trusted machines`);