querysub 0.591.0 → 0.593.0

package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts

@@ -5,7 +5,6 @@ import { getOwnNodeId, getOwnNodeIdAssert } from "../../-f-node-discovery/NodeDi
 import { pathValueArchives } from "../pathValueArchives";
 import { ArchiveLocker, ArchiveTransaction } from "./ArchiveLocks";
 import { copyArchiveFile, IArchives } from "sliftutils/storage/IArchives";
-import debugbreak from "debugbreak";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import { blue, green, magenta, red } from "socket-function/src/formatting/logColors";
 import { devDebugbreak, getDomain } from "../../config";
@@ -15,7 +14,7 @@ import { getNodeId } from "socket-function/src/nodeCache";
 import { logNodeStateStats, logNodeStats } from "../../-0-hooks/hooks";
 import { parsePath, toFileNameKVP, parseFileNameKVP } from "../../misc";
 import { logDisk } from "../../diagnostics/logs/diskLogger";
-import { retryFunctional, runInParallel } from "socket-function/src/batching";
+import { delay, retryFunctional, runInParallel } from "socket-function/src/batching";
 
 /*
     Data starts a .data files, from servers creating files.
@@ -30,8 +29,11 @@ const ARCHIVE_PROPAGATION_TIME = timeInSecond * 5;
 //  a few times though, just in case the storage system was having some issues. After enough
 //  tries though... we HAVE to just skip it, otherwise we will be stuck forever.
 const MAX_APPLY_TRIES = 10;
+// Confirms used to be zero-byte files, which the storage layer now treats as deletes, wiping every historical confirmation. Locked files created before this date are assumed confirmed, as anything invalid that old would have already been cleaned up.
+const ASSUME_CONFIRMED_BEFORE = Date.parse("2026-07-23T00:00:00.000Z");
 const CONCURRENT_READ_COUNT = 32;
 const CONCURRENT_WRITE_COUNT = 16;
+const CONCURRENT_TRANSACTION_READS = 8;
 
 /** Moves by copy-then-delete (IArchives has no move). copyArchiveFile stamps the target with the source's write time, so ordering-by-writeTime survives the move. A missing source means another process already moved it, which is fine. */
 async function moveArchiveFile(from: IArchives, to: IArchives, path: string): Promise<void> {
@@ -60,7 +62,8 @@ export function createArchiveLocker2(config: {
         async getKeys() {
             let filesRawValues = await archiveValues.findInfo("", { type: "files" });
             let filesRawLocks = await archiveLocks.findInfo("", { type: "files" });
-            let files: FileInfo[] = [...filesRawValues, ...filesRawLocks].map(x => ({
+            // Zero-byte files are deletes on new storage servers. Old storage servers still list them, so we filter here to make them behave the same.
+            let files: FileInfo[] = [...filesRawValues, ...filesRawLocks].filter(x => x.size > 0).map(x => ({
                 file: x.path,
                 createTime: x.createTime,
                 size: x.size,
@@ -77,12 +80,6 @@ export function createArchiveLocker2(config: {
         },
         async deleteKey(key) {
             let archives = await getArchives(key);
-            if (
-                !await archives.getInfo(key)
-                && !await archives.get(key)
-            ) {
-                return;
-            }
             // ACTUALLY delete confirms, as they don't contain that much information, and just clutter up the recycle bin.
             //  - DO archive transactions, as it might be useful to view the old transaction history.
             if (archives === archiveLocks && key.endsWith(".confirm")) {
@@ -216,7 +213,8 @@ export function createArchiveLocker2(config: {
                     while (pendingFiles.length > 0) {
                         let file = pendingFiles.pop()!;
                         let value = await storage.getValue(file.file);
-                        readResults.set(file.file, value);
+                        // NOTE: The way we've updated our archives is it's going to return a result unless it throws. And it shouldn't throw very often. The only reason it would return undefined is if the file was empty. After I do the deploy, get keys won't return empty files, but I need to fix this to do the deploy. So we're just going to default any empty returns to have a value. So we stop saying that files are missing and rereading infinitely. 
+                        readResults.set(file.file, value ?? Buffer.alloc(0));
                     }
                 }
                 await Promise.all(list(CONCURRENT_READ_COUNT).map(runReadThread));
@@ -391,7 +389,7 @@ class TransactionLocker {
         let path = this.getConfirmKey(key);
         console.info(`Creating confirmation for ${key}, ${countLeft} left`);
         if (!await this.storage.getValue(path)) {
-            await this.storage.setValue(path, Buffer.from(""));
+            await this.storage.setValue(path, Buffer.from(JSON.stringify({ time: Date.now(), node: getOwnNodeId() })));
         }
         return path;
     }
@@ -410,11 +408,28 @@ class TransactionLocker {
                 if (!op) return;
                 if (op.type === "create") {
                     let key = op.key;
-                    await this.storage.setValue(key, op.value || Buffer.from(""));
+                    const value = op.value;
+                    if (!value || value.length === 0) {
+                        throw new Error(`Create op for ${key} has an empty value. Empty files are treated as deletes by the storage layer, so this create would vanish and the transaction could never be confirmed.`);
+                    }
+                    await this.storage.setValue(key, value);
                 }
             }
         };
         await Promise.all(list(CONCURRENT_WRITE_COUNT).map(runThread));
+
+        // Verify the files still exist. There shouldn't be any valid case where they disappear. If they do disappear, it's our storage's fault. 
+        let keys = await this.storage.getKeys();
+        let keysSet = new Set(keys.map(a => a.file));
+        let creates = transaction.ops.filter(a => a.type === "create") as { type: "create"; key: string; value: Buffer | undefined }[];
+        let missingCreates = creates.filter(a => !keysSet.has(a.key));
+        if (missingCreates.length > 0) {
+            console.error(`We are missing files that we just wrote to storage. This shouldn't be possible because the files can't be cleaned up until they get old enough, so this means our storage is broken`, {
+                missingCount: missingCreates.length,
+                firstMissing: missingCreates[0]?.key,
+                firstMissingSize: missingCreates[0]?.value?.length,
+            });
+        }
     }
     // Just writes the transaction (in a format readDataState can read)
     private async writeTransaction(transaction: Transaction & { seqNum: number; }): Promise<void> {
@@ -494,11 +509,16 @@ class TransactionLocker {
                 createTime: number;
                 source: FileInfo;
             })[] = [];
-            for (let file of files) {
-                if (!file.file.endsWith(".transaction")) continue;
-                let tFile = file;
+            let transactionFiles = files.filter(x => x.file.endsWith(".transaction"));
+            // Set when any reader finds a missing value, so the other parallel readers stop early.
+            let missingValue = false;
+            const readTransaction = async (tFile: FileInfo) => {
+                if (missingValue) return;
                 let buffer = await this.storage.getValue(tFile.file);
-                if (!buffer) return false;
+                if (!buffer) {
+                    missingValue = true;
+                    return;
+                }
                 bufferCache.set(tFile.file, buffer);
                 let transaction: Transaction | undefined;
                 try {
@@ -507,7 +527,7 @@ class TransactionLocker {
                     // Maybe it is actively being written to? Or it is just broken. Skip it.
                     console.error(`Failed to parse transaction. Ignoring, ${tFile.file}, ${e.stack}`);
                 }
-                if (!transaction) continue;
+                if (!transaction) return;
 
                 let { name } = parsePath(tFile.file);
                 let kvp = parseFileNameKVP(name);
@@ -518,8 +538,11 @@ class TransactionLocker {
                     createTime: tFile.createTime,
                     source: tFile,
                 });
-            }
+            };
+            let readTransactionParallel = runInParallel({ parallelCount: CONCURRENT_TRANSACTION_READS }, readTransaction);
+            await Promise.all(transactionFiles.map(readTransactionParallel));
             this.perf.checkpoint("read transaction files");
+            if (missingValue) return false;
 
             // Check all of our files to see if any have changed (or if there a new files).
             //  Because we don't reuse file names, or change them, it means if a file exists
@@ -579,7 +602,7 @@ class TransactionLocker {
                 if (!file.file.endsWith(".locked")) continue;
                 let confirmKey = this.getConfirmKey(file.file);
                 let confirmFile = existingFiles.get(confirmKey);
-                if (confirmFile) {
+                if (confirmFile || file.createTime < ASSUME_CONFIRMED_BEFORE) {
                     currentDataFiles.set(file.file, file);
                 } else {
                     uncomfirmedCount++;
@@ -614,25 +637,67 @@ class TransactionLocker {
     }
     // #endregion
 
-    private isTransactionValid(transaction: Transaction, dataFiles: FileInfo[], rawDataFiles: FileInfo[]): boolean {
+    private async isTransactionValid(transaction: Transaction, dataFiles: FileInfo[], rawDataFiles: FileInfo[]): Promise<boolean> {
         let override = this.isTransactionValidOverride;
         if (override) return override(transaction, dataFiles, rawDataFiles);
         if (transaction.lockedFilesMustEqual) {
             let newLockedFiles = new Set(dataFiles.map(a => a.file).filter(a => a.endsWith(".locked")));
             let mustEqualSet = new Set(transaction.lockedFilesMustEqual);
-            if (mustEqualSet.size !== newLockedFiles.size) return false;
-            if (Array.from(mustEqualSet).some(a => !newLockedFiles.has(a))) return false;
+            let missingFiles = Array.from(mustEqualSet).filter(a => !newLockedFiles.has(a));
+            let extraFiles = Array.from(newLockedFiles).filter(a => !mustEqualSet.has(a));
+            let missingFile = missingFiles[0];
+            if (missingFile) {
+                console.error(`Missing file, cannot apply transaction`, {
+                    missing: missingFile,
+                    totalMissing: missingFiles.length,
+                    totalExtra: extraFiles.length,
+                });
+                return false;
+            }
+            let extraFile = extraFiles[0];
+            if (extraFile) {
+                let extra = await this.storage.getValue(extraFile);
+                console.error(`Extra file, cannot apply transaction`, {
+                    extra: extraFile,
+                    extraLength: extra?.length,
+                    extraFirst1024: extra?.slice(0, 1024).toString("hex"),
+                    totalMissing: missingFiles.length,
+                    totalExtra: extraFiles.length,
+                });
+                return false;
+            }
+            if (mustEqualSet.size !== newLockedFiles.size) {
+                console.error(`Transaction not applied (locked files must equal but size mismatch)`, {
+                    mustEqualSetSize: mustEqualSet.size,
+                    newLockedFilesSize: newLockedFiles.size,
+                });
+                return false;
+            }
         }
         let existingFiles = new Set(dataFiles.map(a => a.file));
         let existingRawFiles = new Set(rawDataFiles.map(a => a.file));
-        return transaction.ops.every(a => {
-            const type = a.type;
-            // Create files might disappear if we hang and they get cleaned up
-            if (type === "create") return existingRawFiles.has(a.key);
-            if (type === "delete") return existingFiles.has(a.key);
-            let unhandled: never = type;
-            throw new Error(`Unhandled type: ${unhandled}`);
-        });
+        let missingCreates = transaction.ops.filter(a => a.type === "create" && !existingRawFiles.has(a.key)) as { type: "create"; key: string; value: Buffer | undefined }[];
+        let missingDeletes = transaction.ops.filter(a => a.type === "delete" && !existingFiles.has(a.key)) as { type: "delete"; key: string }[];
+        if (missingCreates.length > 0 || missingDeletes.length > 0) {
+            console.error(`Transaction not applied (missing creates or deletes)`, {
+                missingCreateFirst: missingCreates[0]?.key,
+                missingDeleteFirst: missingDeletes[0]?.key,
+                missingCreateFirstSize: missingCreates[0]?.value?.length,
+                totalMissingCreates: missingCreates.length,
+                totalMissingDeletes: missingDeletes.length,
+            });
+            return false;
+        }
+        for (let op of transaction.ops) {
+            const type = op.type;
+            if (type === "create") {
+            } else if (type === "delete") {
+            } else {
+                let unhandled: never = type;
+                throw new Error(`Unhandled op type: ${unhandled}`);
+            }
+        }
+        return true;
     }
 
     // NOTE: If the transaction was applied long in the past, this will return false. HOWEVER, this is fine,
@@ -664,11 +729,11 @@ class TransactionLocker {
         }
         return true;
     }
-    private async applyTransaction(transaction: Transaction): Promise<void> {
+    private async applyTransaction(transaction: Transaction & { source: FileInfo }): Promise<void> {
         let createCount = transaction.ops.filter(a => a.type === "create").length;
         let deleteCount = transaction.ops.filter(a => a.type === "delete").length;
         let lockedFiles = transaction.lockedFilesMustEqual?.length;
-        console.info(`Applying transaction with ${createCount} file creates and ${deleteCount} file deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`, {
+        console.log(`Applying transaction with ${createCount} file creates and ${deleteCount} file deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`, {
             transactions: transaction.ops.map(x => JSON.stringify(x)),
         });
         logNodeStats(`archives|TΔ Apply`, formatNumber, 1);
@@ -685,7 +750,7 @@ class TransactionLocker {
                 if (op.type === "create") {
                     await this.createConfirm(op.key, opsRemaining.length);
                 } else if (op.type === "delete") {
-                    await this.deleteDataFile(op.key, `transaction (${getOwnNodeId()})`, opsRemaining.length);
+                    await this.deleteDataFile(op.key, `transaction for ${transaction.source.file}`, opsRemaining.length);
                 } else {
                     let unhandled: never = op;
                     throw new Error(`Unhandled type: ${unhandled}`);
@@ -694,9 +759,29 @@ class TransactionLocker {
         };
         await Promise.all(list(CONCURRENT_WRITE_COUNT).map(runThread));
 
-        console.info(`Applied transaction with ${createCount} file creates and file ${deleteCount} deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`, {
+        console.log(`Applied transaction with ${createCount} file creates and file ${deleteCount} deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`, {
             transactions: transaction.ops.map(x => JSON.stringify(x)),
         });
+
+        let newKeys = await this.storage.getKeys();
+        let newKeysSet = new Set(newKeys.map(a => a.file));
+        let createFiles = transaction.ops.filter(a => a.type === "create") as { type: "create"; key: string; value: Buffer | undefined }[];
+        let deleteFiles = transaction.ops.filter(a => a.type === "delete") as { type: "delete"; key: string }[];
+        let deleteSet = new Set(deleteFiles.map(a => a.key));
+
+        let missingCreates = createFiles.filter(a => !newKeysSet.has(a.key));
+        let extraFiles = newKeys.filter(a => deleteSet.has(a.file));
+
+        if (missingCreates.length > 0 || extraFiles.length > 0) {
+            console.error(`The storage system is broken and did not actually apply our transaction changes (transaction ${transaction.source.file})`, {
+                missingCreateFirst: missingCreates[0]?.key,
+                missingDeleteFirst: extraFiles[0]?.file,
+                missingCreateFirstSize: missingCreates[0]?.value?.length,
+                missingDeleteFirstSize: extraFiles[0]?.size,
+                totalMissingCreates: missingCreates.length,
+                totalExtraFiles: extraFiles.length,
+            });
+        }
     }
 
     /** Only returns data files (no transaction files, or confirmations).
@@ -858,6 +943,13 @@ class TransactionLocker {
         this.perf.checkpoint("apply transaction");
         this.transactionAppliedCount.set(activeT.seqNum, applyCount + 1);
 
+        {
+            let dataState = await this.readDataState();
+            if (!this.wasTransactionApplied(activeT, dataState.confirmedDataFiles, dataState.rawDataFiles)) {
+                console.error(`Transaction ${activeT.seqNum} was not applied, even though we just applied it.`);
+            }
+        }
+
         // Run again, until we can be reasonable sure activeT isn't changing. We can be wrong,
         //      though, which will just result in any transaction depending on activeT being
         //      rejecting, and having to be inserted again.
@@ -869,7 +961,7 @@ class TransactionLocker {
         let dels = transaction.ops.filter(a => a.type === "delete").length;
         let creates = transaction.ops.filter(a => a.type === "create").length;
         let createBytes = transaction.ops.map(a => a.type === "create" && a.value?.length || 0).reduce((a, b) => a + b, 0);
-        console.info(`Starting transaction with ${creates} file creates and ${dels} file deletes, ${formatNumber(createBytes)}B`, {
+        console.log(`Starting transaction with ${creates} file creates and ${dels} file deletes, ${formatNumber(createBytes)}B`, {
             createFilesNames: transaction.ops.filter(a => a.type === "create").map(a => a.key),
             deleteFilesNames: transaction.ops.filter(a => a.type === "delete").map(a => a.key),
         });
@@ -896,9 +988,12 @@ class TransactionLocker {
 
         while (true) {
             let beforeData = await this.getFilesBase();
-            if (!this.isTransactionValid(transaction, beforeData.dataFiles, beforeData.rawDataFiles)) {
+            if (!await this.isTransactionValid(transaction, beforeData.dataFiles, beforeData.rawDataFiles)) {
                 logNodeStats(`archives|TΔ Rejected`, formatNumber, 1);
-                console.info(`Finished transaction with rejection, ${transaction.ops.length} ops`);
+                console.log(`Finished transaction with rejection, ${transaction.ops.length} ops`);
+                await delay(30 * 1000);
+                let valid2 = await this.isTransactionValid(transaction, beforeData.dataFiles, beforeData.rawDataFiles);
+                console.log(`Valid2: ${valid2}`);
                 return "rejected";
             }
 
@@ -907,7 +1002,7 @@ class TransactionLocker {
             let afterData = await this.getFilesBase();
             if (this.wasTransactionApplied(transaction, afterData.dataFiles, afterData.rawDataFiles)) {
                 logNodeStats(`archives|TΔ Accepted`, formatNumber, 1);
-                console.info(`Finished transaction with ${transaction.ops.length} ops`);
+                console.log(`Finished transaction with ${transaction.ops.length} ops`);
                 return "accepted";
             }
         }

package/src/0-path-value-core/archiveLocks/archiveSnapshots.ts

@@ -1,5 +1,4 @@
 import { sha256 } from "js-sha256";
-import { getArchives } from "../../-a-archives/archives";
 import { getAllNodeIds, getOwnThreadId } from "../../-f-node-discovery/NodeDiscovery";
 import { archives, pathValueArchives } from "../pathValueArchives";
 import { ignoreErrors, logErrors, timeoutToUndefinedSilent } from "../../errors";

package/src/0-path-value-core/pathValueArchives.ts

@@ -139,6 +139,7 @@ export class PathValueArchives {
         if (!config?.noFilter) {
             values = await this.filterForDiskStorage(values);
         }
+        if (values.length === 0) return undefined;
 
         let buffers = await pathValueSerializer.serialize(values, {
             noLocks: true,
@@ -180,7 +181,7 @@ export class PathValueArchives {
 
     @measureFnc
     public async archiveValues(values: PathValue[]) {
-        let parts = PathRouter.getPathIdentifierTargets(values, authorityLookup.getOurSpec());
+        let parts = await PathRouter.getPathIdentifierTargets(values, authorityLookup.getOurSpec());
 
         for (let [pathIdentifier, values] of parts) {
             let encodedObj = await this.encodeValuePaths(values);
@@ -315,8 +316,9 @@ export class PathValueArchives {
                     let dataPath = pendingDataPaths.pop()!;
                     if (readCache.has(dataPath)) continue;
                     let data = await archives().get(dataPath);
-                    if (!data) continue;
-                    readCache.set(dataPath, data);
+                    // NOTE: The way we've updated our archives is it's going to return a result unless it throws. And it shouldn't throw very often. The only reason it would return undefined is if the file was empty. After I do the deploy, get keys won't return empty files, but I need to fix this to do the deploy. So we're just going to default any empty returns to have a value. So we stop saying that files are missing and rereading infinitely. 
+                    //if (!data) continue;
+                    readCache.set(dataPath, data || Buffer.alloc(0));
                 }
             }
             await Promise.all(list(32).map(runReadThread));
@@ -358,6 +360,7 @@ export class PathValueArchives {
 
         await safeLoop({ data: dataPaths, name: blue("Load PathValues From Buffer") }, async dataFile => {
             let data = readCache.get(dataFile);
+            if (!data?.length) return;
             if (!data) throw new Error(`Data file ${dataFile} not in lookup which we just checked?`);
             totalSize += data.byteLength;
             fileCount++;
@@ -409,6 +412,7 @@ export class PathValueArchives {
         tooOldValues: number;
     }> {
         const { path, data } = config;
+        if (data.length === 0) return { values: [], rawCount: 0, tooOldValues: 0 };
         let tooOldValues = 0;
 
         let dataValues: PathValue[] = [];
@@ -422,10 +426,10 @@ export class PathValueArchives {
                 skipValues: config.skipValues,
             });
             if (dataValues.length !== decodedObj.valueCount) {
-                console.warn(`Bad archive data file at ${config.path}, Decoded count ${formatNumber(decodedObj.valueCount)} !== count in file name ${formatNumber(dataValues.length)} (${decodedObj.valueCount} !== ${dataValues.length})`);
+                console.warn(`Bad archive data file (size ${data.length}) at ${config.path}, Decoded count ${formatNumber(decodedObj.valueCount)} !== count in file name ${formatNumber(dataValues.length)} (${decodedObj.valueCount} !== ${dataValues.length})`);
             }
         } catch (e: any) {
-            console.warn(`Bad archive data file at ${config.path}, error: ${e.stack}`);
+            console.warn(`Bad archive data file (size ${data.length}) at ${config.path}, error: ${e.stack}`);
         }
         let rawCount = dataValues.length;
 

package/src/2-proxy/archiveMoveHarness.ts

@@ -250,7 +250,7 @@ export async function runArchiveMover(config: {
 
 
             for (let [key, values] of Object.entries(result.newValues)) {
-                let targets = PathRouter.getPathIdentifierTargets(values, authority);
+                let targets = await PathRouter.getPathIdentifierTargets(values, authority);
                 for (let [target, values] of targets) {
                     await addValues(target + "/", values, key);
                 }

package/src/4-deploy/edgeBootstrap.ts

@@ -18,26 +18,28 @@ declare global {
     var getEdgeNodeConfig: undefined | (() => Promise<EdgeNodeConfig>);
 }
 
-let getCachedConfig = cache(async (url: string): Promise<EdgeNodesIndex | undefined> => {
+let getCachedConfig = cache(async (urlsKey: string): Promise<EdgeNodesIndex | undefined> => {
     setTimeout(() => {
-        getCachedConfig.clear(url);
+        getCachedConfig.clear(urlsKey);
         // NOTE: We want deploying to be fast, so we don't cache for long. We also don't
         //  get the new value every 30 seconds, as we don't want to create too many requests.
     }, timeInSecond * 30);
 
-    try {
-        let response = await fetch(url);
-        return await response.json();
-    } catch {
-        return undefined;
+    for (let url of urlsKey.split("\n")) {
+        try {
+            let response = await fetch(url);
+            if (!response.ok) continue;
+            return await response.json();
+        } catch { }
     }
+    return undefined;
 });
 
 export async function getEdgeBootstrapScript(config: {
-    edgeNodeConfigURL: string;
+    edgeNodeConfigURLs: string[];
 }): Promise<string> {
     return await measureBlock(async function getEdgeBootstrapScript() {
-        let cachedConfig = await getCachedConfig(config.edgeNodeConfigURL);
+        let cachedConfig = await getCachedConfig(config.edgeNodeConfigURLs.join("\n"));
         let allowedArgs = ["--local"];
         // TODO: Be smarter about this (getting values and not just flags)
         let argv = process.argv.filter(x => allowedArgs.includes(x));
@@ -72,7 +74,7 @@ declare global {
 */
 
 async function edgeNodeFunction(config: {
-    edgeNodeConfigURL: string;
+    edgeNodeConfigURLs: string[];
     cachedConfig: EdgeNodesIndex | undefined;
     argv: string[];
 }) {
@@ -302,8 +304,45 @@ async function edgeNodeFunction(config: {
             return false;
         }
     }
+    // Fetches all the URLs at the same time and uses whichever responds first, aborting the rest (so slow sources don't hold sockets — chrome limits us to 6 per domain). Racing redundant sources is normally wasteful, but this is the very first request of the page load, so minimizing its latency is a huge win.
+    async function fetchFirstEdgeIndex(urls: string[]): Promise<EdgeNodesIndex> {
+        let controllers = urls.map(() => new AbortController());
+        return await new Promise<EdgeNodesIndex>((resolve, reject) => {
+            let resolved = false;
+            let pendingCount = urls.length;
+            let errors: string[] = [];
+            if (!pendingCount) {
+                reject(new Error(`No edge node config URLs to fetch`));
+                return;
+            }
+            urls.forEach((url, index) => {
+                void (async () => {
+                    try {
+                        let response = await fetch(url, { signal: controllers[index].signal });
+                        if (!response.ok) throw new Error(`Bad status ${response.status} from ${url}`);
+                        let edgeIndex = await response.json() as EdgeNodesIndex;
+                        if (resolved) return;
+                        resolved = true;
+                        for (let otherIndex = 0; otherIndex < controllers.length; otherIndex++) {
+                            if (otherIndex !== index) {
+                                controllers[otherIndex].abort();
+                            }
+                        }
+                        resolve(edgeIndex);
+                    } catch (e: any) {
+                        errors.push(`${url}: ${e?.stack ?? e}`);
+                        pendingCount--;
+                        if (!resolved && pendingCount === 0) {
+                            reject(new Error(`All edge node config URLs failed: ${errors.join(" | ")}`));
+                        }
+                    }
+                })();
+            });
+        });
+    }
+
     async function getEdgeNodeConfig(): Promise<EdgeNodeConfig> {
-        let edgeIndex = cachedConfig || await (await fetch(config.edgeNodeConfigURL)).json() as EdgeNodesIndex;
+        let edgeIndex = cachedConfig || await fetchFirstEdgeIndex(config.edgeNodeConfigURLs);
         cachedConfig = undefined;
 
         edgeIndex.edgeNodes.sort((a, b) => -(a.bootTime - b.bootTime));

package/src/4-deploy/edgeNodes.ts

@@ -3,14 +3,12 @@ import { getOwnMachineId } from "sliftutils/misc/https/certs";
 import { devDebugbreak, getDomain, isLocal, isPublic, noSyncing } from "../config";
 import child_process from "child_process";
 import { getAllNodeIds, getOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
-import { getArchivesBackblazePublic } from "../-a-archives/archivesBackBlaze";
-import { nestArchives } from "../-a-archives/archives";
+import { getArchives2Public } from "../-a-archives/archives2";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { delay, runInSerial, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { compare, compareArray, isNodeTrue, sort, timeInMinute, timeInSecond } from "socket-function/src/misc";
 import { cacheLimited, lazy } from "socket-function/src/caching";
 import { canHaveChildren } from "socket-function/src/types";
-import { hostArchives } from "../-b-authorities/cdnAuthority";
 import { getModuleFromConfig } from "../3-path-functions/pathFunctionLoader";
 import path from "path";
 import debugbreak from "debugbreak";
@@ -31,7 +29,7 @@ import type { FunctionRunnerIndex } from "../4-querysub/FunctionRunnerTracking";
 const UPDATE_POLL_INTERVAL = timeInMinute * 15;
 const DEAD_NODE_COUNT_THRESHOLD = 15;
 
-const edgeNodeStorage = isNodeTrue() && nestArchives("edgenodes/", getArchivesBackblazePublic(getDomain()));
+const edgeNodeStorage = isNodeTrue() && getArchives2Public("edgenodes/");
 const edgeNodeIndexFile = "edge-nodes-index.json";
 
 const getEdgeNodeConfig = cacheLimited(10000, async (fileName: string): Promise<EdgeNodeConfig | undefined> => {
@@ -265,19 +263,15 @@ async function publishOwnScheduledShutdown(time: number) {
     await updateEdgeNodesFile();
 }
 
-export const getEdgeNodeConfigURL = lazy(async () => {
-    let { getURL } = await hostArchives({
-        archives: edgeNodeStorage,
-        subdomain: `edge`,
-        domain: getDomain(),
-    });
-    return await getURL(edgeNodeIndexFile);
-});
+/** Every public URL that can serve the edge node index (write node first, then fastest-first). Not cached: the underlying source config can change (scheduled window switchovers), and after the one initialization await this is synchronous anyway. */
+export async function getEdgeNodeConfigURLs(): Promise<string[]> {
+    return await edgeNodeStorage.getURLs(edgeNodeIndexFile);
+}
 
 const startUpdateLoop = lazy(async () => {
     startEdgeNotifier();
     SocketFunction.expose(EdgeNodeController);
-    await getEdgeNodeConfigURL();
+    await getEdgeNodeConfigURLs();
     await runInfinitePollCallAtStart(UPDATE_POLL_INTERVAL * (1 + Math.random() * 0.1), updateLoop);
 });
 

package/src/4-querysub/Querysub.ts

@@ -825,9 +825,9 @@ export class Querysub {
 
         RequireController.injectHTMLBeforeStartup(async () => {
             const { getEdgeBootstrapScript } = await import("../4-deploy/edgeBootstrap");
-            const { getEdgeNodeConfigURL } = await import("../4-deploy/edgeNodes");
+            const { getEdgeNodeConfigURLs } = await import("../4-deploy/edgeNodes");
             let edgeBootstrapFile = await getEdgeBootstrapScript({
-                edgeNodeConfigURL: await getEdgeNodeConfigURL(),
+                edgeNodeConfigURLs: await getEdgeNodeConfigURLs(),
             });
             return `<script>${edgeBootstrapFile}</script>`;
         });

package/src/archiveapps/archiveJoinEntry.ts

@@ -99,7 +99,7 @@ async function runGenesisJoinIteration(config?: { force?: boolean }) {
                 transaction.deleteFiles.push(file);
             }
 
-            let targets = PathRouter.getPathIdentifierTargets(allCombinedValues, authoritySpec);
+            let targets = await PathRouter.getPathIdentifierTargets(allCombinedValues, authoritySpec);
             for (let [target, values] of targets) {
 
                 let dataObj = await pathValueArchives.encodeValuePaths(values, {

package/src/config2.ts

@@ -1,10 +1,4 @@
-import { deepCloneJSON, isNode } from "socket-function/src/misc";
-import { hasArchivesPermissions } from "./-a-archives/archives";
-import { baseIsClient, getDomain } from "./config";
-import { JSONLACKS } from "socket-function/src/JSONLACKS/JSONLACKS";
-import { rootPathStr, prependToPathStr, getPathDepth } from "./path";
-import fs from "fs";
-import { AuthoritySpec } from "./0-path-value-core/PathRouter";
+import { baseIsClient } from "./config";
 
 export function isClient() {
     return baseIsClient();

package/src/deployManager/components/ServicesListPage.tsx

@@ -17,10 +17,12 @@ import { PendingDeployInfo, UpdateButtons, UpdateServiceButtons } from "./deploy
 import { isDefined } from "../../misc";
 import { formatDateJSX } from "../../misc/formatJSX";
 import { Tools } from "./Tools";
-import { ServiceMeasureBars, ServiceMeasureBreakdown } from "./ServiceMeasureBars";
+import { ServiceMeasureBars, ServiceMeasureBreakdown, ServiceMeasureController } from "./ServiceMeasureBars";
 
 module.hotreload = true;
 
+const INVALIDATE_FLASH_TIME = timeInSecond;
+
 /** What kind of service this is, when we can tell from its command, linking to wherever that kind is administered. Its own box beside the service's, so clicking the service still opens the service. */
 class ServiceCategoryBadge extends qreact.Component<{ config: ServiceConfig }> {
     render() {
@@ -45,6 +47,7 @@ export class ServicesListPage extends qreact.Component {
     state = t.state({
         measurementsEnabled: t.atomic<boolean>(true),
         expandedMeasureNodes: t.lookup(t.string),
+        invalidateClickTime: t.atomic<number>(0),
     });
 
     render() {
@@ -123,13 +126,29 @@ export class ServicesListPage extends qreact.Component {
                 </button>
                 <UpdateButtons services={services.map(x => x[1]).filter(isDefined)} />
             </div>
-            <div
-                className={css.pad2(12, 8).button.bord2(0, 0, 20).hsl(0, 0, 95).alignSelf("flex-start")}
-                onClick={() => {
-                    this.state.measurementsEnabled = !this.state.measurementsEnabled;
-                }}
-            >
-                <span className={css.boldStyle}>{this.state.measurementsEnabled ? "☑" : "☐"} Measurements</span>
+            <div className={css.hbox(8)}>
+                <div
+                    className={css.pad2(12, 8).button.bord2(0, 0, 20).hsl(0, 0, 95)}
+                    onClick={() => {
+                        this.state.measurementsEnabled = !this.state.measurementsEnabled;
+                    }}
+                >
+                    <span className={css.boldStyle}>{this.state.measurementsEnabled ? "☑" : "☐"} Measurements</span>
+                </div>
+                {this.state.measurementsEnabled && (() => {
+                    let justInvalidated = now < this.state.invalidateClickTime + INVALIDATE_FLASH_TIME;
+                    return <div
+                        className={css.pad2(12, 8).button.bord2(0, 0, 20)
+                            + (justInvalidated && css.hsl(120, 50, 80) || css.hsl(0, 0, 95))
+                        }
+                        onClick={() => {
+                            this.state.invalidateClickTime = Date.now();
+                            ServiceMeasureController.resetAll();
+                        }}
+                    >
+                        {justInvalidated ? "Invalidated ✓" : "Invalidate"}
+                    </div>;
+                })()}
             </div>
             <Tools />
             <div className={css.vbox(8)}>

package/src/deployManager/machineDaemonShared.ts

@@ -0,0 +1,2 @@
+export const SERVICE_NAME = "machine-alwaysup";
+export const SERVICE_UNIT_NAME = `${SERVICE_NAME}.service`;