querysub 0.591.0 → 0.593.0

package/.claude/settings.local.json

@@ -31,7 +31,11 @@
       "WebFetch(domain:developers.cloudflare.com)",
       "PowerShell(yarn type *)",
       "PowerShell(Remove-Item -Confirm:$false \"C:\\\\Users\\\\quent\\\\.claude\\\\projects\\\\D--repos-querysub\\\\memory\\\\no-naming-new-bins.md\")",
-      "Bash(python -)"
+      "Bash(python -)",
+      "PowerShell(yarn upreal *)",
+      "PowerShell($env:COMPUTERNAME; $env:USERNAME)",
+      "PowerShell($env:COMPUTERNAME)",
+      "PowerShell($env:USERNAME)"
     ]
   }
 }

package/bin/stop-machine.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+
+require("typenode");
+require("../src/deployManager/stopMachineMain");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.591.0",
+  "version": "0.593.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -26,6 +26,8 @@
     "mc2": "yarn typenode ./src/diagnostics/debugger/mcp-server.ts --cwd D:/repos/qs-cyoa/",
     "autofix": "yarn typenode ./src/diagnostics/logs/errorTickets/autoFixerEntry.ts",
     "parameters-timeline": "yarn typenode ./src/deployManager/parametersTimelineMain.ts",
+    "setup-machine": "yarn typenode ./src/deployManager/setupMachineMain.ts",
+    "stop-machine": "yarn typenode ./src/deployManager/stopMachineMain.ts",
     "ssh-a-claude": "ssh root@a.querysubtest.com"
   },
   "bin": {
@@ -38,6 +40,7 @@
     "machine": "./bin/machine.js",
     "machine-alwaysup": "./bin/machine-alwaysup.js",
     "setup-machine": "./bin/setup-machine.js",
+    "stop-machine": "./bin/stop-machine.js",
     "gc": "./bin/gc.js",
     "gc-watch": "./bin/gc-watch.js",
     "gc-watch-public": "./bin/gc-watch-public.js",
@@ -71,8 +74,8 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
-    "sliftutils": "^1.7.66",
-    "socket-function": "^1.2.27",
+    "sliftutils": "^1.7.72",
+    "socket-function": "^1.2.29",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",
     "typesafecss": "^0.32.0",

package/spec.txt

@@ -1,3 +1,15 @@
+It could be it's just taking a while to get valid. Let's check it two times to see. Although if it is, that's a bug, but it's probably an easy bug to fix. 
+
+Wait, what transaction is it applying now? If it's not a valid transaction and we don't even write it, so what? 
+
+Well, we might have to restore from a snapshot if it breaks it, but I'm pretty confident we're not going to lose all of our data. 
+
+Ugh... I mean it seems like our storage is just really out of sync. Like, we're reading files and it changes every time, etc. 
+
+
+
+
+
 Trigger values
     1) validStateComputer.ingestValuesAndValidStates
     1.5) authorityStorage.ingestValues

package/src/-a-archives/archiveCache.ts

@@ -138,11 +138,17 @@ const getDiskMetricsBase = async () => {
     // We require our input to be a temp file, so our update can be atomic
     async function addCacheFile(archives: CacheKeySource, key: string, sourceTempFile: string): Promise<void> {
         let path = getArchiveCachePath(archives, key);
+        let sourceTempFileStat = await fs.promises.stat(sourceTempFile);
+        if (sourceTempFileStat.size === 0) {
+            try {
+                await fs.promises.unlink(sourceTempFile);
+            } catch { }
+            throw new Error(`Expected non-empty data when caching ${key}, was 0 bytes. Empty files are treated as deletes by the storage layer, so they must never be written or cached.`);
+        }
         let hasFile = fileSizes.some(x => x.path === path);
         if (hasFile) {
             removeFile(path);
         }
-        let sourceTempFileStat = await fs.promises.stat(sourceTempFile);
         availableBytes -= sourceTempFileStat.size;
         availableFiles--;
         sort(fileSizes, x => x.time);
@@ -196,14 +202,19 @@ const getDiskMetricsBase = async () => {
         let path = getArchiveCachePath(archives, key);
         let buffer: Buffer | undefined;
         let handle: fs.promises.FileHandle | undefined;
+        let isEmptyFile = false;
         try {
             handle = await fs.promises.open(path, "r");
             let stats = await handle.stat();
-            let start = config?.range?.start ?? 0;
-            let end = config?.range?.end ?? stats.size;
-            let readBuffer = Buffer.alloc(end - start);
-            let read = await handle.read(readBuffer, 0, end - start, start);
-            buffer = readBuffer.slice(0, read.bytesRead);
+            if (stats.size === 0) {
+                isEmptyFile = true;
+            } else {
+                let start = config?.range?.start ?? 0;
+                let end = config?.range?.end ?? stats.size;
+                let readBuffer = Buffer.alloc(end - start);
+                let read = await handle.read(readBuffer, 0, end - start, start);
+                buffer = readBuffer.slice(0, read.bytesRead);
+            }
         }
         catch { }
         finally {
@@ -211,6 +222,13 @@ const getDiskMetricsBase = async () => {
                 await handle.close();
             }
         }
+        if (isEmptyFile) {
+            removeFile(path);
+            try {
+                await fs.promises.unlink(path);
+            } catch { }
+            return undefined;
+        }
         if (buffer) {
             updateAccessTime(path);
             bumpAtimeIfStale(path);
@@ -355,7 +373,9 @@ export function wrapArchivesWithCache(archives: Archives, rootConfig?: {
             if (rootConfig?.immutable) {
                 try {
                     let stat = await fs.promises.stat(getArchiveCachePath(archives, fileName));
-                    return { writeTime: stat.mtimeMs, size: stat.size };
+                    if (stat.size > 0) {
+                        return { writeTime: stat.mtimeMs, size: stat.size };
+                    }
                 } catch {
                     // Not in our cache yet — fall through to the source.
                 }
@@ -363,6 +383,7 @@ export function wrapArchivesWithCache(archives: Archives, rootConfig?: {
             return archives.getInfo(fileName);
         },
         set: async (fileName: string, data: Buffer) => {
+            if (data.length === 0) throw new Error(`Expected non-empty data when setting ${fileName}, was 0 bytes. Empty writes are treated as deletes by the storage layer.`);
             let metrics = await getDiskMetrics();
             const tempPath = getTempFilePath();
             await fs.promises.writeFile(tempPath, data);

package/src/-a-archives/archiveCache2.ts

@@ -127,7 +127,9 @@ export function wrapArchivesWithCache2(archives: IArchives, rootConfig?: {
             if (rootConfig?.immutable) {
                 try {
                     let stat = await fs.promises.stat(getArchiveCachePath(archives, fileName));
-                    return { writeTime: stat.mtimeMs, size: stat.size };
+                    if (stat.size > 0) {
+                        return { writeTime: stat.mtimeMs, size: stat.size };
+                    }
                 } catch {
                     // Not in our cache yet — fall through to the source.
                 }
@@ -135,6 +137,7 @@ export function wrapArchivesWithCache2(archives: IArchives, rootConfig?: {
             return archives.getInfo(fileName);
         },
         set: async (fileName: string, data: Buffer, config?: SetConfig) => {
+            if (data.length === 0) throw new Error(`Expected non-empty data when setting ${fileName}, was 0 bytes. Empty writes are treated as deletes by the storage layer.`);
             let writtenKey = await archives.set(fileName, data, config);
             // VARIABLE_SHARD keys materialize into a different key on write, in which case
             //  fileName isn't the readable key, so caching under it would be wrong.

package/src/-a-archives/archives.ts

@@ -1,14 +1,4 @@
-import { isNode } from "socket-function/src/misc";
-import { getDomain, isNoNetwork } from "../config";
-import { getStorageDir, getSubFolder } from "../fs";
-import { getArchivesBackblaze, hasBackblazePermissions } from "./archivesBackBlaze";
-import { getArchivesLocal } from "./archivesDisk";
-import { magenta } from "socket-function/src/formatting/logColors";
-import debugbreak from "debugbreak";
-import { isClient } from "../config2";
-import { wrapArchivesWithCache } from "./archiveCache";
 import { Args } from "socket-function/src/types";
-import { IArchives } from "sliftutils/storage/IArchives";
 
 export interface Archives {
     getDebugName(): string;
@@ -105,8 +95,8 @@ export function createArchivesOverride<T extends Partial<Archives>>(
     };
 }
 
-export function nestArchives(path: string, archives: Archives): Archives & IArchives {
-    if (!path) return archives as any;
+export function nestArchives(path: string, archives: Archives): Archives {
+    if (!path) return archives;
     if (!path.endsWith("/")) {
         path = path + "/";
     }
@@ -116,7 +106,6 @@ export function nestArchives(path: string, archives: Archives): Archives & IArch
         }
         return file;
     }
-    // NOTE: Just ignore the errors in here. We're going to be getting rid of this soon enough. 
     return {
         getDebugName: () => archives.getDebugName() + "/" + path,
         get: (fileName: string, config) => archives.get(path + stripFilePrefix(fileName), config),
@@ -153,53 +142,3 @@ export function nestArchives(path: string, archives: Archives): Archives & IArch
     };
 }
 
-let archivesLogging = false;
-export function enableArchivesLogging() {
-    archivesLogging = true;
-}
-
-let cached = new Map<string, Archives>();
-export function getArchives(rootPath = ""): Archives {
-    let value = cached.get(rootPath);
-    if (!value) {
-        value = getArchivesBase(rootPath);
-        cached.set(rootPath, value);
-    }
-    return value;
-}
-export function getArchivesBase(rootPath = ""): Archives {
-    let archives: Archives;
-    let domain = getDomain();
-    if (isNoNetwork()) {
-        archives = getArchivesLocal(domain);
-        archives = nestArchives(rootPath, archives);
-    } else {
-        archives = getArchivesBackblaze(domain);
-        archives = nestArchives(rootPath, archives);
-        // Why did we think we should always cache? We absolutely should not. Even if we were checking
-        //  file hashes on every read (which we do not), it would still potentially fill the cache up
-        //  with large temporary files we have no intention of using (which might evict entries we
-        //  intend to use).
-        //archives = wrapArchivesWithCache(archives);
-    }
-    if (archivesLogging) {
-        archives.enableLogging();
-    }
-    return archives;
-}
-
-export function hasArchivesPermissions() {
-    if (isNoNetwork()) {
-        return true;
-    }
-    return hasBackblazePermissions();
-}
-
-if (isNode()) {
-    if (isNoNetwork()) {
-        console.log("Primary data storage: " + magenta(getStorageDir()));
-    } else {
-        console.log("Config storage path:  " + magenta(getStorageDir()));
-        console.log("Primary data storage: " + magenta(`backblaze:${getDomain()}`));
-    }
-}

package/src/-a-archives/archives2.ts

@@ -4,6 +4,7 @@ import { cache, cacheJSONArgsEqual } from "socket-function/src/caching";
 import { formatDateTime } from "socket-function/src/formatting/format";
 import { getDomain } from "../config";
 import { timeInMinute } from "socket-function/src/misc";
+import { FindConfig } from "sliftutils/storage/IArchives";
 
 export const STORAGE_ACCOUNT = "root";
 
@@ -44,24 +45,27 @@ function archiveBuilder(bucket: string, overrides: Partial<RemoteConfigBase>) {
     const ONTARIO = `https://99-250-124-91.querysubtest.com:5234/file/${STORAGE_ACCOUNT}/${bucket}/storage/storagerouting.json`;
     const HETZNER = `https://65-109-93-113.querysubtest.com:5234/file/${STORAGE_ACCOUNT}/${bucket}/storage/storagerouting.json`;
     const BACKBLAZE = `https://f002.backblazeb2.com/file/${bucket}/storage/storagerouting.json`;
+
+    function stripeSources(urls: string[]) {
+        let sources: Omit<Exclude<RemoteConfigBase, string>, "validWindow">[] = [];
+        for (let j = 0; j < urls.length; j++) {
+            // Shift by j
+            let currentUrls = urls.slice(j).concat(urls.slice(0, j));
+            for (let i = 0; i < urls.length; i++) {
+                sources.push({ type: "remote", url: currentUrls[i], route: [i / urls.length, (i + 1) / urls.length], });
+            }
+        }
+        return sources;
+    }
+
+
     let sources = createSourceWindows(overrides, [
         {
-            startTime: +new Date("2026-07-20 18:00:00-04:00"),
-            sources: [
-                { type: "remote", url: ONTARIO, route: [0.5, 1], },
-                { type: "remote", url: HETZNER, route: [0, 0.5], },
-                { type: "remote", url: ONTARIO, },
-                { type: "remote", url: HETZNER, },
-                { type: "backblaze", url: BACKBLAZE, },
-            ]
-        },
-        {
-            startTime: +new Date("2026-07-21 02:59:00-04:00"),
+            startTime: +new Date("2026-07-20 00:00:00-04:00"),
             sources: [
-                { type: "remote", url: ONTARIO, route: [0, 0.5], },
-                { type: "remote", url: HETZNER, route: [0.5, 1], },
-                { type: "remote", url: ONTARIO, },
-                { type: "remote", url: HETZNER, },
+                // NOTE: Once we add new sources, we need to add something at a new start time in the future so we have time to move the files. 
+                // TODO: EVENTUALLY, Once we have enough data, instead of restriping, we need to bisect the data. That way we can slowly move over the data instead of trying to rewrite all of our data all at once. However, for the moment, moving all the data is probably fine. On the data center, we should be able to move terabytes of data every hour, so we should be able to handle quite a bit, especially if we have a few days to run the restriping. 
+                ...stripeSources([ONTARIO, HETZNER]),
                 { type: "backblaze", url: BACKBLAZE, },
             ]
         },
@@ -89,8 +93,8 @@ function nestBucket(folder: string, archives: ReturnType<typeof createArchives>)
         get: (path: string, config?: GetConfig) => archives.get(folder + path, config),
         set: (path: string, data: Buffer, config?: SetConfig) => archives.set(folder + path, data, config),
         del: (path: string) => archives.del(folder + path),
-        find: async (prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }) => (await archives.find(folder + prefix, config)).map(x => x.slice(folder.length)),
-        findInfo: async (prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }) => (await archives.findInfo(folder + prefix, config)).map(x => ({ ...x, path: x.path.slice(folder.length) })),
+        find: async (prefix: string, config?: FindConfig) => (await archives.find(folder + prefix, config)).map(x => x.slice(folder.length)),
+        findInfo: async (prefix: string, config?: FindConfig) => (await archives.findInfo(folder + prefix, config)).map(x => ({ ...x, path: x.path.slice(folder.length) })),
         get2: (path: string, config?: GetConfig) => archives.get2(folder + path, config),
         setLargeFile: (config: { path: string; lastModified?: number; getNextData(): Promise<Buffer | undefined>; }) => archives.setLargeFile({ ...config, path: folder + config.path }),
         getInfo: (path: string) => archives.getInfo(folder + path),

package/src/-a-archives/archivesDisk.ts

@@ -1,12 +1,12 @@
 import { isNode } from "socket-function/src/misc";
 import os from "os";
-import { fsExistsAsync, getSubFolder } from "../fs";
+import { fsExistsAsync } from "../fs";
 import fs from "fs";
 import { blue, red, yellow } from "socket-function/src/formatting/logColors";
 import { measureFnc } from "socket-function/src/profiling/measure";
 import { delay } from "socket-function/src/batching";
 import debugbreak from "debugbreak";
-import { Archives, getArchives } from "./archives";
+import { Archives } from "./archives";
 import { cache, lazy } from "socket-function/src/caching";
 import { isDefined } from "../misc";
 
@@ -433,17 +433,6 @@ function unescapeFileName(fileName: string): string {
     return fileName;
 }
 
-export const getArchivesLocal = cache((domain: string): Archives => {
-
-    const archivesLocal = new ArchivesDisk();
-
-    if (isNode()) {
-        archivesLocal.LOCAL_ARCHIVE_FOLDER = getSubFolder(domain).replaceAll("\\", "/");
-    }
-
-    return archivesLocal;
-});
-
 export const getArchivesHome = cache((domain: string): Archives => {
     const archivesLocal = new ArchivesDisk();
 

package/src/-a-archives/archivesMemoryCache.ts

@@ -24,21 +24,29 @@ export function testDisableCache() {
     cacheDisabled = true;
 }
 
+export type MemoryCacheConfig = {
+    extraReadSize?: number;
+    maxSize?: number;
+    maxCount?: number;
+    // If the files are guaranteed to be immutable, then all reads, including reading entire file and reading any range, can be cached.
+    //  - Otherwise, we can't cache when the entire file is read, as there might be further appends after it.
+    //  - Also, it means if there's a range read that happens, if that range read doesn't return enough bytes, for example, we try to read from 0 to 100, but we only get 50 bytes back, we can only cache the 0 to 50, and then if they read from 0 to 100 again, we can use the cache, but we have to also read 50 to 100 from the source, because the file might have gotten larger.
+    fullyImmutable?: boolean;
+    stats?: ArchivesMemoryCacheStats;
+    sizeCache?: Map<string, number>;
+};
+
+/** The operations the cache actually needs from its source, so any archives flavor (old Archives or IArchives) can be cached. */
+export type MemoryCacheSource = {
+    get(path: string, config?: { range?: { start: number; end: number; } }): Promise<Buffer | undefined>;
+    getInfo(path: string): Promise<{ size: number } | undefined>;
+};
+
 /** Creates an in-memory cache for the archives. We expect that the values will only have values appended to them or be deleted. Also, if a value is deleted, we won't erase our in-memory cache, and so we will still allow reading from deleted values. */
-export function createArchivesMemoryCache(
-    archives: Archives,
-    config?: {
-        extraReadSize?: number;
-        maxSize?: number;
-        maxCount?: number;
-        // If the files are guaranteed to be immutable, then all reads, including reading entire file and reading any range, can be cached. 
-        //  - Otherwise, we can't cache when the entire file is read, as there might be further appends after it. 
-        //  - Also, it means if there's a range read that happens, if that range read doesn't return enough bytes, for example, we try to read from 0 to 100, but we only get 50 bytes back, we can only cache the 0 to 50, and then if they read from 0 to 100 again, we can use the cache, but we have to also read 50 to 100 from the source, because the file might have gotten larger. 
-        fullyImmutable?: boolean;
-        stats?: ArchivesMemoryCacheStats;
-        sizeCache?: Map<string, number>;
-    }
-): Archives {
+export function createMemoryCachedGet(
+    archives: MemoryCacheSource,
+    config?: MemoryCacheConfig
+) {
     let {
         maxSize = 1024 * 1024 * 1024 * 4,
         maxCount = 1000 * 1000,
@@ -221,8 +229,6 @@ export function createArchivesMemoryCache(
 
     async function cachedGet(path: string, getConfig?: {
         range?: { start: number; end: number; };
-        retryCount?: number;
-        fastRead?: boolean;
     }): Promise<Buffer | undefined> {
         if (cacheDisabled) {
             let result = await archives.get(path, getConfig);
@@ -374,7 +380,11 @@ export function createArchivesMemoryCache(
         return result;
     }
 
+    return cachedGet;
+}
+
+export function createArchivesMemoryCache(archives: Archives, config?: MemoryCacheConfig): Archives {
     return createArchivesOverride(archives, {
-        get: cachedGet
+        get: createMemoryCachedGet(archives, config),
     });
 }

package/src/-a-archives/archivesMemoryCache2.ts

@@ -0,0 +1,23 @@
+import { IArchives } from "sliftutils/storage/IArchives";
+import { createMemoryCachedGet, MemoryCacheConfig } from "./archivesMemoryCache";
+
+/** See createMemoryCachedGet — the same in-memory range cache, wrapping an IArchives (archives2). */
+export function createArchivesMemoryCache2(archives: IArchives, config?: MemoryCacheConfig): IArchives {
+    let cachedGet = createMemoryCachedGet(archives, config);
+    return {
+        get: cachedGet,
+        getDebugName: () => archives.getDebugName(),
+        hasWriteAccess: () => archives.hasWriteAccess(),
+        get2: (fileName, getConfig) => archives.get2(fileName, getConfig),
+        set: (fileName, data, setConfig) => archives.set(fileName, data, setConfig),
+        del: (fileName) => archives.del(fileName),
+        setLargeFile: (largeConfig) => archives.setLargeFile(largeConfig),
+        getInfo: (fileName) => archives.getInfo(fileName),
+        find: (prefix, findConfig) => archives.find(prefix, findConfig),
+        findInfo: (prefix, findConfig) => archives.findInfo(prefix, findConfig),
+        getURL: (path) => archives.getURL(path),
+        getConfig: () => archives.getConfig(),
+        getChangesAfter2: (changesConfig) => archives.getChangesAfter2(changesConfig),
+        getSyncStatus: archives.getSyncStatus && (() => archives.getSyncStatus!()),
+    };
+}

package/src/-d-trust/NetworkTrust2.ts

@@ -1,6 +1,6 @@
 import { measureWrap } from "socket-function/src/profiling/measure";
 import { getIdentityCA, getMachineId, getOwnMachineId } from "sliftutils/misc/https/certs";
-import { getArchives } from "../-a-archives/archives";
+import { getArchives2 } from "../-a-archives/archives2";
 import { isNode, throttleFunction, timeInHour, timeInSecond } from "socket-function/src/misc";
 import { SocketFunctionHook } from "socket-function/SocketFunctionTypes";
 import { SocketFunction } from "socket-function/SocketFunction";
@@ -21,7 +21,7 @@ import { magenta } from "socket-function/src/formatting/logColors";
 const UNTRUST_CACHE_TIME = 30 * timeInSecond;
 const TRUSTED_CACHE_RESET_INTERVAL = timeInHour;
 
-const archives = lazy(() => getArchives("trust2/"));
+const archives = lazy(() => getArchives2("trust2/"));
 
 export const requiresNetworkTrustHook: SocketFunctionHook = async config => {
     // HACK: On the clientside we strip the domain process and machine id, so we can no longer determine
@@ -87,7 +87,7 @@ export const isTrusted = measureWrap(async function isTrusted(machineId: string)
     }
 });
 let populateTrustedCache = lazy(async () => {
-    let trustedMachineIds = await archives().find("");
+    let trustedMachineIds = await archives().find("", { fallbacks: true });
     lastArchivesTrusted = trustedMachineIds.slice();
     for (let trustedMachineId of trustedMachineIds) {
         trustedCache.add(trustedMachineId);

package/src/-e-certs/certAuthority.ts

@@ -5,13 +5,13 @@ import { cache, lazy } from "socket-function/src/caching";
 import { hasDNSWritePermissions, setRecord } from "../-b-authorities/dnsAuthority";
 import { magenta } from "socket-function/src/formatting/logColors";
 import { createKeyStore } from "../persistentLocalStore";
-import { getArchives } from "../-a-archives/archives";
+import { getArchives2 } from "../-a-archives/archives2";
 import { isNoNetwork } from "../config";
 import { formatDateTime, formatTime } from "socket-function/src/formatting/format";
 import { delay } from "socket-function/src/batching";
 import { timeInMinute } from "socket-function/src/misc";
 
-const archives = lazy(() => getArchives(`https_certs_3/`));
+const archives = lazy(() => getArchives2(`https_certs_3/`));
 // Expire EXPIRATION_THRESHOLD% of the way through the certificate's lifetime
 const EXPIRATION_THRESHOLD = 0.4;
 // Let's Encrypt only accepts RSA or ECDSA account keys (not Ed25519, which is all sliftutils/certs generates).

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -1,5 +1,5 @@
 import { SocketFunction } from "socket-function/SocketFunction";
-import { getArchives } from "../-a-archives/archives";
+import { getArchives2 } from "../-a-archives/archives2";
 import { getDomain, isDevDebugbreak, isNoNetwork, isPublic } from "../config";
 import { measureBlock } from "socket-function/src/profiling/measure";
 import { isNode, keyByArray, sha256Hash, throttleFunction, timeInMinute, timeInSecond } from "socket-function/src/misc";
@@ -56,7 +56,7 @@ let DEAD_NODE_POLL_COOLDOWN = timeInMinute * 5;
 
 let shutdown = false;
 
-const archives = lazy(() => getArchives("nodes/"));
+const archives = lazy(() => getArchives2("nodes/"));
 
 let logging = true;
 export function enableNodeDiscoveryLogging() {
@@ -342,7 +342,7 @@ async function syncArchives() {
     if (isServer()) {
         // Make sure we are present
         await writeHeartbeat();
-        let nodeIds = await archives().find("");
+        let nodeIds = await archives().find("", { fallbacks: true });
         console.info(`Synced node ids from archives`, { nodeIds });
         await setNodeIds(nodeIds);
     } else {

package/src/0-path-value-core/AuthorityLookup.ts

@@ -1,8 +1,4 @@
 import { sort, timeInMinute, timeInSecond } from "socket-function/src/misc";
-import { nestArchives } from "../-a-archives/archives";
-import { getArchivesBackblaze } from "../-a-archives/archivesBackBlaze";
-import { archiveJSONT } from "../-a-archives/archivesJSONT";
-import { getDomain, isPublic } from "../config";
 import { cache, lazy } from "socket-function/src/caching";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { delay, runInSerial, runInfinitePollCallAtStart } from "socket-function/src/batching";

package/src/0-path-value-core/PathRouter.ts

@@ -13,6 +13,7 @@ import { sha256 } from "js-sha256";
 import { rangesOverlap, removeRange } from "../rangeMath";
 import { decodeParentFilter } from "./hackedPackedPathParentFiltering";
 import { getBufferInt } from "socket-function/src/bits";
+import { safeLoop } from "socket-function/src/batching";
 
 import { LOCAL_DOMAIN, LOCAL_DOMAIN_PATH } from "./PathRouterConstants";
 export { LOCAL_DOMAIN, LOCAL_DOMAIN_PATH };
@@ -320,7 +321,7 @@ export class PathRouter {
     // NOTE: Encodes all the data, even if it matches our spec or not. If you don't want this, you have to filter first
     //  - If this becomes an issue we COULD filter, as we can do it quickly, but I don't think it is required, as all the present usecases prefilter anyways.
     @measureFnc
-    public static getPathIdentifierTargets(values: PathValue[], ourSpec: AuthoritySpec): Map<string, PathValue[]> {
+    public static async getPathIdentifierTargets(values: PathValue[], ourSpec: AuthoritySpec): Promise<Map<string, PathValue[]>> {
         // NOTE: The file size limit is 1024 bytes. But we also have our folder, etc, so we want to add enough buffer
         //  - Shorter hashes means we can store more, but there's a point when the collisions make it less useful.
         const MAX_PREFIXES_PER_FILE = 50;
@@ -333,17 +334,17 @@ export class PathRouter {
         let prefixes = ourSpec.prefixes.slice();
         sort(prefixes, x => x.prefixPartLength);
 
-        // NOTE: If there are few enough path values for a prefix, we don't even need to calculate the routing hash. 
+        // NOTE: If there are few enough path values for a prefix, we don't even need to calculate the routing hash.
         let byPrefix = new Map<PrefixMatcher | undefined, PathValue[]>();
-        for (let value of values) {
+        await safeLoop({ data: values, name: "PathRouter.getPathIdentifierTargets|groupByPrefix" }, value => {
             let prefix = getMatchingPrefix(ourSpec, value.path);
-            let values = byPrefix.get(prefix);
-            if (!values) {
-                values = [];
-                byPrefix.set(prefix, values);
+            let prefixValues = byPrefix.get(prefix);
+            if (!prefixValues) {
+                prefixValues = [];
+                byPrefix.set(prefix, prefixValues);
             }
-            values.push(value);
-        }
+            prefixValues.push(value);
+        });
 
         let prefixGroups = Array.from(byPrefix.entries()).map(([prefix, values]) => ({
             prefix,
@@ -391,17 +392,17 @@ export class PathRouter {
         }
 
         let finalFiles = new Map<string, PathValue[]>();
-        for (let group of groups) {
+        await safeLoop({ data: groups, name: "PathRouter.getPathIdentifierTargets|shardGroups" }, async group => {
             if (group.count < SHARD_THRESHOLD) {
                 let identifier = this.encodeIdentifier({ prefixes: group.prefixes, rangeStart: 0, rangeEnd: 1 });
                 finalFiles.set(identifier, group.values.flat());
-                continue;
+                return;
             }
             // Split by routing hash
-            let values = group.values.flat();
-            let splitCount = Math.max(MIN_SHARD_FILE_COUNT, Math.ceil(values.length / TARGET_SHARD_SIZE));
+            let groupValues = group.values.flat();
+            let splitCount = Math.max(MIN_SHARD_FILE_COUNT, Math.ceil(groupValues.length / TARGET_SHARD_SIZE));
             let byRouteGroup = new Map<number, PathValue[]>();
-            for (let value of values) {
+            await safeLoop({ data: groupValues, name: "PathRouter.getPathIdentifierTargets|routeValues" }, value => {
                 let route = this.getRouteFull({
                     path: value.path,
                     spec: {
@@ -418,14 +419,14 @@ export class PathRouter {
                     byRouteGroup.set(routeIndex, routeValues);
                 }
                 routeValues.push(value);
-            }
+            });
             for (let [routeIndex, routeValues] of byRouteGroup) {
                 let rangeStart = routeIndex / splitCount;
                 let rangeEnd = (routeIndex + 1) / splitCount;
                 let identifier = this.encodeIdentifier({ prefixes: group.prefixes, rangeStart, rangeEnd });
                 finalFiles.set(identifier, routeValues);
             }
-        }
+        });
 
         // NOTE: There could be a huge number of prefixes and we can't pack them all into one file because of the prefix limit, so this will write any remaining values. 
         if (remainingValues.length > 0) {

package/src/0-path-value-core/ShardPrefixes.ts

@@ -10,10 +10,8 @@
 */
 
 import { isNode } from "typesafecss";
-import { nestArchives } from "../-a-archives/archives";
-import { getArchivesBackblaze } from "../-a-archives/archivesBackBlaze";
+import { getArchives2 } from "../-a-archives/archives2";
 import { archiveJSONT } from "../-a-archives/archivesJSONT";
-import { getDomain } from "../config";
 import { sort } from "socket-function/src/misc";
 import { getPathFromStr, getPathStr } from "../path";
 import { green, red } from "socket-function/src/formatting/logColors";
@@ -22,7 +20,7 @@ type PrefixObj = {
     prefixes: string[];
 };
 
-let prefixes = archiveJSONT<PrefixObj>(() => nestArchives("shard-prefixes/", getArchivesBackblaze(getDomain())));
+let prefixes = archiveJSONT<PrefixObj>(() => getArchives2("shard-prefixes/"));
 // We don't actually need a collection, we only need a single value. However, the collection interface is a lot more convenient than manually dealing with backplays files. 
 const key = "all";