querysub 0.617.0 → 0.620.0

package/appSecrets.ts

@@ -2,7 +2,6 @@ import fs from "fs";
 import { lazy } from "socket-function/src/caching";
 import { ArchivesBackblaze } from "sliftutils/storage/backblaze";
 import { STORAGE_DIR, getDomain, getBackblazePath } from "./src/misc/appPaths";
-import { Querysub } from "./src/4-querysub/Querysub";
 import { formatDateTime } from "socket-function/src/formatting/format";
 
 const keysArchives = lazy(() => new ArchivesBackblaze({ bucketName: getDomain() }));
@@ -26,8 +25,11 @@ export const getCloudflareCreds = lazy(async (): Promise<{ key: string; email: s
 export async function getAppSecret(key: string): Promise<string | undefined> {
     // HACK: This is used by the storage server. Which we want to shut down immediately. That way the next instance can start up. The sword server already looks for notifications about when it's going to be shutting down and it flushes to disk early in this case, so it should be fairly safe to hard kill it.
     doImmediateShutdown();
+
+    // Actually maybe the wrapper should do this instead. 
     // Import query sub so our logs go to our actual log server. Otherwise we won't know if we get any errors. 
-    Querysub;
+    //await import("./src/4-querysub/Querysub");
+
     if (key.startsWith("backblaze.json.")) {
         let creds = JSON.parse(fs.readFileSync(getBackblazePath(), "utf8")) as { applicationKeyId?: string; applicationKey?: string };
         if (key === "backblaze.json.applicationKeyId") return creds.applicationKeyId;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.617.0",
+  "version": "0.620.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -25,6 +25,9 @@
     "mcp2": "yarn typenode ./src/diagnostics/debugger/mcp-server.ts",
     "mc2": "yarn typenode ./src/diagnostics/debugger/mcp-server.ts --cwd D:/repos/qs-cyoa/",
     "autofix": "yarn typenode ./src/diagnostics/logs/errorTickets/autoFixerEntry.ts",
+    "thin": "yarn typenode ./src/archiveapps/snapshotThinningEntry.ts",
+    "thin-watch": "yarn typenode ./src/archiveapps/snapshotThinningEntry.ts --watch",
+    "thin-dry": "yarn typenode ./src/archiveapps/snapshotThinningEntry.ts --dry-run",
     "parameters-timeline": "yarn typenode ./src/deployManager/parametersTimelineMain.ts",
     "setup-machine": "yarn typenode ./src/deployManager/setupMachineMain.ts",
     "stop-machine": "yarn typenode ./src/deployManager/stopMachineMain.ts",
@@ -75,7 +78,7 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
-    "sliftutils": "^1.7.88",
+    "sliftutils": "^1.7.89",
     "socket-function": "^1.2.30",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",

package/src/-a-archives/archives2.ts

@@ -44,13 +44,22 @@ function archiveBuilder(bucket: string, overrides: Partial<RemoteConfigBase>) {
     const HETZNER = `https://65-109-93-113.querysubtest.com:5234/file/${STORAGE_ACCOUNT}/${bucket}/storage/storagerouting.json`;
     const BACKBLAZE = `https://f002.backblazeb2.com/file/${bucket}/storage/storagerouting.json`;
 
-    function stripeSources(urls: string[]) {
+    function stripeSources(urls: string[], config: { routeVersion: number }) {
         let sources: Omit<Exclude<RemoteConfigBase, string>, "validWindow">[] = [];
         for (let j = 0; j < urls.length; j++) {
             // Shift by j
             let currentUrls = urls.slice(j).concat(urls.slice(0, j));
             for (let i = 0; i < urls.length; i++) {
-                sources.push({ type: "remote", url: currentUrls[i], route: [i / urls.length, (i + 1) / urls.length], });
+                let url = new URL(currentUrls[i]);
+                let routeStart = i / urls.length;
+                let routeEnd = (i + 1) / urls.length;
+                sources.push({
+                    type: "remote",
+                    url: currentUrls[i],
+                    route: [routeStart, routeEnd],
+                    // IMPORTANT! name decides where our files are stored. We can't have two things writing to the same folder at the same time. But it also means if we reuse the name, then it has to be the same data. 
+                    name: url.hostname.replaceAll(/[^a-z0-9-]/g, "_") + "_v" + config.routeVersion + "_" + routeStart + "_" + routeEnd,
+                });
             }
         }
         return sources;
@@ -66,8 +75,9 @@ function archiveBuilder(bucket: string, overrides: Partial<RemoteConfigBase>) {
             sources: [
                 // NOTE: Once we add new sources, we need to add something at a new start time in the future so we have time to move the files. 
                 // TODO: EVENTUALLY, Once we have enough data, instead of restriping, we need to bisect the data. That way we can slowly move over the data instead of trying to rewrite all of our data all at once. However, for the moment, moving all the data is probably fine. On the data center, we should be able to move terabytes of data every hour, so we should be able to handle quite a bit, especially if we have a few days to run the restriping. 
-                ...stripeSources([ONTARIO, HETZNER]),
-                { type: "backblaze", url: BACKBLAZE, },
+                // IMPORTANT! Every time we re-stripe, we need to increment the route version. Otherwise, we could reuse a folder that we previously used that's no longer in sync. which will cause us to resurrect old data, which is very bad. We don't want to do that...
+                ...stripeSources([ONTARIO, HETZNER], { routeVersion: 1 }),
+                { type: "backblaze", url: BACKBLAZE, name: "backblaze" },
             ]
         },
     ]);
@@ -159,6 +169,29 @@ export function getArchives2Public(folder: string) {
     return nestBucket(folder, archivesBuilderCache(domain + "-public", { public: true, allowedOrigins: getAllowedOrigins(getDomain()) }));
 }
 
+export function getArchives2Debug(folder: string) {
+    let routeVersion = 1;
+    let archives = createArchives({
+        version: 20,
+        sources: [
+            // IMPORTANT! If we start changing around routes, it could resurrect old data. We can fix this via incrementing the route version.
+            {
+                type: "remote", url: `https://99-250-124-91.querysubtest.com:5239/file/${STORAGE_ACCOUNT}/${folder}/storage/storagerouting.json`,
+                validWindow: [0, Number.MAX_SAFE_INTEGER],
+                route: [0, 0.5],
+                name: "storagea_v" + routeVersion,
+            },
+            {
+                type: "remote", url: `https://65-109-93-113.querysubtest.com:5240/file/${STORAGE_ACCOUNT}/${folder}/storage/storagerouting.json`,
+                validWindow: [0, Number.MAX_SAFE_INTEGER],
+                route: [0.5, 1],
+                name: "storageb_v" + routeVersion,
+            },
+        ],
+    });
+    return nestBucket(folder, archives);
+}
+
 
 export async function initArchives2() {
     let buckets = [

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -541,9 +541,9 @@ async function runServerSyncLoops() {
             let suicideThreshold = Date.now() - SUICIDE_HEARTBEAT_THRESHOLD;
             if (!lastTime || lastTime < suicideThreshold) {
                 if (!lastTime) {
-                    console.error(red(`Self node was removed due to not heartbeating. Terminating self process, as it likely has very stale data.`));
+                    console.error(red(`Self node was removed due to heartbeat file not existing (${selfNodeId}). Terminating self process, as it likely has very stale data.`));
                 } else {
-                    console.error(red(`Self node was has very old heartbeat. Terminating self process, as it likely has very stale data.`));
+                    console.error(red(`Self node was has very old heartbeat. Terminating self process, as it likely has very stale data. ${formatDateTime(lastTime)} < ${formatDateTime(suicideThreshold)}, now is ${formatDateTime(Date.now())}`));
                 }
                 process.exit();
             }

package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts

@@ -4,7 +4,8 @@ import { decodeNodeId } from "sliftutils/misc/https/certs";
 import { getOwnNodeId, getOwnNodeIdAssert } from "../../-f-node-discovery/NodeDiscovery";
 import { pathValueArchives } from "../pathValueArchives";
 import { ArchiveLocker, ArchiveTransaction } from "./ArchiveLocks";
-import { ArchivesSyncStatus, copyArchiveFile, IArchives } from "sliftutils/storage/IArchives";
+import { ArchivesSyncStatus, IArchives } from "sliftutils/storage/IArchives";
+import { copyArchiveFile, moveArchiveFile } from "sliftutils/storage/archiveHelpers";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import { blue, green, magenta, red } from "socket-function/src/formatting/logColors";
 import { devDebugbreak, getDomain } from "../../config";
@@ -35,15 +36,7 @@ const ASSUME_CONFIRMED_BEFORE = Date.parse("2026-07-23T00:00:00.000Z");
 const CONCURRENT_READ_COUNT = 128;
 const CONCURRENT_WRITE_COUNT = 128;
 const CONCURRENT_TRANSACTION_READS = 8;
-// When getFiles fails, how long we wait on the lock store's sync status before assuming the store is unreachable and falling back to non-authoritative (backblaze) reads.
-const FALLBACK_SYNC_STATUS_TIMEOUT = timeInSecond * 20;
 
-/** Moves by copy-then-delete (IArchives has no move). copyArchiveFile stamps the target with the source's write time, so ordering-by-writeTime survives the move. A missing source means another process already moved it, which is fine. */
-async function moveArchiveFile(from: IArchives, to: IArchives, path: string): Promise<void> {
-    let copied = await copyArchiveFile({ from, to, path, noFallbacks: true });
-    if (!copied) return;
-    await from.del(path);
-}
 
 export function createArchiveLocker2(config: {
     archiveValues: IArchives;
@@ -97,7 +90,7 @@ export function createArchiveLocker2(config: {
                 return;
             }
             try {
-                await moveArchiveFile(archives, archiveRecycleBin, key);
+                await moveArchiveFile({ from: archives, to: archiveRecycleBin, path: key, noFallbacks: true });
                 logNodeStats(`archives|Deleted TΔ`, formatNumber, 1);
             } catch {
                 // It was probably just moved by another process
@@ -134,7 +127,7 @@ export function createArchiveLocker2(config: {
                 if (correctFiles.has(file)) return;
                 let info = await archiveValues.getInfo(file);
                 console.log(`Moving file ${file}, ${moveCount}/${valuePaths.size}, ${formatNumber(info?.size || 0)} bytes`);
-                await moveArchiveFile(archiveValues, archiveRecycleBin, file);
+                await moveArchiveFile({ from: archiveValues, to: archiveRecycleBin, path: file, noFallbacks: true });
                 console.log(`Moved file ${file}, ${moveCount}/${valuePaths.size}`);
             }
             let moveFileParallel = runInParallel({ parallelCount: 8 }, moveFile);
@@ -155,7 +148,8 @@ export function createArchiveLocker2(config: {
             let isInRecycleBin = !!(await archiveRecycleBin.getInfo(file));
             if (isInRecycleBin) {
                 console.log(`Restoring ${file}, ${restoreCount++}/${files.length}`);
-                await moveArchiveFile(archiveRecycleBin, archiveValues, file);
+                // We are really careful to never delete from the recycle bin. Restores use copy instead of move, because move could theoretically have a bug where it deletes the file instead of moving it, and then our backups are gone. Copying barely costs any extra storage and isn't much slower, so move is reserved for the other direction (moving files INTO the recycle bin), where the source is still recoverable if something goes wrong.
+                await copyArchiveFile({ from: archiveRecycleBin, to: archiveValues, path: file, noFallbacks: true });
             } else {
                 console.log(`File ${file} is not in the recycle bin, skipping, ${restoreCount++}/${files.length}`);
             }

package/src/0-path-value-core/archiveLocks/archiveSnapshots.ts

@@ -215,9 +215,4 @@ export async function loadSnapshot(config: {
     for (let i = 0; i < 10; i++) {
         console.log(green(`!!!Finished loading snapshot!!!`));
     }
-
-    if (!config.noExit && !isPublic()) {
-        // Exit so we know when this is done. Really, all the servers need to be restarted after we load the snapshot, anyway. 
-        process.exit();
-    }
 }

package/src/1-path-client/RemoteWatcher.ts

@@ -35,7 +35,9 @@ const SHUTDOWN_REHOME_WINDOW = timeInMinute * 3;
 const SHUTDOWN_REHOME_POLL_INTERVAL = timeInSecond * 30;
 
 // Rehome the watches on an authority when an equivalent authority has this factor lower latency. At 2 the router's full-confidence cutoff factor (also 2) then reliably routes everything to the better node on the rewatch.
-const LATENCY_REHOME_FACTOR = 2;
+const LATENCY_REHOME_FACTOR = 1.5;
+// Jitter floor added to the candidate's latency before the factor comparison. Below roughly this many milliseconds the difference between two nodes is noise, not speed (1ms vs 2ms isn't "twice as fast"), and without the floor a candidate that momentarily reads near-zero always looks infinitely faster, so we rehome on pure jitter. Added only to the candidate, keeping us biased against needless moves.
+const LATENCY_REHOME_JITTER_MS = 50;
 // Both latencies must be medians over this many samples — with fewer we don't trust either number enough to move watches over it.
 const LATENCY_REHOME_HISTORY_COUNT = 10;
 const LATENCY_REHOME_POLL_INTERVAL = timeInMinute;
@@ -186,7 +188,7 @@ export class RemoteWatcher {
                 if (!areAuthoritySpecsEquivalent(entry.authoritySpec, candidate.authoritySpec)) continue;
                 let candidateLatency = getNodeLatencyMedian({ nodeId: candidate.nodeId, historyCount: LATENCY_REHOME_HISTORY_COUNT });
                 if (!candidateLatency || candidateLatency.historyUsed < LATENCY_REHOME_HISTORY_COUNT) continue;
-                if (candidateLatency.latency * LATENCY_REHOME_FACTOR > current.latency) continue;
+                if ((candidateLatency.latency + LATENCY_REHOME_JITTER_MS) * LATENCY_REHOME_FACTOR > current.latency) continue;
                 console.log(yellow(`Authority ${authorityId} has high latency (${formatTime(current.latency)}), and the equivalent authority ${candidate.nodeId} is much faster (${formatTime(candidateLatency.latency)}), so we are rehoming all paths watched on it`));
                 logErrors(this.refreshAllWatches(authorityId));
                 break;

package/src/archiveapps/snapshotThinningEntry.ts

@@ -0,0 +1,364 @@
+import "../inject";
+
+import { Querysub } from "../4-querysub/Querysub";
+import { logErrors } from "../errors";
+import { disablePathAuditer } from "../diagnostics/pathAuditerCallback";
+import { isNodeTrue, sort, timeInDay, timeInHour, timeInWeek, timeInYear } from "socket-function/src/misc";
+import { runInfinitePollCallAtStart, runInParallel } from "socket-function/src/batching";
+import { green, magenta, red, yellow } from "socket-function/src/formatting/logColors";
+import { formatNumber, formatTime } from "socket-function/src/formatting/format";
+import { archiveJSONT } from "../-a-archives/archivesJSONT";
+import { archives, archivesRecycleBin } from "../0-path-value-core/pathValueArchives";
+import { lazy } from "socket-function/src/caching";
+import yargs from "yargs";
+import { getArchives2 } from "../-a-archives/archives2";
+
+const timeInMonth = timeInDay * 30;
+
+const POLL_INTERVAL = timeInHour * 24;
+
+const MIN_PER_RANGE = 10;
+const thinningRanges: {
+    timeRange: number;
+    modulus: number;
+}[] = [
+        { timeRange: timeInDay * 3, modulus: 1 },
+        { timeRange: timeInWeek * 2, modulus: 2 },
+        { timeRange: timeInMonth * 2, modulus: 8 },
+        { timeRange: timeInYear * 100, modulus: 64 },
+    ];
+
+// Unreferenced recycle bin files younger than this are kept. Deleting an unreferenced recycle file too eagerly would destroy the backup of anything recycled since the last kept snapshot, so orphans only die once they have sat unreferenced long enough that nothing could still want them back.
+const RECYCLE_ORPHAN_MIN_AGE = timeInMonth * 2;
+
+const INDEX_KEY = "index";
+const PARALLEL_DELETE_COUNT = 32;
+const PARALLEL_READ_COUNT = 32;
+const PROGRESS_LOG_INTERVAL = 10 * 1000;
+
+const snapshotsArchive = lazy(() => getArchives2("snapshots"));
+const snapshotThinningArchive = lazy(() => getArchives2("snapshot-thinning-index"));
+
+type IndexMappings = {
+    nextIndex: number;
+    indexes: { [snapshotFile: string]: number };
+};
+
+const indexMappingStore = archiveJSONT<IndexMappings>(snapshotThinningArchive);
+
+let yargObj = isNodeTrue() && yargs(process.argv)
+    .option("watch", { type: "boolean", desc: "Watch, and run thinning every hour." })
+    .option("dry-run", { type: "boolean", desc: "Don't actually delete anything, just log what would happen." })
+    .argv || {};
+
+type SnapshotInfo = {
+    file: string;
+    time: number;
+    size: number;
+};
+
+function parseSnapshotTime(file: string): number {
+    let parts = file.split("   ");
+    for (let part of parts) {
+        let [key, value] = part.split("=");
+        if (key === "time") return +value;
+    }
+    throw new Error(`Snapshot file name has no time field: ${file}`);
+}
+
+async function listSnapshots(): Promise<SnapshotInfo[]> {
+    let infos = await snapshotsArchive().findInfo("");
+    let result: SnapshotInfo[] = [];
+    for (let info of infos) {
+        result.push({
+            file: info.path,
+            time: parseSnapshotTime(info.path),
+            size: info.size,
+        });
+    }
+    return result;
+}
+
+async function loadIndexMappings(): Promise<IndexMappings> {
+    let existing = await indexMappingStore.get(INDEX_KEY);
+    if (existing) return existing;
+    return { nextIndex: 0, indexes: {} };
+}
+
+function assignIndexes(snapshots: SnapshotInfo[], mappings: IndexMappings): {
+    mappings: IndexMappings;
+    changed: boolean;
+} {
+    let changed = false;
+    let nextMappings: IndexMappings = {
+        nextIndex: mappings.nextIndex,
+        indexes: { ...mappings.indexes },
+    };
+
+    let currentFiles = new Set(snapshots.map(s => s.file));
+    for (let file of Object.keys(nextMappings.indexes)) {
+        if (!currentFiles.has(file)) {
+            delete nextMappings.indexes[file];
+            changed = true;
+        }
+    }
+
+    let unassigned = snapshots.filter(s => !(s.file in nextMappings.indexes));
+    sort(unassigned, s => s.time);
+    for (let snap of unassigned) {
+        nextMappings.indexes[snap.file] = nextMappings.nextIndex++;
+        changed = true;
+    }
+    return { mappings: nextMappings, changed };
+}
+
+function pickThinningRange(ageMs: number) {
+    for (let range of thinningRanges) {
+        if (ageMs <= range.timeRange) return range;
+    }
+    return thinningRanges[thinningRanges.length - 1];
+}
+
+function decideThinning(snapshots: SnapshotInfo[], mappings: IndexMappings, now: number): {
+    keep: SnapshotInfo[];
+    remove: SnapshotInfo[];
+} {
+    let bucketed = new Map<typeof thinningRanges[number], SnapshotInfo[]>();
+    for (let snap of snapshots) {
+        let range = pickThinningRange(now - snap.time);
+        let list = bucketed.get(range);
+        if (!list) {
+            list = [];
+            bucketed.set(range, list);
+        }
+        list.push(snap);
+    }
+
+    let keep: SnapshotInfo[] = [];
+    let remove: SnapshotInfo[] = [];
+    for (let [range, list] of bucketed) {
+        if (list.length <= MIN_PER_RANGE || range.modulus <= 1) {
+            keep.push(...list);
+            continue;
+        }
+        let rangeKeep: SnapshotInfo[] = [];
+        let rangeRemove: SnapshotInfo[] = [];
+        for (let snap of list) {
+            let idx = mappings.indexes[snap.file];
+            if (idx % range.modulus === 0) {
+                rangeKeep.push(snap);
+            } else {
+                rangeRemove.push(snap);
+            }
+        }
+        sort(rangeRemove, s => s.time);
+        while (rangeRemove.length > 0 && rangeKeep.length < MIN_PER_RANGE) {
+            let restored = rangeRemove.pop();
+            if (!restored) break;
+            rangeKeep.push(restored);
+        }
+        keep.push(...rangeKeep);
+        remove.push(...rangeRemove);
+    }
+    return { keep, remove };
+}
+
+async function readSnapshotFiles(file: string): Promise<string[]> {
+    let buf = await snapshotsArchive().get(file);
+    if (!buf) {
+        throw new Error(`Snapshot file unreadable: ${file}`);
+    }
+    let text = buf.toString();
+    if (!text) return [];
+    return text.split("\n");
+}
+
+async function runThinningIteration(config: { dryRun: boolean }) {
+    let { dryRun } = config;
+    let prefix = dryRun ? yellow(`[DRY RUN] `) : ``;
+    let now = Date.now();
+
+    console.log(magenta(`${prefix}Starting snapshot thinning iteration (dryRun=${dryRun})`));
+
+    let snapshots = await listSnapshots();
+    sort(snapshots, s => s.time);
+
+    let mappingsBase = await loadIndexMappings();
+    let { mappings, changed } = assignIndexes(snapshots, mappingsBase);
+
+    if (changed && !dryRun) {
+        await indexMappingStore.set(INDEX_KEY, mappings);
+    } else if (changed) {
+        console.log(`${prefix}Would update index mappings (next index ${mappings.nextIndex}, ${Object.keys(mappings.indexes).length} entries)`);
+    }
+
+    let totalSnapshotBytes = snapshots.reduce((s, x) => s + x.size, 0);
+    console.log(`${prefix}Snapshots: ${formatNumber(snapshots.length)}, total snapshot file bytes: ${formatNumber(totalSnapshotBytes)}`);
+
+    let { keep, remove } = decideThinning(snapshots, mappings, now);
+    let removedBytes = remove.reduce((s, x) => s + x.size, 0);
+    console.log(`${prefix}Snapshots to keep: ${formatNumber(keep.length)}, to remove: ${formatNumber(remove.length)} (${formatNumber(removedBytes)} bytes)`);
+
+    for (let range of thinningRanges) {
+        let inRangeAll = snapshots.filter(s => pickThinningRange(now - s.time) === range);
+        let inRangeKeep = keep.filter(s => pickThinningRange(now - s.time) === range);
+        console.log(`  Range age<=${formatTime(range.timeRange)} modulus=${range.modulus}: ${formatNumber(inRangeAll.length)} => ${formatNumber(inRangeKeep.length)}`);
+    }
+
+    let recycleListStart = Date.now();
+    let recycleInfos = await archivesRecycleBin().findInfo("");
+    let recycleListDuration = Date.now() - recycleListStart;
+    let recycleSet = new Map<string, { size: number; createTime: number }>();
+    for (let info of recycleInfos) {
+        recycleSet.set(info.path, { size: info.size, createTime: info.createTime });
+    }
+    let totalRecycleBytes = 0;
+    for (let info of recycleSet.values()) totalRecycleBytes += info.size;
+    console.log(`${prefix}Recycle bin files: ${formatNumber(recycleSet.size)}, total bytes: ${formatNumber(totalRecycleBytes)}`);
+
+    let parallelRead = runInParallel({ parallelCount: PARALLEL_READ_COUNT }, readSnapshotFiles);
+    let referencedFiles = new Set<string>();
+    let keepFileLists = await Promise.all(keep.map(s => parallelRead(s.file)));
+    for (let list of keepFileLists) {
+        for (let f of list) referencedFiles.add(f);
+    }
+
+    let liveInfos = await archives().findInfo("");
+    let totalLiveBytes = 0;
+    for (let info of liveInfos) totalLiveBytes += info.size;
+    console.log(`${prefix}Production archives (live path-values): ${formatNumber(liveInfos.length)} files, ${formatNumber(totalLiveBytes)} bytes`);
+
+    for (let info of liveInfos) {
+        referencedFiles.add(info.path);
+    }
+
+    let recycleToDelete: { file: string; size: number }[] = [];
+    let youngOrphanCount = 0;
+    let youngOrphanBytes = 0;
+    for (let [file, info] of recycleSet) {
+        if (referencedFiles.has(file)) continue;
+        if (now - info.createTime < RECYCLE_ORPHAN_MIN_AGE) {
+            youngOrphanCount++;
+            youngOrphanBytes += info.size;
+            continue;
+        }
+        recycleToDelete.push({ file, size: info.size });
+    }
+    let recycleDeleteBytes = recycleToDelete.reduce((s, x) => s + x.size, 0);
+    console.log(`${prefix}Planned: delete ${formatNumber(remove.length)} snapshots (${formatNumber(removedBytes)} bytes), ${formatNumber(recycleToDelete.length)} recycle bin files (${formatNumber(recycleDeleteBytes)} bytes). Keeping ${formatNumber(youngOrphanCount)} unreferenced recycle bin files (${formatNumber(youngOrphanBytes)} bytes) that are younger than ${formatTime(RECYCLE_ORPHAN_MIN_AGE)}`);
+
+    function logSummary() {
+        let snapshotCountAfter = keep.length;
+        let snapshotBytesAfter = totalSnapshotBytes - removedBytes;
+        let recycleCountAfter = recycleSet.size - recycleToDelete.length;
+        let recycleBytesAfter = totalRecycleBytes - recycleDeleteBytes;
+        let beforeTotalBytes = totalSnapshotBytes + totalRecycleBytes;
+        let afterTotalBytes = snapshotBytesAfter + recycleBytesAfter;
+        let beforeTotalFiles = snapshots.length + recycleSet.size;
+        let afterTotalFiles = snapshotCountAfter + recycleCountAfter;
+
+        function pct(after: number, before: number) {
+            if (!before) return `100%`;
+            return `${(after / before * 100).toFixed(1)}%`;
+        }
+
+        console.log(magenta(`${prefix}===== Summary =====`));
+        console.log(`${prefix}Snapshots:   ${formatNumber(snapshots.length)} => ${formatNumber(snapshotCountAfter)} files (${pct(snapshotCountAfter, snapshots.length)} kept), ${formatNumber(totalSnapshotBytes)} => ${formatNumber(snapshotBytesAfter)} bytes (${pct(snapshotBytesAfter, totalSnapshotBytes)} kept)`);
+        console.log(`${prefix}Recycle bin: ${formatNumber(recycleSet.size)} => ${formatNumber(recycleCountAfter)} files (${pct(recycleCountAfter, recycleSet.size)} kept), ${formatNumber(totalRecycleBytes)} => ${formatNumber(recycleBytesAfter)} bytes (${pct(recycleBytesAfter, totalRecycleBytes)} kept)`);
+        console.log(`${prefix}Total:       ${formatNumber(beforeTotalFiles)} => ${formatNumber(afterTotalFiles)} files (${pct(afterTotalFiles, beforeTotalFiles)} kept), ${formatNumber(beforeTotalBytes)} => ${formatNumber(afterTotalBytes)} bytes (${pct(afterTotalBytes, beforeTotalBytes)} kept)`);
+        console.log(`${prefix}Live archives (unchanged): ${formatNumber(liveInfos.length)} files, ${formatNumber(totalLiveBytes)} bytes`);
+        console.log(`${prefix}Recycle bin findInfo("") took: ${formatTime(recycleListDuration)} (for ${formatNumber(recycleSet.size)} files)`);
+    }
+
+    if (dryRun) {
+        logSummary();
+        console.log(green(`${prefix}Dry run complete. No changes made.`));
+        return;
+    }
+
+    async function deleteWithProgress(label: string, files: string[], deleter: (file: string) => Promise<void>) {
+        if (files.length === 0) {
+            console.log(`No ${label} to delete`);
+            return;
+        }
+        let done = 0;
+        let failed = 0;
+        let startTime = Date.now();
+        let lastLog = startTime;
+        let lastLogDone = 0;
+        let peakRate = 0;
+        function logProgress() {
+            let nowTs = Date.now();
+            let elapsed = nowTs - startTime;
+            let avgRate = done / (elapsed / 1000);
+            let intervalMs = nowTs - lastLog;
+            let intervalRate = intervalMs > 0 ? (done - lastLogDone) / (intervalMs / 1000) : 0;
+            if (intervalRate > peakRate) peakRate = intervalRate;
+            let remaining = files.length - done;
+            let etaMs = avgRate > 0 ? remaining / avgRate * 1000 : 0;
+            console.log(`  ${label}: ${formatNumber(done)}/${formatNumber(files.length)} deleted (${(done / files.length * 100).toFixed(1)}%), ${formatNumber(failed)} failed, elapsed ${formatTime(elapsed)}, ETA ${formatTime(etaMs)}, avg ${avgRate.toFixed(1)}/s, peak ${peakRate.toFixed(1)}/s`);
+            lastLog = nowTs;
+            lastLogDone = done;
+        }
+        let parallel = runInParallel({ parallelCount: PARALLEL_DELETE_COUNT }, async (file: string) => {
+            try {
+                await deleter(file);
+            } catch (e) {
+                failed++;
+                console.error(`Failed to delete ${file}: ${String((e as Error)?.stack || e)}`);
+            }
+            done++;
+            if (Date.now() - lastLog >= PROGRESS_LOG_INTERVAL) {
+                logProgress();
+            }
+        });
+        console.log(`Deleting ${formatNumber(files.length)} ${label}...`);
+        await Promise.all(files.map(f => parallel(f)));
+        logProgress();
+        console.log(green(`Deleted ${formatNumber(done - failed)} ${label} (${formatNumber(failed)} failed) in ${formatTime(Date.now() - startTime)}`));
+    }
+
+    await deleteWithProgress("snapshots", remove.map(s => s.file), async file => {
+        await snapshotsArchive().del(file);
+    });
+
+    if (remove.length > 0) {
+        let mappingsAfter: IndexMappings = {
+            nextIndex: mappings.nextIndex,
+            indexes: { ...mappings.indexes },
+        };
+        for (let snap of remove) {
+            delete mappingsAfter.indexes[snap.file];
+        }
+        await indexMappingStore.set(INDEX_KEY, mappingsAfter);
+    }
+
+    await deleteWithProgress("recycle bin files", recycleToDelete.map(x => x.file), async file => {
+        await archivesRecycleBin().del(file);
+    });
+
+    logSummary();
+    console.log(green(`Snapshot thinning complete.`));
+}
+
+async function main() {
+    disablePathAuditer();
+    await Querysub.hostService("snapshot-thinning");
+
+    let dryRun = !!yargObj["dry-run"];
+
+    if (yargObj.watch) {
+        console.log("Running in watch mode.");
+        await runInfinitePollCallAtStart(POLL_INTERVAL, () => runThinningIteration({ dryRun }));
+    } else {
+        try {
+            await runThinningIteration({ dryRun });
+        } catch (error) {
+            console.error((error as Error).stack ?? error);
+        } finally {
+            process.exit();
+        }
+    }
+}
+
+main().catch(logErrors);

package/src/deployManager/MachinesPage.tsx

@@ -10,6 +10,7 @@ import { MachineDetailPage } from "./components/MachineDetailPage";
 import { Anchor } from "../library-components/ATag";
 import { DeployPage } from "./components/DeployPage";
 import { StoragePage } from "./components/storage/StoragePage";
+import { RoutingTablePage } from "../diagnostics/misc-pages/RoutingTablePage";
 
 export class MachinesPage extends qreact.Component {
     private renderTabs() {
@@ -22,6 +23,7 @@ export class MachinesPage extends qreact.Component {
                 { key: "services", label: "Services", otherKeys: ["service-detail"] },
                 { key: "deploy", label: "Deploy", otherKeys: [] },
                 { key: "storage", label: "Storage", otherKeys: [] },
+                { key: "routing", label: "PathValues / Functions Routing", otherKeys: [] },
             ].map(tab => {
                 let isActive = currentViewParam.value === tab.key || tab.otherKeys.includes(currentViewParam.value);
                 return <Anchor noStyles key={tab.key}
@@ -50,6 +52,7 @@ export class MachinesPage extends qreact.Component {
                 {currentViewParam.value === "machine-detail" && <MachineDetailPage />}
                 {currentViewParam.value === "deploy" && <DeployPage />}
                 {currentViewParam.value === "storage" && <StoragePage />}
+                {currentViewParam.value === "routing" && <RoutingTablePage />}
             </div>
         </div>;
     }

package/src/deployManager/components/storage/RouteConfigView.tsx

@@ -45,7 +45,7 @@ const JSON_INDENT = 4;
 /** String sources are the shorthand for an always-valid, unsharded backblaze bucket. */
 function normalizeSource(source: RemoteConfigBase): Source {
     if (typeof source !== "string") return source;
-    return { type: "backblaze", url: source, validWindow: FULL_VALID_WINDOW };
+    return { type: "backblaze", url: source, validWindow: FULL_VALID_WINDOW, name: "from url" };
 }
 
 /** Where a source lives, with the bucket the config belongs to masked out so buckets that differ only by their own name group together. */

package/src/deployManager/components/storage/Tag.tsx

@@ -22,7 +22,6 @@ export function getSourceTags(source: HostedConfig | BackblazeConfig): SourceTag
     if (source.intermediate) tags.push({ icon: "⏳", text: "intermediate" });
     if (source.type === "remote") {
         if (source.fast) tags.push({ icon: "⚡", text: "fast" });
-        if (source.rawDisk) tags.push({ icon: "💽", text: "rawDisk" });
         if (source.writeDelay !== undefined) tags.push({ icon: "⏱", text: `writeDelay ${formatTime(source.writeDelay)}` });
     }
     if (source.readerDiskLimit !== undefined) tags.push({ icon: "📦", text: `readCache ${formatNumber(source.readerDiskLimit)}B` });

package/src/deployManager/urlParams.ts

@@ -1,6 +1,6 @@
 import { URLParam } from "../library-components/URLParam";
 
-export type ViewType = "services" | "machines" | "service-detail" | "machine-detail" | "deploy" | "storage";
+export type ViewType = "services" | "machines" | "service-detail" | "machine-detail" | "deploy" | "storage" | "routing";
 
 export const currentViewParam = new URLParam<ViewType>("machineview", "machines");
 export const selectedServiceIdParam = new URLParam("serviceId", "");

package/src/diagnostics/logs/diskLogger.ts

@@ -1,3 +1,6 @@
+require("debugbreak")(2);
+debugger;
+
 import { isNode } from "typesafecss";
 import { partialCopyObject } from "../../misc";
 import { canHaveChildren } from "socket-function/src/types";

package/src/diagnostics/managementPages.tsx

@@ -80,12 +80,6 @@ export async function registerManagementPages2(config: {
     let inputPages: typeof config.pages = [];
 
 
-    inputPages.push({
-        title: "Routing Table",
-        componentName: "RoutingTablePage",
-        controllerName: "RoutingTablePageController",
-        getModule: () => import("./misc-pages/RoutingTablePage"),
-    });
     inputPages.push({
         title: "LOG VIEWER",
         componentName: "LogViewer3",
@@ -404,6 +398,10 @@ class ManagementRoot extends qreact.Component {
                             managementPageURL.getOverride("MachinesPage"),
                             currentViewParam.getOverride("storage"),
                         ]}>Storage</ATag>
+                        <ATag values={[
+                            managementPageURL.getOverride("MachinesPage"),
+                            currentViewParam.getOverride("routing"),
+                        ]}>PathValues / Functions Routing</ATag>
                         {pages.map(page =>
                             <ATag values={[{ param: managementPageURL, value: page.componentName }]}>{page.title}</ATag>
                         )}

package/src/diagnostics/misc-pages/PathValueServersSection.tsx

@@ -45,19 +45,18 @@ class PathSummaryTable extends qreact.Component<{ nodeId: string; direction: Pat
 
 // One row per path-value server, grouped by machine. Reads/writes lead the line; expanding reveals that server's
 // per-path breakdown for both directions.
-class AuthorityNodeRow extends qreact.Component<{ info: NodeAuthorityInfo; firstOfMachine: boolean }> {
+class AuthorityNodeRow extends qreact.Component<{ info: NodeAuthorityInfo }> {
     state = {
         expanded: false,
     };
     render() {
         let info = this.props.info;
-        let spec = info.spec!;
+        let spec = info.spec;
         let expanded = this.state.expanded;
         let totals = RoutingTableSynced(getBrowserUrlNode()).getNodePathTotals(info.nodeId);
         let machineId = machineIdOf(info.nodeId);
         return <div
-            className={css.button.vbox(6).pad2(10).fillWidth.hsl(0, 0, 99)
-                .borderTop(this.props.firstOfMachine ? "2px solid hsl(0, 0%, 78%)" : "1px solid hsl(0, 0%, 90%)")}
+            className={css.button.vbox(6).pad2(10).fillWidth.hsl(0, 0, 99).borderTop("1px solid hsl(0, 0%, 88%)")}
             onClick={() => this.state.expanded = !expanded}
         >
             <div className={css.hbox(10).fillWidth}>
@@ -66,12 +65,12 @@ class AuthorityNodeRow extends qreact.Component<{ info: NodeAuthorityInfo; first
                     ↓{totals ? formatNumber(totals.received) : "…"} ↑{totals ? formatNumber(totals.sent) : "…"}
                 </span>
                 <span className={css.width(`${MACHINE_CELL_WIDTH_CH}ch`).flexShrink0.colorhsl(0, 0, 45)} title={machineId}>
-                    {this.props.firstOfMachine ? machineId.slice(0, ID_CHARS) : ""}
+                    {machineId.slice(0, ID_CHARS)}
                 </span>
                 <span className={css.boldStyle} title={info.nodeId}>{threadLabel(info.nodeId)}</span>
-                <span className={css.colorhsl(0, 0, 45)}>{spec.routeStart.toFixed(4)} - {spec.routeEnd.toFixed(4)}</span>
-                <AuthorityRangeBar start={spec.routeStart} end={spec.routeEnd} />
-                {spec.excludeDefault && <span className={css.colorhsl(0, 70, 35)}>(excludes default)</span>}
+                {spec && <span className={css.colorhsl(0, 0, 45)}>{spec.routeStart.toFixed(4)} - {spec.routeEnd.toFixed(4)}</span>}
+                {spec && <AuthorityRangeBar start={spec.routeStart} end={spec.routeEnd} />}
+                {spec?.excludeDefault && <span className={css.colorhsl(0, 70, 35)}>(excludes default)</span>}
             </div>
             {expanded &&
                 <div className={css.hbox(16).wrap.fillWidth.alignItems("flex-start")} onClick={e => e.stopPropagation()}>
@@ -89,42 +88,67 @@ class AuthorityNodeRow extends qreact.Component<{ info: NodeAuthorityInfo; first
     }
 }
 
-// Invalidate: drop every cached synced call for this node so the whole page's data disappears and refetches. Clear:
-// wipe the in-memory path-value stats (totals and trees) on every node, then invalidate so the now-empty stats reload.
+// Invalidate: drop every cached synced call for this node so the whole page's data disappears and refetches.
 function invalidateRoutingTable(): void {
-    RoutingTableSynced(getBrowserUrlNode()).resetAll();
+    let synced = RoutingTableSynced(getBrowserUrlNode());
+    synced.getNodePathTotals.resetAll();
+    synced.getNodePathSummary.resetAll();
 }
+// Clear: wipe the in-memory path-value stats (totals and trees) on every node, then drop only the stats caches so the
+// now-empty stats reload — leaving the specs/latency/traffic caches untouched.
 async function clearAllPathStats(nodeIds: string[]): Promise<void> {
     let synced = RoutingTableSynced(getBrowserUrlNode());
     await Promise.all(nodeIds.map(nodeId => synced.clearNodePathStats.promise(nodeId).catch(() => { })));
-    synced.resetAll();
+    synced.getNodePathTotals.resetAll();
+    synced.getNodePathSummary.resetAll();
+}
+
+// A list of server rows, most-active first (by total path values sent + received).
+class PathServerList extends qreact.Component<{ infos: NodeAuthorityInfo[]; emptyText: string }> {
+    render() {
+        let synced = RoutingTableSynced(getBrowserUrlNode());
+        let sumOf = (nodeId: string) => {
+            let totals = synced.getNodePathTotals(nodeId);
+            return (totals?.sent ?? 0) + (totals?.received ?? 0);
+        };
+        let infos = this.props.infos.slice();
+        // Two stable passes: machine/thread as a tiebreak, then sent+received descending as the primary order (so rows
+        // with equal — or still-loading, hence 0 — totals keep a stable machine/thread order instead of jittering).
+        let sep = String.fromCharCode(1);
+        sort(infos, x => `${machineIdOf(x.nodeId)}${sep}${threadLabel(x.nodeId)}`);
+        sort(infos, x => -sumOf(x.nodeId));
+        if (!infos.length) return <div className={css.colorhsl(0, 0, 50)}>{this.props.emptyText}</div>;
+        return <div className={css.vbox(0).fillWidth.bord2(0, 0, 88)}>
+            {infos.map(info => <AuthorityNodeRow key={info.nodeId} info={info} />)}
+        </div>;
+    }
 }
 
 export class PathValueServersSection extends qreact.Component<{ infos: NodeAuthorityInfo[] }> {
     render() {
         let infos = this.props.infos;
         let allNodeIds = infos.map(x => x.nodeId);
-        let routableInfos = infos.filter(x => x.spec && x.spec.routeStart >= 0 && x.spec.routeEnd >= 0);
-        // Group each server's threads under their machine (machine, then thread), so a machine's servers sit together.
-        let sep = String.fromCharCode(1);
-        sort(routableInfos, x => `${machineIdOf(x.nodeId)}${sep}${threadLabel(x.nodeId)}`);
+        // Path value servers are the authorities (they own a route range). Every other node can still send/receive
+        // path values, so it gets its own list below.
+        let serverInfos = infos.filter(x => x.spec && x.spec.routeStart >= 0 && x.spec.routeEnd >= 0);
+        let otherInfos = infos.filter(x => !(x.spec && x.spec.routeStart >= 0 && x.spec.routeEnd >= 0));
 
         let buttonClass = css.pad2(12, 8).button.bord2(0, 0, 20).hsl(0, 0, 100).whiteSpace("nowrap");
         return <div className={css.vbox(8).fillWidth}>
             <div className={css.hbox(12).alignItems("center")}>
-                <h2 className={css.margin(0)}>Path Value Servers ({routableInfos.length})</h2>
+                <h2 className={css.margin(0)}>Path Value Servers ({serverInfos.length})</h2>
                 <button className={buttonClass} onClick={() => invalidateRoutingTable()}>Invalidate</button>
                 <button className={buttonClass} onClick={() => void clearAllPathStats(allNodeIds)}>Clear stats</button>
             </div>
             <div className={css.fontSize(13).colorhsl(0, 0, 45)}>
                 Path values each server has received (↓) and sent (↑) since startup or the last clear. Expand a server for its top {PATH_SUMMARY_LIMIT} paths in each direction.
             </div>
-            <div className={css.vbox(0).fillWidth.bord2(0, 0, 88)}>
-                {routableInfos.map((info, i) => {
-                    let firstOfMachine = i === 0 || machineIdOf(routableInfos[i - 1].nodeId) !== machineIdOf(info.nodeId);
-                    return <AuthorityNodeRow key={info.nodeId} info={info} firstOfMachine={firstOfMachine} />;
-                })}
+            <PathServerList infos={serverInfos} emptyText="(no path value servers)" />
+            <h2 className={css.margin(0)}>Other Senders ({otherInfos.length})</h2>
+            <div className={css.fontSize(13).colorhsl(0, 0, 45)}>
+                Nodes that aren't path value servers but still send/receive path values.
             </div>
+            <PathServerList infos={otherInfos} emptyText="(no other senders)" />
         </div>;
     }
 }

package/src/diagnostics/misc-pages/RoutingTablePage.tsx

@@ -107,9 +107,6 @@ export const RoutingTablePageController = SocketFunction.register(
     () => ({
         hooks: [assertIsManagementUser],
     }),
-    {
-        noAutoExpose: true,
-    }
 );
 
 export const RoutingTableSynced = getSyncedController(RoutingTablePageController, {

package/src/diagnostics/misc-pages/SnapshotViewer.tsx

@@ -124,9 +124,23 @@ export class SnapshotViewer extends qreact.Component {
                         <div>Byte Count: {formatNumber(entry.byteCount)}</div>
                     </div>
                 }
+                {snapshotInfo &&
+                    <div class={css.vbox(4)}>
+                        <div class={css.boldStyle}>By status</div>
+                        <Table
+                            rows={summarizeSnapshotByStatus(snapshotInfo.files)}
+                            columns={{
+                                status: { title: "Status" },
+                                files: { title: "Files" },
+                                values: { title: "Values" },
+                                bytes: { title: "Bytes" },
+                            }}
+                        />
+                    </div>
+                }
                 <Button onClick={async () => {
                     let didConfirm = await seriousConfirm({
-                        message: `Restoring will break all servers. This won't break your data, but you will have to restart all servers before the system will be stable again. Before then data will be random, and invalid writes might be allowed. It is recommend to run this with only 1 querysub server running, on a developers machine. Consider writing a database iterate script instead of restoring (similar to how gc and join work).`,
+                        message: `Restoring will put the servers in a bad state. You're gonna need to restart all services after this.`,
                     });
                     if (!didConfirm) {
                         console.log(red("Aborted restore"));
@@ -137,7 +151,6 @@ export class SnapshotViewer extends qreact.Component {
                         overview: entry
                     });
                     console.log(green("Restored snapshot"));
-                    window.location.reload();
                 }}>
                     Restore
                 </Button>
@@ -169,6 +182,26 @@ export class SnapshotViewer extends qreact.Component {
     }
 }
 
+// Canonical status order, so the per-status summary rows stay in a stable, readable order.
+const SNAPSHOT_STATUS_ORDER = ["live", "zombie", "recycled", "missing"];
+type SnapshotStatusRow = { status: string; files: number; values: number; bytes: number };
+function summarizeSnapshotByStatus(files: ArchiveSnapshotRead["files"]): SnapshotStatusRow[] {
+    let byStatus = new Map<string, SnapshotStatusRow>();
+    for (let file of files) {
+        let row = byStatus.get(file.state);
+        if (!row) {
+            row = { status: file.state, files: 0, values: 0, bytes: 0 };
+            byStatus.set(file.state, row);
+        }
+        row.files++;
+        row.values += file.valueCount;
+        row.bytes += file.byteCount;
+    }
+    let rows = Array.from(byStatus.values());
+    sort(rows, row => SNAPSHOT_STATUS_ORDER.indexOf(row.status));
+    return rows;
+}
+
 async function downloadSnapshotFile(component: SnapshotViewer, file: string) {
     if (component.state.downloading[file]) return;
     Querysub.localCommit(() => {

package/src/misc/appPaths.ts

@@ -13,6 +13,7 @@ export const STORAGE_DIR = "./database-storage/";
 
 let yargObj = parseArgsFactory()
     .option("domain", { type: "string", desc: `Sets the domain` })
+    .option("port", { type: "number", desc: `Sets the port` })
     .argv
     ;
 
@@ -42,6 +43,10 @@ export function getDomain() {
     return yargObj.domain || querysubConfig().domain || "querysub.com";
 }
 
+export function getPort(): number | undefined {
+    return yargObj.port;
+}
+
 export function getBackblazePath() {
     let testPaths = [
         STORAGE_DIR + "backblaze.json",

package/src/storageSetup.ts

@@ -1,23 +1,15 @@
 process.env.PRODUCTION = "true";
-import "./inject";
-import { getDomain, getPort } from "./config";
-import { getTrustedMachineIds } from "./-d-trust/NetworkTrust2";
-import { getExternalIP } from "socket-function/src/networking";
+// TODO: Add this back
+//import "./inject";
 
 // The storage server entry point (see bin/storageserve.js): sets up the CLI args the sliftutils storage server expects, synchronizes our network trust list into its trust store, and runs its CLI.
 
 async function main() {
-    if (!process.argv.includes("--url")) {
-        let ip = await getExternalIP();
-        let ipDomain = ip.replaceAll(".", "-");
-        process.argv.push("--url", `https://${ipDomain}.${getDomain()}:${getPort()}`);
-    }
-
-    let trustedMachineIds = await getTrustedMachineIds();
-
-    let { setTrustedMachines } = await import("sliftutils/storage/remoteStorage/serverConfig");
-    await setTrustedMachines({ account: "root", machineIds: trustedMachineIds });
-    console.log(`Synchronized ${trustedMachineIds.length} trusted machines`);
+    // TODO: Add this back
+    // let trustedMachineIds = await getTrustedMachineIds();
+    // let { setTrustedMachines } = await import("sliftutils/storage/remoteStorage/serverConfig");
+    // await setTrustedMachines({ account: "root", machineIds: trustedMachineIds });
+    // console.log(`Synchronized ${trustedMachineIds.length} trusted machines`);
 
     await import("sliftutils/storage/remoteStorage/storageServerCli");
 }