querysub 0.619.0 → 0.621.0

package/appSecrets.ts

@@ -2,7 +2,6 @@ import fs from "fs";
 import { lazy } from "socket-function/src/caching";
 import { ArchivesBackblaze } from "sliftutils/storage/backblaze";
 import { STORAGE_DIR, getDomain, getBackblazePath } from "./src/misc/appPaths";
-import { Querysub } from "./src/4-querysub/Querysub";
 import { formatDateTime } from "socket-function/src/formatting/format";
 
 const keysArchives = lazy(() => new ArchivesBackblaze({ bucketName: getDomain() }));
@@ -26,8 +25,11 @@ export const getCloudflareCreds = lazy(async (): Promise<{ key: string; email: s
 export async function getAppSecret(key: string): Promise<string | undefined> {
     // HACK: This is used by the storage server. Which we want to shut down immediately. That way the next instance can start up. The sword server already looks for notifications about when it's going to be shutting down and it flushes to disk early in this case, so it should be fairly safe to hard kill it.
     doImmediateShutdown();
+
+    // Actually maybe the wrapper should do this instead. 
     // Import query sub so our logs go to our actual log server. Otherwise we won't know if we get any errors. 
-    Querysub;
+    //await import("./src/4-querysub/Querysub");
+
     if (key.startsWith("backblaze.json.")) {
         let creds = JSON.parse(fs.readFileSync(getBackblazePath(), "utf8")) as { applicationKeyId?: string; applicationKey?: string };
         if (key === "backblaze.json.applicationKeyId") return creds.applicationKeyId;

package/bin/emergency-update.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+
+require("typenode");
+require("../src/deployManager/emergencyUpdateMain");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.619.0",
+  "version": "0.621.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -25,6 +25,9 @@
     "mcp2": "yarn typenode ./src/diagnostics/debugger/mcp-server.ts",
     "mc2": "yarn typenode ./src/diagnostics/debugger/mcp-server.ts --cwd D:/repos/qs-cyoa/",
     "autofix": "yarn typenode ./src/diagnostics/logs/errorTickets/autoFixerEntry.ts",
+    "thin": "yarn typenode ./src/archiveapps/snapshotThinningEntry.ts",
+    "thin-watch": "yarn typenode ./src/archiveapps/snapshotThinningEntry.ts --watch",
+    "thin-dry": "yarn typenode ./src/archiveapps/snapshotThinningEntry.ts --dry-run",
     "parameters-timeline": "yarn typenode ./src/deployManager/parametersTimelineMain.ts",
     "setup-machine": "yarn typenode ./src/deployManager/setupMachineMain.ts",
     "stop-machine": "yarn typenode ./src/deployManager/stopMachineMain.ts",
@@ -52,6 +55,7 @@
     "error-watch-public": "./bin/error-watch-public.js",
     "audit-imports": "./bin/audit-imports.js",
     "audit-disk-values": "./bin/audit-disk-values.js",
+    "emergency-update": "./bin/emergency-update.js",
     "mcp-indexed-logs": "./bin/mcp-indexed-logs.js",
     "autofix": "./bin/autofix.js",
     "storageserve": "./bin/storageserve.js"
@@ -75,7 +79,7 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
-    "sliftutils": "^1.7.88",
+    "sliftutils": "^1.7.90",
     "socket-function": "^1.2.30",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",

package/src/-a-archives/archives2.ts

@@ -44,13 +44,22 @@ function archiveBuilder(bucket: string, overrides: Partial<RemoteConfigBase>) {
     const HETZNER = `https://65-109-93-113.querysubtest.com:5234/file/${STORAGE_ACCOUNT}/${bucket}/storage/storagerouting.json`;
     const BACKBLAZE = `https://f002.backblazeb2.com/file/${bucket}/storage/storagerouting.json`;
 
-    function stripeSources(urls: string[]) {
+    function stripeSources(urls: string[], config: { routeVersion: number }) {
         let sources: Omit<Exclude<RemoteConfigBase, string>, "validWindow">[] = [];
         for (let j = 0; j < urls.length; j++) {
             // Shift by j
             let currentUrls = urls.slice(j).concat(urls.slice(0, j));
             for (let i = 0; i < urls.length; i++) {
-                sources.push({ type: "remote", url: currentUrls[i], route: [i / urls.length, (i + 1) / urls.length], });
+                let url = new URL(currentUrls[i]);
+                let routeStart = i / urls.length;
+                let routeEnd = (i + 1) / urls.length;
+                sources.push({
+                    type: "remote",
+                    url: currentUrls[i],
+                    route: [routeStart, routeEnd],
+                    // IMPORTANT! name decides where our files are stored. We can't have two things writing to the same folder at the same time. But it also means if we reuse the name, then it has to be the same data. 
+                    name: url.hostname.replaceAll(/[^a-z0-9-]/g, "_") + "_v" + config.routeVersion + "_" + routeStart + "_" + routeEnd,
+                });
             }
         }
         return sources;
@@ -66,14 +75,15 @@ function archiveBuilder(bucket: string, overrides: Partial<RemoteConfigBase>) {
             sources: [
                 // NOTE: Once we add new sources, we need to add something at a new start time in the future so we have time to move the files. 
                 // TODO: EVENTUALLY, Once we have enough data, instead of restriping, we need to bisect the data. That way we can slowly move over the data instead of trying to rewrite all of our data all at once. However, for the moment, moving all the data is probably fine. On the data center, we should be able to move terabytes of data every hour, so we should be able to handle quite a bit, especially if we have a few days to run the restriping. 
-                ...stripeSources([ONTARIO, HETZNER]),
-                { type: "backblaze", url: BACKBLAZE, },
+                // IMPORTANT! Every time we re-stripe, we need to increment the route version. Otherwise, we could reuse a folder that we previously used that's no longer in sync. which will cause us to resurrect old data, which is very bad. We don't want to do that...
+                ...stripeSources([ONTARIO, HETZNER], { routeVersion: 1 }),
+                { type: "backblaze", url: BACKBLAZE, name: "backblaze" },
             ]
         },
     ]);
 
     return createArchives({
-        version: 22,
+        version: 23,
         sources,
     });
 }
@@ -159,6 +169,29 @@ export function getArchives2Public(folder: string) {
     return nestBucket(folder, archivesBuilderCache(domain + "-public", { public: true, allowedOrigins: getAllowedOrigins(getDomain()) }));
 }
 
+export function getArchives2Debug(folder: string) {
+    let routeVersion = 1;
+    let archives = createArchives({
+        version: 20,
+        sources: [
+            // IMPORTANT! If we start changing around routes, it could resurrect old data. We can fix this via incrementing the route version.
+            {
+                type: "remote", url: `https://99-250-124-91.querysubtest.com:5239/file/${STORAGE_ACCOUNT}/${folder}/storage/storagerouting.json`,
+                validWindow: [0, Number.MAX_SAFE_INTEGER],
+                route: [0, 0.5],
+                name: "storagea_v" + routeVersion,
+            },
+            {
+                type: "remote", url: `https://65-109-93-113.querysubtest.com:5240/file/${STORAGE_ACCOUNT}/${folder}/storage/storagerouting.json`,
+                validWindow: [0, Number.MAX_SAFE_INTEGER],
+                route: [0.5, 1],
+                name: "storageb_v" + routeVersion,
+            },
+        ],
+    });
+    return nestBucket(folder, archives);
+}
+
 
 export async function initArchives2() {
     let buckets = [

package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts

@@ -4,7 +4,8 @@ import { decodeNodeId } from "sliftutils/misc/https/certs";
 import { getOwnNodeId, getOwnNodeIdAssert } from "../../-f-node-discovery/NodeDiscovery";
 import { pathValueArchives } from "../pathValueArchives";
 import { ArchiveLocker, ArchiveTransaction } from "./ArchiveLocks";
-import { ArchivesSyncStatus, copyArchiveFile, IArchives } from "sliftutils/storage/IArchives";
+import { ArchivesSyncStatus, IArchives } from "sliftutils/storage/IArchives";
+import { copyArchiveFile, moveArchiveFile } from "sliftutils/storage/archiveHelpers";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import { blue, green, magenta, red } from "socket-function/src/formatting/logColors";
 import { devDebugbreak, getDomain } from "../../config";
@@ -35,15 +36,7 @@ const ASSUME_CONFIRMED_BEFORE = Date.parse("2026-07-23T00:00:00.000Z");
 const CONCURRENT_READ_COUNT = 128;
 const CONCURRENT_WRITE_COUNT = 128;
 const CONCURRENT_TRANSACTION_READS = 8;
-// When getFiles fails, how long we wait on the lock store's sync status before assuming the store is unreachable and falling back to non-authoritative (backblaze) reads.
-const FALLBACK_SYNC_STATUS_TIMEOUT = timeInSecond * 20;
 
-/** Moves by copy-then-delete (IArchives has no move). copyArchiveFile stamps the target with the source's write time, so ordering-by-writeTime survives the move. A missing source means another process already moved it, which is fine. */
-async function moveArchiveFile(from: IArchives, to: IArchives, path: string): Promise<void> {
-    let copied = await copyArchiveFile({ from, to, path, noFallbacks: true });
-    if (!copied) return;
-    await from.del(path);
-}
 
 export function createArchiveLocker2(config: {
     archiveValues: IArchives;
@@ -97,7 +90,7 @@ export function createArchiveLocker2(config: {
                 return;
             }
             try {
-                await moveArchiveFile(archives, archiveRecycleBin, key);
+                await moveArchiveFile({ from: archives, to: archiveRecycleBin, path: key, noFallbacks: true });
                 logNodeStats(`archives|Deleted TΔ`, formatNumber, 1);
             } catch {
                 // It was probably just moved by another process
@@ -134,7 +127,7 @@ export function createArchiveLocker2(config: {
                 if (correctFiles.has(file)) return;
                 let info = await archiveValues.getInfo(file);
                 console.log(`Moving file ${file}, ${moveCount}/${valuePaths.size}, ${formatNumber(info?.size || 0)} bytes`);
-                await moveArchiveFile(archiveValues, archiveRecycleBin, file);
+                await moveArchiveFile({ from: archiveValues, to: archiveRecycleBin, path: file, noFallbacks: true });
                 console.log(`Moved file ${file}, ${moveCount}/${valuePaths.size}`);
             }
             let moveFileParallel = runInParallel({ parallelCount: 8 }, moveFile);
@@ -155,7 +148,8 @@ export function createArchiveLocker2(config: {
             let isInRecycleBin = !!(await archiveRecycleBin.getInfo(file));
             if (isInRecycleBin) {
                 console.log(`Restoring ${file}, ${restoreCount++}/${files.length}`);
-                await moveArchiveFile(archiveRecycleBin, archiveValues, file);
+                // We are really careful to never delete from the recycle bin. Restores use copy instead of move, because move could theoretically have a bug where it deletes the file instead of moving it, and then our backups are gone. Copying barely costs any extra storage and isn't much slower, so move is reserved for the other direction (moving files INTO the recycle bin), where the source is still recoverable if something goes wrong.
+                await copyArchiveFile({ from: archiveRecycleBin, to: archiveValues, path: file, noFallbacks: true });
             } else {
                 console.log(`File ${file} is not in the recycle bin, skipping, ${restoreCount++}/${files.length}`);
             }

package/src/archiveapps/snapshotThinningEntry.ts

@@ -0,0 +1,364 @@
+import "../inject";
+
+import { Querysub } from "../4-querysub/Querysub";
+import { logErrors } from "../errors";
+import { disablePathAuditer } from "../diagnostics/pathAuditerCallback";
+import { isNodeTrue, sort, timeInDay, timeInHour, timeInWeek, timeInYear } from "socket-function/src/misc";
+import { runInfinitePollCallAtStart, runInParallel } from "socket-function/src/batching";
+import { green, magenta, red, yellow } from "socket-function/src/formatting/logColors";
+import { formatNumber, formatTime } from "socket-function/src/formatting/format";
+import { archiveJSONT } from "../-a-archives/archivesJSONT";
+import { archives, archivesRecycleBin } from "../0-path-value-core/pathValueArchives";
+import { lazy } from "socket-function/src/caching";
+import yargs from "yargs";
+import { getArchives2 } from "../-a-archives/archives2";
+
+const timeInMonth = timeInDay * 30;
+
+const POLL_INTERVAL = timeInHour * 24;
+
+const MIN_PER_RANGE = 10;
+const thinningRanges: {
+    timeRange: number;
+    modulus: number;
+}[] = [
+        { timeRange: timeInDay * 3, modulus: 1 },
+        { timeRange: timeInWeek * 2, modulus: 2 },
+        { timeRange: timeInMonth * 2, modulus: 8 },
+        { timeRange: timeInYear * 100, modulus: 64 },
+    ];
+
+// Unreferenced recycle bin files younger than this are kept. Deleting an unreferenced recycle file too eagerly would destroy the backup of anything recycled since the last kept snapshot, so orphans only die once they have sat unreferenced long enough that nothing could still want them back.
+const RECYCLE_ORPHAN_MIN_AGE = timeInMonth * 2;
+
+const INDEX_KEY = "index";
+const PARALLEL_DELETE_COUNT = 32;
+const PARALLEL_READ_COUNT = 32;
+const PROGRESS_LOG_INTERVAL = 10 * 1000;
+
+const snapshotsArchive = lazy(() => getArchives2("snapshots"));
+const snapshotThinningArchive = lazy(() => getArchives2("snapshot-thinning-index"));
+
+type IndexMappings = {
+    nextIndex: number;
+    indexes: { [snapshotFile: string]: number };
+};
+
+const indexMappingStore = archiveJSONT<IndexMappings>(snapshotThinningArchive);
+
+let yargObj = isNodeTrue() && yargs(process.argv)
+    .option("watch", { type: "boolean", desc: "Watch, and run thinning every hour." })
+    .option("dry-run", { type: "boolean", desc: "Don't actually delete anything, just log what would happen." })
+    .argv || {};
+
+type SnapshotInfo = {
+    file: string;
+    time: number;
+    size: number;
+};
+
+function parseSnapshotTime(file: string): number {
+    let parts = file.split("   ");
+    for (let part of parts) {
+        let [key, value] = part.split("=");
+        if (key === "time") return +value;
+    }
+    throw new Error(`Snapshot file name has no time field: ${file}`);
+}
+
+async function listSnapshots(): Promise<SnapshotInfo[]> {
+    let infos = await snapshotsArchive().findInfo("");
+    let result: SnapshotInfo[] = [];
+    for (let info of infos) {
+        result.push({
+            file: info.path,
+            time: parseSnapshotTime(info.path),
+            size: info.size,
+        });
+    }
+    return result;
+}
+
+async function loadIndexMappings(): Promise<IndexMappings> {
+    let existing = await indexMappingStore.get(INDEX_KEY);
+    if (existing) return existing;
+    return { nextIndex: 0, indexes: {} };
+}
+
+function assignIndexes(snapshots: SnapshotInfo[], mappings: IndexMappings): {
+    mappings: IndexMappings;
+    changed: boolean;
+} {
+    let changed = false;
+    let nextMappings: IndexMappings = {
+        nextIndex: mappings.nextIndex,
+        indexes: { ...mappings.indexes },
+    };
+
+    let currentFiles = new Set(snapshots.map(s => s.file));
+    for (let file of Object.keys(nextMappings.indexes)) {
+        if (!currentFiles.has(file)) {
+            delete nextMappings.indexes[file];
+            changed = true;
+        }
+    }
+
+    let unassigned = snapshots.filter(s => !(s.file in nextMappings.indexes));
+    sort(unassigned, s => s.time);
+    for (let snap of unassigned) {
+        nextMappings.indexes[snap.file] = nextMappings.nextIndex++;
+        changed = true;
+    }
+    return { mappings: nextMappings, changed };
+}
+
+function pickThinningRange(ageMs: number) {
+    for (let range of thinningRanges) {
+        if (ageMs <= range.timeRange) return range;
+    }
+    return thinningRanges[thinningRanges.length - 1];
+}
+
+function decideThinning(snapshots: SnapshotInfo[], mappings: IndexMappings, now: number): {
+    keep: SnapshotInfo[];
+    remove: SnapshotInfo[];
+} {
+    let bucketed = new Map<typeof thinningRanges[number], SnapshotInfo[]>();
+    for (let snap of snapshots) {
+        let range = pickThinningRange(now - snap.time);
+        let list = bucketed.get(range);
+        if (!list) {
+            list = [];
+            bucketed.set(range, list);
+        }
+        list.push(snap);
+    }
+
+    let keep: SnapshotInfo[] = [];
+    let remove: SnapshotInfo[] = [];
+    for (let [range, list] of bucketed) {
+        if (list.length <= MIN_PER_RANGE || range.modulus <= 1) {
+            keep.push(...list);
+            continue;
+        }
+        let rangeKeep: SnapshotInfo[] = [];
+        let rangeRemove: SnapshotInfo[] = [];
+        for (let snap of list) {
+            let idx = mappings.indexes[snap.file];
+            if (idx % range.modulus === 0) {
+                rangeKeep.push(snap);
+            } else {
+                rangeRemove.push(snap);
+            }
+        }
+        sort(rangeRemove, s => s.time);
+        while (rangeRemove.length > 0 && rangeKeep.length < MIN_PER_RANGE) {
+            let restored = rangeRemove.pop();
+            if (!restored) break;
+            rangeKeep.push(restored);
+        }
+        keep.push(...rangeKeep);
+        remove.push(...rangeRemove);
+    }
+    return { keep, remove };
+}
+
+async function readSnapshotFiles(file: string): Promise<string[]> {
+    let buf = await snapshotsArchive().get(file);
+    if (!buf) {
+        throw new Error(`Snapshot file unreadable: ${file}`);
+    }
+    let text = buf.toString();
+    if (!text) return [];
+    return text.split("\n");
+}
+
+async function runThinningIteration(config: { dryRun: boolean }) {
+    let { dryRun } = config;
+    let prefix = dryRun ? yellow(`[DRY RUN] `) : ``;
+    let now = Date.now();
+
+    console.log(magenta(`${prefix}Starting snapshot thinning iteration (dryRun=${dryRun})`));
+
+    let snapshots = await listSnapshots();
+    sort(snapshots, s => s.time);
+
+    let mappingsBase = await loadIndexMappings();
+    let { mappings, changed } = assignIndexes(snapshots, mappingsBase);
+
+    if (changed && !dryRun) {
+        await indexMappingStore.set(INDEX_KEY, mappings);
+    } else if (changed) {
+        console.log(`${prefix}Would update index mappings (next index ${mappings.nextIndex}, ${Object.keys(mappings.indexes).length} entries)`);
+    }
+
+    let totalSnapshotBytes = snapshots.reduce((s, x) => s + x.size, 0);
+    console.log(`${prefix}Snapshots: ${formatNumber(snapshots.length)}, total snapshot file bytes: ${formatNumber(totalSnapshotBytes)}`);
+
+    let { keep, remove } = decideThinning(snapshots, mappings, now);
+    let removedBytes = remove.reduce((s, x) => s + x.size, 0);
+    console.log(`${prefix}Snapshots to keep: ${formatNumber(keep.length)}, to remove: ${formatNumber(remove.length)} (${formatNumber(removedBytes)} bytes)`);
+
+    for (let range of thinningRanges) {
+        let inRangeAll = snapshots.filter(s => pickThinningRange(now - s.time) === range);
+        let inRangeKeep = keep.filter(s => pickThinningRange(now - s.time) === range);
+        console.log(`  Range age<=${formatTime(range.timeRange)} modulus=${range.modulus}: ${formatNumber(inRangeAll.length)} => ${formatNumber(inRangeKeep.length)}`);
+    }
+
+    let recycleListStart = Date.now();
+    let recycleInfos = await archivesRecycleBin().findInfo("");
+    let recycleListDuration = Date.now() - recycleListStart;
+    let recycleSet = new Map<string, { size: number; createTime: number }>();
+    for (let info of recycleInfos) {
+        recycleSet.set(info.path, { size: info.size, createTime: info.createTime });
+    }
+    let totalRecycleBytes = 0;
+    for (let info of recycleSet.values()) totalRecycleBytes += info.size;
+    console.log(`${prefix}Recycle bin files: ${formatNumber(recycleSet.size)}, total bytes: ${formatNumber(totalRecycleBytes)}`);
+
+    let parallelRead = runInParallel({ parallelCount: PARALLEL_READ_COUNT }, readSnapshotFiles);
+    let referencedFiles = new Set<string>();
+    let keepFileLists = await Promise.all(keep.map(s => parallelRead(s.file)));
+    for (let list of keepFileLists) {
+        for (let f of list) referencedFiles.add(f);
+    }
+
+    let liveInfos = await archives().findInfo("");
+    let totalLiveBytes = 0;
+    for (let info of liveInfos) totalLiveBytes += info.size;
+    console.log(`${prefix}Production archives (live path-values): ${formatNumber(liveInfos.length)} files, ${formatNumber(totalLiveBytes)} bytes`);
+
+    for (let info of liveInfos) {
+        referencedFiles.add(info.path);
+    }
+
+    let recycleToDelete: { file: string; size: number }[] = [];
+    let youngOrphanCount = 0;
+    let youngOrphanBytes = 0;
+    for (let [file, info] of recycleSet) {
+        if (referencedFiles.has(file)) continue;
+        if (now - info.createTime < RECYCLE_ORPHAN_MIN_AGE) {
+            youngOrphanCount++;
+            youngOrphanBytes += info.size;
+            continue;
+        }
+        recycleToDelete.push({ file, size: info.size });
+    }
+    let recycleDeleteBytes = recycleToDelete.reduce((s, x) => s + x.size, 0);
+    console.log(`${prefix}Planned: delete ${formatNumber(remove.length)} snapshots (${formatNumber(removedBytes)} bytes), ${formatNumber(recycleToDelete.length)} recycle bin files (${formatNumber(recycleDeleteBytes)} bytes). Keeping ${formatNumber(youngOrphanCount)} unreferenced recycle bin files (${formatNumber(youngOrphanBytes)} bytes) that are younger than ${formatTime(RECYCLE_ORPHAN_MIN_AGE)}`);
+
+    function logSummary() {
+        let snapshotCountAfter = keep.length;
+        let snapshotBytesAfter = totalSnapshotBytes - removedBytes;
+        let recycleCountAfter = recycleSet.size - recycleToDelete.length;
+        let recycleBytesAfter = totalRecycleBytes - recycleDeleteBytes;
+        let beforeTotalBytes = totalSnapshotBytes + totalRecycleBytes;
+        let afterTotalBytes = snapshotBytesAfter + recycleBytesAfter;
+        let beforeTotalFiles = snapshots.length + recycleSet.size;
+        let afterTotalFiles = snapshotCountAfter + recycleCountAfter;
+
+        function pct(after: number, before: number) {
+            if (!before) return `100%`;
+            return `${(after / before * 100).toFixed(1)}%`;
+        }
+
+        console.log(magenta(`${prefix}===== Summary =====`));
+        console.log(`${prefix}Snapshots:   ${formatNumber(snapshots.length)} => ${formatNumber(snapshotCountAfter)} files (${pct(snapshotCountAfter, snapshots.length)} kept), ${formatNumber(totalSnapshotBytes)} => ${formatNumber(snapshotBytesAfter)} bytes (${pct(snapshotBytesAfter, totalSnapshotBytes)} kept)`);
+        console.log(`${prefix}Recycle bin: ${formatNumber(recycleSet.size)} => ${formatNumber(recycleCountAfter)} files (${pct(recycleCountAfter, recycleSet.size)} kept), ${formatNumber(totalRecycleBytes)} => ${formatNumber(recycleBytesAfter)} bytes (${pct(recycleBytesAfter, totalRecycleBytes)} kept)`);
+        console.log(`${prefix}Total:       ${formatNumber(beforeTotalFiles)} => ${formatNumber(afterTotalFiles)} files (${pct(afterTotalFiles, beforeTotalFiles)} kept), ${formatNumber(beforeTotalBytes)} => ${formatNumber(afterTotalBytes)} bytes (${pct(afterTotalBytes, beforeTotalBytes)} kept)`);
+        console.log(`${prefix}Live archives (unchanged): ${formatNumber(liveInfos.length)} files, ${formatNumber(totalLiveBytes)} bytes`);
+        console.log(`${prefix}Recycle bin findInfo("") took: ${formatTime(recycleListDuration)} (for ${formatNumber(recycleSet.size)} files)`);
+    }
+
+    if (dryRun) {
+        logSummary();
+        console.log(green(`${prefix}Dry run complete. No changes made.`));
+        return;
+    }
+
+    async function deleteWithProgress(label: string, files: string[], deleter: (file: string) => Promise<void>) {
+        if (files.length === 0) {
+            console.log(`No ${label} to delete`);
+            return;
+        }
+        let done = 0;
+        let failed = 0;
+        let startTime = Date.now();
+        let lastLog = startTime;
+        let lastLogDone = 0;
+        let peakRate = 0;
+        function logProgress() {
+            let nowTs = Date.now();
+            let elapsed = nowTs - startTime;
+            let avgRate = done / (elapsed / 1000);
+            let intervalMs = nowTs - lastLog;
+            let intervalRate = intervalMs > 0 ? (done - lastLogDone) / (intervalMs / 1000) : 0;
+            if (intervalRate > peakRate) peakRate = intervalRate;
+            let remaining = files.length - done;
+            let etaMs = avgRate > 0 ? remaining / avgRate * 1000 : 0;
+            console.log(`  ${label}: ${formatNumber(done)}/${formatNumber(files.length)} deleted (${(done / files.length * 100).toFixed(1)}%), ${formatNumber(failed)} failed, elapsed ${formatTime(elapsed)}, ETA ${formatTime(etaMs)}, avg ${avgRate.toFixed(1)}/s, peak ${peakRate.toFixed(1)}/s`);
+            lastLog = nowTs;
+            lastLogDone = done;
+        }
+        let parallel = runInParallel({ parallelCount: PARALLEL_DELETE_COUNT }, async (file: string) => {
+            try {
+                await deleter(file);
+            } catch (e) {
+                failed++;
+                console.error(`Failed to delete ${file}: ${String((e as Error)?.stack || e)}`);
+            }
+            done++;
+            if (Date.now() - lastLog >= PROGRESS_LOG_INTERVAL) {
+                logProgress();
+            }
+        });
+        console.log(`Deleting ${formatNumber(files.length)} ${label}...`);
+        await Promise.all(files.map(f => parallel(f)));
+        logProgress();
+        console.log(green(`Deleted ${formatNumber(done - failed)} ${label} (${formatNumber(failed)} failed) in ${formatTime(Date.now() - startTime)}`));
+    }
+
+    await deleteWithProgress("snapshots", remove.map(s => s.file), async file => {
+        await snapshotsArchive().del(file);
+    });
+
+    if (remove.length > 0) {
+        let mappingsAfter: IndexMappings = {
+            nextIndex: mappings.nextIndex,
+            indexes: { ...mappings.indexes },
+        };
+        for (let snap of remove) {
+            delete mappingsAfter.indexes[snap.file];
+        }
+        await indexMappingStore.set(INDEX_KEY, mappingsAfter);
+    }
+
+    await deleteWithProgress("recycle bin files", recycleToDelete.map(x => x.file), async file => {
+        await archivesRecycleBin().del(file);
+    });
+
+    logSummary();
+    console.log(green(`Snapshot thinning complete.`));
+}
+
+async function main() {
+    disablePathAuditer();
+    await Querysub.hostService("snapshot-thinning");
+
+    let dryRun = !!yargObj["dry-run"];
+
+    if (yargObj.watch) {
+        console.log("Running in watch mode.");
+        await runInfinitePollCallAtStart(POLL_INTERVAL, () => runThinningIteration({ dryRun }));
+    } else {
+        try {
+            await runThinningIteration({ dryRun });
+        } catch (error) {
+            console.error((error as Error).stack ?? error);
+        } finally {
+            process.exit();
+        }
+    }
+}
+
+main().catch(logErrors);

package/src/deployManager/components/storage/RouteConfigView.tsx

@@ -45,7 +45,7 @@ const JSON_INDENT = 4;
 /** String sources are the shorthand for an always-valid, unsharded backblaze bucket. */
 function normalizeSource(source: RemoteConfigBase): Source {
     if (typeof source !== "string") return source;
-    return { type: "backblaze", url: source, validWindow: FULL_VALID_WINDOW };
+    return { type: "backblaze", url: source, validWindow: FULL_VALID_WINDOW, name: "from url" };
 }
 
 /** Where a source lives, with the bucket the config belongs to masked out so buckets that differ only by their own name group together. */

package/src/deployManager/components/storage/Tag.tsx

@@ -22,7 +22,6 @@ export function getSourceTags(source: HostedConfig | BackblazeConfig): SourceTag
     if (source.intermediate) tags.push({ icon: "⏳", text: "intermediate" });
     if (source.type === "remote") {
         if (source.fast) tags.push({ icon: "⚡", text: "fast" });
-        if (source.rawDisk) tags.push({ icon: "💽", text: "rawDisk" });
         if (source.writeDelay !== undefined) tags.push({ icon: "⏱", text: `writeDelay ${formatTime(source.writeDelay)}` });
     }
     if (source.readerDiskLimit !== undefined) tags.push({ icon: "📦", text: `readCache ${formatNumber(source.readerDiskLimit)}B` });

package/src/deployManager/emergencyUpdateMain.ts

@@ -0,0 +1,78 @@
+import "../inject";
+
+import { Querysub } from "../4-querysub/Querysub";
+import { logErrors } from "../errors";
+import { disablePathAuditer } from "../diagnostics/pathAuditerCallback";
+import { formatDateTime } from "socket-function/src/formatting/format";
+import { green, magenta } from "socket-function/src/formatting/logColors";
+import { delay } from "socket-function/src/batching";
+import { MachineServiceControllerBase, ServiceConfig } from "./machineSchema";
+
+// setServiceConfigs notifies the machines without awaiting the notifications, so we linger before exiting to let them go out. If they are lost anyway (ex, the machines are unreachable), the machines still pick up the new configs on their fallback resync poll.
+const NOTIFY_GRACE_PERIOD = 10 * 1000;
+
+// The emergency workflow from the services list page, as one command: commit/publish/push querysub, commit/push the service repo, then point every service config at the new commit with no overlap. Everything here writes to the service configs archives directly, which falls back to backblaze, so this works while the storage servers (and therefore the deploy pages) are down.
+async function main() {
+    disablePathAuditer();
+    await Querysub.hostService("emergency-update");
+
+    let controller = new MachineServiceControllerBase();
+    let commitMessage = `emergency update ${formatDateTime(Date.now())}`;
+
+    let gitInfo = await controller.getGitInfo();
+    console.log(magenta(`Starting emergency update of ${gitInfo.repoUrl}`), {
+        commitMessage,
+        querysubUncommittedFiles: gitInfo.querysubUncommitted.length,
+        uncommittedFiles: gitInfo.uncommitted.length,
+    });
+
+    if (gitInfo.querysubUncommitted.length > 0) {
+        console.log(magenta(`Committing, publishing, and pushing querysub (${gitInfo.querysubUncommitted.length} files changed)`));
+        await controller.commitPushAndPublishQuerysub(commitMessage);
+    } else {
+        console.log(`Querysub has no uncommitted changes, skipping the querysub publish`);
+    }
+
+    // Re-read, as publishing querysub updates our package.json, which then needs to be committed here
+    gitInfo = await controller.getGitInfo();
+    if (gitInfo.uncommitted.length > 0) {
+        console.log(magenta(`Committing and pushing the service repo (${gitInfo.uncommitted.length} files changed)`));
+        await controller.commitPushService(commitMessage);
+    } else {
+        console.log(`The service repo has no uncommitted changes, skipping the commit`);
+    }
+
+    // Re-read, as latestRef must include the commit we just pushed
+    gitInfo = await controller.getGitInfo();
+
+    let serviceIds = await controller.getServiceList();
+    let configs: ServiceConfig[] = [];
+    for (let serviceId of serviceIds) {
+        let config = await controller.getServiceConfig(serviceId);
+        if (config) configs.push(config);
+    }
+    let outdated = configs.filter(config =>
+        config.parameters.repoUrl === gitInfo.repoUrl
+        && config.parameters.gitRef !== gitInfo.latestRef
+    );
+    console.log(`Services: ${configs.length} total, ${outdated.length} on an old commit (latest is ${gitInfo.latestRef})`);
+    if (outdated.length === 0) {
+        console.log(green(`All services are already on the latest commit, nothing to deploy`));
+        return;
+    }
+
+    let now = Date.now();
+    for (let config of outdated) {
+        console.log(`Updating ${config.parameters.key}: ${config.parameters.gitRef} => ${gitInfo.latestRef} (release at ${formatDateTime(now)}, no overlap)`);
+        config.parameters.gitRef = gitInfo.latestRef;
+        config.parameters.releaseTime = now;
+        config.parameters.overlapTime = 0;
+    }
+    await controller.setServiceConfigs(outdated);
+    console.log(`Wrote ${outdated.length} service configs, waiting for machine notifications to go out`);
+    await delay(NOTIFY_GRACE_PERIOD);
+
+    console.log(green(`Emergency update complete: ${outdated.length} services updating to ${gitInfo.latestRef}`));
+}
+
+main().catch(logErrors).finally(() => process.exit());

package/src/diagnostics/misc-pages/SnapshotViewer.tsx

@@ -124,6 +124,20 @@ export class SnapshotViewer extends qreact.Component {
                         <div>Byte Count: {formatNumber(entry.byteCount)}</div>
                     </div>
                 }
+                {snapshotInfo &&
+                    <div class={css.vbox(4)}>
+                        <div class={css.boldStyle}>By status</div>
+                        <Table
+                            rows={summarizeSnapshotByStatus(snapshotInfo.files)}
+                            columns={{
+                                status: { title: "Status" },
+                                files: { title: "Files" },
+                                values: { title: "Values" },
+                                bytes: { title: "Bytes" },
+                            }}
+                        />
+                    </div>
+                }
                 <Button onClick={async () => {
                     let didConfirm = await seriousConfirm({
                         message: `Restoring will put the servers in a bad state. You're gonna need to restart all services after this.`,
@@ -168,6 +182,26 @@ export class SnapshotViewer extends qreact.Component {
     }
 }
 
+// Canonical status order, so the per-status summary rows stay in a stable, readable order.
+const SNAPSHOT_STATUS_ORDER = ["live", "zombie", "recycled", "missing"];
+type SnapshotStatusRow = { status: string; files: number; values: number; bytes: number };
+function summarizeSnapshotByStatus(files: ArchiveSnapshotRead["files"]): SnapshotStatusRow[] {
+    let byStatus = new Map<string, SnapshotStatusRow>();
+    for (let file of files) {
+        let row = byStatus.get(file.state);
+        if (!row) {
+            row = { status: file.state, files: 0, values: 0, bytes: 0 };
+            byStatus.set(file.state, row);
+        }
+        row.files++;
+        row.values += file.valueCount;
+        row.bytes += file.byteCount;
+    }
+    let rows = Array.from(byStatus.values());
+    sort(rows, row => SNAPSHOT_STATUS_ORDER.indexOf(row.status));
+    return rows;
+}
+
 async function downloadSnapshotFile(component: SnapshotViewer, file: string) {
     if (component.state.downloading[file]) return;
     Querysub.localCommit(() => {

package/src/misc/appPaths.ts

@@ -13,6 +13,7 @@ export const STORAGE_DIR = "./database-storage/";
 
 let yargObj = parseArgsFactory()
     .option("domain", { type: "string", desc: `Sets the domain` })
+    .option("port", { type: "number", desc: `Sets the port` })
     .argv
     ;
 
@@ -42,6 +43,10 @@ export function getDomain() {
     return yargObj.domain || querysubConfig().domain || "querysub.com";
 }
 
+export function getPort(): number | undefined {
+    return yargObj.port;
+}
+
 export function getBackblazePath() {
     let testPaths = [
         STORAGE_DIR + "backblaze.json",

package/src/storageSetup.ts

@@ -1,23 +1,15 @@
 process.env.PRODUCTION = "true";
-import "./inject";
-import { getDomain, getPort } from "./config";
-import { getTrustedMachineIds } from "./-d-trust/NetworkTrust2";
-import { getExternalIP } from "socket-function/src/networking";
+// TODO: Add this back
+//import "./inject";
 
 // The storage server entry point (see bin/storageserve.js): sets up the CLI args the sliftutils storage server expects, synchronizes our network trust list into its trust store, and runs its CLI.
 
 async function main() {
-    if (!process.argv.includes("--url")) {
-        let ip = await getExternalIP();
-        let ipDomain = ip.replaceAll(".", "-");
-        process.argv.push("--url", `https://${ipDomain}.${getDomain()}:${getPort()}`);
-    }
-
-    let trustedMachineIds = await getTrustedMachineIds();
-
-    let { setTrustedMachines } = await import("sliftutils/storage/remoteStorage/serverConfig");
-    await setTrustedMachines({ account: "root", machineIds: trustedMachineIds });
-    console.log(`Synchronized ${trustedMachineIds.length} trusted machines`);
+    // TODO: Add this back
+    // let trustedMachineIds = await getTrustedMachineIds();
+    // let { setTrustedMachines } = await import("sliftutils/storage/remoteStorage/serverConfig");
+    // await setTrustedMachines({ account: "root", machineIds: trustedMachineIds });
+    // console.log(`Synchronized ${trustedMachineIds.length} trusted machines`);
 
     await import("sliftutils/storage/remoteStorage/storageServerCli");
 }