sliftutils 1.7.123 → 1.7.125

package/security/notifications/discord.ts

@@ -0,0 +1,190 @@
+import fs from "fs/promises";
+
+// PORTED CODE: security/authorizedKeys/daemon/portsecureDaemon.js embeds a hand port of this file in plain JS, so the
+// daemon can run with no dependencies. The two are expected to behave identically - if you change
+// one, make the matching change in the other.
+
+export const DEFAULT_WEBHOOK_FILE_PATH = "/etc/portsecure/discord-webhook";
+const DEFAULT_CHECK_INTERVAL = 5 * 60 * 1000;
+const VALID_WEBHOOK_PREFIXES = [
+    "https://discord.com/api/webhooks/",
+    "https://discordapp.com/api/webhooks/",
+    "https://canary.discord.com/api/webhooks/",
+];
+const DISCORD_MESSAGE_LIMIT = 2000;
+const MAX_ERROR_BODY_LENGTH = 500;
+const MAX_SEND_ATTEMPTS = 3;
+const REDACTED_TOKEN_VISIBLE = 8;
+const DEFAULT_RATE_LIMIT_WAIT = 2 * 1000;
+
+let notificationState: {
+    filePath: string;
+    webhookURL: string;
+    checkTimer: NodeJS.Timeout;
+} | undefined;
+
+// Discord webhooks are rate limited (roughly 5 requests per 2 seconds), so every send
+// goes through one chain instead of racing.
+let sendChain: Promise<unknown> = Promise.resolve();
+
+/** Pulls the webhook URL out of webhook file contents. `sourceName` only appears in errors, so
+    callers can name a remote path. */
+export function parseWebhookFile(config: { contents: string; sourceName: string }) {
+    let { contents, sourceName } = config;
+    let lines = contents.split("\n").map(line => line.trim()).filter(line => line && !line.startsWith("#"));
+    let webhookURL = lines[0];
+    if (!webhookURL) {
+        throw new Error(`Expected a Discord webhook URL in ${sourceName}, the file has no usable lines`);
+    }
+    if (!VALID_WEBHOOK_PREFIXES.some(prefix => webhookURL.startsWith(prefix))) {
+        throw new Error(
+            `Expected a Discord webhook URL starting with one of ${VALID_WEBHOOK_PREFIXES.join(", ")}, `
+            + `was ${webhookURL.slice(0, MAX_ERROR_BODY_LENGTH)} (in ${sourceName})`
+        );
+    }
+    return webhookURL;
+}
+
+/** Keeps the id and both ends of the token, so a reader can confirm which webhook replaced theirs
+    without receiving one they could post to. */
+export function redactWebhookURL(webhookURL: string) {
+    let separatorIndex = webhookURL.lastIndexOf("/");
+    let base = webhookURL.slice(0, separatorIndex + 1);
+    let token = webhookURL.slice(separatorIndex + 1);
+    if (token.length <= REDACTED_TOKEN_VISIBLE * 2) {
+        // Too short to show both ends without giving away the whole token.
+        return `${base}${token.slice(0, REDACTED_TOKEN_VISIBLE)}...`;
+    }
+    return `${base}${token.slice(0, REDACTED_TOKEN_VISIBLE)}...${token.slice(-REDACTED_TOKEN_VISIBLE)}`;
+}
+
+export function formatWebhookFile(webhookURL: string) {
+    return `# portsecure Discord webhook. First non-comment line is used.\n${webhookURL}\n`;
+}
+
+async function readWebhookFile(filePath: string) {
+    let contents;
+    try {
+        contents = await fs.readFile(filePath, "utf8");
+    } catch (e) {
+        throw new Error(`Expected a readable Discord webhook file at ${filePath}, ${e}`);
+    }
+    return parseWebhookFile({ contents, sourceName: filePath });
+}
+
+async function postToWebhook(webhookURL: string, message: string) {
+    let content = message;
+    if (content.length > DISCORD_MESSAGE_LIMIT) {
+        content = content.slice(0, DISCORD_MESSAGE_LIMIT - 3) + "...";
+    }
+    for (let attempt = 1; attempt <= MAX_SEND_ATTEMPTS; attempt++) {
+        let response = await fetch(webhookURL, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({ content }),
+        });
+        if (response.ok) {
+            return;
+        }
+        let body = (await response.text()).slice(0, MAX_ERROR_BODY_LENGTH);
+        if (response.status === 429 && attempt < MAX_SEND_ATTEMPTS) {
+            let retryAfter = Number(response.headers.get("retry-after"));
+            let waitTime = retryAfter && retryAfter * 1000 || DEFAULT_RATE_LIMIT_WAIT;
+            await new Promise(resolve => setTimeout(resolve, waitTime));
+            continue;
+        }
+        throw new Error(
+            `Expected a 2xx response from the Discord webhook, was ${response.status} ${response.statusText}, body ${body}`
+        );
+    }
+}
+
+function queueSend(webhookURL: string, message: string) {
+    let send = () => postToWebhook(webhookURL, message);
+    let result = sendChain.then(send, send);
+    sendChain = result.catch(() => undefined);
+    return result;
+}
+
+/** Sends to an explicit webhook URL, for tooling that acts on a webhook before (or instead of)
+    configureDiscordNotifications - setup, migrations, connectivity checks. */
+export function sendToWebhookURL(config: { webhookURL: string; message: string }) {
+    return queueSend(config.webhookURL, config.message);
+}
+
+async function checkWebhookFileChanged() {
+    let state = notificationState;
+    if (!state) {
+        return;
+    }
+    let newWebhookURL;
+    try {
+        newWebhookURL = await readWebhookFile(state.filePath);
+    } catch (e) {
+        console.error(`portsecure: Discord webhook file is no longer readable, still using the loaded webhook. ${e}`);
+        return;
+    }
+    if (newWebhookURL === state.webhookURL) {
+        return;
+    }
+    // Warn the old channel first, with the new webhook redacted, so a stolen old webhook does not
+    // hand the attacker a usable new one.
+    try {
+        await queueSend(
+            state.webhookURL,
+            `**portsecure**: the Discord webhook in \`${state.filePath}\` changed to`
+            + ` \`${redactWebhookURL(newWebhookURL)}\`.`
+            + ` Notifications are moving to the new webhook and this channel will stop receiving them.`
+        );
+    } catch (e) {
+        console.error(`portsecure: failed to warn the old Discord webhook about the change. ${e}`);
+    }
+    state.webhookURL = newWebhookURL;
+}
+
+/** Must be called once on startup, before any notification is sent. Aborts the process if the
+    webhook file is missing or invalid, then re-checks the file on an interval and warns the old
+    webhook whenever it changes. */
+export async function configureDiscordNotifications(config?: {
+    filePath?: string;
+    checkInterval?: number;
+}) {
+    if (notificationState) {
+        throw new Error(`Expected configureDiscordNotifications to be called once, was called again (already using ${notificationState.filePath})`);
+    }
+    let filePath = config?.filePath || DEFAULT_WEBHOOK_FILE_PATH;
+    let checkInterval = config?.checkInterval || DEFAULT_CHECK_INTERVAL;
+    let webhookURL;
+    try {
+        webhookURL = await readWebhookFile(filePath);
+    } catch (e) {
+        console.error(`portsecure: refusing to start without a valid Discord webhook file.\n${e}`);
+        process.exit(1);
+    }
+    let checkTimer = setInterval(() => {
+        checkWebhookFileChanged().catch(e => console.error(`portsecure: Discord webhook file check failed. ${e}`));
+    }, checkInterval);
+    // The check should never be the reason the process stays alive.
+    checkTimer.unref();
+    notificationState = { filePath, webhookURL, checkTimer };
+    return { filePath, checkInterval };
+}
+
+// DO NOT add new calls to this. Every message goes to a real Discord server someone reads, so a
+// notification is only ever added when the user explicitly asks for that specific case. Startup,
+// success, errors, retries and recoveries all belong in a log instead.
+export async function sendDiscordNotification(message: string) {
+    let state = notificationState;
+    if (!state) {
+        throw new Error(`Expected configureDiscordNotifications to be called before sending, was called with message ${message.slice(0, MAX_ERROR_BODY_LENGTH)}`);
+    }
+    await queueSend(state.webhookURL, message);
+}
+
+export function stopDiscordNotifications() {
+    if (!notificationState) {
+        return;
+    }
+    clearInterval(notificationState.checkTimer);
+    notificationState = undefined;
+}

package/security/notifications/remoteWebhook.ts

@@ -0,0 +1,85 @@
+import { DEFAULT_WEBHOOK_FILE_PATH, formatWebhookFile, parseWebhookFile, redactWebhookURL, sendToWebhookURL } from "./discord";
+import { readRemoteFile, writeRemoteFile } from "../helpers/remoteSSH";
+
+export const REPLACE_KEYWORD = "replace";
+const UNPARSEABLE_LABEL = "(unparseable)";
+
+/** Puts a webhook on a remote host, refusing to clobber a different existing one unless asked.
+    Returns what happened, so callers can decide whether to keep going. */
+export async function ensureRemoteWebhook(config: {
+    host: string;
+    webhookURL: string;
+    replace: boolean;
+    filePath?: string;
+}) {
+    let { host, webhookURL, replace } = config;
+    let filePath = config.filePath || DEFAULT_WEBHOOK_FILE_PATH;
+
+    let replacedURL = "";
+    let existingContents = await readRemoteFile({ host, filePath });
+    if (existingContents) {
+        // A corrupt existing file still counts as a conflict, so "replace" can repair it instead
+        // of the setup dead ending on a parse error.
+        let existingURL = UNPARSEABLE_LABEL;
+        try {
+            existingURL = parseWebhookFile({ contents: existingContents, sourceName: `${host}:${filePath}` });
+        } catch (e) {
+            console.error(`${host}:${filePath} exists but could not be parsed. ${e}`);
+        }
+        if (existingURL === webhookURL) {
+            return { outcome: "unchanged" as const, filePath };
+        }
+        if (!replace) {
+            throw new Error(
+                `Expected no conflicting webhook on ${host}, but ${filePath} already holds a different one.\n`
+                + `Existing: ${existingURL}\n`
+                + `New:      ${webhookURL}\n`
+                + `Pass "${REPLACE_KEYWORD}" to overwrite it.`
+            );
+        }
+        replacedURL = existingURL;
+    }
+
+    await writeRemoteFile({
+        host,
+        filePath,
+        contents: formatWebhookFile(webhookURL),
+        fileMode: "600",
+        directoryMode: "700",
+    });
+
+    let writtenContents = await readRemoteFile({ host, filePath });
+    if (!writtenContents) {
+        throw new Error(`Expected ${filePath} to exist on ${host} after writing, no such file exists`);
+    }
+    let writtenURL = parseWebhookFile({ contents: writtenContents, sourceName: `${host}:${filePath}` });
+    if (writtenURL !== webhookURL) {
+        throw new Error(`Expected ${host}:${filePath} to hold the new webhook, was ${writtenURL}`);
+    }
+
+    if (replacedURL && replacedURL !== UNPARSEABLE_LABEL) {
+        // The new webhook is redacted, so this channel can identify the replacement without
+        // receiving a webhook it could post to.
+        try {
+            await sendToWebhookURL({
+                webhookURL: replacedURL,
+                message: `**portsecure**: the Discord webhook for \`${host}\` was replaced with`
+                    + ` \`${redactWebhookURL(webhookURL)}\`.`
+                    + ` This channel will stop receiving notifications for that host.`,
+            });
+        } catch (e) {
+            // The replacement already happened, so a dead old webhook must not fail the setup.
+            console.error(`Could not notify the old webhook that it was replaced. ${e}`);
+        }
+    }
+
+    // Failing here means the file is in place but the webhook itself does not work, which is
+    // exactly what the operator needs to hear about.
+    await sendToWebhookURL({
+        webhookURL,
+        message: `**portsecure**: notifications are now configured for \`${host}\`.`
+            + ` This channel will receive its security notifications.`,
+    });
+
+    return { outcome: replacedURL && "replaced" as const || "created" as const, filePath };
+}

package/security/notifications/setupNotify.ts

@@ -0,0 +1,29 @@
+import { parseWebhookFile } from "./discord";
+import { ensureRemoteWebhook, REPLACE_KEYWORD } from "./remoteWebhook";
+
+const USAGE = `Usage: yarn setupnotify <host> <discord-webhook-url> [${REPLACE_KEYWORD}]`;
+
+function parseArgs(argv: string[]) {
+    let replace = argv.includes(REPLACE_KEYWORD);
+    let positional = argv.filter(arg => arg !== REPLACE_KEYWORD);
+    if (positional.length !== 2) {
+        throw new Error(`Expected a host and a webhook URL, was ${positional.length} argument(s): ${positional.join(" ") || "(none)"}\n${USAGE}`);
+    }
+    let [host, webhookURL] = positional;
+    // Validates the URL shape up front, so we never ssh anywhere with a bad webhook.
+    parseWebhookFile({ contents: webhookURL, sourceName: "the command line" });
+    return { host, webhookURL, replace };
+}
+
+export async function main() {
+    let { host, webhookURL, replace } = parseArgs(process.argv.slice(2));
+    let { outcome, filePath } = await ensureRemoteWebhook({ host, webhookURL, replace });
+    if (outcome === "unchanged") {
+        console.log(`${host} already has this exact webhook in ${filePath}, nothing to do.`);
+        return;
+    }
+    if (outcome === "replaced") {
+        console.log(`Replaced the webhook on ${host} and notified the old one.`);
+    }
+    console.log(`Wrote ${filePath} on ${host} (mode 600), and confirmed on the new webhook.`);
+}

package/security/signedFiles/manifest.ts

@@ -0,0 +1,56 @@
+import crypto from "crypto";
+import fs from "fs/promises";
+import path from "path";
+import { spawnPromise } from "../helpers/spawn";
+
+// PORTED CODE: security/authorizedKeys/daemon/portsecureDaemon.js contains a plain JS port of the
+// verifying half of this file, so it can check a signature with no dependencies. Both sides must
+// agree on the manifest shape and on which files it covers - if you change one, make the matching
+// change in the other.
+
+export const MANIFEST_NAME = "signedfiles.json";
+export const SIGNATURE_NAME = "signedfiles.json.sig";
+// ssh signatures are namespaced, so a signature made for one purpose cannot be replayed as another.
+export const SIGN_NAMESPACE = "signfiles";
+export const MANIFEST_VERSION = 1;
+
+export type Manifest = {
+    version: number;
+    files: { path: string; size: number; sha256: string }[];
+};
+
+/** Everything in the repo that is not ignored, which is exactly what a clone of it will contain.
+    The manifest and its signature are left out, since they cannot describe themselves. */
+export async function listRepoFiles(repoPath: string) {
+    let result = await spawnPromise({
+        command: "git",
+        args: ["ls-files", "--cached", "--others", "--exclude-standard"],
+        cwd: repoPath,
+    });
+    if (result.status !== 0) {
+        throw new Error(`Expected to list the files in ${repoPath}, git ls-files exited ${result.status}. ${result.stderr}`);
+    }
+    return result.stdout.split("\n")
+        .map(line => line.trim())
+        .filter(line => line && line !== MANIFEST_NAME && line !== SIGNATURE_NAME)
+        .sort();
+}
+
+export async function hashFile(filePath: string) {
+    return crypto.createHash("sha256").update(await fs.readFile(filePath)).digest("hex");
+}
+
+export async function buildManifest(repoPath: string) {
+    let files: Manifest["files"] = [];
+    for (let relativePath of await listRepoFiles(repoPath)) {
+        let fullPath = path.join(repoPath, relativePath);
+        let stats = await fs.stat(fullPath);
+        files.push({ path: relativePath, size: stats.size, sha256: await hashFile(fullPath) });
+    }
+    return { version: MANIFEST_VERSION, files };
+}
+
+/** Sorted keys and a trailing newline, so the same tree always produces the same bytes to sign. */
+export function formatManifest(manifest: Manifest) {
+    return JSON.stringify(manifest, undefined, 4) + "\n";
+}

package/security/signedFiles/signFiles.ts

@@ -0,0 +1,116 @@
+import fs from "fs/promises";
+import os from "os";
+import path from "path";
+import { expandHome } from "../helpers/paths";
+import { spawnPromise } from "../helpers/spawn";
+import { buildManifest, formatManifest, MANIFEST_NAME, SIGNATURE_NAME, SIGN_NAMESPACE } from "./manifest";
+
+// A hardware backed key is the entire point. A key sitting on disk is compromised the moment the
+// machine is, and then the signature proves nothing, so this is what we make when asked to make one.
+const DEFAULT_KEY_TYPE = "ed25519-sk";
+const DEFAULT_KEY_PATH = "~/.ssh/signfiles_ed25519_sk";
+const GIT_KEYWORD = "git";
+const COMMIT_MESSAGE = "deploying signed files";
+const MAX_ERROR_BODY_LENGTH = 500;
+const USAGE = `Usage: yarn signfiles [signing-key] [${GIT_KEYWORD}]
+
+Signs the files of the repo in the current directory. With no key, a hardware backed
+${DEFAULT_KEY_TYPE} key at ${DEFAULT_KEY_PATH} is used, and created if it does not exist.
+Pass ${GIT_KEYWORD} to also commit and push the result.`;
+
+async function pathExists(filePath: string) {
+    try {
+        await fs.access(filePath);
+        return true;
+    } catch (e) {
+        return false;
+    }
+}
+
+async function run(config: { command: string; args: string[]; cwd?: string; interactive?: boolean }) {
+    let { command, args, cwd, interactive } = config;
+    let result = await spawnPromise({ command, args, cwd, inheritStderr: interactive });
+    if (result.error) {
+        throw new Error(`Expected ${command} to run, failed with ${result.error.message}`);
+    }
+    if (result.status !== 0) {
+        throw new Error(
+            `Expected ${command} ${args.join(" ")} to exit 0, was ${result.status}. `
+            + `${(result.stderr || "").slice(0, MAX_ERROR_BODY_LENGTH)}`
+        );
+    }
+    return result;
+}
+
+/** Creating this needs the security key plugged in, and a touch, so its output goes straight to
+    the terminal rather than being captured. */
+async function ensureDefaultKey() {
+    let keyPath = expandHome(DEFAULT_KEY_PATH);
+    if (await pathExists(keyPath)) {
+        return keyPath;
+    }
+    console.log(`No signing key at ${keyPath}, creating an ${DEFAULT_KEY_TYPE} one.`);
+    console.log(`Plug your security key in - you will be asked to touch it.`);
+    await fs.mkdir(path.dirname(keyPath), { recursive: true, mode: 0o700 });
+    await run({
+        command: "ssh-keygen",
+        args: ["-t", DEFAULT_KEY_TYPE, "-f", keyPath, "-N", "", "-C", "signfiles"],
+        interactive: true,
+    });
+    return keyPath;
+}
+
+async function fingerprintOf(keyPath: string) {
+    let result = await run({ command: "ssh-keygen", args: ["-lf", `${keyPath}.pub`] });
+    let match = result.stdout.match(/(SHA256:[A-Za-z0-9+/=]+)/);
+    if (!match) {
+        throw new Error(`Expected a fingerprint for ${keyPath}.pub, was ${result.stdout.slice(0, MAX_ERROR_BODY_LENGTH)}`);
+    }
+    return match[1];
+}
+
+function parseArgs(argv: string[]) {
+    let pushToGit = argv.includes(GIT_KEYWORD);
+    let positional = argv.filter(arg => arg !== GIT_KEYWORD);
+    if (positional.length > 1) {
+        throw new Error(`Expected at most a signing key, was ${positional.length} argument(s): ${positional.join(" ")}\n${USAGE}`);
+    }
+    return { keyPath: positional[0], pushToGit };
+}
+
+export async function main() {
+    let { keyPath, pushToGit } = parseArgs(process.argv.slice(2));
+
+    let topLevel = await spawnPromise({ command: "git", args: ["rev-parse", "--show-toplevel"] });
+    if (topLevel.status !== 0) {
+        throw new Error(`Expected the current directory to be inside a git repo, it is not.\n${USAGE}`);
+    }
+    let repoPath = topLevel.stdout.trim();
+
+    let signingKey = keyPath && expandHome(keyPath) || await ensureDefaultKey();
+    if (!await pathExists(signingKey)) {
+        throw new Error(`Expected a signing key at ${signingKey}, no such file exists`);
+    }
+
+    let manifest = await buildManifest(repoPath);
+    let manifestPath = path.join(repoPath, MANIFEST_NAME);
+    await fs.writeFile(manifestPath, formatManifest(manifest));
+    console.log(`${MANIFEST_NAME} covers ${manifest.files.length} file(s) in ${repoPath}`);
+
+    // Signing happens before any git work, so a failed push never costs a second touch of the key.
+    await run({
+        command: "ssh-keygen",
+        args: ["-Y", "sign", "-f", signingKey, "-n", SIGN_NAMESPACE, manifestPath],
+        interactive: true,
+    });
+    console.log(`Signed with ${await fingerprintOf(signingKey)}`);
+
+    if (!pushToGit) {
+        console.log(`Commit and push ${MANIFEST_NAME} and ${SIGNATURE_NAME} for anything to see them.`);
+        return;
+    }
+    await run({ command: "git", args: ["add", "-A"], cwd: repoPath, interactive: true });
+    await run({ command: "git", args: ["commit", "-m", COMMIT_MESSAGE], cwd: repoPath, interactive: true });
+    await run({ command: "git", args: ["push"], cwd: repoPath, interactive: true });
+    console.log(`Committed and pushed.`);
+}

package/storage/BulkDatabase2/BulkDatabaseBase.ts

@@ -40,6 +40,10 @@ const ROLLOVER_BYTES = 5 * 1024 * 1024;
 const MEMORY_WATCHDOG_INTERVAL_MS = 60 * 1000;
 const STALE_DELETE_MS = 24 * 60 * 60 * 1000;
 const MAX_INDEX_RELOAD_ATTEMPTS = 3;
+// A bulk file under this is "loose" - still worth rolling up into a bigger one. Half the target file size, because that is exactly where combining stops paying: merging two files that are each over half the target just splits back into two files again, so the file count (and with it the per-file key list every read holds in memory) does not drop. Under half, any two inputs fit in one output, so phase 2 always makes the count go down.
+//
+// NOT the target size itself. runPlannedMerge cuts a chunk BEFORE adding the key that would exceed the target, so every file it writes is under TARGET_FILE_BYTES - testing against the target would classify the entire collection as loose and re-merge all of it, forever.
+const LOOSE_BULK_MAX_BYTES = TARGET_FILE_BYTES / 2;
 const KEY_GROUP_BYTES = 800 * 1024 * 1024;
 const DUP_THRESHOLD = 0.4;
 // Whole-tier dedup short-circuit (start of phase 3): when the combined tier is over this size AND the overall key duplication fraction is over this threshold, fold every combined file in one merge instead of the per-key-group walk (which spaces merges 5 min apart — 16 h for 200 groups).
@@ -363,16 +367,13 @@ export class BulkDatabaseBase<T extends { key: string }> {
         return this.streamBytesOnDisk > bulkDatabase2Timing.streamFoldTriggerBytes;
     }
 
-    // Stream files nobody can still be appending to, split by who owned them:
-    //   retirable - a fold may delete these the moment it has consumed them, instead of waiting out streamSealAgeMs. Includes our own sealed files and merge-carry output.
-    //   ownerGone - nobody is left to fold these (a closed tab's leftovers, or a legacy name with no owner stamp), and until one is folded every reader has to decode all of it just to build the index. Worth folding at any size. Our own files are deliberately NOT in here: every merge pass seals ours, so counting them would fold on every tick; the streamFileMaxBytes rollover and the streamFoldTriggerBytes gate cover ours instead.
+    // Stream files nobody can still be appending to: our own sealed files, merge-carry output, and files whose owner is gone (a closed tab's leftovers, or a legacy name with no owner stamp). A fold may delete these the moment it has consumed them, instead of waiting out streamSealAgeMs.
     //
     // Foreign owners are probed over the sync channel. In Node there is no channel, so we cannot know - every foreign owner is assumed alive and the streamSealAgeMs rule stands.
     //
     // assumeSealed is for planning: a merge pass broadcasts a seal before it starts, so by the time it merges our current file IS final. The planner passes isSyncSupported() to predict that; anything deciding a real deletion passes false and goes by streamFileName as it actually stands.
-    private async findAbandonedStreams(streamFiles: StreamFileInfo[], assumeSealed: boolean): Promise<{ retirable: Set<string>; ownerGone: Set<string> }> {
+    private async findAbandonedStreams(streamFiles: StreamFileInfo[], assumeSealed: boolean): Promise<Set<string>> {
         const retirable = new Set<string>();
-        const ownerGone = new Set<string>();
         const hasForeignOwner = streamFiles.some(f => f.ownerId && f.ownerId !== writerId && f.ownerId !== MERGE_OUTPUT_OWNER);
         const live = hasForeignOwner && await queryLiveWriters(this.name, bulkDatabase2Timing.liveWriterProbeMs) || undefined;
         for (const f of streamFiles) {
@@ -382,12 +383,9 @@ export class BulkDatabaseBase<T extends { key: string }> {
                 if (assumeSealed || f.fileName !== this.streamFileName) retirable.add(f.fileName);
                 continue;
             }
-            if (!f.ownerId || live && !live.has(f.ownerId)) {
-                retirable.add(f.fileName);
-                ownerGone.add(f.fileName);
-            }
+            if (!f.ownerId || live && !live.has(f.ownerId)) retirable.add(f.fileName);
         }
-        return { retirable, ownerGone };
+        return retirable;
     }
 
     private async automaticCompactionAllowed(): Promise<boolean> {
@@ -858,7 +856,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
         const streamReader = ordered.length ? streamReaderFromEntries(ordered, 0).reader : undefined;
 
         // An abandoned stream that yielded no entries (zero bytes, or nothing but torn bytes) holds no data and has no writer left. Retire it here: the merge below never lists it as a used source, so the normal retirement path would skip it and it would re-trigger the abandoned-stream fold on every pass. replacedBy is empty because nothing supersedes it - the marker hides it from reads immediately and processMarkers deletes it once aged.
-        const { retirable: retirableStreams } = await this.findAbandonedStreams(streamFiles, false);
+        const retirableStreams = await this.findAbandonedStreams(streamFiles, false);
         const contributingStreams = new Set(streamData.entries.map(e => e.fileName));
         const emptyAbandoned = streamFiles.filter(f => retirableStreams.has(f.fileName) && !contributingStreams.has(f.fileName)).map(f => f.fileName);
         if (emptyAbandoned.length) await writeDeleteMarker(storage, { deleteFiles: emptyAbandoned, replacedBy: [] });
@@ -966,7 +964,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
         return true;
     }
 
-    // Splits the bulk tier the way the merge writer does. A merge cuts an output file once it reaches TARGET_FILE_BYTES, so only its LAST output can be under that — meaning a sub-target file is either a single stream fold or the tail of an earlier merge, and is still waiting to be rolled up ("loose"). Anything at or over the target is done growing ("combined") and only phase 3 touches it again.
+    // Splits the bulk tier into files still worth rolling up ("loose", under LOOSE_BULK_MAX_BYTES - a single stream fold, or the tail end of an earlier merge) and files that are done growing ("combined", which only phase 3 touches again).
     //
     // A file whose size won't read is reported as combined: phase 2 can't consume one (its reader won't load, so the merge won't retire it), and calling it loose would re-trigger phase 2 on every pass until handleUnreadableFile finally deletes it.
     private async splitBulkTier(bulkFiles: BulkFileInfo[]): Promise<{ loose: BulkFileInfo[]; looseBytes: number; combined: BulkFileInfo[]; sizes: Map<string, number> }> {
@@ -978,7 +976,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
         for (let i = 0; i < bulkFiles.length; i++) {
             const bytes = logicalSizes[i];
             sizes.set(bulkFiles[i].fileName, bytes ?? 0);
-            if (bytes === undefined || bytes >= TARGET_FILE_BYTES) {
+            if (bytes === undefined || bytes >= LOOSE_BULK_MAX_BYTES) {
                 combined.push(bulkFiles[i]);
                 continue;
             }
@@ -1064,28 +1062,26 @@ export class BulkDatabaseBase<T extends { key: string }> {
         });
 
         // A pass seals before it merges, so by then our own current file is final too - predict that rather than reporting it as still-open.
-        const { retirable, ownerGone } = await this.findAbandonedStreams(streamFiles, isSyncSupported());
+        const retirable = await this.findAbandonedStreams(streamFiles, isSyncSupported());
         // Only fold what we can also retire. Folding a stream a live foreign owner may still append to would copy it into bulk without removing it, so the bytes would stay in memory and just get re-folded next pass; that owner rolls its own file over at streamFileMaxBytes instead.
         // Aged past streamSealAgeMs counts as retirable too (canDeleteStream's own first rule): no writer appends past the seal age, and this is the only thing that frees the tier in Node, where liveness cannot be probed at all.
         const foldable = streamFiles.filter(f => f.ownerId !== MERGE_OUTPUT_OWNER
             && (retirable.has(f.fileName) || time - f.timestamp >= bulkDatabase2Timing.streamSealAgeMs));
         // Merge-carry files hold nothing but tombstones and are never a REASON to fold - folding one alone would just rewrite it into another carry file, forever. They ride along whenever something else folds, which collapses however many have piled up into one.
         const carry = streamFiles.filter(f => f.ownerId === MERGE_OUTPUT_OWNER);
-        // A stream whose writer is gone will never grow and nothing else will ever fold it, so fold it at any size - even one row. Zero-byte ones don't count: they hold no data to fold, and mergeFileSetInner retires them directly.
-        const abandoned = foldable.filter(f => ownerGone.has(f.fileName) && streamSizes.get(f.fileName));
+        // Size is the only reason to fold. An abandoned stream is NOT one on its own: it is already retirable, so it is counted here and gets swept up the moment the tier is worth folding, and until then it is bounded by this very threshold. Triggering on one would fold on essentially every pass - a browser leaves a file behind whose writer never answers the liveness probe on every reload - which mints a small bulk file each time and pushes the fragmentation into phase 2.
         const foldTriggers = [
             makeTrigger({ name: "foldableBytes", value: streamBytes(foldable), threshold: bulkDatabase2Timing.streamFoldTriggerBytes, unit: "bytes" }),
-            makeTrigger({ name: "abandonedFiles", value: abandoned.length, threshold: 1, unit: "count" }),
         ];
         steps.push({
-            phase: 1, kind: "streamFold", requires: "any", triggers: foldTriggers,
+            phase: 1, kind: "streamFold", requires: "all", triggers: foldTriggers,
             // Skipped when the hard limit already folds everything this would have.
-            ready: !hardLimit.met && foldTriggers.some(t => t.met),
+            ready: !hardLimit.met && foldTriggers.every(t => t.met),
             bulkFiles: [], streamFiles: [...foldable, ...carry], bytes: streamBytes([...foldable, ...carry]),
         });
 
         // ── Phase 2: loose bulk -> combined bulk ─────────────────────────────────────────────────────
-        // Phase 1 emits one small bulk file per fold. Each is cheap to read (index only) but holds its whole key list in memory and joins into every read, so they have to be rolled up. Merging just the loose ones also dedupes them for free - a rewrite-heavy workload collapses a gigabyte of near-identical folds into almost nothing - and it always terminates, because a merge cuts every output but the last at TARGET_FILE_BYTES, so one pass can leave at most one loose file behind.
+        // Phase 1 emits one small bulk file per fold. Each is cheap to read (index only) but holds its whole key list in memory and joins into every read, so they have to be rolled up. Merging just the loose ones also dedupes them for free - a rewrite-heavy workload collapses a gigabyte of near-identical folds into almost nothing - and it always terminates: a chunk is only cut once the next key would take it past TARGET_FILE_BYTES, so every output but the last is over half the target and lands in the combined tier, leaving at most one loose file behind.
         const { loose, looseBytes, combined, sizes } = await this.splitBulkTier(bulkFiles);
         const looseTriggers = [
             makeTrigger({ name: "looseBytes", value: looseBytes, threshold: bulkDatabase2Timing.looseBulkTriggerBytes, unit: "bytes" }),