signalk-questdb 1.5.2 → 1.5.4

package/dist/index.js

@@ -42,7 +42,9 @@ const history_v2_1 = require("./history-v2");
 const history_v1_1 = require("./history-v1");
 const retention_1 = require("./retention");
 const full_export_range_1 = require("./full-export-range");
+const wal_monitor_1 = require("./wal-monitor");
 const questdb_endpoint_1 = require("./questdb-endpoint");
+const host_limits_1 = require("./host-limits");
 // The managed QuestDB container's name. signalk-container prefixes it with
 // `sk-` for the actual container and its DNS name (e.g. `sk-signalk-questdb`).
 const QUESTDB_CONTAINER_NAME = "signalk-questdb";
@@ -88,6 +90,16 @@ const QUESTDB_ULIMITS = { nofile: 1048576 };
 // schema (and rebuild any ILP auto-created with the wrong shape). 60s is far
 // faster than a human would notice "recording broke" yet negligible load.
 const SCHEMA_HEAL_INTERVAL_MS = 60_000;
+// How long the guided-skip endpoint waits after a RESUME WAL before
+// re-reading wal_tables(). An apply attempt against unreadable segment data
+// fails within milliseconds (observed ~3ms in the field), so a few seconds
+// cleanly separates "re-suspended at the same txn" from "healthily replaying".
+const SKIP_RECHECK_DELAY_MS = 3_000;
+// Deadline for the engine-log fetch inside /api/wal-diagnosis. The container
+// API exposes no abort signal, so this bounds how long the diagnosis waits —
+// a runtime that stalls streaming logs must degrade the response (applyError
+// null), not hang it.
+const ENGINE_LOG_TIMEOUT_MS = 10_000;
 function buildResourceLimits(config) {
     return {
         memory: config.questdbMemoryLimit?.trim() || null,
@@ -116,6 +128,10 @@ module.exports = (app) => {
     // Probed once via the richer status query and cached so an old build does
     // not throw-and-fall-back on every /api/status call. Null = not yet probed.
     let walTablesHasErrorColumns = null;
+    // Watches for WAL suspensions, auto-applies the lossless remedy once per
+    // stall point, and raises/clears the Signal K notification. Lives from a
+    // completed start to the next stop/purge.
+    let walMonitor = null;
     // Serializes the lifecycle operations that create or destroy QuestDB
     // resources — asyncStart, /api/update/apply, and /api/purge-data — so they
     // can never interleave. Without this, a purge (or update) issued while a
@@ -131,6 +147,12 @@ module.exports = (app) => {
     // the chain — or one parked in a long await — would run to completion after
     // teardown and resurrect the resources that were just torn down.
     let lifecycleGeneration = 0;
+    // Bumped by /api/update/apply when it recreates the QuestDB container.
+    // stop/purge bump lifecycleGeneration, but an update deliberately does
+    // not (its own generation check detects stop preemption) — so a request
+    // that must not span a container swap (the WAL skip's final guard) needs
+    // this separate epoch to notice one.
+    let containerEpoch = 0;
     // True only between a FULLY completed start (providers, stream
     // subscription, and timers all registered) and the next stop/purge.
     // queryClient/writer are not usable as a running sentinel: they are
@@ -360,6 +382,139 @@ module.exports = (app) => {
         }
         return false;
     }
+    const WAL_NOTIFICATION_PATH = "notifications.signalk-questdb.walSuspended";
+    // Dedupe key of the last emitted notification. The monitor reports every
+    // check cycle; the delta only goes out when the key changes, so clients
+    // aren't spammed with a fresh alert every minute. The key must be built
+    // from STABLE facts (state, table@stall-point, outcome) — the message
+    // itself contains txnLag, which grows every cycle while ingestion
+    // continues and would defeat the dedupe.
+    let lastWalNotification = null;
+    function emitWalNotification(state, message, key) {
+        if (key === lastWalNotification)
+            return;
+        lastWalNotification = key;
+        app.handleMessage("signalk-questdb", {
+            updates: [
+                {
+                    values: [
+                        {
+                            path: WAL_NOTIFICATION_PATH,
+                            value: {
+                                state,
+                                method: ["visual"],
+                                message,
+                                timestamp: new Date().toISOString(),
+                            },
+                        },
+                    ],
+                },
+            ],
+        });
+    }
+    // A latched alert must not outlive the monitoring that backs it: after
+    // stop/purge nobody would ever clear it, and a permanent stale alarm
+    // teaches operators to ignore the path. Downgrade honestly (monitoring
+    // stopped ≠ recovered) and reset the dedupe so the next start re-alerts
+    // if the suspension is still there.
+    function clearWalAlertOnTeardown() {
+        if (lastWalNotification?.startsWith("alert:")) {
+            emitWalNotification("normal", "QuestDB WAL monitoring stopped (plugin stopped or data removed); " +
+                "suspension state is no longer tracked.", "normal:teardown");
+        }
+        lastWalNotification = null;
+    }
+    // wal_tables() probe shared by /api/status, the WAL monitor, and the
+    // resume endpoints. errorTag/errorMessage carry QuestDB's reason for a
+    // suspension when it has one; the columns are absent on an older pinned
+    // QuestDB, so probe for them once (the richer query throws if missing),
+    // cache the result in `walTablesHasErrorColumns`, and thereafter run the
+    // query the build actually supports. Detection never regresses; only the
+    // diagnostic columns are dropped on old builds. Throws on query failure —
+    // callers decide whether that means "treat as not suspended" (status) or
+    // "skip this cycle" (monitor).
+    const RICH_WAL_QUERY = "SELECT name, writerTxn, sequencerTxn, errorTag, errorMessage FROM wal_tables() WHERE suspended = true";
+    const BASIC_WAL_QUERY = "SELECT name, writerTxn, sequencerTxn FROM wal_tables() WHERE suspended = true";
+    async function listSuspendedTables(client) {
+        let walResult;
+        if (walTablesHasErrorColumns === null) {
+            try {
+                walResult = await client.exec(RICH_WAL_QUERY);
+                walTablesHasErrorColumns = true;
+            }
+            catch (err) {
+                walResult = await client.exec(BASIC_WAL_QUERY);
+                // Only a definitive rejection (a 4xx — unknown column on an older
+                // QuestDB) proves the columns are missing. A transient failure
+                // (timeout, engine mid-restart) that happens to spare the cheaper
+                // basic query must not permanently disable the diagnostic columns —
+                // leave the flag unprobed so the next call retries the richer query.
+                const msg = err instanceof Error ? err.message : String(err);
+                if (/^QuestDB query failed \(4\d\d\)/.test(msg)) {
+                    walTablesHasErrorColumns = false;
+                }
+            }
+        }
+        else {
+            walResult = await client.exec(walTablesHasErrorColumns ? RICH_WAL_QUERY : BASIC_WAL_QUERY);
+        }
+        return client.toObjects(walResult).map((row) => {
+            const writerTxn = Number(row.writerTxn ?? 0);
+            const sequencerTxn = Number(row.sequencerTxn ?? 0);
+            return {
+                name: String(row.name),
+                writerTxn,
+                sequencerTxn,
+                txnLag: Math.max(0, sequencerTxn - writerTxn),
+                errorTag: String(row.errorTag ?? ""),
+                errorMessage: String(row.errorMessage ?? ""),
+            };
+        });
+    }
+    async function pendingSegments(client, table) {
+        const result = await client.exec((0, wal_monitor_1.buildPendingSegmentsSQL)(table.name, table.writerTxn), 15_000);
+        return client.toObjects(result).map((row) => ({
+            walId: Number(row.walId ?? 0),
+            segmentId: Number(row.segmentId ?? 0),
+            txns: Number(row.txns ?? 0),
+            minTxn: Number(row.minTxn ?? 0),
+            maxTxn: Number(row.maxTxn ?? 0),
+            minTimestamp: String(row.minTimestamp ?? ""),
+            maxTimestamp: String(row.maxTimestamp ?? ""),
+        }));
+    }
+    // Engine log lines for the apply-failure scrape; extractApplyError then
+    // runs per suspended table over the one fetch. Best effort: external mode,
+    // an older signalk-container without getLogs, or a fetch failure all
+    // degrade to null (the diagnosis then shows only what wal_tables()
+    // reports).
+    async function fetchEngineLogLines() {
+        if (currentConfig?.managedContainer === false)
+            return null;
+        const containers = globalThis.__signalk_containerManager;
+        if (!containers?.getLogs)
+            return null;
+        let timeoutTimer;
+        try {
+            return await Promise.race([
+                // The trailing catch keeps a fetch that fails AFTER the timeout won
+                // the race from surfacing as an unhandled rejection.
+                containers
+                    .getLogs(QUESTDB_CONTAINER_NAME, { tail: 3000 })
+                    .catch(() => null),
+                new Promise((resolve) => {
+                    timeoutTimer = setTimeout(() => resolve(null), ENGINE_LOG_TIMEOUT_MS);
+                }),
+            ]);
+        }
+        catch {
+            return null;
+        }
+        finally {
+            if (timeoutTimer)
+                clearTimeout(timeoutTimer);
+        }
+    }
     // Runs the actual startup. Always invoked through the lifecycle lock (see
     // asyncStart) so a purge/update can't interleave with it.
     //
@@ -623,6 +778,66 @@ module.exports = (app) => {
         schemaHealTimer = setInterval(() => {
             void healSchemaTables();
         }, SCHEMA_HEAL_INTERVAL_MS);
+        // Watch for WAL suspensions. Detection alone is not enough — a suspended
+        // table keeps accepting ILP writes while committing nothing, and the only
+        // built-in recovery QuestDB itself attempts is at engine start. The
+        // monitor closes that gap: loud persistent logging, a Signal K alert
+        // (recording silently stalling for weeks is exactly what notifications
+        // exist for), and one automatic lossless resume per stall point.
+        const client = queryClient;
+        walMonitor = new wal_monitor_1.WalMonitor({
+            // Owned tables only: in external mode the plugin shares a QuestDB with
+            // whatever else the operator runs there, and auto-resuming (or
+            // alerting on) someone else's suspended table is not this plugin's
+            // call to make.
+            listSuspended: async () => (await listSuspendedTables(client)).filter((t) => FULL_EXPORT_TABLE_SET.has(t.name)),
+            resumeTable: (name) => client
+                .exec(`ALTER TABLE "${name.replace(/"/g, '""')}" RESUME WAL`, 10_000)
+                .then(() => undefined),
+            onSuspended: (tables, anyAutoResumeFailed) => {
+                const summary = tables
+                    .map((t) => `${t.name} (${t.txnLag} txns behind)`)
+                    .join(", ");
+                app.setPluginError(`QuestDB WAL suspended: ${summary}`);
+                const key = "alert:" +
+                    tables
+                        .map((t) => `${t.name}@${t.writerTxn}`)
+                        .sort()
+                        .join(",") +
+                    `:${anyAutoResumeFailed}`;
+                emitWalNotification("alert", anyAutoResumeFailed
+                    ? `QuestDB history recording is stalled: ${summary}. Automatic ` +
+                        `resume failed — the data at the stall point is likely ` +
+                        `unreadable. Open the QuestDB History plugin panel to repair.`
+                    : `QuestDB history recording is stalled: ${summary}. Automatic ` +
+                        `recovery is being attempted.`, key);
+            },
+            onResolved: () => {
+                emitWalNotification("normal", "QuestDB history recording has recovered.", "normal:recovered");
+                app.setPluginStatus(`Recording to QuestDB at ${ilpHost}:${ilpPort}`);
+            },
+            debug: (msg) => app.debug(msg),
+            error: (msg) => app.error(msg),
+        });
+        walMonitor.start();
+        // vm.max_map_count is kernel-global (not namespaced), so in managed mode
+        // this local probe reflects the limit the QuestDB container actually runs
+        // under. QuestDB memory-maps every partition column file and WAL segment;
+        // on a grown database the stock 65530 exhausts and mmap fails with
+        // out-of-memory errors — failed queries and suspended WAL apply while
+        // plenty of RAM is free. Warn once here; /api/status re-reads it live so
+        // the panel banner clears as soon as the operator raises the sysctl.
+        if (config.managedContainer !== false) {
+            const mapCount = await (0, host_limits_1.readMaxMapCount)();
+            if (mapCount?.tooLow) {
+                app.error(`Host vm.max_map_count is ${mapCount.current}; QuestDB recommends ` +
+                    `${mapCount.recommended}. A grown database can exhaust this and ` +
+                    `fail queries or suspend recording. Fix in a host shell (not ` +
+                    `inside the container — the limit is kernel-global): ` +
+                    `echo 'vm.max_map_count=${mapCount.recommended}' | sudo tee ` +
+                    `/etc/sysctl.d/99-questdb.conf && sudo sysctl --system`);
+            }
+        }
         app.setPluginStatus(`Recording to QuestDB at ${ilpHost}:${ilpPort}`);
         // Everything is registered — only now does the plugin count as running.
         // Any earlier return/throw leaves the flag false, so a half-started
@@ -682,6 +897,11 @@ module.exports = (app) => {
                 clearInterval(schemaHealTimer);
                 schemaHealTimer = null;
             }
+            if (walMonitor) {
+                walMonitor.stop();
+                walMonitor = null;
+            }
+            clearWalAlertOnTeardown();
             schemaMismatch = false;
             // Clear the clamp warning so it doesn't survive into the next start
             // (e.g. a switch to external/unmanaged mode that never calls ensureRunning).
@@ -714,9 +934,21 @@ module.exports = (app) => {
                         res.status(503).json({ status: "not_running" });
                         return;
                     }
+                    // Probe the host limit BEFORE the health gate: mmap exhaustion is
+                    // one of the ways QuestDB becomes unhealthy in the first place, so
+                    // the remediation hint must survive into the unhealthy response
+                    // instead of disappearing exactly when it matters. Live /proc read
+                    // (cheap) rather than a cached startup probe, so the panel warning
+                    // clears on the next poll once the operator applies the sysctl —
+                    // raising it takes effect immediately, no container restart needed.
+                    // Managed mode only: an external QuestDB may run on another machine
+                    // whose kernel this probe cannot see.
+                    const hostMaxMapCount = currentConfig?.managedContainer !== false
+                        ? await (0, host_limits_1.readMaxMapCount)()
+                        : null;
                     const healthy = await queryClient.isHealthy();
                     if (!healthy) {
-                        res.status(503).json({ status: "unhealthy" });
+                        res.status(503).json({ status: "unhealthy", hostMaxMapCount });
                         return;
                     }
                     let totalRows = 0;
@@ -741,49 +973,16 @@ module.exports = (app) => {
                     // still reads "running". Surface it explicitly so the panel can warn
                     // instead of looking healthy. `txnLag` = sequencer ahead of writer =
                     // the backlog that will never drain until the WAL is resumed.
+                    // `autoResume` is the monitor's verdict for the current stall point:
+                    // "pending" while the automatic lossless resume may still succeed,
+                    // "failed" once replay re-hit the same failure — the panel then
+                    // offers the guided skip instead of just the resume button.
                     let suspendedTables = [];
-                    // errorTag/errorMessage carry QuestDB's reason for the suspension
-                    // (e.g. an out-of-memory or disk error during WAL apply). They are
-                    // empty on a healthy table; surfacing them lets the panel — and a
-                    // bug report — show WHY ingestion stalled instead of just THAT it
-                    // did, without needing to catch the live wal_tables() state by hand
-                    // before the data is reset.
-                    //
-                    // The columns are absent on an older pinned QuestDB. Probe for them
-                    // ONCE (the richer query throws if they're missing), cache the
-                    // result in `walTablesHasErrorColumns`, and thereafter run the query
-                    // the build actually supports — so an old build does not throw and
-                    // fall back on every /api/status call. Detection never regresses;
-                    // only the diagnostic columns are dropped on old builds.
-                    const RICH_WAL_QUERY = "SELECT name, writerTxn, sequencerTxn, errorTag, errorMessage FROM wal_tables() WHERE suspended = true";
-                    const BASIC_WAL_QUERY = "SELECT name, writerTxn, sequencerTxn FROM wal_tables() WHERE suspended = true";
                     try {
-                        let walResult;
-                        if (walTablesHasErrorColumns === null) {
-                            try {
-                                walResult = await queryClient.exec(RICH_WAL_QUERY);
-                                walTablesHasErrorColumns = true;
-                            }
-                            catch {
-                                walResult = await queryClient.exec(BASIC_WAL_QUERY);
-                                walTablesHasErrorColumns = false;
-                            }
-                        }
-                        else {
-                            walResult = await queryClient.exec(walTablesHasErrorColumns ? RICH_WAL_QUERY : BASIC_WAL_QUERY);
-                        }
-                        suspendedTables = queryClient.toObjects(walResult).map((row) => {
-                            const writerTxn = Number(row.writerTxn ?? 0);
-                            const sequencerTxn = Number(row.sequencerTxn ?? 0);
-                            return {
-                                name: String(row.name),
-                                writerTxn,
-                                sequencerTxn,
-                                txnLag: Math.max(0, sequencerTxn - writerTxn),
-                                errorTag: String(row.errorTag ?? ""),
-                                errorMessage: String(row.errorMessage ?? ""),
-                            };
-                        });
+                        suspendedTables = (await listSuspendedTables(queryClient)).map((t) => ({
+                            ...t,
+                            autoResume: walMonitor?.outcomeFor(t.name) ?? null,
+                        }));
                     }
                     catch {
                         // wal_tables() itself is unavailable on non-WAL/older QuestDB, or
@@ -798,6 +997,7 @@ module.exports = (app) => {
                         suspendedTables,
                         schemaMismatch,
                         ulimitClamp,
+                        hostMaxMapCount,
                         endpoint: questdbEndpoints
                             ? `${questdbEndpoints.http.host}:${questdbEndpoints.http.port}`
                             : null,
@@ -1025,6 +1225,7 @@ module.exports = (app) => {
                         // generation one more time.
                         assertNotStopped();
                         ulimitClamp = null;
+                        containerEpoch++;
                         await containers.ensureRunning(QUESTDB_CONTAINER_NAME, updateConfig, {
                             onUlimitClamped,
                         });
@@ -1125,17 +1326,14 @@ module.exports = (app) => {
                             .json({ error: "QuestDB connection not initialized" });
                         return;
                     }
-                    const walResult = await client.exec("SELECT name, writerTxn, sequencerTxn FROM wal_tables() WHERE suspended = true", 10_000);
-                    const suspended = client
-                        .toObjects(walResult)
-                        .filter((row) => FULL_EXPORT_TABLE_SET.has(String(row.name)));
+                    const suspended = (await listSuspendedTables(client)).filter((row) => FULL_EXPORT_TABLE_SET.has(row.name));
                     const results = [];
                     for (const row of suspended) {
-                        const table = String(row.name);
+                        const table = row.name;
                         const base = {
                             table,
-                            writerTxn: Number(row.writerTxn ?? 0),
-                            sequencerTxn: Number(row.sequencerTxn ?? 0),
+                            writerTxn: row.writerTxn,
+                            sequencerTxn: row.sequencerTxn,
                         };
                         try {
                             // Plain RESUME WAL replays from the next unapplied txn — it
@@ -1170,6 +1368,189 @@ module.exports = (app) => {
                     });
                 }
             });
+            // Per-suspended-table repair diagnosis: the pending-segment map, the
+            // minimal bounded skip that would get past the stuck segment (with the
+            // exact loss it implies), and the apply job's real failure reason
+            // scraped from the engine log. The panel calls this when a suspension's
+            // automatic resume has failed, to present the skip decision with its
+            // cost quantified instead of a blind "RESUME WAL FROM TXN" incantation.
+            router.get("/api/wal-diagnosis", async (_req, res) => {
+                try {
+                    const client = queryClient;
+                    if (!client) {
+                        res
+                            .status(503)
+                            .json({ error: "QuestDB connection not initialized" });
+                        return;
+                    }
+                    const suspended = (await listSuspendedTables(client)).filter((row) => FULL_EXPORT_TABLE_SET.has(row.name));
+                    const logLines = suspended.length > 0 ? await fetchEngineLogLines() : null;
+                    const tables = [];
+                    for (const table of suspended) {
+                        let segments = [];
+                        let segmentError = null;
+                        try {
+                            segments = await pendingSegments(client, table);
+                        }
+                        catch (err) {
+                            // wal_transactions() may be unavailable on an older QuestDB —
+                            // report the rest of the diagnosis without a skip plan.
+                            segmentError = err instanceof Error ? err.message : String(err);
+                        }
+                        tables.push({
+                            ...table,
+                            autoResume: walMonitor?.outcomeFor(table.name) ?? null,
+                            // Commit-time of the first unapplied txn = when apply froze.
+                            suspendedSince: segments.length > 0 ? segments[0].minTimestamp : null,
+                            pendingSegments: segments.length,
+                            skipPlan: (0, wal_monitor_1.computeSkipPlan)(segments),
+                            segmentError,
+                            applyError: logLines
+                                ? (0, wal_monitor_1.extractApplyError)(logLines, table.name)
+                                : null,
+                        });
+                    }
+                    res.json({ tables });
+                }
+                catch (err) {
+                    res.status(500).json({
+                        error: err instanceof Error ? err.message : String(err),
+                    });
+                }
+            });
+            // Bounded lossy skip past an unreadable WAL segment. This is the ONLY
+            // write path that can lose data, so it is deliberately narrow:
+            //   - the table must be one of ours and currently suspended;
+            //   - a lossless resume is attempted first and re-checked — if it
+            //     sticks, nothing is skipped;
+            //   - the skip target is computed server-side from the live segment
+            //     map (never taken from the client), so it can only ever be the
+            //     minimal next-segment boundary.
+            // The client sends the plan it displayed only so a stale UI can be
+            // rejected instead of skipping more than the operator agreed to.
+            router.post("/api/resume-wal/skip", async (req, res) => {
+                try {
+                    const client = queryClient;
+                    if (!client) {
+                        res
+                            .status(503)
+                            .json({ error: "QuestDB connection not initialized" });
+                        return;
+                    }
+                    const table = String(req.body?.table ?? "");
+                    if (!FULL_EXPORT_TABLE_SET.has(table)) {
+                        res.status(400).json({ error: `Unknown table: ${table}` });
+                        return;
+                    }
+                    const quoted = `"${table.replace(/"/g, '""')}"`;
+                    // Captured at entry: stop/purge bump the generation, an update
+                    // recreate bumps the container epoch — the destructive statement
+                    // below must never run against whatever lifecycle replaced the
+                    // state this request was validated on.
+                    const generation = lifecycleGeneration;
+                    const epoch = containerEpoch;
+                    const recheck = async () => (await listSuspendedTables(client)).find((t) => t.name === table);
+                    const before = await recheck();
+                    if (!before) {
+                        res.json({
+                            skipped: false,
+                            healed: true,
+                            message: `${table} is not suspended — nothing to skip.`,
+                        });
+                        return;
+                    }
+                    // Safety ladder step 1: the lossless path. If the suspension cause
+                    // was transient after all, this recovers everything and the lossy
+                    // step below never runs.
+                    await client.exec(`ALTER TABLE ${quoted} RESUME WAL`, 10_000);
+                    await new Promise((resolve) => setTimeout(resolve, SKIP_RECHECK_DELAY_MS));
+                    const afterResume = await recheck();
+                    if (!afterResume) {
+                        res.json({
+                            skipped: false,
+                            healed: true,
+                            message: `Lossless resume succeeded on ${table} — no data was skipped. The backlog is now replaying.`,
+                        });
+                        return;
+                    }
+                    if (afterResume.writerTxn !== before.writerTxn) {
+                        res.json({
+                            skipped: false,
+                            healed: false,
+                            progressed: true,
+                            message: `The writer advanced on ${table} before re-suspending — the situation changed. Re-run the diagnosis.`,
+                        });
+                        return;
+                    }
+                    // Step 2: replay re-froze at the same transaction. Everything the
+                    // destructive statement depends on is validated from the state
+                    // read CLOSEST to execution: re-read the suspension state, require
+                    // the writer still frozen at the same txn and the lifecycle
+                    // untouched by stop/update/purge, recompute the plan from that
+                    // state, and require it to match the operator's confirmation in
+                    // every field. Only that final validated plan is executed.
+                    const preExec = await recheck();
+                    if (generation !== lifecycleGeneration ||
+                        epoch !== containerEpoch ||
+                        !preExec ||
+                        preExec.writerTxn !== before.writerTxn) {
+                        res.status(409).json({
+                            error: "The table's state changed while confirming the skip — nothing was skipped. Re-run the diagnosis.",
+                        });
+                        return;
+                    }
+                    const plan = (0, wal_monitor_1.computeSkipPlan)(await pendingSegments(client, preExec));
+                    if (!plan) {
+                        res.status(500).json({
+                            error: `Cannot compute a skip plan for ${table} — no pending transactions found.`,
+                        });
+                        return;
+                    }
+                    if (!(0, wal_monitor_1.skipPlansEqual)(plan, req.body?.confirmPlan)) {
+                        res.status(409).json({
+                            error: "The skip plan changed since the diagnosis was displayed. Re-run the diagnosis and confirm again.",
+                            skipPlan: plan,
+                        });
+                        return;
+                    }
+                    // Last state read before the destructive statement — only
+                    // synchronous code between this validation and the ALTER, so no
+                    // await window remains in which the writer could move or the
+                    // lifecycle could be swapped. (The two HTTP calls are still
+                    // distinct reads; that residue is irreducible client-side.)
+                    const finalState = await recheck();
+                    if (generation !== lifecycleGeneration ||
+                        epoch !== containerEpoch ||
+                        !finalState ||
+                        finalState.writerTxn !== before.writerTxn) {
+                        res.status(409).json({
+                            error: "The table's state changed while confirming the skip — nothing was skipped. Re-run the diagnosis.",
+                            skipPlan: plan,
+                        });
+                        return;
+                    }
+                    app.error(`Skipping ${plan.skippedTxns} unreadable WAL txn(s) on ${table} ` +
+                        `(walId=${plan.walId} segmentId=${plan.segmentId}, ` +
+                        `${plan.skipWindowStart} → ${plan.skipWindowEnd}) — ` +
+                        `RESUME WAL FROM TXN ${plan.skipToTxn}, requested via panel`);
+                    await client.exec(`ALTER TABLE ${quoted} RESUME WAL FROM TXN ${plan.skipToTxn}`, 10_000);
+                    await new Promise((resolve) => setTimeout(resolve, SKIP_RECHECK_DELAY_MS));
+                    const afterSkip = await recheck();
+                    res.json({
+                        skipped: true,
+                        skipPlan: plan,
+                        stillSuspended: Boolean(afterSkip),
+                        message: afterSkip
+                            ? `Skipped ${plan.skippedTxns} txn(s) on ${table}, but the table re-suspended at txn ${afterSkip.writerTxn + 1} — another segment appears unreadable. Re-run the diagnosis to skip it too.`
+                            : `Skipped ${plan.skippedTxns} txn(s) on ${table} (${plan.skipWindowStart} → ${plan.skipWindowEnd}). The remaining backlog is replaying.`,
+                    });
+                }
+                catch (err) {
+                    res.status(500).json({
+                        error: err instanceof Error ? err.message : String(err),
+                    });
+                }
+            });
             router.post("/api/purge-data", async (_req, res) => {
                 try {
                     // External mode is a config fact independent of the runtime, so
@@ -1212,6 +1593,11 @@ module.exports = (app) => {
                             clearInterval(schemaHealTimer);
                             schemaHealTimer = null;
                         }
+                        if (walMonitor) {
+                            walMonitor.stop();
+                            walMonitor = null;
+                        }
+                        clearWalAlertOnTeardown();
                         if (retentionTimer) {
                             clearInterval(retentionTimer);
                             retentionTimer = null;