signalk-questdb 2.3.2 → 2.5.0

package/dist/index.js

@@ -1,5 +1,8 @@
-import { waitForContainerManager } from "signalk-container-helper";
+import { readFileSync } from "fs";
+import { errMsg, updateImageAfterPluginUpdate, waitForContainerManager, } from "signalk-container-helper";
+import { waitForQuestdb } from "./questdb-ready.js";
 import { ILPWriter } from "./ilp-writer.js";
+import { refreshImage } from "./image-refresh.js";
 import { QueryClient, isReadOnlySQL } from "./query-client.js";
 import { ConfigSchema, normalizeConfig } from "./config/schema.js";
 import { extractVesselName, flattenObjectValue, routeDeltaValue, UnstorableTracker, } from "./delta-routing.js";
@@ -19,11 +22,16 @@ import { FileCheckpointStore, migrationIdentity, sameIdentity, } from "./migrati
 import { countLegacyImportRows, removeLegacyImportRows, } from "./migration-cleanup.js";
 import { adoptDatabaseFromVolumeRoot, wipeDataInSignalk, resolveQuestdbMount, } from "./questdb-mount.js";
 import { WalMonitor, buildPendingSegmentsSQL, computeSkipPlan, extractApplyError, isPartitionOpenFailure, skipPlansEqual, } from "./wal-monitor.js";
+import { QuestdbWatchdog, questdbErrorLines } from "./questdb-watchdog.js";
+import { WatchdogChecks } from "./watchdog-checks.js";
+import { drained } from "./drained.js";
 import { QUESTDB_INTERNAL_HTTP_PORT, QUESTDB_INTERNAL_ILP_PORT, QUESTDB_INTERNAL_PG_PORT, QUESTDB_ACCESSIBLE_PORTS, resolveManagedEndpoints, resolveLanExposureHost, lanExposureEndpoints, } from "./questdb-endpoint.js";
 import { nofileClampSatisfied, readMaxMapCount } from "./host-limits.js";
 import { buildContainerEnv } from "./container-env.js";
 import { PathMatcher, RateMatcher, Throttle } from "./path-matcher.js";
 import { ChangeGate } from "./change-gate.js";
+/** This plugin's own version, which an update of it changes. */
+const PLUGIN_VERSION = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")).version;
 // signalk-container's API, imported rather than mirrored. The 156-line
 // hand-written copy that used to live here drifted by construction: it was
 // only ever updated when this plugin happened to need a new member, so
@@ -45,15 +53,10 @@ const FULL_EXPORT_TABLES = [
     ARCHIVE_TABLES.signalk_position,
 ];
 const FULL_EXPORT_TABLE_SET = new Set(FULL_EXPORT_TABLES);
-// Healthcheck for the QuestDB container. The `questdb/questdb` image ships no
-// HEALTHCHECK of its own, so under Podman the container would otherwise sit in
-// `starting` forever (Podman reports a probeless container as perpetually
-// starting, never healthy). We give it an explicit probe — `curl` is present
-// in the image — hitting QuestDB's purpose-built `/ping` liveness endpoint,
-// which returns an empty `204` immediately. (Probing `/` instead makes curl
-// hang on the web-console `301` redirect until it times out.) signalk-container
-// emits this as `--health-*` run flags (see its `healthcheck` ContainerConfig
-// field).
+/** An update refused while a start holds the lifecycle lock. */
+const STILL_STARTING = "The plugin is still starting QuestDB; try again once it is recording";
+/** How long a retry probes for the LAN-exposure host; the wait itself polls. */
+const RETRY_PROBE_DEADLINE_MS = 5_000;
 const QUESTDB_HEALTHCHECK = {
     test: [
         "CMD",
@@ -153,6 +156,8 @@ export default (app) => {
     // OLD config/version, so we must not report success or persist a new version
     // as if the change applied. Cleared at the top of each ensureRunning attempt.
     let containerWedged = null;
+    // The image check after a plugin update, while it waits for a free link.
+    let imageCheckAbort = null;
     // Whether this QuestDB's wal_tables() exposes the errorTag/errorMessage
     // columns (present on current builds, absent on an older pinned version).
     // Probed once via the richer status query and cached so an old build does
@@ -162,6 +167,35 @@ export default (app) => {
     // stall point, and raises/clears the Signal K notification. Lives from a
     // completed start to the next stop/purge.
     let walMonitor = null;
+    // Restarts a managed QuestDB that ran out of memory or stopped answering
+    // (questdb-watchdog.ts). Lives from a completed start to the next
+    // stop/purge, managed mode only.
+    let watchdog = null;
+    // The watchdog's probe and restart, and what they learn from the plugin's
+    // own queries (watchdog-checks.ts).
+    let watchdogChecks = null;
+    // A client reports to the watchdog of the start that created it: a request
+    // of an earlier start that ends late tells nothing about the container now.
+    const questdbClientOptions = (generation) => ({
+        onErrorResponse: (body) => {
+            if (generation === lifecycleGeneration)
+                watchdogChecks?.noteErrorResponse(body);
+        },
+        onResponse: () => {
+            if (generation === lifecycleGeneration)
+                watchdogChecks?.noteResponse();
+        },
+    });
+    // The watchdog's error is the one on the status line, so its recovery may
+    // replace it with the recording line. Any other error published since
+    // stays.
+    let watchdogStatusShown = false;
+    // Parquet exports streaming from QuestDB. One can keep QuestDB's single
+    // worker busy for minutes, so health checks time out while it streams: the
+    // watchdog does not judge QuestDB meanwhile. Counted only once QuestDB has
+    // answered, so an export waiting on a QuestDB that answers nothing does
+    // not hide it.
+    let questdbExportsRunning = 0;
     // Serializes the lifecycle operations that create or destroy QuestDB
     // resources — asyncStart, /api/update/apply, and /api/purge-data — so they
     // can never interleave. Without this, a purge (or update) issued while a
@@ -183,6 +217,9 @@ export default (app) => {
     // that must not span a container swap (the WAL skip's final guard) needs
     // this separate epoch to notice one.
     let containerEpoch = 0;
+    // The watchdog's container stops still in flight, across plugin restarts
+    // (WatchdogChecksDeps.stops).
+    const watchdogStops = { pending: 0 };
     // True only between a FULLY completed start (providers, stream
     // subscription, and timers all registered) and the next stop/purge.
     // queryClient/writer are not usable as a running sentinel: they are
@@ -209,6 +246,7 @@ export default (app) => {
     let pluginErrorActive = false;
     const publishError = (msg) => {
         pluginErrorActive = true;
+        watchdogStatusShown = false;
         app.setPluginError(msg);
     };
     // Republish the recording line, but never at the cost of an active error.
@@ -440,12 +478,16 @@ export default (app) => {
             // by probing, which only works once QuestDB is up. The resolver runs
             // after ensureRunning() on both call sites, so defer the probe into
             // the closure (and memoize: the endpoint can't change while the
-            // container config that produced it is live).
+            // container config that produced it is live). Asked `again`, it probes
+            // anew: a QuestDB that was not answering yet left the gateway, the
+            // fallback, which need not be where it is.
             let endpoints = null;
-            return async () => {
-                if (!endpoints) {
+            return async (options) => {
+                if (!endpoints || options?.again) {
                     const skHost = config.exposeToContainers
-                        ? await resolveLanExposureHost(containers, httpPort, (msg) => app.debug(msg))
+                        ? await resolveLanExposureHost(containers, httpPort, (msg) => app.debug(msg), options?.again
+                            ? { deadlineMs: RETRY_PROBE_DEADLINE_MS }
+                            : undefined)
                         : "127.0.0.1";
                     endpoints = lanExposureEndpoints(skHost, httpPort, ilpPort);
                 }
@@ -514,10 +556,10 @@ export default (app) => {
         const matches = matcher.matches(path);
         return mode === "exclude" ? !matches : matches;
     }
-    function isThrottled(path, context, defaultRate) {
+    function isThrottled(path, context, defaultRate, source) {
         // Per-path override wins when one matches; otherwise the default rate.
         const overrideRate = rateMatcher?.rateFor(path) ?? null;
-        return throttle.shouldDrop(path, context, overrideRate ?? defaultRate, Date.now());
+        return throttle.shouldDrop(path, context, overrideRate ?? defaultRate, Date.now(), source);
     }
     const WAL_NOTIFICATION_PATH = "notifications.signalk-questdb.walSuspended";
     // Dedupe key of the last emitted notification. The monitor reports every
@@ -561,6 +603,41 @@ export default (app) => {
         }
         lastWalNotification = null;
     }
+    const WATCHDOG_NOTIFICATION_PATH = "notifications.signalk-questdb.watchdog";
+    // The watchdog sends each state change once (an alert when it restarts or
+    // holds a restart back, normal once QuestDB answers again), so no dedupe.
+    function emitWatchdogNotification(state, message) {
+        app.handleMessage("signalk-questdb", {
+            updates: [
+                {
+                    values: [
+                        {
+                            path: WATCHDOG_NOTIFICATION_PATH,
+                            value: {
+                                state,
+                                method: ["visual"],
+                                message,
+                                timestamp: new Date().toISOString(),
+                            },
+                        },
+                    ],
+                },
+            ],
+        });
+    }
+    // Same reason as clearWalAlertOnTeardown: an alert nobody will clear must
+    // not outlive the watchdog.
+    function stopWatchdog() {
+        if (!watchdog)
+            return;
+        watchdog.stop();
+        if (watchdog.alertOutstanding) {
+            emitWatchdogNotification("normal", "QuestDB watchdog stopped (plugin stopped or data removed); QuestDB " +
+                "is no longer watched.");
+        }
+        watchdog = null;
+        watchdogChecks = null;
+    }
     // wal_tables() probe shared by /api/status, the WAL monitor, and the
     // resume endpoints. errorTag/errorMessage carry QuestDB's reason for a
     // suspension when it has one; the columns are absent on an older pinned
@@ -652,6 +729,193 @@ export default (app) => {
                 clearTimeout(timeoutTimer);
         }
     }
+    // The update button's recreate, wait and reconnect, and the image check's
+    // after a plugin update. Runs under the lifecycle lock; `generation` is the
+    // one the caller started from, and a stop() or purge that ended it fails
+    // the rest. A wedged container defers the recreate: `containerWedged` then
+    // says so. `background`, as for the image check, sends progress to the
+    // debug log, as no one is waiting on it. With `refresh`, the container is
+    // recreated on the image a pull just brought for the tag it already runs,
+    // where that differs from the one it runs.
+    async function recreateQuestdb(tag, generation, containers, { background, refresh }) {
+        const assertNotStopped = () => {
+            if (generation !== lifecycleGeneration)
+                throw new Error("plugin stopped while the update was running");
+        };
+        const progress = (line) => {
+            if (background)
+                app.debug(line);
+            else
+                app.setPluginStatus(line);
+        };
+        const updateMount = await resolveQuestdbDataMount(containers);
+        progress(`Starting QuestDB ${tag}...`);
+        const updateConfig = {
+            image: "questdb/questdb",
+            tag,
+            volumes: updateMount.volumes,
+            env: {
+                ...buildContainerEnv(currentConfig ?? {}),
+                ...updateMount.env,
+            },
+            restart: "unless-stopped",
+            resources: currentConfig ? buildResourceLimits(currentConfig) : undefined,
+            ulimits: QUESTDB_ULIMITS,
+            healthcheck: QUESTDB_HEALTHCHECK,
+            // The tag names what it did before, so only this recreates on the image
+            // the pull brought. Through ensureRunning rather than recreate(), which
+            // drops the container's addresses and may pick another host port.
+            ...(refresh ? { autoUpdateOnFloatingTag: true } : {}),
+        };
+        const resolveUpdateEndpoints = await applyQuestdbNetworking(currentConfig ?? {}, containers, QUESTDB_CONTAINER_NAME, updateConfig);
+        // Clear any prior clamp before re-running so a no-longer-clamping
+        // update doesn't leave a stale warning. The recreate must not
+        // happen after a stop/purge teardown, so gate it on the
+        // generation one more time.
+        assertNotStopped();
+        ulimitClamp = null;
+        containerWedged = null;
+        containerEpoch++;
+        await containers.ensureRunning(QUESTDB_CONTAINER_NAME, updateConfig, {
+            onUlimitClamped,
+            onContainerWedged,
+        });
+        assertNotStopped();
+        // The QuestDB version just changed, so the cached wal_tables()
+        // column shape may no longer match (the errorTag/errorMessage
+        // columns could appear on an upgrade or disappear on a downgrade).
+        // Force the next /api/status to re-probe instead of trusting a
+        // stale capability flag.
+        walTablesHasErrorColumns = null;
+        // Re-resolve so the export endpoints and status line reflect the
+        // current endpoint. The version bump keeps the same container name
+        // and networking, so the endpoint is stable — the existing
+        // QueryClient/ILPWriter stay valid (and the registered history
+        // providers keep their reference). On the LAN path this await
+        // probes for the reachable host, so it's long enough for a stop()
+        // to land mid-flight — re-check before touching plugin state.
+        const updatedEndpoints = await resolveUpdateEndpoints();
+        assertNotStopped();
+        questdbEndpoints = updatedEndpoints;
+        const { host: ilpHost, port: ilpPort } = questdbEndpoints.ilp;
+        // Wait for the recreated container to answer, then reconnect ILP.
+        // Locals are captured because stop() nulls the module bindings:
+        // reading them again mid-loop would throw, and the generation
+        // checks after every await are what keep a concurrent teardown
+        // from being resurrected by the reconnect.
+        const client = queryClient;
+        if (client) {
+            const deadline = Date.now() + 30000;
+            while (Date.now() < deadline) {
+                assertNotStopped();
+                if (await client.isHealthy())
+                    break;
+                await new Promise((resolve) => setTimeout(resolve, 500));
+            }
+        }
+        assertNotStopped();
+        const ilpWriter = writer;
+        if (ilpWriter) {
+            try {
+                await ilpWriter.disconnect();
+            }
+            catch {
+                /* ignore */
+            }
+            assertNotStopped();
+            await ilpWriter.connect();
+            // A stop() that landed during connect() has already nulled
+            // `writer`, so nothing would ever disconnect this fresh
+            // connection — close it ourselves before failing the update.
+            if (generation !== lifecycleGeneration) {
+                try {
+                    await ilpWriter.disconnect();
+                }
+                catch {
+                    /* ignore */
+                }
+                assertNotStopped();
+            }
+        }
+        return { ilpHost, ilpPort };
+    }
+    /** The running QuestDB's version, from `build()`; null where it does not say. */
+    async function runningQuestdbVersion(client) {
+        const result = await client.exec("SELECT build()");
+        const build = result.dataset.length > 0 ? String(result.dataset[0][0]) : "";
+        return build.match(/QuestDB\s+([\d.]+)/)?.[1] ?? null;
+    }
+    // After the plugin itself was updated, its first start moves QuestDB on a
+    // floating tag such as `latest` to the latest release, once, on a free
+    // link. signalk-container pulls an image only when it is missing, so such
+    // a tag otherwise stays on whatever release the first install found.
+    function checkImageAfterPluginUpdate(containers) {
+        // One check at a time; stop() aborts it too.
+        imageCheckAbort?.abort();
+        const abort = new AbortController();
+        imageCheckAbort = abort;
+        const generation = lifecycleGeneration;
+        const tag = currentConfig?.questdbVersion ?? "latest";
+        updateImageAfterPluginUpdate({
+            manager: containers,
+            pluginVersion: PLUGIN_VERSION,
+            // Beside the data directory, which is QuestDB's own.
+            stateFile: `${app.getDataDirPath()}.image-check.json`,
+            tag,
+            versionSource: { githubReleases: "questdb/questdb" },
+            currentVersion: async () => queryClient ? runningQuestdbVersion(queryClient) : null,
+            update: (signal) => refreshImage({
+                containers,
+                name: QUESTDB_CONTAINER_NAME,
+                image: "questdb/questdb",
+                withLock: withLifecycleLock,
+                current: () => generation === lifecycleGeneration && pluginRunning,
+                configuredTag: () => currentConfig?.questdbVersion ?? "latest",
+                // On a link that was metered, the check may come hours later, in
+                // the middle of work the recreate would break.
+                blocked: () => legacyCleanupRunning
+                    ? "the string table is being rebuilt"
+                    : containerWedged
+                        ? "the container is wedged"
+                        : activeMigration?.state === "running"
+                            ? "an InfluxDB import is running"
+                            : gpxImports.active
+                                ? "a GPX import is running"
+                                : questdbExportsRunning > 0
+                                    ? "an export is running"
+                                    : null,
+                recreate: async (newTag) => {
+                    await recreateQuestdb(newTag, generation, containers, {
+                        background: true,
+                        refresh: true,
+                    });
+                },
+                restore: async (newTag) => {
+                    await recreateQuestdb(newTag, generation, containers, {
+                        background: true,
+                        refresh: false,
+                    });
+                },
+                debug: (msg) => app.debug(msg),
+                error: (msg) => app.error(msg),
+                reportDown: (msg) => {
+                    app.error(msg);
+                    publishError(msg);
+                },
+            }, tag, signal),
+            signal: abort.signal,
+        })
+            .then((outcome) => app.debug(`QuestDB image check after the plugin update to ${PLUGIN_VERSION}: ${outcome}`))
+            .catch((err) => {
+            if (abort.signal.aborted)
+                return;
+            app.error(`QuestDB image check after the plugin update to ${PLUGIN_VERSION} failed: ${errMsg(err)}`);
+        })
+            .finally(() => {
+            if (imageCheckAbort === abort)
+                imageCheckAbort = null;
+        });
+    }
     // Runs the actual startup. Always invoked through the lifecycle lock (see
     // asyncStart) so a purge/update can't interleave with it.
     //
@@ -711,6 +975,8 @@ export default (app) => {
                 port: config.questdbIlpPort ?? QUESTDB_INTERNAL_ILP_PORT,
             },
         };
+        // Managed mode: where the container is now, which a recreate can change.
+        let refreshEndpoints = null;
         if (config.managedContainer !== false) {
             // Two phases in one call: poll for the manager global, then wait for
             // runtime detection to settle. The previous code read the global ONCE,
@@ -794,6 +1060,7 @@ export default (app) => {
                 const resolveEndpoints = await applyQuestdbNetworking(config, containers, QUESTDB_CONTAINER_NAME, containerConfig);
                 if (signal.aborted)
                     return;
+                refreshEndpoints = resolveEndpoints;
                 app.setPluginStatus("Starting QuestDB container...");
                 // Clear any prior clamp so a run that no longer clamps (e.g. the host
                 // limit was raised) doesn't leave a stale warning; onUlimitClamped
@@ -824,29 +1091,47 @@ export default (app) => {
                 return;
             }
         }
-        const { host: httpHost, port: httpPort } = questdbEndpoints.http;
-        const { host: ilpHost, port: ilpPort } = questdbEndpoints.ilp;
+        let { host: httpHost, port: httpPort } = questdbEndpoints.http;
         app.debug("connecting to QuestDB at %s:%d", httpHost, httpPort);
-        queryClient = new QueryClient(httpHost, httpPort);
+        queryClient = new QueryClient(httpHost, httpPort, questdbClientOptions(lifecycleGeneration));
         app.setPluginStatus("Waiting for QuestDB to become ready...");
-        const deadline = Date.now() + 30000;
-        while (Date.now() < deadline) {
-            if (signal.aborted) {
-                queryClient = null;
-                return;
-            }
-            if (await queryClient.isHealthy())
-                break;
-            await new Promise((resolve) => setTimeout(resolve, 500));
+        let questdb = queryClient;
+        let notResponding = false;
+        const refresh = refreshEndpoints;
+        try {
+            await waitForQuestdb(() => questdb.isHealthy(), {
+                signal,
+                // A container recreated during the wait can come back on another port.
+                beforeRetry: refresh
+                    ? async () => {
+                        const resolved = await refresh({ again: true });
+                        if (signal.aborted)
+                            return;
+                        questdbEndpoints = resolved;
+                        ({ host: httpHost, port: httpPort } = resolved.http);
+                        questdb = queryClient = new QueryClient(httpHost, httpPort, questdbClientOptions(lifecycleGeneration));
+                    }
+                    : undefined,
+                onAttemptFailed: (err, nextDelayMs) => {
+                    app.debug(`QuestDB readiness attempt failed: ${errMsg(err)}`);
+                    notResponding = true;
+                    publishError(`QuestDB not responding at ${httpHost}:${httpPort} — retrying in ${Math.round(nextDelayMs / 1000)}s`);
+                },
+            });
+        }
+        catch (err) {
+            if (!signal.aborted)
+                throw err;
         }
         if (signal.aborted) {
             queryClient = null;
             return;
         }
-        if (!(await queryClient.isHealthy())) {
-            publishError(`QuestDB not responding at ${httpHost}:${httpPort}`);
-            return;
-        }
+        // It answers now: the error was this wait's own, and must not hold back
+        // the recording line once the writer is up.
+        if (notResponding)
+            pluginErrorActive = false;
+        const { host: ilpHost, port: ilpPort } = questdbEndpoints.ilp;
         app.setPluginStatus("Creating tables...");
         // A migration that degrades (dedup off on an external table) is worth a
         // line in the server log, but must never abort startup — see ensureTables.
@@ -872,6 +1157,9 @@ export default (app) => {
             // stuck on a cheerful "Recording" while every sample is dropped.
             onUnhealthy: (msg) => publishError(msg),
             onHealthy: () => {
+                // The watchdog's error stays until the watchdog clears it.
+                if (watchdogStatusShown)
+                    return;
                 // The writer recovered: the error is no longer current, so the
                 // recording line may take the status entry back.
                 pluginErrorActive = false;
@@ -905,7 +1193,14 @@ export default (app) => {
         // restore sees every vessel that transmitted during startup.
         liveContexts.clear();
         trackLiveContexts = config.restoreOnStart;
-        const bus = app.streambundle.getBus();
+        // getBus() carries only the source that Signal K's source priorities
+        // prefer on each path, the value instruments show. recordAllSources
+        // reads the unfiltered bus instead, so a source that loses the ranking
+        // is recorded as well: the raw paddlewheel next to a corrected STW
+        // (issue #235).
+        const bus = config.recordAllSources
+            ? app.streambundle.getUnfilteredBus()
+            : app.streambundle.getBus();
         const unsub = bus.onValue((delta) => {
             if (!writer)
                 return;
@@ -924,9 +1219,26 @@ export default (app) => {
             // syntax. Undefined when the delta carries none — the column stays
             // null, same as rows written before it existed.
             const source = typeof delta.$source === "string" ? delta.$source : undefined;
+            // Recording every source keeps each one apart. It gets its own
+            // sampling window: the shared one would keep whichever source arrives
+            // first, so the fastest receiver would crowd out the ones recorded to
+            // compare against it. And its nulls forget only its own last values:
+            // a gateway sending "not available" several times a second would
+            // otherwise make every other source's unchanged values due again at
+            // each sampling window instead of each heartbeat.
+            const perSource = config.recordAllSources ? source : undefined;
+            // Meta updates ride the same bus as values, flagged isMeta. They
+            // describe a path (units, description, displayName), they are not a
+            // reading of it. Routed like a value object, their keys were stored
+            // as rows such as `….units` and `….displayName`, again each time a
+            // plugin republished its meta. Nor do they show that a vessel is
+            // transmitting: a local plugin publishes its meta on its own start,
+            // and counting that would cost the restore a vessel's saved position.
+            if (delta.isMeta)
+                return;
             // Note the context BEFORE any early return or filter: identity-only
             // deltas return below, and path filters and throttles decide what gets
-            // STORED — but any delta at all proves the vessel is transmitting now,
+            // STORED — but any value at all proves the vessel is transmitting now,
             // which is what makes its stored position obsolete for the restore.
             if (trackLiveContexts) {
                 liveContexts.add(context === app.selfContext ? "self" : context);
@@ -969,7 +1281,7 @@ export default (app) => {
             // the change gate must forget the path, so the first reading after the
             // gap is recorded even when it equals the last one before it.
             if (path && value === null) {
-                changeGate?.forget(path, context === app.selfContext ? "self" : context);
+                changeGate?.forget(path, context === app.selfContext ? "self" : context, perSource);
             }
             if (!path || value === undefined || value === null)
                 return;
@@ -1016,13 +1328,14 @@ export default (app) => {
                     return;
                 // Throttled per path AND context: the sampling rate bounds each
                 // vessel's stream, not the fleet's (issue #93). The stored context
-                // ("self" vs raw) is the key, matching what the rows carry. The key
-                // deliberately does NOT include source: the sampling rate bounds the
-                // per-path row volume, and keying per source would multiply it by the
-                // number of receivers. With two sources alternating, each window
-                // keeps whichever arrived first — a source-filtered read then sees
-                // roughly every other sample, which the rate already allows for.
-                if (isThrottled(path, isSelf ? "self" : context, config.defaultSamplingRate ?? 2000))
+                // ("self" vs raw) is the key, matching what the rows carry. Unless
+                // recordAllSources asks for every source, the key deliberately does
+                // NOT include source: the sampling rate bounds the per-path row
+                // volume, and keying per source would multiply it by the number of
+                // receivers. With two sources alternating, each window keeps
+                // whichever arrived first — a source-filtered read then sees roughly
+                // every other sample, which the rate already allows for.
+                if (isThrottled(path, isSelf ? "self" : context, config.defaultSamplingRate ?? 2000, perSource))
                     return;
             }
             // Rows are stamped with the server receive time, deliberately NOT the
@@ -1077,7 +1390,7 @@ export default (app) => {
                         continue;
                     if (changeGate?.isRepeat(leaf.path, ctx, source, leaf.value, now))
                         continue;
-                    if (isThrottled(leaf.path, ctx, config.defaultSamplingRate ?? 2000))
+                    if (isThrottled(leaf.path, ctx, config.defaultSamplingRate ?? 2000, perSource))
                         continue;
                     changeGate?.wrote(leaf.path, ctx, source, leaf.value, now);
                     if (typeof leaf.value === "number") {
@@ -1129,6 +1442,7 @@ export default (app) => {
                 .exec(`ALTER TABLE "${name.replace(/"/g, '""')}" RESUME WAL`, 10_000)
                 .then(() => undefined),
             onSuspended: (tables, anyAutoResumeFailed) => {
+                watchdogChecks?.noteSuspended(tables);
                 const summary = tables
                     .map((t) => `${t.name} (${t.txnLag} txns behind)`)
                     .join(", ");
@@ -1148,12 +1462,78 @@ export default (app) => {
             },
             onResolved: () => {
                 emitWalNotification("normal", "QuestDB history recording has recovered.", "normal:recovered");
+                if (watchdogStatusShown)
+                    return;
                 app.setPluginStatus(recordingStatus(ilpHost, ilpPort));
             },
             debug: (msg) => app.debug(msg),
             error: (msg) => app.error(msg),
         });
         walMonitor.start();
+        // Restart QuestDB when it runs out of memory or stops answering: neither
+        // ends the JVM, so the container's restart policy never acts, and both
+        // cost every sample from then on (questdb-watchdog.ts). Managed mode
+        // only: an external QuestDB is not the plugin's to restart.
+        const managedContainers = config.managedContainer !== false
+            ? globalThis.__signalk_containerManager
+            : undefined;
+        if (managedContainers) {
+            // Captured like an update's: stop() and purge bump it, and a restart
+            // that spans one must not start the container the teardown stopped.
+            const generation = lifecycleGeneration;
+            const ended = () => generation !== lifecycleGeneration;
+            const checks = new WatchdogChecks({
+                containers: managedContainers,
+                name: QUESTDB_CONTAINER_NAME,
+                client: () => queryClient,
+                busy: () => legacyCleanupRunning
+                    ? "the string table is being rebuilt"
+                    : questdbExportsRunning > 0
+                        ? "an export is running"
+                        : null,
+                epoch: () => containerEpoch,
+                bumpEpoch: () => {
+                    containerEpoch++;
+                },
+                ended,
+                endpoint: () => {
+                    const http = questdbEndpoints?.http;
+                    return http ? `${http.host}:${http.port}` : null;
+                },
+                stops: watchdogStops,
+                reconnectWriter: () => {
+                    // A connect that fails leaves the writer's own reconnect with
+                    // backoff to it.
+                    if (writer && !ended())
+                        writer.connect().catch(() => { });
+                },
+            });
+            watchdogChecks = checks;
+            watchdog = new QuestdbWatchdog({
+                exclusive: (fn) => withLifecycleLock(fn),
+                probe: () => checks.probe(),
+                recentErrors: async () => questdbErrorLines((await fetchEngineLogLines()) ?? []),
+                restart: () => checks.restart(),
+                log: (msg) => app.error(msg),
+                status: (msg) => {
+                    publishError(msg);
+                    watchdogStatusShown = true;
+                },
+                recovered: () => {
+                    // Another error published meanwhile is still current.
+                    if (pluginErrorActive && !watchdogStatusShown)
+                        return;
+                    pluginErrorActive = false;
+                    watchdogStatusShown = false;
+                    const ilp = questdbEndpoints?.ilp;
+                    if (ilp)
+                        app.setPluginStatus(recordingStatus(ilp.host, ilp.port));
+                },
+                notify: (state, message) => emitWatchdogNotification(state, message),
+                debug: (msg) => app.debug(msg),
+            });
+            watchdog.start();
+        }
         // vm.max_map_count is kernel-global (not namespaced), so in managed mode
         // this local probe reflects the limit the QuestDB container actually runs
         // under. QuestDB memory-maps every partition column file and WAL segment;
@@ -1184,6 +1564,10 @@ export default (app) => {
         // Any earlier return/throw leaves the flag false, so a half-started
         // plugin rejects lifecycle-dependent requests like /api/update/apply.
         pluginRunning = true;
+        // Not awaited: it waits for a free link, and its recreate for the
+        // lifecycle lock this start still holds.
+        if (managedContainers)
+            checkImageAfterPluginUpdate(managedContainers);
         // Repopulate the model from history — only now, because the guard below
         // reads pluginRunning and everything before this point can still throw
         // and abandon the start.
@@ -1295,6 +1679,7 @@ export default (app) => {
             // resurrecting a writer/client after this teardown ran.
             lifecycleGeneration++;
             startAbort?.abort();
+            imageCheckAbort?.abort();
             pluginRunning = false;
             // A start aborted before its restore ran would otherwise leave tracking
             // armed, accumulating contexts for a restore that never happens.
@@ -1332,6 +1717,7 @@ export default (app) => {
                 walMonitor = null;
             }
             clearWalAlertOnTeardown();
+            stopWatchdog();
             schemaMismatch = false;
             // Clear the clamp warning so it doesn't survive into the next start
             // (e.g. a switch to external/unmanaged mode that never calls ensureRunning).
@@ -1616,13 +2002,7 @@ export default (app) => {
                         res.status(503).json({ error: "QuestDB not connected" });
                         return;
                     }
-                    // Get running QuestDB version via SQL
-                    const buildResult = await queryClient.exec("SELECT build()");
-                    const buildStr = buildResult.dataset.length > 0
-                        ? buildResult.dataset[0][0]
-                        : "";
-                    const versionMatch = buildStr.match(/QuestDB\s+([\d.]+)/);
-                    const currentVersion = versionMatch ? versionMatch[1] : "unknown";
+                    const currentVersion = (await runningQuestdbVersion(queryClient)) ?? "unknown";
                     // Get latest stable release from GitHub
                     const ghRes = await fetch("https://api.github.com/repos/questdb/questdb/releases?per_page=5", {
                         headers: { Accept: "application/vnd.github+json" },
@@ -1673,6 +2053,13 @@ export default (app) => {
                         });
                         return;
                     }
+                    // A start holds the lifecycle lock until QuestDB answers, which it
+                    // waits for as long as the plugin runs: queued behind it, this
+                    // request would hang until then.
+                    if (startAbort) {
+                        res.status(409).json({ error: STILL_STARTING });
+                        return;
+                    }
                     // Captured at route entry, before ANY await: a stop() that lands
                     // during the release fetch below must already invalidate this
                     // update, not just one that lands after the lock is acquired.
@@ -1698,6 +2085,11 @@ export default (app) => {
                         return;
                     }
                     const newTag = stable.tag_name;
+                    // Asked again: a restart during the release fetch above starts anew.
+                    if (startAbort) {
+                        res.status(409).json({ error: STILL_STARTING });
+                        return;
+                    }
                     // Run the mutating update under the lifecycle lock so a concurrent
                     // purge/start can't interleave with the pull + recreate + reconnect.
                     // The route-entry generation is re-checked at lock entry and after
@@ -1725,93 +2117,7 @@ export default (app) => {
                         // recreate without throwing, and persisting newTag now would make
                         // the plugin claim a version the still-running old container is not
                         // actually on, surviving restarts.
-                        const updateMount = await resolveQuestdbDataMount(containers);
-                        app.setPluginStatus(`Starting QuestDB ${newTag}...`);
-                        const updateConfig = {
-                            image: "questdb/questdb",
-                            tag: newTag,
-                            volumes: updateMount.volumes,
-                            env: {
-                                ...buildContainerEnv(currentConfig ?? {}),
-                                ...updateMount.env,
-                            },
-                            restart: "unless-stopped",
-                            resources: currentConfig
-                                ? buildResourceLimits(currentConfig)
-                                : undefined,
-                            ulimits: QUESTDB_ULIMITS,
-                            healthcheck: QUESTDB_HEALTHCHECK,
-                        };
-                        const resolveUpdateEndpoints = await applyQuestdbNetworking(currentConfig ?? {}, containers, QUESTDB_CONTAINER_NAME, updateConfig);
-                        // Clear any prior clamp before re-running so a no-longer-clamping
-                        // update doesn't leave a stale warning. The recreate must not
-                        // happen after a stop/purge teardown, so gate it on the
-                        // generation one more time.
-                        assertNotStopped();
-                        ulimitClamp = null;
-                        containerWedged = null;
-                        containerEpoch++;
-                        await containers.ensureRunning(QUESTDB_CONTAINER_NAME, updateConfig, {
-                            onUlimitClamped,
-                            onContainerWedged,
-                        });
-                        assertNotStopped();
-                        // The QuestDB version just changed, so the cached wal_tables()
-                        // column shape may no longer match (the errorTag/errorMessage
-                        // columns could appear on an upgrade or disappear on a downgrade).
-                        // Force the next /api/status to re-probe instead of trusting a
-                        // stale capability flag.
-                        walTablesHasErrorColumns = null;
-                        // Re-resolve so the export endpoints and status line reflect the
-                        // current endpoint. The version bump keeps the same container name
-                        // and networking, so the endpoint is stable — the existing
-                        // QueryClient/ILPWriter stay valid (and the registered history
-                        // providers keep their reference). On the LAN path this await
-                        // probes for the reachable host, so it's long enough for a stop()
-                        // to land mid-flight — re-check before touching plugin state.
-                        const updatedEndpoints = await resolveUpdateEndpoints();
-                        assertNotStopped();
-                        questdbEndpoints = updatedEndpoints;
-                        const { host: ilpHost, port: ilpPort } = questdbEndpoints.ilp;
-                        // Wait for the recreated container to answer, then reconnect ILP.
-                        // Locals are captured because stop() nulls the module bindings:
-                        // reading them again mid-loop would throw, and the generation
-                        // checks after every await are what keep a concurrent teardown
-                        // from being resurrected by the reconnect.
-                        const client = queryClient;
-                        if (client) {
-                            const deadline = Date.now() + 30000;
-                            while (Date.now() < deadline) {
-                                assertNotStopped();
-                                if (await client.isHealthy())
-                                    break;
-                                await new Promise((resolve) => setTimeout(resolve, 500));
-                            }
-                        }
-                        assertNotStopped();
-                        const ilpWriter = writer;
-                        if (ilpWriter) {
-                            try {
-                                await ilpWriter.disconnect();
-                            }
-                            catch {
-                                /* ignore */
-                            }
-                            assertNotStopped();
-                            await ilpWriter.connect();
-                            // A stop() that landed during connect() has already nulled
-                            // `writer`, so nothing would ever disconnect this fresh
-                            // connection — close it ourselves before failing the update.
-                            if (updateGeneration !== lifecycleGeneration) {
-                                try {
-                                    await ilpWriter.disconnect();
-                                }
-                                catch {
-                                    /* ignore */
-                                }
-                                assertNotStopped();
-                            }
-                        }
+                        const { ilpHost, ilpPort } = await recreateQuestdb(newTag, updateGeneration, containers, { background: false, refresh: false });
                         // Persist the new version only now that the recreate is confirmed
                         // (a wedge deferred it — the old container is still running).
                         if (!containerWedged && currentConfig) {
@@ -2187,6 +2493,7 @@ export default (app) => {
                             walMonitor = null;
                         }
                         clearWalAlertOnTeardown();
+                        stopWatchdog();
                         if (retention) {
                             retention.stop();
                             retention = null;
@@ -2241,6 +2548,11 @@ export default (app) => {
                     // later wakes up returns early instead of recreating the container.
                     lifecycleGeneration++;
                     startAbort?.abort();
+                    // A check that outlived the purge would download the image again.
+                    imageCheckAbort?.abort();
+                    // Silences a restart in flight now; the teardown below clears the
+                    // alert once it has the lock.
+                    watchdog?.stop();
                     let lockedTeardownSettled = false;
                     let lockedTeardownError;
                     const lockedTeardown = withLifecycleLock(teardown).then(() => {
@@ -3165,8 +3477,18 @@ export default (app) => {
                                 expUrl.searchParams.set("compression_level", String(currentConfig?.compressionLevel ?? 3));
                             }
                         }
+                        // Also bounds the streaming below: a client that stops reading
+                        // must not hold the export past it (see drained()). A client that
+                        // leaves ends it at once, as in /api/full-export: during a slow
+                        // query QuestDB may send nothing for minutes.
+                        const clientLeft = new AbortController();
+                        res.once("close", () => clientLeft.abort());
+                        const exportSignal = AbortSignal.any([
+                            AbortSignal.timeout(300000),
+                            clientLeft.signal,
+                        ]);
                         const qdbRes = await fetch(expUrl.toString(), {
-                            signal: AbortSignal.timeout(300000),
+                            signal: exportSignal,
                         });
                         if (!qdbRes.ok || !qdbRes.body) {
                             const body = await qdbRes.text().catch(() => "");
@@ -3177,20 +3499,41 @@ export default (app) => {
                         res.setHeader("Content-Disposition", `attachment; filename="signalk-export-${dateSlug}.parquet"`);
                         // Stream the response through
                         const reader = qdbRes.body.getReader();
+                        // Same as /api/full-export: QuestDB is busy until the stream ends.
+                        questdbExportsRunning++;
+                        let complete = false;
                         try {
                             while (true) {
                                 const { done, value } = await reader.read();
-                                if (done)
+                                if (done) {
+                                    complete = true;
                                     break;
-                                if (!res.write(value)) {
-                                    await new Promise((resolve) => res.once("drain", resolve));
                                 }
+                                if (!res.write(value) && !(await drained(res, exportSignal)))
+                                    break;
                             }
-                            res.end();
                         }
                         catch (streamErr) {
                             app.debug("export stream error:", streamErr);
-                            res.end();
+                        }
+                        finally {
+                            questdbExportsRunning--;
+                            // A stream cut short must not end like a complete one: the
+                            // response is chunked, so end() would hand the client a
+                            // truncated parquet file under a clean 200.
+                            if (complete)
+                                res.end();
+                            else
+                                res.destroy();
+                            // A client that left mid-stream ends the loop early: close
+                            // QuestDB's side too rather than leave it streaming to nobody.
+                            // cancel() throws if the stream already ended; that is fine.
+                            try {
+                                await reader.cancel();
+                            }
+                            catch {
+                                // already finished
+                            }
                         }
                     }
                     else {
@@ -3306,25 +3649,31 @@ export default (app) => {
                     }
                     // Streaming phase: headers are committed at the first write,
                     // so we can no longer switch to a JSON error. Best we can do
-                    // on stream failure is end the response and let the client
-                    // notice the truncation.
+                    // on stream failure is break the connection, so the client sees
+                    // the truncation.
                     res.setHeader("Content-Type", "application/vnd.apache.parquet");
                     res.setHeader("Content-Disposition", `attachment; filename="${table}.parquet"`);
                     const reader = qdbRes.body.getReader();
+                    // QuestDB answered and streams: its one worker is busy until the
+                    // stream ends (see questdbExportsRunning).
+                    questdbExportsRunning++;
+                    let complete = false;
                     try {
                         while (true) {
                             const { done, value } = await reader.read();
-                            if (done)
+                            if (done) {
+                                complete = true;
                                 break;
-                            if (!res.write(value)) {
-                                await new Promise((resolve) => res.once("drain", resolve));
                             }
+                            if (!res.write(value) && !(await drained(res, controller.signal)))
+                                break;
                         }
                     }
                     catch (streamErr) {
                         app.debug("full-export stream error:", streamErr);
                     }
                     finally {
+                        questdbExportsRunning--;
                         clearTimeout(timeoutId);
                         res.removeListener("close", onClientClose);
                         // Cancel the upstream reader so QuestDB's connection is
@@ -3337,7 +3686,11 @@ export default (app) => {
                             // already finished
                         }
                         if (!res.writableEnded) {
-                            res.end();
+                            // Same as /api/export: a truncated file must not end cleanly.
+                            if (complete)
+                                res.end();
+                            else
+                                res.destroy();
                         }
                     }
                 }