signalk-questdb 2.0.1 → 2.1.0

package/dist/index.js

@@ -9,6 +9,8 @@ import { startRetention } from "./retention.js";
 import { RESTORE_SOURCE, restoreFromHistory } from "./restore.js";
 import { createConsoleProxy } from "./console-proxy.js";
 import { buildFullExportWhere } from "./full-export-range.js";
+import { detectInflux, validateInfluxUrl } from "./influx-detect.js";
+import { listBuckets, listMeasurements, runMigration, MigrationRun, } from "./migration.js";
 import { WalMonitor, buildPendingSegmentsSQL, computeSkipPlan, extractApplyError, isPartitionOpenFailure, skipPlansEqual, } from "./wal-monitor.js";
 import { QUESTDB_INTERNAL_HTTP_PORT, QUESTDB_INTERNAL_ILP_PORT, QUESTDB_INTERNAL_PG_PORT, QUESTDB_ACCESSIBLE_PORTS, resolveManagedEndpoints, resolveLanExposureHost, lanExposureEndpoints, } from "./questdb-endpoint.js";
 import { nofileClampSatisfied, readMaxMapCount } from "./host-limits.js";
@@ -94,6 +96,13 @@ function buildResourceLimits(config) {
 }
 export default (app) => {
     let writer = null;
+    // The most recent (or currently running) InfluxDB import. Deliberately
+    // in-memory and single-slot: an import is an operator-driven one-off, and
+    // surviving a plugin restart would mean persisting a half-finished run with
+    // no writer to resume it onto. A restart therefore forgets it, and re-running
+    // is safe because imported rows upsert on (ts, path, context, source).
+    let activeMigration = null;
+    let migrationCounter = 0;
     let queryClient = null;
     let retentionTimer = null;
     let schemaHealTimer = null;
@@ -107,6 +116,12 @@ export default (app) => {
     // capped QuestDB's requested value. Null until a clamp happens (or on a
     // signalk-container older than 1.18.0, which doesn't emit the event).
     let ulimitClamp = null;
+    // Set by onContainerWedged: the runtime couldn't stop/remove the container to
+    // apply a change, so signalk-container deferred the recreate (it does NOT
+    // throw). Sticky within a start/update cycle — the container is running the
+    // OLD config/version, so we must not report success or persist a new version
+    // as if the change applied. Cleared at the top of each ensureRunning attempt.
+    let containerWedged = null;
     // Whether this QuestDB's wal_tables() exposes the errorTag/errorMessage
     // columns (present on current builds, absent on an older pinned version).
     // Probed once via the richer status query and cached so an old build does
@@ -211,6 +226,18 @@ export default (app) => {
         ulimitClamp = event;
         app.debug(event.reason);
     };
+    // A wedged container (the runtime can't stop/remove it — usually an orphaned
+    // rootless-Podman user namespace after a Podman service restart) cannot be
+    // recovered from the plugin; signalk-container defers the recreate and fires
+    // this once. Surface the remedy as a plugin error rather than letting the
+    // operator see only a silent drift loop in the server log. Requires
+    // signalk-container >= 1.29.0; on older versions the callback never fires.
+    const onContainerWedged = (event) => {
+        containerWedged = event.reason;
+        // publishError → setPluginError already lands in the server log at error
+        // level and drives the admin-UI status; matches every other error site.
+        publishError(event.reason);
+    };
     // Tables the plugin owns; each is rebuilt if ILP auto-created it with the
     // wrong designated-timestamp schema. (signalk_position uses `ts` too.)
     const OWNED_TABLES = ["signalk", "signalk_str", "signalk_position"];
@@ -714,8 +741,10 @@ export default (app) => {
                 // limit was raised) doesn't leave a stale warning; onUlimitClamped
                 // re-sets it if this run clamps again.
                 ulimitClamp = null;
+                containerWedged = null;
                 await containers.ensureRunning(QUESTDB_CONTAINER_NAME, containerConfig, {
                     onUlimitClamped,
+                    onContainerWedged,
                 });
                 // ensureRunning() just (re)created the container. If purge preempted us
                 // while it ran, return now so we don't proceed to resolve endpoints and
@@ -795,7 +824,7 @@ export default (app) => {
             queryClient = null;
             return;
         }
-        const v2Provider = createHistoryProviderV2(queryClient, app.selfContext);
+        const v2Provider = createHistoryProviderV2(queryClient, app.selfContext, config.historySourcePolicyAll ?? false, (msg) => app.debug(msg));
         app.registerHistoryApiProvider(v2Provider);
         const v1Provider = createHistoryProviderV1(queryClient, app.selfContext, (msg) => app.debug(msg));
         app.registerHistoryProvider(v1Provider);
@@ -1034,7 +1063,14 @@ export default (app) => {
                     `/etc/sysctl.d/99-questdb.conf && sudo sysctl --system`);
             }
         }
-        app.setPluginStatus(recordingStatus(ilpHost, ilpPort));
+        // If the container wedged, ensureRunning deferred rather than threw, and the
+        // "Waiting…/Creating tables…" progress lines above have since overwritten
+        // the error onContainerWedged set. Re-assert it as the final status so the
+        // operator sees the remedy, not a false "Recording". publishRecordingStatus
+        // is guarded on pluginErrorActive, so it is a no-op while the error stands.
+        if (containerWedged)
+            publishError(containerWedged);
+        publishRecordingStatus(ilpHost, ilpPort);
         // Everything is registered — only now does the plugin count as running.
         // Any earlier return/throw leaves the flag false, so a half-started
         // plugin rejects lifecycle-dependent requests like /api/update/apply.
@@ -1189,6 +1225,14 @@ export default (app) => {
             // Clear the clamp warning so it doesn't survive into the next start
             // (e.g. a switch to external/unmanaged mode that never calls ensureRunning).
             ulimitClamp = null;
+            // Stop any import before the writer goes away: runMigration holds a
+            // direct reference to it, so a run left going would keep enqueueing
+            // lines onto a disconnected writer for as long as its read loop lasts.
+            // The flag is checked between windows and between rows, so this takes
+            // effect within one window rather than at the end of the whole import.
+            if (activeMigration && activeMigration.state === "running") {
+                activeMigration.cancel();
+            }
             if (writer) {
                 await writer.disconnect();
                 writer = null;
@@ -1552,17 +1596,11 @@ export default (app) => {
                         app.setPluginStatus(`Pulling QuestDB ${newTag}...`);
                         await containers.pullImage(`questdb/questdb:${newTag}`);
                         assertNotStopped();
-                        if (currentConfig) {
-                            currentConfig.questdbVersion = newTag;
-                            await new Promise((resolve, reject) => {
-                                app.savePluginOptions({ ...currentConfig }, (err) => {
-                                    if (err)
-                                        reject(err);
-                                    else
-                                        resolve();
-                                });
-                            });
-                        }
+                        // NOTE: questdbVersion is persisted only AFTER a confirmed
+                        // recreate (below), not here — a wedged container defers the
+                        // recreate without throwing, and persisting newTag now would make
+                        // the plugin claim a version the still-running old container is not
+                        // actually on, surviving restarts.
                         const updateVolumeSource = await resolveQuestdbVolumeSource(containers);
                         app.setPluginStatus(`Starting QuestDB ${newTag}...`);
                         const updateConfig = {
@@ -1586,9 +1624,11 @@ export default (app) => {
                         // generation one more time.
                         assertNotStopped();
                         ulimitClamp = null;
+                        containerWedged = null;
                         containerEpoch++;
                         await containers.ensureRunning(QUESTDB_CONTAINER_NAME, updateConfig, {
                             onUlimitClamped,
+                            onContainerWedged,
                         });
                         assertNotStopped();
                         // The QuestDB version just changed, so the cached wal_tables()
@@ -1647,8 +1687,30 @@ export default (app) => {
                                 assertNotStopped();
                             }
                         }
+                        // Persist the new version only now that the recreate is confirmed
+                        // (a wedge deferred it — the old container is still running).
+                        if (!containerWedged && currentConfig) {
+                            currentConfig.questdbVersion = newTag;
+                            await new Promise((resolve, reject) => {
+                                app.savePluginOptions({ ...currentConfig }, (err) => {
+                                    if (err)
+                                        reject(err);
+                                    else
+                                        resolve();
+                                });
+                            });
+                        }
                         return { ilpHost, ilpPort };
                     });
+                    if (containerWedged) {
+                        // The recreate was deferred: the container is still on the OLD
+                        // version. Do not claim success — report the wedge and its remedy.
+                        res.status(409).json({
+                            error: "container wedged",
+                            message: containerWedged,
+                        });
+                        return;
+                    }
                     app.setPluginStatus(`Recording to QuestDB ${newTag} at ${ilp.ilpHost}:${ilp.ilpPort}`);
                     res.json({
                         status: "updated",
@@ -2067,64 +2129,270 @@ export default (app) => {
                     });
                 }
             });
+            // Strip the class down to the wire shape. `MigrationRun` carries a
+            // `cancel()` method and a private flag that must not be serialised.
+            const migrationRunView = (run) => ({
+                id: run.id,
+                state: run.state,
+                url: run.url,
+                bucket: run.bucket,
+                startedAt: run.startedAt,
+                finishedAt: run.finishedAt,
+                progress: { ...run.progress },
+                error: run.error,
+            });
+            // Auth travels in the request BODY rather than plugin config: the token
+            // is only needed for the duration of an import and storing it in the
+            // plugin's settings file would persist a credential the user never
+            // asked us to keep. Never the query string — Signal K logs full request
+            // URLs (morgan "combined"), so a token there reaches the access log.
+            const readAuth = (src) => {
+                const token = typeof src.token === "string" ? src.token : undefined;
+                const username = typeof src.username === "string" ? src.username : undefined;
+                const password = typeof src.password === "string" ? src.password : undefined;
+                const org = typeof src.org === "string" ? src.org : undefined;
+                if (!token && !username && !org)
+                    return undefined;
+                return { token, username, password, org };
+            };
             router.get("/api/migration/detect", async (req, res) => {
-                const baseUrl = req.query.url || "http://localhost:8086";
+                // `?url=a&url=b` arrives as an ARRAY, not a string — a cast here
+                // would hand a non-string to validateInfluxUrl and throw inside the
+                // handler rather than answering 400.
+                const requestedUrl = req.query.url;
+                if (requestedUrl !== undefined && typeof requestedUrl !== "string") {
+                    res.status(400).json({
+                        error: "url must be a single value",
+                        sources: [],
+                    });
+                    return;
+                }
+                const baseUrl = validateInfluxUrl(requestedUrl || "http://localhost:8086");
+                if (!baseUrl) {
+                    res.status(400).json({
+                        error: "Only localhost and private network http(s) URLs allowed",
+                        sources: [],
+                    });
+                    return;
+                }
                 try {
-                    const urlObj = new URL(baseUrl);
-                    const host = urlObj.hostname;
-                    const isLocal = host === "localhost" || host === "127.0.0.1" || host === "::1";
-                    const isPrivate = /^(10\.|172\.(1[6-9]|2\d|3[01])\.|192\.168\.)/.test(host);
-                    if (!isLocal && !isPrivate) {
-                        res.status(400).json({
-                            error: "Only localhost and private network URLs allowed",
-                        });
-                        return;
-                    }
+                    const sources = await detectInflux(baseUrl);
+                    res.json({ sources });
                 }
-                catch {
-                    res.status(400).json({ error: "Invalid URL" });
+                catch (err) {
+                    res.status(500).json({
+                        error: err instanceof Error ? err.message : "Unknown error",
+                        sources: [],
+                    });
+                }
+            });
+            // POST, not GET, purely because of the credentials. Signal K mounts
+            // morgan("combined"), which writes the full request URL — query string
+            // included — to the access log, and browsers keep it in history. An
+            // InfluxDB API token in a query parameter therefore ends up in both.
+            // Discovery is a read, but the token has to travel in the body.
+            router.post("/api/migration/buckets", async (req, res) => {
+                const body = (req.body ?? {});
+                const baseUrl = validateInfluxUrl(typeof body.url === "string" ? body.url : "http://localhost:8086");
+                if (!baseUrl) {
+                    res.status(400).json({
+                        error: "Invalid URL",
+                        buckets: [],
+                    });
                     return;
                 }
-                const sources = [];
-                // Detect InfluxDB 1.x
                 try {
-                    const r = await fetch(`${baseUrl}/ping`, {
-                        method: "HEAD",
-                        signal: AbortSignal.timeout(3000),
+                    const buckets = await listBuckets({
+                        url: baseUrl,
+                        type: typeof body.type === "string" ? body.type : "influxdb2",
+                        auth: readAuth(body),
                     });
-                    if (r.status === 204) {
-                        sources.push({
-                            type: "influxdb1",
-                            url: baseUrl,
-                            status: "found",
-                            version: r.headers.get("X-Influxdb-Version") || "unknown",
-                        });
-                    }
+                    res.json({ buckets });
                 }
-                catch {
-                    // not running
+                catch (err) {
+                    res.status(502).json({
+                        error: err instanceof Error ? err.message : "Unknown error",
+                        buckets: [],
+                    });
+                }
+            });
+            // POST for the same reason as /buckets: the token must not reach the
+            // access log or the browser's history via the query string.
+            router.post("/api/migration/measurements", async (req, res) => {
+                const body = (req.body ?? {});
+                const baseUrl = validateInfluxUrl(typeof body.url === "string" ? body.url : "http://localhost:8086");
+                const bucket = typeof body.bucket === "string" ? body.bucket : "";
+                if (!baseUrl || !bucket) {
+                    res.status(400).json({
+                        error: "Missing or invalid url/bucket",
+                        measurements: [],
+                    });
+                    return;
                 }
-                // Detect InfluxDB 2.x
                 try {
-                    const r = await fetch(`${baseUrl}/health`, {
-                        signal: AbortSignal.timeout(3000),
+                    const measurements = await listMeasurements({
+                        url: baseUrl,
+                        type: typeof body.type === "string" ? body.type : "influxdb2",
+                        bucket,
+                        auth: readAuth(body),
                     });
-                    if (r.ok) {
-                        const data = (await r.json());
-                        if (data.status === "pass") {
-                            sources.push({
-                                type: "influxdb2",
-                                url: baseUrl,
-                                status: "found",
-                                version: data.version || "unknown",
-                            });
-                        }
-                    }
+                    res.json({ measurements });
                 }
-                catch {
-                    // not running
+                catch (err) {
+                    res.status(502).json({
+                        error: err instanceof Error ? err.message : "Unknown error",
+                        measurements: [],
+                    });
+                }
+            });
+            router.post("/api/migration/start", async (req, res) => {
+                // One import at a time. Two concurrent runs would interleave on the
+                // single ILP writer and race each other for its buffer, and there is
+                // no sane way to show two progress bars against one writer.
+                if (activeMigration && activeMigration.state === "running") {
+                    res.status(409).json({
+                        error: "A migration is already running",
+                        run: migrationRunView(activeMigration),
+                    });
+                    return;
+                }
+                if (!writer) {
+                    res.status(503).json({
+                        error: "QuestDB writer not connected",
+                    });
+                    return;
+                }
+                const body = (req.body ?? {});
+                const baseUrl = validateInfluxUrl(typeof body.url === "string" ? body.url : "http://localhost:8086");
+                const bucket = typeof body.bucket === "string" ? body.bucket : "";
+                if (!baseUrl || !bucket) {
+                    res.status(400).json({
+                        error: "Missing or invalid url/bucket",
+                    });
+                    return;
                 }
-                res.json({ sources });
+                const from = typeof body.from === "string" ? body.from : "";
+                const to = typeof body.to === "string" ? body.to : "";
+                if (!from ||
+                    !to ||
+                    Number.isNaN(Date.parse(from)) ||
+                    Number.isNaN(Date.parse(to))) {
+                    res.status(400).json({
+                        error: "Missing or invalid from/to",
+                    });
+                    return;
+                }
+                // An inverted range is rejected here rather than left to fail inside
+                // the run: the caller gets a 400 it can act on, instead of a 200
+                // followed by a run that reports "failed" a moment later.
+                if (Date.parse(to) <= Date.parse(from)) {
+                    res.status(400).json({
+                        error: "`to` must be after `from`",
+                    });
+                    return;
+                }
+                // "self" and "vessels.self" both mean this vessel; anything else must
+                // be a fully-qualified Signal K context (vessels.urn:mrn:..., or
+                // atons./aircraft./sar. for other targets).
+                //
+                // The own vessel is stored as the LITERAL string "self", not as
+                // `vessels.<uuid>` — that is what the live recorder writes (see the
+                // `isSelf ? "self" : context` in the delta handler) and what both
+                // history providers normalise incoming contexts to. Importing under
+                // the expanded uuid would file the history in a context no query ever
+                // looks at: the rows would be there and the API would return nothing.
+                const rawContext = typeof body.context === "string" ? body.context.trim() : "";
+                let importContext;
+                if (!rawContext ||
+                    rawContext === "self" ||
+                    rawContext === "vessels.self" ||
+                    rawContext === `vessels.${app.selfId}`) {
+                    importContext = "self";
+                }
+                else if (/^(vessels|atons|aircraft|sar)\.[A-Za-z0-9:._-]+$/.test(rawContext)) {
+                    importContext = rawContext;
+                }
+                else {
+                    res.status(400).json({
+                        error: "Invalid context: expected `self` or a Signal K context such as `vessels.urn:mrn:imo:mmsi:123456789`",
+                    });
+                    return;
+                }
+                // Kept deliberately narrow: a label is a short identifier, not free
+                // text. Anything else falls back to the default.
+                const rawLabel = typeof body.sourceLabel === "string" ? body.sourceLabel.trim() : "";
+                const migrationSourceLabel = rawLabel && /^[A-Za-z0-9._:-]{1,64}$/.test(rawLabel)
+                    ? rawLabel
+                    : "influxdb-import";
+                const request = {
+                    url: baseUrl,
+                    type: typeof body.type === "string" ? body.type : "influxdb2",
+                    bucket,
+                    auth: readAuth(body),
+                    from,
+                    to,
+                    // Default to the server's own vessel: that is what a boat migrating
+                    // its own history means by "my data". A supplied context is checked
+                    // against the Signal K shape rather than passed through — it becomes
+                    // a SYMBOL value on every imported row and is what the history API
+                    // queries by, so a malformed one writes history nothing can read.
+                    context: importContext,
+                    measurements: Array.isArray(body.measurements)
+                        ? body.measurements.filter((m) => typeof m === "string")
+                        : undefined,
+                    // The label is written as the `source` tag on every imported row,
+                    // which is both a dedup key component and how an operator tells
+                    // imported history from live recording. A blank or whitespace-only
+                    // value would leave the tag off entirely (making imported rows
+                    // indistinguishable from recorded ones, and changing the dedup
+                    // key so a re-run duplicates instead of upserting), so it falls
+                    // back to the default rather than being trusted as given.
+                    sourceLabel: migrationSourceLabel,
+                };
+                // The id only has to be unique within this process, where the counter
+                // already guarantees it; the timestamp just makes logs readable.
+                const run = new MigrationRun(`mig-${Date.now()}-${++migrationCounter}`, baseUrl, bucket);
+                activeMigration = run;
+                const startedWriter = writer;
+                // Deliberately not awaited: an import runs for minutes to hours, so
+                // the HTTP call returns immediately and the panel polls /status.
+                void runMigration(request, startedWriter, run, {
+                    debug: (msg) => app.debug(msg),
+                })
+                    .catch((err) => {
+                    // runMigration traps its own failures, but a few statements run
+                    // BEFORE its try block (auth header construction). A rejection
+                    // from there would be an unhandled promise rejection on a
+                    // deliberately un-awaited call — fatal to the Signal K process
+                    // under Node's default policy — and would leave the run stuck
+                    // reading "running" forever.
+                    run.state = "failed";
+                    run.error = err instanceof Error ? err.message : String(err);
+                    run.finishedAt = new Date().toISOString();
+                })
+                    .finally(() => {
+                    app.debug(`migration ${run.id} ${run.state}: ${run.progress.written} written, ${run.progress.skipped} skipped`);
+                });
+                res.json({
+                    run: migrationRunView(run),
+                });
+            });
+            router.get("/api/migration/status", (_req, res) => {
+                res.json({
+                    run: activeMigration ? migrationRunView(activeMigration) : undefined,
+                });
+            });
+            router.post("/api/migration/cancel", (_req, res) => {
+                if (!activeMigration || activeMigration.state !== "running") {
+                    res.status(409).json({
+                        error: "No migration running",
+                    });
+                    return;
+                }
+                activeMigration.cancel();
+                res.json({
+                    run: migrationRunView(activeMigration),
+                });
             });
             router.get("/api/export", async (req, res) => {
                 try {