signalk-questdb 2.1.4 → 2.1.5

package/dist/index.js

@@ -10,7 +10,10 @@ import { RESTORE_SOURCE, restoreFromHistory } from "./restore.js";
 import { createConsoleProxy } from "./console-proxy.js";
 import { buildFullExportWhere } from "./full-export-range.js";
 import { detectInflux, validateInfluxUrl } from "./influx-detect.js";
-import { listBuckets, listMeasurements, runMigration, MigrationRun, } from "./migration.js";
+import { DEFAULT_WINDOW_MS, isSignalKContext, listBuckets, listContexts, listMeasurements, runMigration, MigrationRun, } from "./migration.js";
+import { FileCheckpointStore, migrationIdentity, sameIdentity, } from "./migration-checkpoint.js";
+import { countLegacyImportRows, removeLegacyImportRows, } from "./migration-cleanup.js";
+import { adoptDatabaseFromVolumeRoot, wipeDataInSignalk, resolveQuestdbMount, } from "./questdb-mount.js";
 import { WalMonitor, buildPendingSegmentsSQL, computeSkipPlan, extractApplyError, isPartitionOpenFailure, skipPlansEqual, } from "./wal-monitor.js";
 import { QUESTDB_INTERNAL_HTTP_PORT, QUESTDB_INTERNAL_ILP_PORT, QUESTDB_INTERNAL_PG_PORT, QUESTDB_ACCESSIBLE_PORTS, resolveManagedEndpoints, resolveLanExposureHost, lanExposureEndpoints, } from "./questdb-endpoint.js";
 import { nofileClampSatisfied, readMaxMapCount } from "./host-limits.js";
@@ -102,6 +105,23 @@ export default (app) => {
     // no writer to resume it onto. A restart therefore forgets it, and re-running
     // is safe because imported rows upsert on (ts, path, context, source).
     let activeMigration = null;
+    // While the string table is being rebuilt — or a rebuild that failed after
+    // its swap is being finished — nothing may import into it, and the schema
+    // heal must keep its hands off it.
+    let legacyCleanupRunning = false;
+    // The rebuild in flight, so stop() can wait for it: disconnecting the
+    // writer while it is held would drop the held lines, and a swap left half
+    // done is work for the next run rather than a loss, but still work.
+    let legacyCleanup = null;
+    // A heal in progress drops and recreates a mismatched table; the rebuild
+    // waits it out rather than rename underneath it.
+    const awaitHealIdle = async () => {
+        for (let waited = 0; healing && waited < 30_000; waited += 100) {
+            await new Promise((r) => setTimeout(r, 100));
+        }
+        if (healing)
+            throw new Error("A schema repair is in progress; try again");
+    };
     let migrationCounter = 0;
     let queryClient = null;
     let retentionTimer = null;
@@ -217,6 +237,10 @@ export default (app) => {
     // anyway. After this it forces its teardown through even if a hung start
     // never released the lock (see PURGE_LOCK_TIMEOUT_MS use below).
     const PURGE_LOCK_TIMEOUT_MS = 30000;
+    // A rebuild of the string table waits this long for the lifecycle lock. A
+    // wedged start pins the lock (see PURGE_LOCK_TIMEOUT_MS); unlike a purge
+    // the rebuild then gives up rather than proceed, since it is optional.
+    const LEGACY_CLEANUP_LOCK_TIMEOUT_MS = 30000;
     // Record a clamp event so /api/status and the config-panel banner can
     // surface it. (The plugin status line is not used — it is driven by the
     // recording state and would immediately overwrite a warning set here.)
@@ -250,7 +274,10 @@ export default (app) => {
     // is rebuilt with the correct `ts` schema. Best-effort: introspection/heal
     // errors are logged, not thrown, so they never break the lifecycle.
     const healSchemaTables = async () => {
-        if (!queryClient || healing)
+        // Not while the string table is being rebuilt: the rebuild renames it
+        // away for a moment, and a heal finding it absent would create a fresh
+        // one for the rebuilt table to then collide with.
+        if (!queryClient || healing || legacyCleanupRunning)
             return;
         healing = true;
         try {
@@ -319,34 +346,9 @@ export default (app) => {
         debug: (msg) => app.debug(msg),
         mountPath: "/plugins/signalk-questdb/console",
     });
-    /**
-     * Compute the bind-mount source for QuestDB's /var/lib/questdb volume.
-     *
-     * `app.getDataDirPath()` returns the path from SK's own perspective. On
-     * bare-metal that is the host path and the runtime can use it directly.
-     * When SK runs inside a container the same string is the SK-container-
-     * internal path, and the host's runtime daemon — which is on the host,
-     * not inside SK — cannot resolve it. signalk-container 1.9.0+ exposes
-     * `resolveHostPath()` to translate such paths back to the host source;
-     * if it returns null (older signalk-container or no covering mount) we
-     * fall back to the original path, preserving bare-metal behaviour.
-     */
-    async function resolveQuestdbVolumeSource(containers) {
-        const dataPath = app.getDataDirPath();
-        if (typeof containers.resolveHostPath !== "function")
-            return dataPath;
-        // signalk-container's resolveHostPath is documented as non-throwing, but
-        // we consume it through a runtime cross-plugin API (cast through `any`),
-        // so an unexpected throw from a future or older version must not abort
-        // startup — fall back to the original path instead.
-        try {
-            const resolved = await containers.resolveHostPath(dataPath);
-            return resolved?.source ?? dataPath;
-        }
-        catch (err) {
-            app.debug("resolveHostPath threw, falling back to dataPath:", err);
-            return dataPath;
-        }
+    /** The mount for QuestDB's data directory — see questdb-mount.ts. */
+    async function resolveQuestdbDataMount(containers) {
+        return resolveQuestdbMount(containers, app.getDataDirPath(), (msg) => app.debug(msg));
     }
     /**
      * Apply the chosen networking to a managed-QuestDB ContainerConfig and
@@ -718,16 +720,36 @@ export default (app) => {
             app.debug("container runtime ready, starting QuestDB");
             try {
                 const containerEnv = buildContainerEnv(config);
-                const volumeSource = await resolveQuestdbVolumeSource(containers);
+                const mount = await resolveQuestdbDataMount(containers);
                 if (signal.aborted)
                     return;
+                // A volume-backed data directory: what QuestDB wrote at the volume's
+                // root while the offset went unhonoured is its database, and would be
+                // left behind. Both places are one volume, so it is renamed across.
+                if (mount.volumeRootInSignalk) {
+                    const fs = await import("fs/promises");
+                    const dataFs = {
+                        exists: (p) => fs.access(p).then(() => true, () => false),
+                        rename: (from, to) => fs.rename(from, to),
+                        mkdir: (p) => fs.mkdir(p, { recursive: true }).then(() => { }),
+                    };
+                    const moved = await adoptDatabaseFromVolumeRoot(dataFs, mount.volumeRootInSignalk, app.getDataDirPath(), 
+                    // The container from before may still be running on that root,
+                    // and QuestDB writes to paths under its root, so it is stopped
+                    // before its directories move — once the move is known to go
+                    // ahead; ensureRunning below brings it back on the new mount.
+                    () => containers.stop(QUESTDB_CONTAINER_NAME));
+                    if (signal.aborted)
+                        return;
+                    if (moved.length > 0) {
+                        app.debug(`moved QuestDB's database from the volume root into the data directory: ${moved.join(", ")}`);
+                    }
+                }
                 const containerConfig = {
                     image: "questdb/questdb",
                     tag: config.questdbVersion ?? "latest",
-                    volumes: {
-                        "/var/lib/questdb": volumeSource,
-                    },
-                    env: containerEnv,
+                    volumes: mount.volumes,
+                    env: { ...containerEnv, ...mount.env },
                     restart: "unless-stopped",
                     resources: buildResourceLimits(config),
                     ulimits: QUESTDB_ULIMITS,
@@ -1225,6 +1247,10 @@ export default (app) => {
             // Clear the clamp warning so it doesn't survive into the next start
             // (e.g. a switch to external/unmanaged mode that never calls ensureRunning).
             ulimitClamp = null;
+            // A rebuild of the string table first: its hold keeps the writer's last
+            // lines back, and disconnecting under it would lose them.
+            if (legacyCleanup)
+                await legacyCleanup.catch(() => { });
             // Stop any import before the writer goes away: runMigration holds a
             // direct reference to it, so a run left going would keep enqueueing
             // lines onto a disconnected writer for as long as its read loop lasts.
@@ -1549,6 +1575,14 @@ export default (app) => {
             });
             router.post("/api/update/apply", async (_req, res) => {
                 try {
+                    // Not under a rebuild of the string table: it holds the writer
+                    // across a swap, and an update tears the writer down.
+                    if (legacyCleanupRunning) {
+                        res.status(409).json({
+                            error: "The string table is being rebuilt; wait for it to finish first",
+                        });
+                        return;
+                    }
                     // Captured at route entry, before ANY await: a stop() that lands
                     // during the release fetch below must already invalidate this
                     // update, not just one that lands after the lock is acquired.
@@ -1601,15 +1635,16 @@ export default (app) => {
                         // recreate without throwing, and persisting newTag now would make
                         // the plugin claim a version the still-running old container is not
                         // actually on, surviving restarts.
-                        const updateVolumeSource = await resolveQuestdbVolumeSource(containers);
+                        const updateMount = await resolveQuestdbDataMount(containers);
                         app.setPluginStatus(`Starting QuestDB ${newTag}...`);
                         const updateConfig = {
                             image: "questdb/questdb",
                             tag: newTag,
-                            volumes: {
-                                "/var/lib/questdb": updateVolumeSource,
+                            volumes: updateMount.volumes,
+                            env: {
+                                ...buildContainerEnv(currentConfig ?? {}),
+                                ...updateMount.env,
                             },
-                            env: buildContainerEnv(currentConfig ?? {}),
                             restart: "unless-stopped",
                             resources: currentConfig
                                 ? buildResourceLimits(currentConfig)
@@ -1999,8 +2034,21 @@ export default (app) => {
                     });
                 }
             });
+            // One import runs at a time, so one checkpoint is all there is to keep.
+            // Beside the data directory, not in it: QuestDB's entrypoint chowns its
+            // data root to its own uid on start, after which the Signal K user can
+            // no longer create a file there.
+            const checkpoints = new FileCheckpointStore(`${app.getDataDirPath()}.influx-import-checkpoint.json`);
             router.post("/api/purge-data", async (_req, res) => {
                 try {
+                    // Not under a rebuild of the string table: it holds the writer
+                    // across a swap, and this tears the writer down.
+                    if (legacyCleanupRunning) {
+                        res.status(409).json({
+                            error: "The string table is being rebuilt; wait for it to finish first",
+                        });
+                        return;
+                    }
                     // External mode is a config fact independent of the runtime, so
                     // answer it first — an external-mode install without signalk-container
                     // should get the clear 400, not a 503 about a missing container
@@ -2016,7 +2064,10 @@ export default (app) => {
                         res.status(503).json({ error: "Container manager not available" });
                         return;
                     }
-                    if (!containers.removeManagedData) {
+                    // A volume is purged from this process; only a host path needs
+                    // signalk-container's in-userns wipe.
+                    const mount = await resolveQuestdbDataMount(containers);
+                    if (mount.wipe === "runtime" && !containers.removeManagedData) {
                         res.status(501).json({
                             error: "Data removal requires signalk-container 1.19.0 or newer. Update it, or delete the QuestDB data directory manually.",
                         });
@@ -2060,9 +2111,29 @@ export default (app) => {
                             writer = null;
                         }
                         queryClient = null;
-                        const hostPath = await resolveQuestdbVolumeSource(containers);
                         app.setPluginStatus("Removing QuestDB container and data...");
-                        await containers.removeManagedData(QUESTDB_CONTAINER_NAME, hostPath, { ownerPluginId: "signalk-questdb" });
+                        try {
+                            if (mount.wipe === "runtime") {
+                                await containers.removeManagedData(QUESTDB_CONTAINER_NAME, mount.wipePath, { ownerPluginId: "signalk-questdb" });
+                            }
+                            else {
+                                // A volume: the runtime cannot mount it by Signal K's path,
+                                // so signalk-container's in-userns wipe would mount the wrong
+                                // directory. Delete from here, the one process that reaches
+                                // it.
+                                await containers.remove(QUESTDB_CONTAINER_NAME);
+                                const fs = await import("fs/promises");
+                                await wipeDataInSignalk((p) => fs.rm(p, { recursive: true, force: true }), mount.wipePath);
+                            }
+                        }
+                        finally {
+                            // The checkpoint names a position in the database being
+                            // deleted; a fresh one must not be offered a resume into it.
+                            // Cleared even when the delete is refused: the purge was asked
+                            // for, and the position would outlive the database once it is
+                            // deleted by hand.
+                            await checkpoints.clear();
+                        }
                     };
                     // Purge is the RECOVERY action, so it must make progress even when a
                     // start is wedged on a never-settling container call. Step 1: signal
@@ -2140,6 +2211,52 @@ export default (app) => {
                 finishedAt: run.finishedAt,
                 progress: { ...run.progress },
                 error: run.error,
+                resumedFrom: run.resumedFrom,
+            });
+            const CONFIRM_STORED_TIMEOUT_MS = 5_000;
+            // QuestDB's word that the rows behind a position are committed and
+            // applied: the newest row written to each table can be read back. ILP
+            // carries nanoseconds and QuestDB stores microseconds, so the row is
+            // looked up at the truncated instant.
+            const instantLiteral = (tsNanos) => {
+                const micros = tsNanos / 1000n;
+                const iso = new Date(Number(micros / 1000n)).toISOString();
+                return `'${iso.replace("Z", `${String(micros % 1000n).padStart(3, "0")}Z`)}'`;
+            };
+            const stringLiteral = (s) => `'${s.replace(/'/g, "''")}'`;
+            const confirmStored = async (tail) => {
+                if (!queryClient)
+                    return false;
+                const source = stringLiteral(tail.source);
+                const lookups = [];
+                if (tail.numeric) {
+                    lookups.push(`SELECT count() FROM signalk WHERE ts = ${instantLiteral(tail.numeric.tsNanos)} AND path = ${stringLiteral(tail.numeric.path)} AND context = ${stringLiteral(tail.numeric.context)} AND source = ${source}`);
+                }
+                if (tail.string) {
+                    lookups.push(`SELECT count() FROM signalk_str WHERE ts = ${instantLiteral(tail.string.tsNanos)} AND path = ${stringLiteral(tail.string.path)} AND context = ${stringLiteral(tail.string.context)} AND source = ${source}`);
+                }
+                if (tail.position) {
+                    lookups.push(`SELECT count() FROM signalk_position WHERE ts = ${instantLiteral(tail.position.tsNanos)} AND context = ${stringLiteral(tail.position.context)} AND source = ${source}`);
+                }
+                // Short-fused: a QuestDB that does not answer is a "no" — the position
+                // waits — rather than a stall of the import behind an open request.
+                for (const sql of lookups) {
+                    const result = await queryClient.exec(sql, CONFIRM_STORED_TIMEOUT_MS);
+                    if (Number(result.dataset[0]?.[0] ?? 0) < 1)
+                        return false;
+                }
+                return true;
+            };
+            const interruptedView = (cp) => ({
+                url: cp.identity.url,
+                type: cp.identity.type,
+                bucket: cp.identity.bucket,
+                from: cp.identity.from,
+                to: cp.identity.to,
+                measurement: cp.current?.measurement,
+                windowStart: cp.current && new Date(cp.current.windowStart).toISOString(),
+                measurementsDone: cp.done.length,
+                updatedAt: cp.updatedAt,
             });
             // Auth travels in the request BODY rather than plugin config: the token
             // is only needed for the duration of an import and storing it in the
@@ -2245,6 +2362,41 @@ export default (app) => {
                     });
                 }
             });
+            // POST like /buckets and /measurements: the credentials travel in the
+            // body. Answers with the vessels the source holds, and which of them is
+            // this server's own vessel, if any is.
+            router.post("/api/migration/contexts", async (req, res) => {
+                const body = (req.body ?? {});
+                const baseUrl = validateInfluxUrl(typeof body.url === "string" ? body.url : "http://localhost:8086");
+                const bucket = typeof body.bucket === "string" ? body.bucket : "";
+                if (!baseUrl || !bucket) {
+                    res.status(400).json({
+                        error: "Missing or invalid url/bucket",
+                        contexts: [],
+                    });
+                    return;
+                }
+                try {
+                    const contexts = await listContexts({
+                        url: baseUrl,
+                        type: typeof body.type === "string" ? body.type : "influxdb2",
+                        bucket,
+                        auth: readAuth(body),
+                    });
+                    res.json({
+                        contexts,
+                        self: contexts.includes(app.selfContext)
+                            ? app.selfContext
+                            : undefined,
+                    });
+                }
+                catch (err) {
+                    res.status(502).json({
+                        error: err instanceof Error ? err.message : "Unknown error",
+                        contexts: [],
+                    });
+                }
+            });
             router.post("/api/migration/start", async (req, res) => {
                 // One import at a time. Two concurrent runs would interleave on the
                 // single ILP writer and race each other for its buffer, and there is
@@ -2262,7 +2414,35 @@ export default (app) => {
                     });
                     return;
                 }
-                const body = (req.body ?? {});
+                if (legacyCleanupRunning) {
+                    res.status(409).json({
+                        error: "The string table is being rebuilt; try again in a moment",
+                    });
+                    return;
+                }
+                const posted = (req.body ?? {});
+                const stored = await checkpoints.load();
+                // Resuming takes the import's parameters from the checkpoint and only
+                // the credentials from the request: after a restart the panel no
+                // longer knows what range was entered, and credentials are never
+                // stored. The stored parameters then go through the same validation
+                // as posted ones — the file is on disk and could have been edited.
+                let body = posted;
+                if (posted.resume === true) {
+                    if (!stored) {
+                        res.status(409).json({
+                            error: "There is no interrupted import to resume",
+                        });
+                        return;
+                    }
+                    body = {
+                        ...stored.identity,
+                        token: posted.token,
+                        username: posted.username,
+                        password: posted.password,
+                        org: posted.org,
+                    };
+                }
                 const baseUrl = validateInfluxUrl(typeof body.url === "string" ? body.url : "http://localhost:8086");
                 const bucket = typeof body.bucket === "string" ? body.bucket : "";
                 if (!baseUrl || !bucket) {
@@ -2309,7 +2489,7 @@ export default (app) => {
                     rawContext === `vessels.${app.selfId}`) {
                     importContext = "self";
                 }
-                else if (/^(vessels|atons|aircraft|sar)\.[A-Za-z0-9:._-]+$/.test(rawContext)) {
+                else if (isSignalKContext(rawContext)) {
                     importContext = rawContext;
                 }
                 else {
@@ -2318,6 +2498,66 @@ export default (app) => {
                     });
                     return;
                 }
+                // Which of the source's vessels is the own one. The source tags every
+                // point with the context the recording server used, and that is not
+                // necessarily this server's — a new install has a new identity — so
+                // it is chosen rather than matched. Absent, a single-vessel source is
+                // unambiguous; a source holding several vessels is refused rather
+                // than filed wholesale under one of them.
+                // A supplied choice is checked against the source too: one that is
+                // not a vessel there would make nothing the own vessel, and the whole
+                // import would land under other contexts without a word.
+                const rawSelf = typeof body.sourceSelfContext === "string"
+                    ? body.sourceSelfContext.trim()
+                    : "";
+                // A resume of a checkpoint from before vessels were told apart keeps
+                // its meaning — every row the own vessel — rather than being given a
+                // choice it never made, which would make it a different import and
+                // start it over.
+                const legacyResume = posted.resume === true &&
+                    stored !== null &&
+                    stored.identity.sourceSelfContext === undefined;
+                let found = [];
+                if (!legacyResume) {
+                    try {
+                        found = await listContexts({
+                            url: baseUrl,
+                            type: typeof body.type === "string" ? body.type : "influxdb2",
+                            bucket,
+                            auth: readAuth(body),
+                        });
+                    }
+                    catch (err) {
+                        res.status(502).json({
+                            error: err instanceof Error ? err.message : "Unknown error",
+                        });
+                        return;
+                    }
+                }
+                let sourceSelfContext;
+                if (found.length === 0) {
+                    // No vessels to tell apart: every row is the own vessel.
+                    sourceSelfContext = undefined;
+                }
+                else if (rawSelf) {
+                    if (!found.includes(rawSelf)) {
+                        res.status(400).json({
+                            error: `\`${rawSelf}\` is not a vessel in the source; it holds ${found.join(", ")}`,
+                        });
+                        return;
+                    }
+                    sourceSelfContext = rawSelf;
+                }
+                else if (found.length === 1) {
+                    sourceSelfContext = found[0];
+                }
+                else {
+                    res.status(400).json({
+                        error: "The source holds several vessels; say which is yours with `sourceSelfContext`",
+                    });
+                    return;
+                }
+                const others = body.others === "skip" ? "skip" : "keep";
                 // Kept deliberately narrow: a label is a short identifier, not free
                 // text. Anything else falls back to the default.
                 const rawLabel = typeof body.sourceLabel === "string" ? body.sourceLabel.trim() : "";
@@ -2337,6 +2577,8 @@ export default (app) => {
                     // a SYMBOL value on every imported row and is what the history API
                     // queries by, so a malformed one writes history nothing can read.
                     context: importContext,
+                    sourceSelfContext,
+                    others,
                     measurements: Array.isArray(body.measurements)
                         ? body.measurements.filter((m) => typeof m === "string")
                         : undefined,
@@ -2351,6 +2593,43 @@ export default (app) => {
                 };
                 // The id only has to be unique within this process, where the counter
                 // already guarantees it; the timestamp just makes logs readable.
+                // The same import started again continues, whether it was asked to or
+                // not: the rows upsert, so the result is the same and only the time
+                // differs. A different import replaces the old checkpoint, and it is
+                // removed now rather than when the new run first saves — otherwise a
+                // new run that stopped early would offer to resume the old one.
+                const resumeFrom = stored &&
+                    sameIdentity(stored.identity, migrationIdentity(request, DEFAULT_WINDOW_MS))
+                    ? stored
+                    : undefined;
+                if (stored && !resumeFrom) {
+                    // Not worth refusing the import over: the new run overwrites the
+                    // file with its own first position anyway.
+                    await checkpoints.clear().catch((err) => {
+                        app.debug(`migration: old checkpoint not removed: ${err instanceof Error ? err.message : String(err)}`);
+                    });
+                }
+                // The awaits above yielded; another start, or a rebuild of the string
+                // table, may have got in.
+                if (activeMigration && activeMigration.state === "running") {
+                    res.status(409).json({
+                        error: "A migration is already running",
+                        run: migrationRunView(activeMigration),
+                    });
+                    return;
+                }
+                if (legacyCleanupRunning) {
+                    res.status(409).json({
+                        error: "The string table is being rebuilt; try again in a moment",
+                    });
+                    return;
+                }
+                if (!writer) {
+                    res.status(503).json({
+                        error: "QuestDB writer not connected",
+                    });
+                    return;
+                }
                 const run = new MigrationRun(`mig-${Date.now()}-${++migrationCounter}`, baseUrl, bucket);
                 activeMigration = run;
                 const startedWriter = writer;
@@ -2358,6 +2637,9 @@ export default (app) => {
                 // the HTTP call returns immediately and the panel polls /status.
                 void runMigration(request, startedWriter, run, {
                     debug: (msg) => app.debug(msg),
+                    checkpoints,
+                    resumeFrom,
+                    confirmStored,
                 })
                     .catch((err) => {
                     // runMigration traps its own failures, but a few statements run
@@ -2377,7 +2659,157 @@ export default (app) => {
                     run: migrationRunView(run),
                 });
             });
-            router.get("/api/migration/status", (_req, res) => {
+            router.get("/api/migration/status", async (_req, res) => {
+                // Not while a run is active: its checkpoint is that run's own, and
+                // offering to resume an import that is still going makes no sense.
+                const stored = activeMigration?.state === "running"
+                    ? null
+                    : await checkpoints.load();
+                res.json({
+                    run: activeMigration ? migrationRunView(activeMigration) : undefined,
+                    interrupted: stored ? interruptedView(stored) : undefined,
+                });
+            });
+            // What an import made before 2.1.5 left under suffixed paths. Counted
+            // on request rather than at start: it is a scan of the string table,
+            // and a one-time question.
+            router.get("/api/migration/legacy-rows", async (_req, res) => {
+                if (!queryClient) {
+                    res.status(503).json({
+                        error: "QuestDB not connected",
+                        rows: 0,
+                    });
+                    return;
+                }
+                // Counting also finishes a rebuild that failed after its swap, which
+                // must not run alongside one in progress — so it takes the same flag.
+                if (legacyCleanupRunning) {
+                    res.status(409).json({
+                        error: "The rows are being removed",
+                        rows: 0,
+                    });
+                    return;
+                }
+                legacyCleanupRunning = true;
+                try {
+                    await awaitHealIdle();
+                    res.json({
+                        rows: await countLegacyImportRows(queryClient),
+                    });
+                }
+                catch (err) {
+                    res.status(500).json({
+                        error: err instanceof Error ? err.message : "Unknown error",
+                        rows: 0,
+                    });
+                }
+                finally {
+                    legacyCleanupRunning = false;
+                }
+            });
+            // Rebuilds the string table without those rows. Not while an import is
+            // writing to it: the rebuild would race the import for the table.
+            router.post("/api/migration/legacy-rows/remove", async (_req, res) => {
+                if (!pluginRunning || !queryClient || !writer) {
+                    res.status(503).json({
+                        error: "QuestDB not connected",
+                        rows: 0,
+                    });
+                    return;
+                }
+                if (activeMigration && activeMigration.state === "running") {
+                    res.status(409).json({
+                        error: "An import is running; wait for it or cancel it first",
+                        rows: 0,
+                    });
+                    return;
+                }
+                if (legacyCleanupRunning) {
+                    res.status(409).json({
+                        error: "The rows are already being removed",
+                        rows: 0,
+                    });
+                    return;
+                }
+                legacyCleanupRunning = true;
+                // Under the lifecycle lock, like the update and the purge teardown:
+                // neither can then disconnect the writer while a swap holds it. The
+                // route-entry checks above and in those handlers only fail fast; this
+                // is the exclusion. (A purge that has waited its 30 s for the lock
+                // proceeds regardless, by design — it discards the data anyway.)
+                //
+                // Bounded: a wedged start pins the lock, and a rebuild queued behind
+                // it would hang this request and keep the flag set for good. Past the
+                // timeout the request fails, and the queued callback is abandoned so
+                // it cannot run later, unasked, once the lock frees.
+                let acquired = false;
+                let abandoned = false;
+                const rebuild = withLifecycleLock(async () => {
+                    if (abandoned)
+                        return null;
+                    acquired = true;
+                    if (!queryClient || !writer) {
+                        throw new Error("QuestDB not connected");
+                    }
+                    await awaitHealIdle();
+                    return removeLegacyImportRows(queryClient, writer);
+                });
+                const lockTimeout = new Promise((_, reject) => {
+                    const timer = setTimeout(() => {
+                        if (acquired)
+                            return;
+                        abandoned = true;
+                        reject(new Error("The plugin is busy starting or updating; try again in a moment"));
+                    }, LEGACY_CLEANUP_LOCK_TIMEOUT_MS);
+                    void rebuild.then(() => clearTimeout(timer), () => clearTimeout(timer));
+                });
+                // What stop() waits for is the bounded request, not the queued
+                // callback: behind a wedged start, the latter would hold stop() too.
+                const bounded = Promise.race([rebuild, lockTimeout]);
+                legacyCleanup = bounded;
+                try {
+                    const result = await bounded;
+                    if (result === null)
+                        throw new Error("The rebuild did not run");
+                    res.json({
+                        rows: result.removed,
+                        dropped: result.dropped,
+                    });
+                }
+                catch (err) {
+                    // Busy is a 409, like the other lifecycle refusals; a rebuild that
+                    // ran and failed is a 500.
+                    res.status(abandoned ? 409 : 500).json({
+                        error: err instanceof Error ? err.message : "Unknown error",
+                        rows: 0,
+                    });
+                }
+                finally {
+                    legacyCleanupRunning = false;
+                    if (legacyCleanup === bounded)
+                        legacyCleanup = null;
+                }
+            });
+            // Start over: forget the stopped import, so the next start is a new one.
+            router.post("/api/migration/discard", async (_req, res) => {
+                if (activeMigration && activeMigration.state === "running") {
+                    res.status(409).json({
+                        error: "A migration is running",
+                        run: migrationRunView(activeMigration),
+                    });
+                    return;
+                }
+                try {
+                    await checkpoints.clear();
+                }
+                catch (err) {
+                    // Said, not swallowed: the position is still there, and the next
+                    // start of the same import would continue from it.
+                    res.status(500).json({
+                        error: `Could not remove the saved position: ${err instanceof Error ? err.message : String(err)}`,
+                    });
+                    return;
+                }
                 res.json({
                     run: activeMigration ? migrationRunView(activeMigration) : undefined,
                 });