signalk-questdb 2.1.4 → 2.1.5

package/dist/migration-checkpoint.js

@@ -0,0 +1,276 @@
+// Where an InfluxDB import has got to, kept on disk so it survives a restart.
+//
+// An import runs for hours, measurement by measurement, and a re-run begins
+// again at the first one. Interrupted at the same point each time, it never
+// reaches the measurements late in the alphabet — `navigation.position` among
+// them. The checkpoint is what lets a re-run pick up where the last one stopped.
+//
+// The one rule it must never break: a checkpoint may not run ahead of the
+// data. Rows the importer has "written" sit in the ILP writer's buffer, then in
+// the socket, then in QuestDB's page cache, before they are on disk. A
+// checkpoint claiming them done while they are still in any of those would,
+// after a crash, make the resumed run skip rows that were never stored — a
+// silent hole in the history. CheckpointTracker is the guard against that.
+import { mkdir, open, readFile, rename, rm } from "node:fs/promises";
+import path from "node:path";
+export function migrationIdentity(req, windowMs) {
+    return {
+        url: req.url,
+        type: req.type,
+        bucket: req.bucket,
+        // Normalised, so the same instant written two ways is the same import.
+        from: new Date(req.from).toISOString(),
+        to: new Date(req.to).toISOString(),
+        context: req.context,
+        sourceSelfContext: req.sourceSelfContext,
+        others: req.others ?? "keep",
+        sourceLabel: req.sourceLabel ?? "influxdb-import",
+        measurements: req.measurements && req.measurements.length > 0
+            ? [...req.measurements].sort()
+            : undefined,
+        windowMs,
+    };
+}
+export function sameIdentity(a, b) {
+    return (a.url === b.url &&
+        a.type === b.type &&
+        a.bucket === b.bucket &&
+        a.from === b.from &&
+        a.to === b.to &&
+        a.context === b.context &&
+        a.sourceSelfContext === b.sourceSelfContext &&
+        (a.others ?? "keep") === (b.others ?? "keep") &&
+        a.sourceLabel === b.sourceLabel &&
+        a.windowMs === b.windowMs &&
+        JSON.stringify(a.measurements ?? null) ===
+            JSON.stringify(b.measurements ?? null));
+}
+/** The checkpoint of the one import this plugin runs at a time, as a file. */
+export class FileCheckpointStore {
+    file;
+    constructor(file) {
+        this.file = file;
+    }
+    /**
+     * The stored checkpoint, or null. A file that cannot be read or does not
+     * hold a checkpoint is treated as absent: the cost is an import starting
+     * over, which is always safe, where trusting a damaged one is not.
+     */
+    async load() {
+        let text;
+        try {
+            text = await readFile(this.file, "utf8");
+        }
+        catch {
+            return null;
+        }
+        try {
+            const parsed = JSON.parse(text);
+            return isCheckpoint(parsed) ? parsed : null;
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Written beside the target, synced, and renamed over it, so a crash or a
+     * power cut leaves either the previous checkpoint or the new one — never a
+     * truncated one. The directory is synced too, so the rename itself is on
+     * disk; where a directory cannot be opened for that, the rename is left to
+     * the file system's own ordering.
+     */
+    async save(checkpoint) {
+        const dir = path.dirname(this.file);
+        await mkdir(dir, { recursive: true });
+        const tmp = `${this.file}.tmp`;
+        const fh = await open(tmp, "w");
+        try {
+            await fh.writeFile(JSON.stringify(checkpoint), "utf8");
+            await fh.sync();
+        }
+        finally {
+            await fh.close();
+        }
+        await rename(tmp, this.file);
+        await syncDirectory(dir);
+    }
+    /** Synced like a save: a cleared checkpoint must not come back after a power cut. */
+    async clear() {
+        await rm(this.file, { force: true });
+        await syncDirectory(path.dirname(this.file));
+    }
+}
+/**
+ * Put a directory's entries on disk, so a rename or removal in it survives a
+ * power cut. A directory that does not exist has nothing to sync, and not
+ * every platform lets one be opened and synced; any other failure is a real
+ * one and is not hidden.
+ */
+async function syncDirectory(dir) {
+    try {
+        const dh = await open(dir, "r");
+        try {
+            await dh.sync();
+        }
+        finally {
+            await dh.close();
+        }
+    }
+    catch (err) {
+        const code = typeof err === "object" &&
+            err !== null &&
+            "code" in err &&
+            typeof err.code === "string"
+            ? err.code
+            : "";
+        if (code !== "ENOENT" && !DIRECTORY_SYNC_UNSUPPORTED.has(code))
+            throw err;
+    }
+}
+/**
+ * Errors that mean "this platform or file system does not sync directories":
+ * Windows refuses to open one, and some file systems refuse to fsync one.
+ */
+const DIRECTORY_SYNC_UNSUPPORTED = new Set([
+    "EISDIR",
+    "EPERM",
+    "EINVAL",
+    "ENOTSUP",
+    "EOPNOTSUPP",
+]);
+function isCheckpoint(value) {
+    if (typeof value !== "object" || value === null)
+        return false;
+    const cp = value;
+    const id = cp.identity;
+    const progress = cp.progress;
+    return (cp.version === 1 &&
+        typeof id === "object" &&
+        id !== null &&
+        typeof id.url === "string" &&
+        typeof id.type === "string" &&
+        typeof id.bucket === "string" &&
+        typeof id.from === "string" &&
+        typeof id.to === "string" &&
+        typeof id.context === "string" &&
+        (id.sourceSelfContext === undefined ||
+            typeof id.sourceSelfContext === "string") &&
+        (id.others === undefined || id.others === "keep" || id.others === "skip") &&
+        typeof id.sourceLabel === "string" &&
+        typeof id.windowMs === "number" &&
+        (id.measurements === undefined ||
+            (Array.isArray(id.measurements) &&
+                id.measurements.every((m) => typeof m === "string"))) &&
+        Array.isArray(cp.done) &&
+        cp.done.every((m) => typeof m === "string") &&
+        (cp.current === undefined ||
+            (typeof cp.current === "object" &&
+                cp.current !== null &&
+                typeof cp.current.measurement === "string" &&
+                typeof cp.current.windowStart === "number")) &&
+        typeof progress === "object" &&
+        progress !== null &&
+        typeof progress.read === "number" &&
+        typeof progress.written === "number" &&
+        typeof progress.skipped === "number" &&
+        typeof cp.updatedAt === "string");
+}
+/**
+ * How old a position must be before it is persisted.
+ *
+ * Leaving the writer is not the same as being stored. QuestDB takes ILP rows
+ * into memory and commits them on an interval of seconds, so a crash of either
+ * process loses the rows of the last interval. The managed container runs
+ * QuestDB with cairo.commit.mode=sync, which puts a commit on disk; an external
+ * one may be on QuestDB's default nosync, where what survives a power cut is
+ * what the kernel had already written back — with default settings anything
+ * older than about 35 s (30 s dirty-page expiry plus the 5 s flusher period).
+ * A minute clears both with room to spare, on the assumption of those
+ * defaults: an external QuestDB is the operator's, and the README asks them to
+ * run it with sync, which is the real answer for a power cut. The price is
+ * that a resumed import redoes up to a minute of work, which costs nothing but
+ * the time: rows upsert.
+ */
+export const CHECKPOINT_LAG_MS = 60_000;
+/**
+ * Persists positions only once they are safe to resume from.
+ *
+ * A position offered now is held as the candidate. It is written later, by a
+ * subsequent offer, once three things hold: every line enqueued up to it has
+ * left the writer, it is at least `lagMs` old, and — where a `confirmStored`
+ * is given — QuestDB can read back the newest row written before it. The
+ * first two are local knowledge and bound only the writer and the commit
+ * delay; the third is QuestDB's own word that the rows are committed and
+ * applied, which no lag can stand in for when the server is stalled. A
+ * position that cannot be confirmed simply waits for a later offer. Positions
+ * offered while a candidate is waiting are passed over, so what reaches disk
+ * is always a position that was reached at least `lagMs` ago.
+ *
+ * If the writer drops lines the tracker stops for good. The cap discards the
+ * OLDEST buffered lines, which may predate any position taken since, so none
+ * can be trusted; the last one persisted before the drop still can. The run
+ * itself fails at its end for the same drops, and the re-run resumes from
+ * there.
+ */
+export class CheckpointTracker {
+    store;
+    writer;
+    lagMs;
+    now;
+    confirmStored;
+    candidate = null;
+    droppedAtStart;
+    constructor(store, writer, lagMs = CHECKPOINT_LAG_MS, now = Date.now, confirmStored) {
+        this.store = store;
+        this.writer = writer;
+        this.lagMs = lagMs;
+        this.now = now;
+        this.confirmStored = confirmStored;
+        this.droppedAtStart = writer.droppedLineCount ?? 0;
+    }
+    /**
+     * Offer the position just reached, with the newest rows written before it.
+     * `snapshot` is only called if the position is taken up, so passing over
+     * one costs nothing.
+     */
+    async offer(snapshot, tail) {
+        const dropped = this.writer.droppedLineCount ?? 0;
+        if (dropped !== this.droppedAtStart) {
+            this.candidate = null;
+            return;
+        }
+        const enqueued = this.writer.enqueuedLineCount;
+        const settled = this.writer.settledLineCount;
+        if (enqueued === undefined || settled === undefined)
+            return;
+        if (this.candidate) {
+            // Lines leave the writer in the order they came, so the ones that have
+            // left — accepted by the socket, or dropped before this run began — are
+            // the first `settled + dropped` of the sequence.
+            const left = settled + dropped;
+            const ripe = this.now() - this.candidate.takenAt >= this.lagMs;
+            if (!ripe || left < this.candidate.mark)
+                return;
+            if (this.confirmStored && !(await this.stored(this.candidate.tail)))
+                return;
+            await this.store.save(this.candidate.checkpoint);
+            this.candidate = null;
+        }
+        this.candidate = {
+            checkpoint: snapshot(),
+            tail: { ...tail },
+            mark: enqueued,
+            takenAt: this.now(),
+        };
+    }
+    /** A confirmation that fails to answer is a "no": the position waits. */
+    async stored(tail) {
+        try {
+            return await this.confirmStored(tail);
+        }
+        catch {
+            return false;
+        }
+    }
+}
+//# sourceMappingURL=migration-checkpoint.js.map

package/dist/migration-checkpoint.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"migration-checkpoint.js","sourceRoot":"","sources":["../src/migration-checkpoint.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,EAAE;AACF,4EAA4E;AAC5E,4EAA4E;AAC5E,8EAA8E;AAC9E,iFAAiF;AACjF,EAAE;AACF,0EAA0E;AAC1E,gFAAgF;AAChF,uEAAuE;AACvE,4EAA4E;AAC5E,2EAA2E;AAC3E,2EAA2E;AAE3E,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACrE,OAAO,IAAI,MAAM,WAAW,CAAC;AAyC7B,MAAM,UAAU,iBAAiB,CAC/B,GAAqB,EACrB,QAAgB;IAEhB,OAAO;QACL,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,uEAAuE;QACvE,IAAI,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE;QACtC,EAAE,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE;QAClC,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,iBAAiB,EAAE,GAAG,CAAC,iBAAiB;QACxC,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,MAAM;QAC5B,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,iBAAiB;QACjD,YAAY,EACV,GAAG,CAAC,YAAY,IAAI,GAAG,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC;YAC7C,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC,IAAI,EAAE;YAC9B,CAAC,CAAC,SAAS;QACf,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,CAAoB,EACpB,CAAoB;IAEpB,OAAO,CACL,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG;QACf,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI;QACjB,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;QACrB,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI;QACjB,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE;QACb,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO;QACvB,CAAC,CAAC,iBAAiB,KAAK,CAAC,CAAC,iBAAiB;QAC3C,CAAC,CAAC,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,IAAI,MAAM,CAAC;QAC7C,CAAC,CAAC,WAAW,KAAK,CAAC,CAAC,WAAW;QAC/B,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,QAAQ;QACzB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,IAAI,IAAI,CAAC;YACpC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,IAAI,IAAI,CAAC,CACzC,CAAC;AACJ,CAAC;AA+BD,8EAA8E;AAC9E,MAAM,OAAO,mBAAmB;IACD;IAA7B,YAA6B,IAAY;QAAZ,SAAI,GAAJ,IAAI,CAAQ;IAAG,CAAC;IAE7C;;;;OAIG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,IAAY,CAAC;QACjB,IAAI,CAAC;YACH,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,CAAC;YACH,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACzC,OAAO,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;QAC9C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,IAAI,CAAC,UAA+B;QACxC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpC,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACtC,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,IAAI,MAAM,CAAC;QAC/B,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,CAAC;YACvD,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;QAClB,CAAC;gBAAS,CAAC;YACT,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;QACnB,CAAC;QACD,MAAM,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,aAAa,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAED,qFAAqF;IACrF,KAAK,CAAC,KAAK;QACT,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrC,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAC/C,CAAC;CACF;AAED;;;;;GAKG;AACH,KAAK,UAAU,aAAa,CAAC,GAAW;IACtC,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;QAClB,CAAC;gBAAS,CAAC;YACT,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;QACnB,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,GACR,OAAO,GAAG,KAAK,QAAQ;YACvB,GAAG,KAAK,IAAI;YACZ,MAAM,IAAI,GAAG;YACb,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ;YAC1B,CAAC,CAAC,GAAG,CAAC,IAAI;YACV,CAAC,CAAC,EAAE,CAAC;QACT,IAAI,IAAI,KAAK,QAAQ,IAAI,CAAC,0BAA0B,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,MAAM,GAAG,CAAC;IAC5E,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,0BAA0B,GAAwB,IAAI,GAAG,CAAC;IAC9D,QAAQ;IACR,OAAO;IACP,QAAQ;IACR,SAAS;IACT,YAAY;CACb,CAAC,CAAC;AAEH,SAAS,YAAY,CAAC,KAAc;IAClC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC9D,MAAM,EAAE,GAAG,KAAqC,CAAC;IACjD,MAAM,EAAE,GAAG,EAAE,CAAC,QAAkD,CAAC;IACjE,MAAM,QAAQ,GAAG,EAAE,CAAC,QAAQ,CAAC;IAC7B,OAAO,CACL,EAAE,CAAC,OAAO,KAAK,CAAC;QAChB,OAAO,EAAE,KAAK,QAAQ;QACtB,EAAE,KAAK,IAAI;QACX,OAAO,EAAE,CAAC,GAAG,KAAK,QAAQ;QAC1B,OAAO,EAAE,CAAC,IAAI,KAAK,QAAQ;QAC3B,OAAO,EAAE,CAAC,MAAM,KAAK,QAAQ;QAC7B,OAAO,EAAE,CAAC,IAAI,KAAK,QAAQ;QAC3B,OAAO,EAAE,CAAC,EAAE,KAAK,QAAQ;QACzB,OAAO,EAAE,CAAC,OAAO,KAAK,QAAQ;QAC9B,CAAC,EAAE,CAAC,iBAAiB,KAAK,SAAS;YACjC,OAAO,EAAE,CAAC,iBAAiB,KAAK,QAAQ,CAAC;QAC3C,CAAC,EAAE,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,CAAC,MAAM,KAAK,MAAM,IAAI,EAAE,CAAC,MAAM,KAAK,MAAM,CAAC;QACzE,OAAO,EAAE,CAAC,WAAW,KAAK,QAAQ;QAClC,OAAO,EAAE,CAAC,QAAQ,KAAK,QAAQ;QAC/B,CAAC,EAAE,CAAC,YAAY,KAAK,SAAS;YAC5B,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,YAAY,CAAC;gBAC7B,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC;QACzD,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC;QACtB,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC;QAC3C,CAAC,EAAE,CAAC,OAAO,KAAK,SAAS;YACvB,CAAC,OAAO,EAAE,CAAC,OAAO,KAAK,QAAQ;gBAC7B,EAAE,CAAC,OAAO,KAAK,IAAI;gBACnB,OAAO,EAAE,CAAC,OAAO,CAAC,WAAW,KAAK,QAAQ;gBAC1C,OAAO,EAAE,CAAC,OAAO,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC;QAChD,OAAO,QAAQ,KAAK,QAAQ;QAC5B,QAAQ,KAAK,IAAI;QACjB,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ;QACjC,OAAO,QAAQ,CAAC,OAAO,KAAK,QAAQ;QACpC,OAAO,QAAQ,CAAC,OAAO,KAAK,QAAQ;QACpC,OAAO,EAAE,CAAC,SAAS,KAAK,QAAQ,CACjC,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,MAAM,CAAC;AAcxC;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,OAAO,iBAAiB;IAUT;IACA;IACA;IACA;IACA;IAbX,SAAS,GAKN,IAAI,CAAC;IACC,cAAc,CAAS;IAExC,YACmB,KAAsB,EACtB,MAAwB,EACxB,QAAgB,iBAAiB,EACjC,MAAoB,IAAI,CAAC,GAAG,EAC5B,aAA6B;QAJ7B,UAAK,GAAL,KAAK,CAAiB;QACtB,WAAM,GAAN,MAAM,CAAkB;QACxB,UAAK,GAAL,KAAK,CAA4B;QACjC,QAAG,GAAH,GAAG,CAAyB;QAC5B,kBAAa,GAAb,aAAa,CAAgB;QAE9C,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,gBAAgB,IAAI,CAAC,CAAC;IACrD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,KAAK,CACT,QAAmC,EACnC,IAAiB;QAEjB,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,CAAC,CAAC;QAClD,IAAI,OAAO,KAAK,IAAI,CAAC,cAAc,EAAE,CAAC;YACpC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;YACtB,OAAO;QACT,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC;QAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC;QAC7C,IAAI,QAAQ,KAAK,SAAS,IAAI,OAAO,KAAK,SAAS;YAAE,OAAO;QAC5D,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,uEAAuE;YACvE,wEAAwE;YACxE,iDAAiD;YACjD,MAAM,IAAI,GAAG,OAAO,GAAG,OAAO,CAAC;YAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC;YAC/D,IAAI,CAAC,IAAI,IAAI,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI;gBAAE,OAAO;YAChD,IAAI,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;gBACjE,OAAO;YACT,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YACjD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;QACD,IAAI,CAAC,SAAS,GAAG;YACf,UAAU,EAAE,QAAQ,EAAE;YACtB,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE;YACjB,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE;SACpB,CAAC;IACJ,CAAC;IAED,yEAAyE;IACjE,KAAK,CAAC,MAAM,CAAC,IAAiB;QACpC,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,CAAC,aAAc,CAAC,IAAI,CAAC,CAAC;QACzC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;CACF"}

package/dist/migration-cleanup.d.ts

@@ -0,0 +1,48 @@
+import type { QueryClient } from "./query-client.js";
+/**
+ * The rows in question: imported, and under a suffixed path. A live row has
+ * no source when its delta had none, and the predicate is spelled so that
+ * such a row is kept whatever the database makes of a comparison with null —
+ * verified on QuestDB, which keeps it either way, but not left to it.
+ */
+export declare const LEGACY_IMPORT_ROWS_WHERE: string;
+export interface CleanupWriter {
+    hold(): void;
+    release(): void;
+    startTimestampWatch(): void;
+    endTimestampWatch(): bigint | null;
+    writeStringAtNanos(path: string, context: string, value: string, tsNanos: bigint, kind?: "boolean" | "identity", source?: string): void;
+    readonly droppedLineCount?: number;
+}
+/**
+ * How many legacy rows there are — after finishing what a rebuild that failed
+ * after its swap left undone, so the count never reads zero while a retired
+ * table still holds rows to carry over.
+ */
+export declare function countLegacyImportRows(query: Pick<QueryClient, "exec">): Promise<number>;
+/**
+ * Rebuild the string table without the legacy rows.
+ *
+ * The copy runs with the writer working as usual. Two markers through the
+ * writer's own stream bound what it wrote meanwhile: one before the copy,
+ * waited for, so that everything sent earlier is in the snapshot; and one
+ * last before the hold, waited for, so that everything sent before the swap
+ * is in the old table. Between the two the writer notes the oldest timestamp
+ * it wrote, and rows of the old table from there on are carried over before
+ * it is dropped — a source with a wrong clock records the present under any
+ * date. Only the swap holds the writer. A swap that fails before the rebuilt
+ * table is in place puts the old one back; a failure after that leaves the
+ * rebuilt table in place, since it has accepted writes since, and the old one
+ * is carried over and dropped by the next run.
+ *
+ * Reports the lines the writer's cap discarded during the hold, if any: the
+ * hold lasts seconds, but a recorder fed fast enough could still fill it.
+ */
+export declare function removeLegacyImportRows(query: Pick<QueryClient, "exec">, writer: CleanupWriter, deps?: {
+    now?: () => number;
+    sleep?: (ms: number) => Promise<void>;
+    barrierTimeoutMs?: number;
+}): Promise<{
+    removed: number;
+    dropped: number;
+}>;

package/dist/migration-cleanup.js

@@ -0,0 +1,155 @@
+// Removing what an import made before 2.1.5 left behind.
+//
+// Against a signalk-to-influxdb 1.x source, those imports filed every string,
+// boolean and object under a path suffixed with the InfluxDB field name —
+// `navigation.state.stringValue`, `steering.autopilot.engaged.boolValue`,
+// `navigation.position.jsonValue` — paths no consumer ever asks for. A re-run
+// of the import writes the rows under their real paths, but cannot remove the
+// old ones: the path is part of the dedup key, and QuestDB has no row DELETE.
+// The only way to drop rows is to rebuild the table without them.
+/** The field names the 1.x writer used, which the old importer appended. */
+const LEGACY_SUFFIXES = [".stringValue", ".boolValue", ".jsonValue"];
+/**
+ * The rows in question: imported, and under a suffixed path. A live row has
+ * no source when its delta had none, and the predicate is spelled so that
+ * such a row is kept whatever the database makes of a comparison with null —
+ * verified on QuestDB, which keeps it either way, but not left to it.
+ */
+export const LEGACY_IMPORT_ROWS_WHERE = `(source IS NOT NULL AND source = 'influxdb-import' AND (` +
+    LEGACY_SUFFIXES.map((s) => `path LIKE '%${s}'`).join(" OR ") +
+    `))`;
+const TABLE = "signalk_str";
+const REBUILT = "signalk_str_clean";
+const RETIRED = "signalk_str_old";
+/** How long the rebuild may take: a table of tens of millions of rows. */
+const REBUILD_TIMEOUT_MS = 30 * 60 * 1000;
+/** How long to wait for the marker, and so for everything sent before it. */
+const BARRIER_TIMEOUT_MS = 60 * 1000;
+/**
+ * Marker rows: one written before the copy, one written last before the
+ * hold. Rows of the old table only — never copied, never carried over,
+ * dropped with it. Their source keeps them out of every query the plugin
+ * makes for real data.
+ */
+const MARKER_SOURCE = "signalk-questdb-cleanup";
+const MARKER_PATH = "cleanup.marker";
+/** Every row the rebuild keeps or carries over: not legacy, not a marker. */
+const KEPT_ROWS_WHERE = `NOT ${LEGACY_IMPORT_ROWS_WHERE} AND (source IS NULL OR source != '${MARKER_SOURCE}')`;
+/**
+ * How many legacy rows there are — after finishing what a rebuild that failed
+ * after its swap left undone, so the count never reads zero while a retired
+ * table still holds rows to carry over.
+ */
+export async function countLegacyImportRows(query) {
+    await recoverRetiredTable(query);
+    const r = await query.exec(`SELECT count() FROM ${TABLE} WHERE ${LEGACY_IMPORT_ROWS_WHERE}`, REBUILD_TIMEOUT_MS);
+    return Number(r.dataset[0]?.[0] ?? 0);
+}
+/**
+ * Rebuild the string table without the legacy rows.
+ *
+ * The copy runs with the writer working as usual. Two markers through the
+ * writer's own stream bound what it wrote meanwhile: one before the copy,
+ * waited for, so that everything sent earlier is in the snapshot; and one
+ * last before the hold, waited for, so that everything sent before the swap
+ * is in the old table. Between the two the writer notes the oldest timestamp
+ * it wrote, and rows of the old table from there on are carried over before
+ * it is dropped — a source with a wrong clock records the present under any
+ * date. Only the swap holds the writer. A swap that fails before the rebuilt
+ * table is in place puts the old one back; a failure after that leaves the
+ * rebuilt table in place, since it has accepted writes since, and the old one
+ * is carried over and dropped by the next run.
+ *
+ * Reports the lines the writer's cap discarded during the hold, if any: the
+ * hold lasts seconds, but a recorder fed fast enough could still fill it.
+ */
+export async function removeLegacyImportRows(query, writer, deps = {}) {
+    const now = deps.now ?? Date.now;
+    const sleep = deps.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+    const barrierTimeoutMs = deps.barrierTimeoutMs ?? BARRIER_TIMEOUT_MS;
+    const before = await countLegacyImportRows(query);
+    if (before === 0)
+        return { removed: 0, dropped: 0 };
+    // A rebuild that failed earlier may have left its table behind.
+    await query.exec(`DROP TABLE IF EXISTS ${REBUILT}`);
+    const writeMarker = () => {
+        const marker = `${now()}-${Math.random().toString(36).slice(2)}`;
+        writer.writeStringAtNanos(MARKER_PATH, "self", marker, BigInt(now()) * 1000000n, undefined, MARKER_SOURCE);
+        return marker;
+    };
+    // The watch first, then the marker: every line from here on is watched,
+    // and once the marker is there, every line from before is in the table
+    // and so in the snapshot the copy takes.
+    writer.startTimestampWatch();
+    await waitForMarker(query, writeMarker(), sleep, barrierTimeoutMs);
+    await query.exec(`CREATE TABLE ${REBUILT} AS (SELECT * FROM ${TABLE} WHERE ${KEPT_ROWS_WHERE}) TIMESTAMP(ts) PARTITION BY DAY WAL`, REBUILD_TIMEOUT_MS);
+    await query.exec(`ALTER TABLE ${REBUILT} DEDUP ENABLE UPSERT KEYS(ts, path, context, source)`);
+    const droppedBefore = writer.droppedLineCount ?? 0;
+    // Written and held in one go, so this marker is the last line sent.
+    const last = writeMarker();
+    writer.hold();
+    // Everything the writer sent into the old table after the copy's snapshot
+    // it sent between the two markers, and the watch says how old the oldest
+    // of it is. Nothing at all means nothing to carry over.
+    const oldestWritten = writer.endTimestampWatch();
+    try {
+        await waitForMarker(query, last, sleep, barrierTimeoutMs);
+        await query.exec(`RENAME TABLE ${TABLE} TO ${RETIRED}`);
+        try {
+            await query.exec(`RENAME TABLE ${REBUILT} TO ${TABLE}`);
+        }
+        catch (err) {
+            await query.exec(`RENAME TABLE ${RETIRED} TO ${TABLE}`);
+            throw err;
+        }
+        await carryOverAndDrop(query, oldestWritten === null
+            ? undefined
+            : new Date(Number(oldestWritten / 1000000n)).toISOString(), oldestWritten === null);
+    }
+    finally {
+        writer.release();
+    }
+    return {
+        removed: before,
+        dropped: (writer.droppedLineCount ?? 0) - droppedBefore,
+    };
+}
+/**
+ * Rows of the retired table that are not legacy rows or markers go into the
+ * live table — dedup makes the overlap harmless — and then it is dropped.
+ * `since` narrows the copy to the stragglers when the cutoff is known; a
+ * recovery of a table left behind takes everything; `nothing` skips the copy
+ * when the writer wrote nothing meanwhile.
+ */
+async function carryOverAndDrop(query, since, nothing = false) {
+    if (!nothing) {
+        const window = since ? `ts >= '${since}' AND ` : "";
+        await query.exec(`INSERT INTO ${TABLE} SELECT * FROM ${RETIRED} WHERE ${window}${KEPT_ROWS_WHERE}`, REBUILD_TIMEOUT_MS);
+    }
+    await query.exec(`DROP TABLE ${RETIRED}`);
+}
+/**
+ * A rebuild that failed after the swap leaves the retired table behind, with
+ * the rows recorded during the copy still in it. Finish that work first.
+ */
+async function recoverRetiredTable(query) {
+    const r = await query.exec(`SELECT count() FROM tables() WHERE table_name = '${RETIRED}'`);
+    if (Number(r.dataset[0]?.[0] ?? 0) === 0)
+        return;
+    await carryOverAndDrop(query);
+}
+/** Until the marker can be read back: everything sent before it has landed. */
+async function waitForMarker(query, marker, sleep, timeoutMs) {
+    let waited = 0;
+    for (;;) {
+        const r = await query.exec(`SELECT count() FROM ${TABLE} WHERE path = '${MARKER_PATH}' AND source = '${MARKER_SOURCE}' AND value_str = '${marker}'`);
+        if (Number(r.dataset[0]?.[0] ?? 0) > 0)
+            return;
+        if (waited >= timeoutMs) {
+            throw new Error(`QuestDB has not taken the last writes to ${TABLE} after ${Math.round(timeoutMs / 1000)}s; try again later`);
+        }
+        await sleep(500);
+        waited += 500;
+    }
+}
+//# sourceMappingURL=migration-cleanup.js.map

package/dist/migration-cleanup.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"migration-cleanup.js","sourceRoot":"","sources":["../src/migration-cleanup.ts"],"names":[],"mappings":"AAAA,yDAAyD;AACzD,EAAE;AACF,8EAA8E;AAC9E,0EAA0E;AAC1E,0EAA0E;AAC1E,8EAA8E;AAC9E,8EAA8E;AAC9E,8EAA8E;AAC9E,kEAAkE;AAIlE,4EAA4E;AAC5E,MAAM,eAAe,GAAG,CAAC,cAAc,EAAE,YAAY,EAAE,YAAY,CAAU,CAAC;AAE9E;;;;;GAKG;AACH,MAAM,CAAC,MAAM,wBAAwB,GACnC,0DAA0D;IAC1D,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;IAC5D,IAAI,CAAC;AAEP,MAAM,KAAK,GAAG,aAAa,CAAC;AAC5B,MAAM,OAAO,GAAG,mBAAmB,CAAC;AACpC,MAAM,OAAO,GAAG,iBAAiB,CAAC;AAElC,0EAA0E;AAC1E,MAAM,kBAAkB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAC1C,6EAA6E;AAC7E,MAAM,kBAAkB,GAAG,EAAE,GAAG,IAAI,CAAC;AACrC;;;;;GAKG;AACH,MAAM,aAAa,GAAG,yBAAyB,CAAC;AAChD,MAAM,WAAW,GAAG,gBAAgB,CAAC;AACrC,6EAA6E;AAC7E,MAAM,eAAe,GAAG,OAAO,wBAAwB,sCAAsC,aAAa,IAAI,CAAC;AAkB/G;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,KAAgC;IAEhC,MAAM,mBAAmB,CAAC,KAAK,CAAC,CAAC;IACjC,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,IAAI,CACxB,uBAAuB,KAAK,UAAU,wBAAwB,EAAE,EAChE,kBAAkB,CACnB,CAAC;IACF,OAAO,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;AACxC,CAAC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,KAAgC,EAChC,MAAqB,EACrB,OAII,EAAE;IAEN,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;IACjC,MAAM,KAAK,GACT,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IACxE,MAAM,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,IAAI,kBAAkB,CAAC;IAErE,MAAM,MAAM,GAAG,MAAM,qBAAqB,CAAC,KAAK,CAAC,CAAC;IAClD,IAAI,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IAEpD,gEAAgE;IAChE,MAAM,KAAK,CAAC,IAAI,CAAC,wBAAwB,OAAO,EAAE,CAAC,CAAC;IAEpD,MAAM,WAAW,GAAG,GAAW,EAAE;QAC/B,MAAM,MAAM,GAAG,GAAG,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QACjE,MAAM,CAAC,kBAAkB,CACvB,WAAW,EACX,MAAM,EACN,MAAM,EACN,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,QAAU,EAC1B,SAAS,EACT,aAAa,CACd,CAAC;QACF,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC;IAEF,wEAAwE;IACxE,uEAAuE;IACvE,yCAAyC;IACzC,MAAM,CAAC,mBAAmB,EAAE,CAAC;IAC7B,MAAM,aAAa,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE,KAAK,EAAE,gBAAgB,CAAC,CAAC;IACnE,MAAM,KAAK,CAAC,IAAI,CACd,gBAAgB,OAAO,sBAAsB,KAAK,UAAU,eAAe,sCAAsC,EACjH,kBAAkB,CACnB,CAAC;IACF,MAAM,KAAK,CAAC,IAAI,CACd,eAAe,OAAO,sDAAsD,CAC7E,CAAC;IAEF,MAAM,aAAa,GAAG,MAAM,CAAC,gBAAgB,IAAI,CAAC,CAAC;IACnD,oEAAoE;IACpE,MAAM,IAAI,GAAG,WAAW,EAAE,CAAC;IAC3B,MAAM,CAAC,IAAI,EAAE,CAAC;IACd,0EAA0E;IAC1E,yEAAyE;IACzE,wDAAwD;IACxD,MAAM,aAAa,GAAG,MAAM,CAAC,iBAAiB,EAAE,CAAC;IACjD,IAAI,CAAC;QACH,MAAM,aAAa,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,gBAAgB,CAAC,CAAC;QAC1D,MAAM,KAAK,CAAC,IAAI,CAAC,gBAAgB,KAAK,OAAO,OAAO,EAAE,CAAC,CAAC;QACxD,IAAI,CAAC;YACH,MAAM,KAAK,CAAC,IAAI,CAAC,gBAAgB,OAAO,OAAO,KAAK,EAAE,CAAC,CAAC;QAC1D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,KAAK,CAAC,IAAI,CAAC,gBAAgB,OAAO,OAAO,KAAK,EAAE,CAAC,CAAC;YACxD,MAAM,GAAG,CAAC;QACZ,CAAC;QACD,MAAM,gBAAgB,CACpB,KAAK,EACL,aAAa,KAAK,IAAI;YACpB,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,GAAG,QAAU,CAAC,CAAC,CAAC,WAAW,EAAE,EAC9D,aAAa,KAAK,IAAI,CACvB,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;IACD,OAAO;QACL,OAAO,EAAE,MAAM;QACf,OAAO,EAAE,CAAC,MAAM,CAAC,gBAAgB,IAAI,CAAC,CAAC,GAAG,aAAa;KACxD,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,KAAK,UAAU,gBAAgB,CAC7B,KAAgC,EAChC,KAAc,EACd,OAAO,GAAG,KAAK;IAEf,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;QACpD,MAAM,KAAK,CAAC,IAAI,CACd,eAAe,KAAK,kBAAkB,OAAO,UAAU,MAAM,GAAG,eAAe,EAAE,EACjF,kBAAkB,CACnB,CAAC;IACJ,CAAC;IACD,MAAM,KAAK,CAAC,IAAI,CAAC,cAAc,OAAO,EAAE,CAAC,CAAC;AAC5C,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,mBAAmB,CAChC,KAAgC;IAEhC,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,IAAI,CACxB,oDAAoD,OAAO,GAAG,CAC/D,CAAC;IACF,IAAI,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC;QAAE,OAAO;IACjD,MAAM,gBAAgB,CAAC,KAAK,CAAC,CAAC;AAChC,CAAC;AAED,+EAA+E;AAC/E,KAAK,UAAU,aAAa,CAC1B,KAAgC,EAChC,MAAc,EACd,KAAoC,EACpC,SAAiB;IAEjB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,SAAS,CAAC;QACR,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,IAAI,CACxB,uBAAuB,KAAK,kBAAkB,WAAW,mBAAmB,aAAa,sBAAsB,MAAM,GAAG,CACzH,CAAC;QACF,IAAI,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC;YAAE,OAAO;QAC/C,IAAI,MAAM,IAAI,SAAS,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CACb,4CAA4C,KAAK,UAAU,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,oBAAoB,CAC5G,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QACjB,MAAM,IAAI,GAAG,CAAC;IAChB,CAAC;AACH,CAAC"}