signalk-questdb 2.1.4 → 2.1.5

package/dist/migration.js

@@ -8,18 +8,21 @@
 // dropped: a migration that quietly loses a third of the history looks
 // identical to one that worked.
 //
-// Reads are windowed by time and drained window by window. A boat's InfluxDB
-// is routinely larger than the Pi's RAM, so neither the query nor the result
-// may be unbounded — and a window that returns nothing still advances, so a
-// multi-year gap costs one empty query per window rather than a stall.
+// Reads are windowed by time, and each window is streamed: read a batch, write
+// it, release it, read the next. A boat's InfluxDB is routinely larger than the
+// Pi's RAM, and a single dense day of one path can be too — so nothing here
+// holds a whole response, and reading stops while QuestDB is catching up. A
+// window that returns nothing still advances, so a multi-year gap costs one
+// empty query per window rather than a stall.
 import { routeDeltaValue, flattenObjectValue } from "./delta-routing.js";
+import { CheckpointTracker, migrationIdentity, sameIdentity, } from "./migration-checkpoint.js";
 /** Nanoseconds per millisecond, the unit gap between `Date` and ILP. */
 const NANOS_PER_MS = 1000000n;
 /**
- * How much time one read covers. Small enough that a dense measurement does
- * not return a result set too large to hold, large enough that a sparse one
- * does not need thousands of round trips. A day is the natural unit for boat
- * data: it bounds a busy passage to a few hundred thousand points.
+ * How much time one query covers. Large enough that a sparse measurement does
+ * not need thousands of round trips, small enough that one query stays cheap
+ * for InfluxDB and progress moves in visible steps. It does not bound memory:
+ * a window is streamed in batches, however many points it holds.
  */
 export const DEFAULT_WINDOW_MS = 24 * 60 * 60 * 1000;
 /**
@@ -193,6 +196,49 @@ schema.measurements(bucket: ${JSON.stringify(req.bucket)}, start: 1970-01-01T00:
     const values = body2.results?.[0]?.series?.[0]?.values ?? [];
     return values.map((row) => ({ name: String(row[0]), fields: [] }));
 }
+/**
+ * The `context` tag values in a bucket/database: the vessels the source holds.
+ * Both Signal K InfluxDB writers tag every point with the delta's context, so
+ * this is how the source's own vessel is told from the AIS targets it recorded.
+ * A source with no such tag — a non-Signal K schema — returns none.
+ */
+export async function listContexts(req, fetchImpl = fetch) {
+    const headers = authHeaders(req.type, req.auth);
+    if (req.type === "influxdb2") {
+        const flux = `import "influxdata/influxdb/schema"
+schema.tagValues(bucket: ${JSON.stringify(req.bucket)}, tag: "context", start: 1970-01-01T00:00:00Z)`;
+        const rows = await runFlux(req, flux, headers, fetchImpl);
+        return [
+            ...new Set(rows.map((r) => r.values["_value"]).filter((v) => !!v)),
+        ].sort();
+    }
+    // Answered per measurement, each series a list of [key, value] rows.
+    const url = `${req.url}/query?db=${encodeURIComponent(req.bucket)}&q=${encodeURIComponent('SHOW TAG VALUES WITH KEY = "context"')}`;
+    const r = await fetchImpl(url, {
+        headers,
+        signal: AbortSignal.timeout(15000),
+    });
+    if (!r.ok)
+        throw new Error(await describeHttpError(r, "list contexts"));
+    const body = (await r.json());
+    if (body.results?.[0]?.error)
+        throw new Error(body.results[0].error);
+    const values = new Set();
+    for (const s of body.results?.[0]?.series ?? []) {
+        for (const row of s.values ?? []) {
+            if (typeof row[1] === "string" && row[1])
+                values.add(row[1]);
+        }
+    }
+    return [...values].sort();
+}
+/**
+ * Whether a string is a Signal K context this plugin will file history under:
+ * a vessel, aton, aircraft or SAR target with a well-formed identifier.
+ */
+export function isSignalKContext(context) {
+    return /^(vessels|atons|aircraft|sar)\.[A-Za-z0-9:._-]+$/.test(context);
+}
 async function describeHttpError(r, what) {
     const text = await r.text().catch(() => "");
     // 401/403 against InfluxDB almost always means a missing or wrong token,
@@ -211,108 +257,234 @@ async function describeHttpError(r, what) {
  * one POST and a CSV parse.
  */
 async function runFlux(req, flux, headers, fetchImpl, timeoutMs = 120_000) {
+    const r = await fetchImpl(fluxQueryUrl(req), {
+        ...fluxQueryInit(flux, headers),
+        signal: AbortSignal.timeout(timeoutMs),
+    });
+    if (!r.ok)
+        throw new Error(await describeHttpError(r, "run query"));
+    return parseAnnotatedCsv(await r.text());
+}
+function fluxQueryUrl(req) {
     const org = req.auth?.org ?? "";
-    const url = `${req.url}/api/v2/query${org ? `?org=${encodeURIComponent(org)}` : ""}`;
-    const r = await fetchImpl(url, {
+    return `${req.url}/api/v2/query${org ? `?org=${encodeURIComponent(org)}` : ""}`;
+}
+/**
+ * A Flux query as a request.
+ *
+ * Sent as JSON, not as a raw `application/vnd.flux` body, because only the JSON
+ * form can carry a dialect — and without one InfluxDB answers with bare CSV, no
+ * annotation lines at all (verified against a live 2.7.12). The reader needs
+ * two of them: `#datatype`, without which a string field holding "3.5" is
+ * indistinguishable from the number and lands in the numeric table, and
+ * `#group`, which is how a pivoted record's tags are told from its fields.
+ */
+function fluxQueryInit(flux, headers) {
+    return {
         method: "POST",
         headers: {
             ...headers,
-            "Content-Type": "application/vnd.flux",
+            "Content-Type": "application/json",
             Accept: "text/csv",
         },
-        body: flux,
-        signal: AbortSignal.timeout(timeoutMs),
+        body: JSON.stringify({
+            query: flux,
+            type: "flux",
+            dialect: { header: true, annotations: ["datatype", "group", "default"] },
+        }),
+    };
+}
+/**
+ * How long InfluxDB may stay silent while a read is outstanding.
+ *
+ * Not a limit on the whole request: a streamed window is read only as fast as
+ * QuestDB accepts it, so a dense one legitimately stays open for many minutes
+ * while the writer drains. The clock runs only while this side is waiting for
+ * bytes it has asked for.
+ */
+const READ_IDLE_TIMEOUT_MS = 120_000;
+/**
+ * Run a request and yield its response body one physical line at a time.
+ *
+ * The body is never held whole — a dense day of one measurement runs to
+ * hundreds of megabytes of CSV, and a heap exhausted by it cannot be caught:
+ * it ends the Signal K process. Reading is demand-driven, so a consumer that
+ * pauses (the import, waiting for the writer to drain) stops the reads, and
+ * TCP flow control carries that back to InfluxDB.
+ *
+ * Stopping early — cancelling a run — aborts the request rather than leaving
+ * InfluxDB streaming into a socket nobody reads.
+ */
+async function* streamLines(fetchImpl, url, init, what, idleTimeoutMs) {
+    const controller = new AbortController();
+    // Rejects on its own clock rather than waiting for the abort to surface
+    // through the pending step: the abort is what frees the connection, but the
+    // run must fail on time even if the step never notices it.
+    const guarded = (step) => new Promise((resolve, reject) => {
+        const timer = setTimeout(() => {
+            controller.abort();
+            reject(new Error(`Cannot ${what}: InfluxDB sent no data for ${Math.round(idleTimeoutMs / 1000)}s`));
+        }, idleTimeoutMs);
+        step().then((value) => {
+            clearTimeout(timer);
+            resolve(value);
+        }, (err) => {
+            clearTimeout(timer);
+            reject(err instanceof Error ? err : new Error(String(err)));
+        });
     });
+    const r = await guarded(() => fetchImpl(url, { ...init, signal: controller.signal }));
     if (!r.ok)
-        throw new Error(await describeHttpError(r, "run query"));
-    return parseAnnotatedCsv(await r.text());
+        throw new Error(await guarded(() => describeHttpError(r, what)));
+    if (!r.body)
+        return;
+    const reader = r.body.getReader();
+    const decoder = new TextDecoder();
+    let carry = "";
+    try {
+        for (;;) {
+            const { done, value } = await guarded(() => reader.read());
+            if (done)
+                break;
+            carry += decoder.decode(value, { stream: true });
+            let from = 0;
+            for (let nl = carry.indexOf("\n"); nl >= 0; nl = carry.indexOf("\n", from)) {
+                yield carry.slice(from, nl);
+                from = nl + 1;
+            }
+            carry = carry.slice(from);
+        }
+        carry += decoder.decode();
+        if (carry !== "")
+            yield carry;
+    }
+    finally {
+        controller.abort();
+        await reader.cancel().catch(() => { });
+    }
 }
 /**
- * Minimal annotated-CSV reader for Flux responses.
+ * Incremental annotated-CSV reader for Flux responses.
+ *
+ * Fed one physical line at a time, so a response is never held whole: the
+ * caller streams the body through it and sees each record as it completes.
  *
  * Flux emits one or more *tables*, each preceded by annotation lines starting
  * with `#` and then a header row. A response can therefore change its column
  * layout partway through, which is why the header is tracked per block rather
  * than read once at the top.
  */
-export function parseAnnotatedCsv(text) {
-    const out = [];
-    let header = null;
+export class AnnotatedCsvReader {
+    header = null;
     // Per-block `#datatype` line. Without it a genuine STRING value of "3.5"
     // is indistinguishable from the number 3.5 in the CSV body — verified
     // against a live 2.9.1 — and would be imported into the numeric table.
-    let datatypes = null;
-    // A quoted CSV value may contain a newline — verified against a live 2.9.1,
-    // which returns `"line1\r\nline2"` for a string value holding one. Splitting
-    // on newlines alone would tear that record in half and produce two garbage
-    // rows, so physical lines are joined until the quotes balance.
-    // \r is stripped per PHYSICAL line before joining: a quoted value spanning
-    // two CRLF lines would otherwise keep the \r of the first inside the joined
-    // value, so the imported string silently carries a stray carriage return.
-    for (const rawLine of joinQuotedLines(text.split("\n").map((l) => l.replace(/\r$/, "")))) {
-        const line = rawLine;
+    datatypes = null;
+    groups = null;
+    // Built once per table and shared by its records: a table can run to
+    // millions of rows, and a copy per record was most of what a window cost.
+    types = {};
+    grouped = new Set();
+    pending = null;
+    /**
+     * Feed one physical line, without its `\n`. Returns the record it
+     * completes, or null for annotations, headers, blanks and partial records.
+     *
+     * A quoted CSV value may contain a newline — verified against a live 2.9.1,
+     * which returns `"line1\r\nline2"` for a string value holding one. Splitting
+     * on newlines alone would tear that record in half and produce two garbage
+     * rows, so physical lines are joined until the quotes balance ("" inside a
+     * quoted value is an escaped quote and does not change the balance).
+     * \r is stripped per PHYSICAL line before joining: a quoted value spanning
+     * two CRLF lines would otherwise keep the \r of the first inside the joined
+     * value, so the imported string silently carries a stray carriage return.
+     */
+    push(physicalLine) {
+        const stripped = physicalLine.endsWith("\r")
+            ? physicalLine.slice(0, -1)
+            : physicalLine;
+        const line = this.pending === null ? stripped : `${this.pending}\n${stripped}`;
+        if (!quotesBalanced(line)) {
+            this.pending = line;
+            return null;
+        }
+        this.pending = null;
+        return this.logicalLine(line);
+    }
+    /** Call once after the last line. */
+    end() {
+        if (this.pending === null)
+            return null;
+        // Unterminated quote at EOF: emit what there is rather than dropping it.
+        const line = this.pending;
+        this.pending = null;
+        return this.logicalLine(line);
+    }
+    logicalLine(line) {
         if (line === "") {
             // Blank line separates tables; the next non-# line is a fresh header.
-            header = null;
-            datatypes = null;
-            continue;
+            this.header = null;
+            this.datatypes = null;
+            this.groups = null;
+            return null;
         }
         if (line.startsWith("#")) {
             // Annotation (#datatype/#group/#default). A new annotation block means
             // the previous header no longer applies.
             if (line.startsWith("#datatype"))
-                datatypes = splitCsvLine(line);
-            header = null;
-            continue;
+                this.datatypes = splitCsvLine(line);
+            else if (line.startsWith("#group"))
+                this.groups = splitCsvLine(line);
+            this.header = null;
+            return null;
         }
         const cells = splitCsvLine(line);
-        if (!header) {
-            header = cells;
-            continue;
+        if (!this.header) {
+            this.header = cells;
+            const types = {};
+            const grouped = new Set();
+            for (let i = 0; i < cells.length; i++) {
+                const key = cells[i];
+                if (key === "")
+                    continue;
+                // Annotations are positionally aligned with the header row.
+                if (this.datatypes?.[i])
+                    types[key] = this.datatypes[i];
+                if (this.groups?.[i] === "true")
+                    grouped.add(key);
+            }
+            this.types = types;
+            this.grouped = grouped;
+            return null;
         }
-        const rec = {};
-        const types = {};
-        for (let i = 0; i < header.length; i++) {
-            const key = header[i];
+        const values = {};
+        for (let i = 0; i < this.header.length; i++) {
+            const key = this.header[i];
             // Flux's leading empty column is the annotation gutter, not data.
             if (key === "")
                 continue;
-            rec[key] = cells[i] ?? "";
-            // #datatype is positionally aligned with the header row.
-            if (datatypes && datatypes[i])
-                types[key] = datatypes[i];
+            values[key] = cells[i] ?? "";
         }
-        out.push({ values: rec, types });
+        return { values, types: this.types, grouped: this.grouped };
     }
-    return out;
 }
-/**
- * Re-join physical lines that belong to one CSV record.
- *
- * A record is complete when its double quotes are balanced ("" inside a quoted
- * value is an escaped quote and does not change the balance). Annotation and
- * blank lines can never be mid-record, so they pass straight through and the
- * caller's boundary handling is unaffected.
- */
-function joinQuotedLines(lines) {
+/** Parse a whole annotated-CSV response. For small results only. */
+export function parseAnnotatedCsv(text) {
+    const reader = new AnnotatedCsvReader();
     const out = [];
-    let pending = null;
-    for (const line of lines) {
-        const candidate = pending === null ? line : `${pending}\n${line}`;
-        if (quotesBalanced(candidate)) {
-            out.push(candidate);
-            pending = null;
-        }
-        else {
-            pending = candidate;
-        }
+    for (const line of text.split("\n")) {
+        const rec = reader.push(line);
+        if (rec)
+            out.push(rec);
     }
-    // Unterminated quote at EOF: emit what there is rather than dropping it.
-    if (pending !== null)
-        out.push(pending);
+    const last = reader.end();
+    if (last)
+        out.push(last);
     return out;
 }
 function quotesBalanced(s) {
+    if (!s.includes('"'))
+        return true;
     let inQuotes = false;
     for (let i = 0; i < s.length; i++) {
         if (s[i] !== '"')
@@ -326,6 +498,8 @@ function quotesBalanced(s) {
     return !inQuotes;
 }
 function splitCsvLine(line) {
+    if (!line.includes('"'))
+        return line.split(",");
     const cells = [];
     let cur = "";
     let inQuotes = false;
@@ -382,19 +556,72 @@ export function rfc3339ToNanos(value) {
     const nanosPart = BigInt((frac + "000000000").slice(0, 9));
     return BigInt(baseMs) * NANOS_PER_MS + nanosPart;
 }
+/**
+ * The field signalk-to-influxdb 1.x uses for a value that is neither a number,
+ * a string nor a boolean: the value itself, JSON-encoded. `navigation.position`
+ * is always stored this way; `lat`/`lon` fields exist only when that plugin's
+ * `separateLatLon` option is on, which it is not by default.
+ */
+const JSON_VALUE_FIELD = "jsonValue";
+/**
+ * Field names that hold the value of the measurement's own path.
+ *
+ * signalk-to-influxdb2 writes every value to `value`. signalk-to-influxdb 1.x
+ * picks the field by the value's type — `value` for numbers, then
+ * `stringValue`, `boolValue` and `jsonValue` — because an InfluxDB 1.x field
+ * cannot change type once it has been written.
+ */
+const PATH_VALUE_FIELDS = new Set([
+    "value",
+    "",
+    "_value",
+    "stringValue",
+    "boolValue",
+    JSON_VALUE_FIELD,
+]);
 /**
  * The Signal K path a measurement+field pair represents.
  *
- * signalk-to-influxdb writes the path as the measurement name and the value in
- * a field called `value`. Anything else (a measurement with several named
- * fields) is a non-Signal K schema, so the field name is appended to keep the
- * two apart instead of overwriting one with the other.
+ * The Signal K InfluxDB plugins write the path as the measurement name and the
+ * value in one of PATH_VALUE_FIELDS. Anything else (a measurement with several
+ * named fields) is a non-Signal K schema, so the field name is appended to keep
+ * the two apart instead of overwriting one with the other.
  */
 export function toSignalKPath(measurement, field) {
-    if (field === "value" || field === "" || field === "_value")
+    if (PATH_VALUE_FIELDS.has(field))
         return measurement;
     return `${measurement}.${field}`;
 }
+/**
+ * Decode `jsonValue` fields back into the values they encode.
+ *
+ * Left as the string it arrives as, a position would be recorded in
+ * `signalk_str` as text and `signalk_position` would stay empty. Decoded, it
+ * routes exactly like the live value did: a position to the position table,
+ * any other object to its scalar leaves.
+ *
+ * A value that does not parse cannot be represented, so it is dropped — and
+ * reported, so it lands in the run's skipped total.
+ */
+export function decodeJsonValueRows(rows) {
+    if (!rows.some((row) => row.field === JSON_VALUE_FIELD))
+        return { rows, dropped: 0 };
+    let dropped = 0;
+    const decoded = [];
+    for (const row of rows) {
+        if (row.field !== JSON_VALUE_FIELD || typeof row.value !== "string") {
+            decoded.push(row);
+            continue;
+        }
+        try {
+            decoded.push({ ...row, value: JSON.parse(row.value) });
+        }
+        catch {
+            dropped++;
+        }
+    }
+    return { rows: decoded, dropped };
+}
 /** Coerce an InfluxDB field value (always text over CSV) to a JS value. */
 export function coerceValue(raw, dataType) {
     if (raw === "")
@@ -429,6 +656,7 @@ export class MigrationRun {
     startedAt = new Date().toISOString();
     finishedAt;
     error;
+    resumedFrom;
     progress = {
         read: 0,
         written: 0,
@@ -462,6 +690,7 @@ export async function runMigration(req, writer, run, deps = {}) {
     const debug = deps.debug ?? (() => { });
     const sleep = deps.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
     const headers = authHeaders(req.type, req.auth);
+    const readIdleTimeoutMs = deps.readIdleTimeoutMs ?? READ_IDLE_TIMEOUT_MS;
     // A non-positive window would make `start += windowMs` never advance and
     // spin forever. Not reachable from the HTTP API (which does not expose the
     // knob), but an infinite loop inside the Signal K process is severe enough
@@ -469,10 +698,39 @@ export async function runMigration(req, writer, run, deps = {}) {
     const requestedWindow = req.windowMs ?? DEFAULT_WINDOW_MS;
     const windowMs = requestedWindow > 0 ? requestedWindow : DEFAULT_WINDOW_MS;
     const context = req.context;
+    const sourceSelf = req.sourceSelfContext;
+    const others = req.others ?? "keep";
     const source = req.sourceLabel ?? "influxdb-import";
     // The writer's drop counter is monotonic and shared with the live recorder,
     // so only the delta across this run is attributable to it.
     const droppedAtStart = writer.droppedLineCount ?? 0;
+    // The writer drops the OLDEST buffered lines when its cap is hit — while
+    // disconnected, or when QuestDB cannot keep up. Those rows were counted as
+    // written but never reached the database, so the run must not go on as if
+    // they had, and must not end as a success: the whole point of an import is
+    // knowing what actually landed.
+    //
+    // Checked before every row is written, not only at the end, and that is
+    // load-bearing for resume. A run that stops at its first drop never writes
+    // anything after a gap, so a window's last row, wherever it came from, is
+    // only ever in QuestDB with the whole window before it — which is what lets
+    // a saved position be confirmed by reading that row back, even one an
+    // earlier run wrote. Before the write rather than after: a drop registered
+    // while the run waited for the writer would otherwise be followed by one
+    // more row.
+    const failOnDrops = () => {
+        const droppedByWriter = (writer.droppedLineCount ?? 0) - droppedAtStart;
+        if (droppedByWriter > 0) {
+            throw new Error(`QuestDB could not keep up: ${droppedByWriter} buffered rows were dropped before reaching the database. ` +
+                `Re-run the import once QuestDB is healthy (already-imported rows are not duplicated).`);
+        }
+        // A write into a full buffer would itself cause the drop — and be the
+        // row behind the gap. Refused instead, before it is enqueued.
+        if (writer.atCapacity) {
+            throw new Error(`QuestDB could not keep up: the writer's buffer is full. ` +
+                `Re-run the import once QuestDB is healthy (already-imported rows are not duplicated).`);
+        }
+    };
     try {
         // Checked BEFORE discovery: listMeasurements is a network round trip, and
         // running it first means a bad range costs a request and — worse — any
@@ -486,6 +744,24 @@ export async function runMigration(req, writer, run, deps = {}) {
         if (toMs <= fromMs) {
             throw new Error("`to` must be after `from`");
         }
+        // Before the first await, so a caller that started this run without
+        // awaiting it — the HTTP handler — already sees it as resumed, with the
+        // earlier totals, in the response it sends straight away.
+        const now = deps.now ?? Date.now;
+        const identity = migrationIdentity(req, windowMs);
+        const resume = deps.resumeFrom && sameIdentity(deps.resumeFrom.identity, identity)
+            ? deps.resumeFrom
+            : undefined;
+        const done = new Set(resume?.done ?? []);
+        if (resume) {
+            run.progress.read = resume.progress.read;
+            run.progress.written = resume.progress.written;
+            run.progress.skipped = resume.progress.skipped;
+            run.resumedFrom = {
+                measurement: resume.current?.measurement,
+                windowStart: resume.current && new Date(resume.current.windowStart).toISOString(),
+            };
+        }
         let measurements = req.measurements ?? [];
         if (measurements.length === 0) {
             measurements = (await listMeasurements(req, fetchImpl)).map((m) => m.name);
@@ -500,6 +776,33 @@ export async function runMigration(req, writer, run, deps = {}) {
             }
         }
         run.progress.measurementsTotal = measurements.length;
+        const tracker = deps.checkpoints
+            ? new CheckpointTracker(deps.checkpoints, writer, deps.checkpointLagMs, now, deps.confirmStored)
+            : undefined;
+        // The newest row written to each table, kept up to date by writeRow: what
+        // a position offered after it is confirmed against.
+        const tail = { source };
+        // A checkpoint that cannot be written costs a later resume, nothing more.
+        // It must not cost the import that is running.
+        const offerCheckpoint = async (current) => {
+            try {
+                await tracker?.offer(() => ({
+                    version: 1,
+                    identity,
+                    done: [...done],
+                    current,
+                    progress: {
+                        read: run.progress.read,
+                        written: run.progress.written,
+                        skipped: run.progress.skipped,
+                    },
+                    updatedAt: new Date(now()).toISOString(),
+                }), tail);
+            }
+            catch (err) {
+                debug(`migration: checkpoint not written: ${err instanceof Error ? err.message : String(err)}`);
+            }
+        };
         // Wait for the writer to fall back to the resume mark.
         //
         // Two marks, not one: resuming at the high-water mark would park the
@@ -524,56 +827,126 @@ export async function runMigration(req, writer, run, deps = {}) {
         for (const measurement of measurements) {
             if (run.isCancelled)
                 break;
+            if (done.has(measurement)) {
+                run.progress.measurementsDone++;
+                continue;
+            }
             run.progress.currentMeasurement = measurement;
-            for (let start = fromMs; start < toMs; start += windowMs) {
+            // Only a start this run would itself arrive at: anything else in the
+            // file is not a position of this import, and the measurement is read
+            // from its beginning instead.
+            let firstStart = fromMs;
+            const resumeAt = resume?.current?.measurement === measurement
+                ? resume.current.windowStart
+                : undefined;
+            if (resumeAt !== undefined &&
+                resumeAt > fromMs &&
+                resumeAt < toMs &&
+                (resumeAt - fromMs) % windowMs === 0) {
+                firstStart = resumeAt;
+            }
+            for (let start = firstStart; start < toMs; start += windowMs) {
                 if (run.isCancelled)
                     break;
                 const end = Math.min(start + windowMs, toMs);
                 run.progress.currentWindowStart = new Date(start).toISOString();
                 // Backpressure: let the writer drain before reading more.
                 await awaitDrain();
-                const { rows, dropped } = req.type === "influxdb2"
-                    ? await readWindowFlux(req, measurement, start, end, headers, fetchImpl)
-                    : await readWindowInfluxQl(req, measurement, start, end, headers, fetchImpl);
-                // Half-pair positions never become rows, so they are counted here or
-                // not at all — otherwise read - written - skipped silently disagrees.
-                run.progress.read += dropped;
-                run.progress.skipped += dropped;
-                // Checked inside the row loop too, not only between windows: a single
-                // dense window can carry far more rows than the writer's cap on its
-                // own, and waiting only at the window boundary would let the buffer
-                // sail past MAX_BUFFER_LINES mid-window — dropping its OLDEST lines,
-                // which is exactly the silent data loss this guard exists to prevent.
+                const batches = req.type === "influxdb2"
+                    ? readWindowFlux(req, measurement, start, end, headers, fetchImpl, readIdleTimeoutMs)
+                    : readWindowInfluxQl(req, measurement, start, end, headers, fetchImpl, readIdleTimeoutMs);
+                // The window is read a batch at a time and each batch is written
+                // before the next is asked for, so what is held never grows with the
+                // window. Leaving the loop early — a cancel — ends the read as well.
                 let sinceDrainCheck = 0;
-                for (const row of rows) {
+                for await (const { rows, dropped } of batches) {
                     if (run.isCancelled)
                         break;
-                    if (++sinceDrainCheck >= 1000) {
-                        sinceDrainCheck = 0;
-                        await awaitDrain();
+                    // Half-pair positions never become rows, so they are counted here or
+                    // not at all — otherwise read - written - skipped silently disagrees.
+                    run.progress.read += dropped;
+                    run.progress.skipped += dropped;
+                    // Checked inside the row loop too, not only between batches: waiting
+                    // only at a boundary would let the buffer sail past
+                    // MAX_BUFFER_LINES mid-batch — dropping its OLDEST lines, which is
+                    // exactly the silent data loss this guard exists to prevent.
+                    for (const row of rows) {
+                        if (run.isCancelled)
+                            break;
+                        if (++sinceDrainCheck >= 1000) {
+                            sinceDrainCheck = 0;
+                            await awaitDrain();
+                        }
+                        failOnDrops();
+                        run.progress.read++;
+                        const target = targetContext(row.context, sourceSelf, context, others);
+                        if (target === null) {
+                            run.progress.skipped++;
+                            continue;
+                        }
+                        const written = writeRow(row, measurement, target, source, writer, tail);
+                        if (written)
+                            run.progress.written++;
+                        else
+                            run.progress.skipped++;
                     }
-                    run.progress.read++;
-                    const written = writeRow(row, measurement, context, source, writer);
-                    if (written)
-                        run.progress.written++;
-                    else
-                        run.progress.skipped++;
+                    // Again after the batch, so a drop on its last row ends the run
+                    // here rather than after the next window has been requested.
+                    failOnDrops();
+                }
+                // A window left early is not an imported window.
+                if (run.isCancelled)
+                    break;
+                if (start + windowMs < toMs) {
+                    await offerCheckpoint({
+                        measurement,
+                        windowStart: start + windowMs,
+                    });
                 }
             }
+            if (run.isCancelled)
+                break;
+            done.add(measurement);
             run.progress.measurementsDone++;
+            await offerCheckpoint();
             debug(`migration: ${measurement} done (${run.progress.written} written, ${run.progress.skipped} skipped)`);
         }
-        // The writer drops the OLDEST buffered lines when its cap is hit — while
-        // disconnected, or when QuestDB cannot keep up. Those rows were counted as
-        // written but never reached the database, so a run that ends there is NOT
-        // a clean success and must not be reported as one: the whole point of an
-        // import is knowing what actually landed.
-        const droppedByWriter = (writer.droppedLineCount ?? 0) - droppedAtStart;
-        if (droppedByWriter > 0) {
-            throw new Error(`QuestDB could not keep up: ${droppedByWriter} buffered rows were dropped before reaching the database. ` +
-                `Re-run the import once QuestDB is healthy (already-imported rows are not duplicated).`);
+        failOnDrops();
+        // "Done" is a claim about QuestDB, not about the writer. The last rows
+        // are still on their way when the loop ends, so wait for them to leave
+        // the writer and, where QuestDB can be asked, for it to read the newest
+        // of them back — the same standard a saved position is held to. A run
+        // that cannot get that within the drain timeout fails and keeps its
+        // checkpoint, so the last windows are done again rather than assumed.
+        if (!run.isCancelled) {
+            const mark = writer.enqueuedLineCount;
+            let waited = 0;
+            while (!run.isCancelled) {
+                failOnDrops();
+                const settled = mark === undefined ||
+                    writer.settledLineCount === undefined ||
+                    writer.settledLineCount + (writer.droppedLineCount ?? 0) >= mark;
+                const confirmed = settled &&
+                    (!deps.confirmStored ||
+                        (await deps.confirmStored(tail).catch(() => false)));
+                if (confirmed)
+                    break;
+                if (waited >= WRITER_DRAIN_TIMEOUT_MS) {
+                    throw new Error(`QuestDB has not confirmed the last rows after ${Math.round(WRITER_DRAIN_TIMEOUT_MS / 1000)}s. ` +
+                        `Re-run the import once QuestDB is healthy (already-imported rows are not duplicated).`);
+                }
+                await sleep(200);
+                waited += 200;
+            }
         }
         run.state = run.isCancelled ? "cancelled" : "done";
+        // Nothing left to resume. Kept on a cancel or a failure, which is exactly
+        // when a later run wants it.
+        if (run.state === "done") {
+            await deps.checkpoints?.clear().catch((err) => {
+                debug(`migration: finished checkpoint not removed: ${err instanceof Error ? err.message : String(err)}`);
+            });
+        }
     }
     catch (err) {
         run.state = "failed";
@@ -585,6 +958,27 @@ export async function runMigration(req, writer, run, deps = {}) {
         run.progress.currentWindowStart = undefined;
     }
 }
+/**
+ * The Signal K context a row is written under, or null for a row that is left
+ * out — counted as skipped by the caller.
+ *
+ * The source's own vessel becomes `self`, whether the point was tagged with
+ * the source's own context, with the literal `vessels.self`, or not at all.
+ * Any other tag is another vessel: kept under its own context if it is a
+ * well-formed Signal K one, since that is what the history API is queried by,
+ * or left out when other vessels are not wanted. With no `sourceSelf` given,
+ * every row is the own vessel — what a single-vessel source amounts to.
+ */
+export function targetContext(rowContext, sourceSelf, self, others) {
+    if (rowContext === undefined ||
+        rowContext === "vessels.self" ||
+        rowContext === sourceSelf ||
+        sourceSelf === undefined)
+        return self;
+    if (others === "skip")
+        return null;
+    return isSignalKContext(rowContext) ? rowContext : null;
+}
 /**
  * Map one source row onto the plugin's tables, reusing the live path's routing
  * so imported data is classified exactly like recorded data.
@@ -592,7 +986,7 @@ export async function runMigration(req, writer, run, deps = {}) {
  * Returns false when the row could not be represented, which the caller counts
  * as skipped.
  */
-function writeRow(row, measurement, context, source, writer) {
+function writeRow(row, measurement, context, source, writer, tail) {
     const path = toSignalKPath(measurement, row.field);
     if (!path)
         return false;
@@ -600,16 +994,20 @@ function writeRow(row, measurement, context, source, writer) {
     switch (route) {
         case "number":
             writer.writeAtNanos(path, context, row.value, row.tsNanos, source);
+            tail.numeric = { path, context, tsNanos: row.tsNanos };
             return true;
         case "string":
             writer.writeStringAtNanos(path, context, row.value, row.tsNanos, undefined, source);
+            tail.string = { path, context, tsNanos: row.tsNanos };
             return true;
         case "boolean":
             writer.writeStringAtNanos(path, context, String(row.value), row.tsNanos, "boolean", source);
+            tail.string = { path, context, tsNanos: row.tsNanos };
             return true;
         case "position": {
             const v = row.value;
             writer.writePositionAtNanos(context, v, row.tsNanos, source);
+            tail.position = { context, tsNanos: row.tsNanos };
             return true;
         }
         case "flatten": {
@@ -623,12 +1021,15 @@ function writeRow(row, measurement, context, source, writer) {
             for (const leaf of leaves) {
                 if (typeof leaf.value === "number") {
                     writer.writeAtNanos(leaf.path, context, leaf.value, row.tsNanos, source);
+                    tail.numeric = { path: leaf.path, context, tsNanos: row.tsNanos };
                 }
                 else if (typeof leaf.value === "boolean") {
                     writer.writeStringAtNanos(leaf.path, context, String(leaf.value), row.tsNanos, "boolean", source);
+                    tail.string = { path: leaf.path, context, tsNanos: row.tsNanos };
                 }
                 else {
                     writer.writeStringAtNanos(leaf.path, context, leaf.value, row.tsNanos, undefined, source);
+                    tail.string = { path: leaf.path, context, tsNanos: row.tsNanos };
                 }
                 any = true;
             }
@@ -639,35 +1040,74 @@ function writeRow(row, measurement, context, source, writer) {
     }
 }
 /**
- * Read one time window of a measurement from InfluxDB 2.x.
+ * How many Flux records go into one batch. Bounds what a window holds at once:
+ * a batch is parsed, written and released before the next is read.
+ */
+const FLUX_BATCH_RECORDS = 5_000;
+/**
+ * Whether a measurement may hold a position split across latitude/longitude
+ * fields — the measurements mergePositionRows reassembles.
+ */
+function isPositionMeasurement(measurement) {
+    return measurement.includes("position");
+}
+/**
+ * Turn the rows gathered so far into a batch.
  *
- * `navigation.position` is stored by the Signal K plugins as two fields on one
- * measurement, so the rows are pivoted back into a single object value —
- * otherwise latitude and longitude would land as two meaningless numeric paths
- * instead of a position.
+ * Decoding and position reassembly work within the batch, so a batch must
+ * hold WHOLE points: every field of an instant together. Both readers
+ * guarantee that — see each for how.
  */
-async function readWindowFlux(req, measurement, startMs, endMs, headers, fetchImpl) {
-    const flux = `from(bucket: ${JSON.stringify(req.bucket)})
+function finishBatch(measurement, rows, unusable) {
+    const decoded = decodeJsonValueRows(rows);
+    const merged = mergePositionRows(measurement, decoded.rows);
+    return {
+        rows: merged.rows,
+        dropped: merged.dropped + decoded.dropped + unusable,
+    };
+}
+/**
+ * Read one time window of a measurement from InfluxDB 2.x, in batches.
+ *
+ * Flux returns one table per field, each running the length of the window. For
+ * most measurements that is fine — every record is a complete value. A
+ * position is not: its latitude and longitude arrive as two separate tables,
+ * the second starting only after the first has ended, so pairing them from the
+ * stream would mean holding a whole table back. Position measurements are
+ * therefore pivoted by InfluxDB, which puts every field of an instant on one
+ * record and lets a position be assembled record by record.
+ */
+async function* readWindowFlux(req, measurement, startMs, endMs, headers, fetchImpl, idleTimeoutMs) {
+    const pivoted = isPositionMeasurement(measurement);
+    const source = `from(bucket: ${JSON.stringify(req.bucket)})
   |> range(start: ${new Date(startMs).toISOString()}, stop: ${new Date(endMs).toISOString()})
-  |> filter(fn: (r) => r._measurement == ${JSON.stringify(measurement)})
-  |> keep(columns: ["_time", "_field", "_value"])`;
-    const records = await runFlux(req, flux, headers, fetchImpl);
-    const rows = [];
+  |> filter(fn: (r) => r._measurement == ${JSON.stringify(measurement)})`;
+    const flux = pivoted
+        ? `${source}
+  |> pivot(rowKey: ["_time"], columnKey: ["_field"], valueColumn: "_value")
+  |> drop(columns: ["_start", "_stop", "_measurement"])`
+        : `${source}
+  |> keep(columns: ["_time", "_field", "_value", "context"])`;
+    const csv = new AnnotatedCsvReader();
+    let rows = [];
+    let records = 0;
     // A record with no usable timestamp cannot be placed in time, so it is not
     // importable — but it IS reported. The module's contract is that nothing is
     // dropped silently, and a `read` total that quietly excludes these makes an
     // import look smaller than the source rather than showing what was lost.
     let unusable = 0;
-    for (const rec of records) {
+    const take = (rec) => {
+        records++;
         const t = rec.values["_time"];
-        if (!t) {
-            unusable++;
-            continue;
-        }
-        const tsNanos = rfc3339ToNanos(t);
+        const tsNanos = t ? rfc3339ToNanos(t) : null;
         if (tsNanos === null) {
             unusable++;
-            continue;
+            return;
+        }
+        const context = rec.values["context"] || undefined;
+        if (pivoted) {
+            pushPivotedFields(rec, tsNanos, context, rows);
+            return;
         }
         rows.push({
             tsNanos,
@@ -675,13 +1115,72 @@ async function readWindowFlux(req, measurement, startMs, endMs, headers, fetchIm
             // The declared type decides: a genuine string "3.5" must stay a string
             // rather than being parsed into the numeric table.
             value: coerceValue(rec.values["_value"] ?? "", rec.types["_value"]),
+            context,
         });
+    };
+    const lines = streamLines(fetchImpl, fluxQueryUrl(req), fluxQueryInit(flux, headers), "read data", idleTimeoutMs);
+    for await (const line of lines) {
+        const rec = csv.push(line);
+        if (!rec)
+            continue;
+        take(rec);
+        if (records >= FLUX_BATCH_RECORDS) {
+            yield finishBatch(measurement, rows, unusable);
+            rows = [];
+            records = 0;
+            unusable = 0;
+        }
     }
-    const merged = mergePositionRows(measurement, rows);
-    return { rows: merged.rows, dropped: merged.dropped + unusable };
+    const last = csv.end();
+    if (last)
+        take(last);
+    if (records > 0)
+        yield finishBatch(measurement, rows, unusable);
 }
-/** Read one time window of a measurement from InfluxDB 1.x. */
-async function readWindowInfluxQl(req, measurement, startMs, endMs, headers, fetchImpl) {
+/** Columns of a pivoted record that are bookkeeping rather than fields. */
+const PIVOT_NON_FIELD_COLUMNS = new Set([
+    "result",
+    "table",
+    "_time",
+]);
+/**
+ * The fields of one pivoted record, as rows at its instant.
+ *
+ * A pivoted record carries the series' tags as columns too. They are what the
+ * table is grouped by, which is how they are told from fields: reading them as
+ * fields would import a bogus `measurement.source` path for every point.
+ *
+ * An empty cell is a field this point does not have, not a value.
+ */
+function pushPivotedFields(rec, tsNanos, context, out) {
+    for (const column of Object.keys(rec.values)) {
+        if (PIVOT_NON_FIELD_COLUMNS.has(column) || rec.grouped.has(column))
+            continue;
+        const raw = rec.values[column];
+        if (raw === "")
+            continue;
+        out.push({
+            tsNanos,
+            field: column,
+            value: coerceValue(raw, rec.types[column]),
+            context,
+        });
+    }
+}
+/**
+ * How many points InfluxDB 1.x puts in one chunk of a chunked response. Each
+ * chunk becomes a batch, so this bounds what a window holds at once.
+ */
+const INFLUXQL_CHUNK_POINTS = 10_000;
+/**
+ * Read one time window of a measurement from InfluxDB 1.x, in batches.
+ *
+ * `chunked=true` makes InfluxDB answer with a series of complete JSON
+ * documents, one per line, instead of one document for the whole window. A
+ * 1.x row already carries every field of its point, so each chunk holds whole
+ * points and can be finished on its own.
+ */
+async function* readWindowInfluxQl(req, measurement, startMs, endMs, headers, fetchImpl, idleTimeoutMs) {
     // Measurement names come from SHOW MEASUREMENTS on this same server, but
     // they still land inside a query string — quote them as identifiers.
     //
@@ -696,28 +1195,46 @@ async function readWindowInfluxQl(req, measurement, startMs, endMs, headers, fet
     // bogus `measurement.source = "n2k"` string path for every single point.
     // Signal K's own InfluxDB writers tag their points, so this is the common
     // case, not an exotic one.
-    const q = `SELECT *::field FROM ${quoted} WHERE time >= ${BigInt(startMs) * NANOS_PER_MS} AND time < ${BigInt(endMs) * NANOS_PER_MS}`;
-    const url = `${req.url}/query?db=${encodeURIComponent(req.bucket)}&epoch=ns&q=${encodeURIComponent(q)}`;
-    const r = await fetchImpl(url, {
-        headers,
-        signal: AbortSignal.timeout(120_000),
-    });
-    if (!r.ok)
-        throw new Error(await describeHttpError(r, "read data"));
-    const body = (await r.json());
-    if (body.results?.[0]?.error)
-        throw new Error(body.results[0].error);
-    const series = body.results?.[0]?.series ?? [];
-    const rows = [];
-    // See readWindowFlux: values that cannot be imported are reported, not
-    // dropped in silence.
+    // The context tag rides along as one more column, named after the tag. A
+    // measurement without it simply has no such column.
+    const q = `SELECT *::field, "context"::tag FROM ${quoted} WHERE time >= ${BigInt(startMs) * NANOS_PER_MS} AND time < ${BigInt(endMs) * NANOS_PER_MS}`;
+    const url = `${req.url}/query?db=${encodeURIComponent(req.bucket)}&epoch=ns&chunked=true&chunk_size=${INFLUXQL_CHUNK_POINTS}&q=${encodeURIComponent(q)}`;
+    for await (const line of streamLines(fetchImpl, url, { headers }, "read data", idleTimeoutMs)) {
+        if (line.trim() === "")
+            continue;
+        let body;
+        try {
+            body = JSON.parse(line);
+        }
+        catch {
+            throw new Error("Cannot read data: InfluxDB returned a malformed response");
+        }
+        if (body.error)
+            throw new Error(body.error);
+        if (body.results?.[0]?.error)
+            throw new Error(body.results[0].error);
+        const rows = [];
+        const unusable = pushInfluxQlSeries(body.results?.[0]?.series ?? [], rows);
+        if (rows.length > 0 || unusable > 0)
+            yield finishBatch(measurement, rows, unusable);
+    }
+}
+/**
+ * The points of one InfluxQL result, as rows. Returns how many values could
+ * not be imported — reported, like readWindowFlux's, not dropped in silence.
+ */
+function pushInfluxQlSeries(series, rows) {
     let unusable = 0;
     for (const s of series) {
         const columns = s.columns ?? [];
         const timeIdx = columns.indexOf("time");
         if (timeIdx < 0)
             continue;
+        const contextIdx = columns.indexOf("context");
         for (const values of s.values ?? []) {
+            const context = contextIdx >= 0 && typeof values[contextIdx] === "string"
+                ? values[contextIdx] || undefined
+                : undefined;
             // epoch=ns returns the time as a number; going through BigInt(String)
             // avoids the precision loss a float64 would suffer past 2^53 ns.
             const rawTime = values[timeIdx];
@@ -731,7 +1248,7 @@ async function readWindowInfluxQl(req, measurement, startMs, endMs, headers, fet
                 continue;
             }
             for (let i = 0; i < columns.length; i++) {
-                if (i === timeIdx)
+                if (i === timeIdx || i === contextIdx)
                     continue;
                 const raw = values[i];
                 // A wide row has a column per field; nulls are fields absent from
@@ -758,39 +1275,55 @@ async function readWindowInfluxQl(req, measurement, startMs, endMs, headers, fet
                     tsNanos,
                     field: columns[i],
                     value: raw,
+                    context,
                 });
             }
         }
     }
-    const merged = mergePositionRows(measurement, rows);
-    return { rows: merged.rows, dropped: merged.dropped + unusable };
+    return unusable;
 }
 /**
  * Recombine latitude/longitude fields of a position measurement into one
  * object value at the same instant.
  *
- * The Signal K InfluxDB plugins store navigation.position as two fields
- * (`latitude`/`longitude`, or `lat`/`lon`). Left as-is they would import as
- * two numeric paths and the position history would be unusable — QuestDB keeps
- * positions in their own table.
+ * signalk-to-influxdb2 stores navigation.position as two fields (`lat`/`lon`),
+ * as does signalk-to-influxdb 1.x with `separateLatLon` on. Left as-is they
+ * would import as two numeric paths and the position history would be unusable
+ * — QuestDB keeps positions in their own table. The 1.x default encoding, a
+ * single `jsonValue`, is already whole by the time it gets here (see
+ * decodeJsonValueRows).
  */
 export function mergePositionRows(measurement, rows) {
     if (!measurement.includes("position"))
         return { rows, dropped: 0 };
     const latKeys = new Set(["latitude", "lat"]);
     const lonKeys = new Set(["longitude", "lon", "lng"]);
+    // Keyed by vessel AND instant: two vessels' fixes at the same instant are
+    // two positions, and a latitude of one must never pair with a longitude of
+    // the other.
     const byTime = new Map();
     const passthrough = [];
+    // Instants that already carry a whole position. signalk-to-influxdb 1.x with
+    // `separateLatLon` on writes `jsonValue` AND `lat`/`lon` for the same fix, so
+    // the pair is a second copy of a row that is already going to be written.
+    const wholePositionTimes = new Set();
     for (const row of rows) {
-        const key = row.tsNanos.toString();
-        if (latKeys.has(row.field) && typeof row.value === "number") {
+        const key = `${row.context ?? ""}|${row.tsNanos}`;
+        // Routed on the mapped path, as writeRow does, so a row is only counted as
+        // a whole position here if it is going to be written as one.
+        if (routeDeltaValue(toSignalKPath(measurement, row.field), row.value) ===
+            "position") {
+            wholePositionTimes.add(key);
+            passthrough.push(row);
+        }
+        else if (latKeys.has(row.field) && typeof row.value === "number") {
             if (!byTime.has(key))
-                byTime.set(key, {});
+                byTime.set(key, { context: row.context, tsNanos: row.tsNanos });
             byTime.get(key).lat = row.value;
         }
         else if (lonKeys.has(row.field) && typeof row.value === "number") {
             if (!byTime.has(key))
-                byTime.set(key, {});
+                byTime.set(key, { context: row.context, tsNanos: row.tsNanos });
             byTime.get(key).lon = row.value;
         }
         else {
@@ -799,7 +1332,9 @@ export function mergePositionRows(measurement, rows) {
     }
     const merged = [];
     let dropped = 0;
-    for (const [key, { lat, lon }] of byTime) {
+    for (const [key, { context, tsNanos, lat, lon }] of byTime) {
+        if (wholePositionTimes.has(key))
+            continue;
         // A half-pair (lat with no lon at the same instant) is not a position.
         // Emitting it as a bare number would be worse than skipping it, so it is
         // dropped — and REPORTED, so it lands in the run's skipped total instead
@@ -809,9 +1344,10 @@ export function mergePositionRows(measurement, rows) {
             continue;
         }
         merged.push({
-            tsNanos: BigInt(key),
+            tsNanos,
             field: "value",
             value: { latitude: lat, longitude: lon },
+            ...(context === undefined ? {} : { context }),
         });
     }
     return { rows: [...passthrough, ...merged], dropped };