signalk-questdb 2.2.2 → 2.3.0

package/dist/gpx-import.js

@@ -0,0 +1,452 @@
+// Importing a GPX track into the archive.
+//
+// A chartplotter's track export runs to gigabytes — a TimeZero export of four
+// years is 1.5 GB and 11 million points — so nothing here holds a document.
+// The browser sends the file in slices, each slice is parsed as it arrives,
+// and its points go to the ILP writer before the next slice is accepted. Memory
+// stays at one slice plus the parser's state, whatever the file's size.
+//
+// Points land in the position archive, the table retention never drops,
+// under their own source, so an import can be told apart from what the
+// server recorded and a second import of the same file upserts onto the
+// first instead of doubling it. They are thinned on the way in to the
+// archive's resolution, by stamping each on the start of its interval and
+// letting the table's dedup keep one per interval: an export's block seams
+// and overlapping tracks then collapse without the importer having to see
+// them.
+import { SaxesParser } from "saxes";
+import { ARCHIVE_TABLES, readFrom } from "./archive.js";
+/** The sourceRef imported positions are stored under. */
+export const IMPORT_SOURCE = "import.gpx";
+// Slices handed to the parser at once. Each runs synchronously, so this
+// bounds how long an import holds the event loop between yields.
+const SLICE_BYTES = 64 * 1024;
+// A point is left out where the vessel already has a recorded position in the
+// same five minutes: interleaving two receivers' fixes draws a sawtooth.
+const COVERAGE_MS = 5 * 60 * 1000;
+const DAY_MS = 24 * 60 * 60 * 1000;
+// Days of coverage kept while an import runs. An export is mostly in time
+// order, so only the days around the current point are ever asked again.
+const COVERAGE_DAYS_CACHED = 8;
+// Track names listed individually in the summary; the rest are counted under
+// one key. The summary goes out with every slice's reply, and a file can hold
+// a whole anchorage's worth of AIS targets.
+const MAX_NAMED = 100;
+const OTHER_NAMES = "(other tracks)";
+const GPX_NAMESPACES = new Set([
+    "",
+    "http://www.topografix.com/GPX/1/1",
+    "http://www.topografix.com/GPX/1/0",
+]);
+/**
+ * An attribute as an `xsd:decimal`, or NaN. `Number()` reads `0x10` and
+ * `1e1` too, which GPX does not allow and no exporter means.
+ */
+export function gpxDecimal(value) {
+    return value !== undefined && /^[+-]?(?:\d+(?:\.\d*)?|\.\d+)$/.test(value)
+        ? Number(value)
+        : Number.NaN;
+}
+/**
+ * A `<time>` as epoch milliseconds, or NaN. GPX types it `xsd:dateTime`;
+ * `Date.parse` alone takes a bare year and rolls February 30th into March.
+ */
+export function gpxDateTime(value) {
+    const match = /^(\d{4})-(\d{2})-(\d{2})T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/.exec(value);
+    if (!match)
+        return Number.NaN;
+    const parsed = Date.parse(value);
+    if (Number.isNaN(parsed))
+        return Number.NaN;
+    const year = Number(match[1]);
+    const month = Number(match[2]);
+    const day = Number(match[3]);
+    const calendar = new Date(Date.UTC(year, month - 1, day));
+    return calendar.getUTCFullYear() === year &&
+        calendar.getUTCMonth() === month - 1 &&
+        calendar.getUTCDate() === day
+        ? parsed
+        : Number.NaN;
+}
+/**
+ * A longitude brought into [-180, 180], or NaN when it is not one.
+ *
+ * Crossing the antimeridian, TimeZero writes the fixes just past it as
+ * -180.00002 rather than 179.99998. Those are real positions, so they are
+ * wrapped; a value beyond a full turn is not a position at all.
+ */
+export function antimeridian(longitude) {
+    if (!(longitude > -360 && longitude < 360))
+        return Number.NaN;
+    if (longitude < -180)
+        return longitude + 360;
+    if (longitude > 180)
+        return longitude - 360;
+    return longitude;
+}
+// Names counted so far per tally, so a point past the cap costs no scan of
+// the keys: in an export full of AIS targets that is most points.
+const NAMED = new WeakMap();
+/** Count one point against a track name, bounded at MAX_NAMED names. */
+function tally(counts, name) {
+    // Own properties only: a track named "constructor" is a name like any
+    // other, not the object's inherited function.
+    if (Object.hasOwn(counts, name)) {
+        counts[name] += 1;
+        return;
+    }
+    const named = NAMED.get(counts) ?? 0;
+    if (named < MAX_NAMED) {
+        counts[name] = 1;
+        NAMED.set(counts, named + 1);
+        return;
+    }
+    counts[OTHER_NAMES] =
+        (Object.hasOwn(counts, OTHER_NAMES) ? counts[OTHER_NAMES] : 0) + 1;
+}
+/** The document is not well-formed XML; rows before the error are kept. */
+export class GpxParseError extends Error {
+}
+/** The import was aborted while a slice was being written. */
+export class GpxImportCancelled extends Error {
+}
+/**
+ * One import, fed the document slice by slice.
+ *
+ * Only GPX's own structure counts: a `trk` under `gpx`, a `trkseg` under it,
+ * a `trkpt` under that. A vendor `<trkseg>` inside `<extensions>` is not a
+ * segment, and a `<name>` inside a point is not the track's name.
+ */
+export class GpxImport {
+    options;
+    summary = {
+        bytes: 0,
+        tracks: 0,
+        // Without a prototype: a track name is the file's to choose, and one
+        // named "__proto__" must be a key like any other, not the setter.
+        importedTracks: Object.create(null),
+        skippedTracks: Object.create(null),
+        points: 0,
+        imported: 0,
+        thinned: 0,
+        alreadyRecorded: 0,
+        invalid: 0,
+        from: null,
+        to: null,
+    };
+    parser = new SaxesParser({ xmlns: true });
+    decoder = new TextDecoder("utf-8", { fatal: true });
+    parseError = null;
+    cancelled = false;
+    // Local names of the open GPX elements, "" for a foreign one, so a
+    // structural element is recognised only at its place in the hierarchy.
+    path = [];
+    trackName = null;
+    nameText = "";
+    selected = false;
+    segment = 0;
+    point = null;
+    timeText = "";
+    pending = [];
+    lastSegment = -1;
+    last = null;
+    lastBucket = Number.NaN;
+    coverage = new Map();
+    coveredDay = Number.NaN;
+    covered = new Set();
+    constructor(options) {
+        this.options = options;
+        this.parser.on("opentag", (tag) => this.open(tag));
+        this.parser.on("closetag", () => this.close());
+        this.parser.on("text", (text) => this.text(text));
+        this.parser.on("cdata", (text) => this.text(text));
+        // saxes reports and carries on; an import stops at the first error.
+        this.parser.on("error", (err) => {
+            this.parseError ??= err;
+        });
+    }
+    /**
+     * The instant of the row written last, or null before the first. Not
+     * `summary.to`: an export out of time order writes its latest row early.
+     */
+    get lastWritten() {
+        return this.last;
+    }
+    /** Stop writing: the slice in progress ends at its next point. */
+    cancel() {
+        this.cancelled = true;
+    }
+    /** Parse the next bytes of the document and write the points they hold. */
+    async write(chunk) {
+        for (let at = 0; at < chunk.length; at += SLICE_BYTES) {
+            const slice = chunk.subarray(at, at + SLICE_BYTES);
+            let text;
+            try {
+                text = this.decoder.decode(slice, { stream: true });
+            }
+            catch {
+                throw new GpxParseError(`The file is not UTF-8 near byte ${this.summary.bytes}`);
+            }
+            this.parse(text);
+            this.summary.bytes += slice.length;
+            // The points read before an error are sound; they go out first.
+            await this.drain();
+            this.throwParseError();
+            await new Promise((resolve) => setImmediate(resolve));
+        }
+    }
+    /** The document has ended: anything left unclosed is an error. */
+    async finish() {
+        let rest;
+        try {
+            rest = this.decoder.decode();
+        }
+        catch {
+            throw new GpxParseError("The file ends inside a UTF-8 character");
+        }
+        this.parse(rest);
+        this.parser.close();
+        await this.drain();
+        this.throwParseError();
+        return this.summary;
+    }
+    parse(text) {
+        if (text !== "" && !this.parseError)
+            this.parser.write(text);
+    }
+    throwParseError() {
+        if (this.parseError) {
+            throw new GpxParseError(`The file is not well-formed XML: ${this.parseError.message}`);
+        }
+    }
+    at(...names) {
+        if (this.path.length !== names.length)
+            return false;
+        return names.every((name, i) => this.path[i] === name);
+    }
+    open(tag) {
+        if (this.parseError)
+            return;
+        this.path.push(GPX_NAMESPACES.has(tag.uri) ? tag.local : "");
+        if (this.at("gpx", "trk")) {
+            this.summary.tracks++;
+            this.trackName = null;
+            this.selected = false;
+        }
+        else if (this.at("gpx", "trk", "name")) {
+            this.nameText = "";
+        }
+        else if (this.at("gpx", "trk", "trkseg")) {
+            // GPX puts <name> before the segments; a track whose name is missing
+            // or comes later is selected as the track without one.
+            if (this.trackName === null)
+                this.name("");
+            this.segment++;
+        }
+        else if (this.at("gpx", "trk", "trkseg", "trkpt")) {
+            this.point = {
+                latitude: gpxDecimal(tag.attributes.lat?.value),
+                longitude: gpxDecimal(tag.attributes.lon?.value),
+            };
+            this.timeText = "";
+        }
+    }
+    text(text) {
+        if (this.at("gpx", "trk", "name"))
+            this.nameText += text;
+        else if (this.at("gpx", "trk", "trkseg", "trkpt", "time")) {
+            this.timeText += text;
+        }
+    }
+    close() {
+        // saxes carries on past an error, so the rest of the slice it was in
+        // still produces events; none of them is trusted.
+        if (this.parseError)
+            return;
+        if (this.at("gpx", "trk", "name") && this.trackName === null) {
+            this.name(this.nameText.trim());
+        }
+        else if (this.at("gpx", "trk", "trkseg", "trkpt") && this.point) {
+            this.summary.points++;
+            const { latitude, longitude } = this.point;
+            this.point = null;
+            const name = this.trackName ?? "";
+            if (!this.selected) {
+                tally(this.summary.skippedTracks, name);
+            }
+            else {
+                tally(this.summary.importedTracks, name);
+                const ms = gpxDateTime(this.timeText.trim());
+                const lon = antimeridian(longitude);
+                if (Number.isNaN(ms) ||
+                    !(latitude >= -90 && latitude <= 90) ||
+                    Number.isNaN(lon)) {
+                    this.summary.invalid++;
+                }
+                else {
+                    this.pending.push({
+                        ms,
+                        latitude,
+                        longitude: lon,
+                        segment: this.segment,
+                    });
+                }
+            }
+        }
+        this.path.pop();
+    }
+    name(name) {
+        this.trackName = name;
+        this.selected = this.options.tracks.has(name);
+    }
+    async recordedIn(ms) {
+        const lookup = this.options.recorded;
+        if (!lookup)
+            return null;
+        const day = Math.floor(ms / DAY_MS) * DAY_MS;
+        if (day === this.coveredDay)
+            return this.covered;
+        let found = this.coverage.get(day);
+        if (!found) {
+            found = lookup(day);
+            this.coverage.set(day, found);
+            if (this.coverage.size > COVERAGE_DAYS_CACHED) {
+                const oldest = this.coverage.keys().next().value;
+                if (oldest !== undefined)
+                    this.coverage.delete(oldest);
+            }
+        }
+        this.covered = await found;
+        this.coveredDay = day;
+        return this.covered;
+    }
+    async drain() {
+        if (this.pending.length === 0)
+            return;
+        const points = this.pending;
+        this.pending = [];
+        const resolution = Math.floor(this.options.resolutionSeconds) * 1000;
+        for (const point of points) {
+            if (this.cancelled)
+                throw new GpxImportCancelled("The import was aborted");
+            const recorded = await this.recordedIn(point.ms);
+            if (recorded?.has(Math.floor(point.ms / COVERAGE_MS))) {
+                this.summary.alreadyRecorded++;
+                continue;
+            }
+            const bucket = resolution > 0 ? Math.floor(point.ms / resolution) : point.ms;
+            if (point.segment === this.lastSegment && bucket === this.lastBucket) {
+                this.summary.thinned++;
+                continue;
+            }
+            this.lastSegment = point.segment;
+            this.lastBucket = bucket;
+            const ms = resolution > 0 ? bucket * resolution : point.ms;
+            this.options.write(ms, point.latitude, point.longitude);
+            this.last = ms;
+            this.summary.imported++;
+            const iso = new Date(ms).toISOString();
+            if (this.summary.from === null || iso < this.summary.from) {
+                this.summary.from = iso;
+            }
+            if (this.summary.to === null || iso > this.summary.to) {
+                this.summary.to = iso;
+            }
+        }
+        await this.options.backpressure();
+    }
+}
+// Same marks as the InfluxDB migration: far below the writer's buffer cap,
+// which drops the OLDEST lines once reached — the very rows being imported.
+const WRITER_HIGH_WATER = 20_000;
+const WRITER_RESUME = 5_000;
+const WRITER_TIMEOUT_MS = 60_000;
+const WRITER_POLL_MS = 200;
+/**
+ * What an import needs from the shared ILP writer: to wait while it is full,
+ * to know when everything written has left it, and to notice when it had to
+ * drop lines, which may have been the import's.
+ */
+export class WriterGate {
+    writer;
+    pollMs;
+    timeoutMs;
+    droppedAtStart;
+    constructor(writer, pollMs = WRITER_POLL_MS, timeoutMs = WRITER_TIMEOUT_MS) {
+        this.writer = writer;
+        this.pollMs = pollMs;
+        this.timeoutMs = timeoutMs;
+        this.droppedAtStart = writer.droppedLineCount;
+    }
+    /** Resolves once the writer has room; rejects if it does not drain. */
+    async room() {
+        this.checkDrops();
+        if (this.writer.pendingLines <= WRITER_HIGH_WATER)
+            return;
+        await this.until(() => this.writer.pendingLines <= WRITER_RESUME, "QuestDB is not accepting writes");
+    }
+    /** Resolves once every line enqueued so far has left the writer. */
+    async settled() {
+        const mark = this.writer.enqueuedLineCount;
+        await this.until(() => this.writer.settledLineCount + this.writer.droppedLineCount >= mark, "QuestDB has not taken the last rows");
+        this.checkDrops();
+    }
+    async until(done, failure) {
+        const deadline = Date.now() + this.timeoutMs;
+        while (!done()) {
+            this.checkDrops();
+            if (Date.now() > deadline) {
+                throw new Error(`${failure} after ${Math.round(this.timeoutMs / 1000)} s. Run the ` +
+                    `import again once QuestDB is healthy: rows already imported are ` +
+                    `not duplicated.`);
+            }
+            await new Promise((resolve) => setTimeout(resolve, this.pollMs));
+        }
+    }
+    checkDrops() {
+        const dropped = this.writer.droppedLineCount - this.droppedAtStart;
+        if (dropped > 0) {
+            throw new Error(`The writer dropped ${dropped} lines while QuestDB was behind, so ` +
+                `part of the import may be missing. Run it again: rows already ` +
+                `imported are not duplicated.`);
+        }
+    }
+}
+/**
+ * The five-minute intervals of a day in which the own vessel has positions
+ * that did not come from an import, read from the base table and, where it
+ * covers the day, the archive.
+ */
+export function recordedIntervals(queryClient, archive) {
+    return async (dayStartMs) => {
+        const from = new Date(dayStartMs).toISOString();
+        const to = new Date(dayStartMs + DAY_MS).toISOString();
+        const table = readFrom("signalk_position", archive.covers("signalk_position", from, to));
+        const result = await queryClient.exec(`SELECT ts, count() FROM ${table} WHERE ts >= '${from}' AND ts < '${to}' ` +
+            `AND context = 'self' AND (source IS NULL OR source != '${IMPORT_SOURCE}') ` +
+            `SAMPLE BY 5m ALIGN TO CALENDAR`);
+        return new Set(result.dataset.map((row) => Math.floor(Date.parse(String(row[0])) / COVERAGE_MS)));
+    };
+}
+/**
+ * Wait until the imported row stamped `ms` reads back from the archive.
+ * Readable means QuestDB committed and applied it, and with it every row the
+ * writer sent before.
+ */
+export async function waitForImportedRow(queryClient, ms, timeoutMs, pollMs = 1000) {
+    const deadline = Date.now() + timeoutMs;
+    for (;;) {
+        const result = await queryClient.exec(`SELECT count() FROM ${ARCHIVE_TABLES.signalk_position} ` +
+            `WHERE ts = '${new Date(ms).toISOString()}' AND context = 'self' ` +
+            `AND source = '${IMPORT_SOURCE}'`);
+        if (Number(result.dataset[0]?.[0] ?? 0) > 0)
+            return;
+        if (Date.now() > deadline) {
+            throw new Error("QuestDB has not stored the last imported row. Run the import " +
+                "again once QuestDB is healthy: rows already imported are not " +
+                "duplicated.");
+        }
+        await new Promise((resolve) => setTimeout(resolve, pollMs));
+    }
+}
+//# sourceMappingURL=gpx-import.js.map

package/dist/gpx-import.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gpx-import.js","sourceRoot":"","sources":["../src/gpx-import.ts"],"names":[],"mappings":"AAAA,0CAA0C;AAC1C,EAAE;AACF,8EAA8E;AAC9E,4EAA4E;AAC5E,4EAA4E;AAC5E,gFAAgF;AAChF,wEAAwE;AACxE,EAAE;AACF,wEAAwE;AACxE,uEAAuE;AACvE,wEAAwE;AACxE,sEAAsE;AACtE,0EAA0E;AAC1E,2EAA2E;AAC3E,0EAA0E;AAC1E,QAAQ;AAER,OAAO,EAAE,WAAW,EAAmB,MAAM,OAAO,CAAC;AAIrD,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAsB,MAAM,cAAc,CAAC;AAE5E,yDAAyD;AACzD,MAAM,CAAC,MAAM,aAAa,GAAG,YAAY,CAAC;AAE1C,wEAAwE;AACxE,iEAAiE;AACjE,MAAM,WAAW,GAAG,EAAE,GAAG,IAAI,CAAC;AAE9B,8EAA8E;AAC9E,yEAAyE;AACzE,MAAM,WAAW,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AAClC,MAAM,MAAM,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AACnC,0EAA0E;AAC1E,yEAAyE;AACzE,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAE/B,6EAA6E;AAC7E,8EAA8E;AAC9E,4CAA4C;AAC5C,MAAM,SAAS,GAAG,GAAG,CAAC;AACtB,MAAM,WAAW,GAAG,gBAAgB,CAAC;AAErC,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC;IAC7B,EAAE;IACF,mCAAmC;IACnC,mCAAmC;CACpC,CAAC,CAAC;AAEH;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,KAAyB;IAClD,OAAO,KAAK,KAAK,SAAS,IAAI,gCAAgC,CAAC,IAAI,CAAC,KAAK,CAAC;QACxE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;QACf,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,KAAa;IACvC,MAAM,KAAK,GACT,4EAA4E,CAAC,IAAI,CAC/E,KAAK,CACN,CAAC;IACJ,IAAI,CAAC,KAAK;QAAE,OAAO,MAAM,CAAC,GAAG,CAAC;IAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACjC,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;QAAE,OAAO,MAAM,CAAC,GAAG,CAAC;IAC5C,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/B,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7B,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IAC1D,OAAO,QAAQ,CAAC,cAAc,EAAE,KAAK,IAAI;QACvC,QAAQ,CAAC,WAAW,EAAE,KAAK,KAAK,GAAG,CAAC;QACpC,QAAQ,CAAC,UAAU,EAAE,KAAK,GAAG;QAC7B,CAAC,CAAC,MAAM;QACR,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;AACjB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,YAAY,CAAC,SAAiB;IAC5C,IAAI,CAAC,CAAC,SAAS,GAAG,CAAC,GAAG,IAAI,SAAS,GAAG,GAAG,CAAC;QAAE,OAAO,MAAM,CAAC,GAAG,CAAC;IAC9D,IAAI,SAAS,GAAG,CAAC,GAAG;QAAE,OAAO,SAAS,GAAG,GAAG,CAAC;IAC7C,IAAI,SAAS,GAAG,GAAG;QAAE,OAAO,SAAS,GAAG,GAAG,CAAC;IAC5C,OAAO,SAAS,CAAC;AACnB,CAAC;AAiCD,2EAA2E;AAC3E,kEAAkE;AAClE,MAAM,KAAK,GAAG,IAAI,OAAO,EAAkC,CAAC;AAE5D,wEAAwE;AACxE,SAAS,KAAK,CAAC,MAA8B,EAAE,IAAY;IACzD,sEAAsE;IACtE,8CAA8C;IAC9C,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;QAChC,MAAM,CAAC,IAAI,CAAE,IAAI,CAAC,CAAC;QACnB,OAAO;IACT,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;QACtB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjB,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;QAC7B,OAAO;IACT,CAAC;IACD,MAAM,CAAC,WAAW,CAAC;QACjB,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AACxE,CAAC;AAED,2EAA2E;AAC3E,MAAM,OAAO,aAAc,SAAQ,KAAK;CAAG;AAE3C,8DAA8D;AAC9D,MAAM,OAAO,kBAAmB,SAAQ,KAAK;CAAG;AAEhD;;;;;;GAMG;AACH,MAAM,OAAO,SAAS;IAwCS;IAvCpB,OAAO,GAAqB;QACnC,KAAK,EAAE,CAAC;QACR,MAAM,EAAE,CAAC;QACT,qEAAqE;QACrE,kEAAkE;QAClE,cAAc,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAA2B;QAC7D,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAA2B;QAC5D,MAAM,EAAE,CAAC;QACT,QAAQ,EAAE,CAAC;QACX,OAAO,EAAE,CAAC;QACV,eAAe,EAAE,CAAC;QAClB,OAAO,EAAE,CAAC;QACV,IAAI,EAAE,IAAI;QACV,EAAE,EAAE,IAAI;KACT,CAAC;IAEe,MAAM,GAAG,IAAI,WAAW,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1C,OAAO,GAAG,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7D,UAAU,GAAiB,IAAI,CAAC;IAChC,SAAS,GAAG,KAAK,CAAC;IAE1B,mEAAmE;IACnE,uEAAuE;IACtD,IAAI,GAAa,EAAE,CAAC;IAC7B,SAAS,GAAkB,IAAI,CAAC;IAChC,QAAQ,GAAG,EAAE,CAAC;IACd,QAAQ,GAAG,KAAK,CAAC;IACjB,OAAO,GAAG,CAAC,CAAC;IACZ,KAAK,GAAmD,IAAI,CAAC;IAC7D,QAAQ,GAAG,EAAE,CAAC;IACd,OAAO,GAAY,EAAE,CAAC;IAEtB,WAAW,GAAG,CAAC,CAAC,CAAC;IACjB,IAAI,GAAkB,IAAI,CAAC;IAC3B,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC;IACf,QAAQ,GAAG,IAAI,GAAG,EAAwC,CAAC;IACpE,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC;IACxB,OAAO,GAAwB,IAAI,GAAG,EAAE,CAAC;IAEjD,YAA6B,OAAyB;QAAzB,YAAO,GAAP,OAAO,CAAkB;QACpD,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACnD,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;QAC/C,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAClD,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACnD,oEAAoE;QACpE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YAC9B,IAAI,CAAC,UAAU,KAAK,GAAG,CAAC;QAC1B,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAED,kEAAkE;IAClE,MAAM;QACJ,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,2EAA2E;IAC3E,KAAK,CAAC,KAAK,CAAC,KAAiB;QAC3B,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,KAAK,CAAC,MAAM,EAAE,EAAE,IAAI,WAAW,EAAE,CAAC;YACtD,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,GAAG,WAAW,CAAC,CAAC;YACnD,IAAI,IAAY,CAAC;YACjB,IAAI,CAAC;gBACH,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YACtD,CAAC;YAAC,MAAM,CAAC;gBACP,MAAM,IAAI,aAAa,CACrB,mCAAmC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CACxD,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACjB,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;YACnC,gEAAgE;YAChE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACnB,IAAI,CAAC,eAAe,EAAE,CAAC;YACvB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IAED,kEAAkE;IAClE,KAAK,CAAC,MAAM;QACV,IAAI,IAAY,CAAC;QACjB,IAAI,CAAC;YACH,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,aAAa,CAAC,wCAAwC,CAAC,CAAC;QACpE,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACjB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,eAAe,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAEO,KAAK,CAAC,IAAY;QACxB,IAAI,IAAI,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/D,CAAC;IAEO,eAAe;QACrB,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,MAAM,IAAI,aAAa,CACrB,oCAAoC,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAC9D,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,EAAE,CAAC,GAAG,KAAe;QAC3B,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC;QACpD,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IACzD,CAAC;IAEO,IAAI,CAAC,GAAe;QAC1B,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO;QAC5B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC7D,IAAI,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE,CAAC;YAC1B,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YACtB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;YACtB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACxB,CAAC;aAAM,IAAI,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC;YACzC,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;QACrB,CAAC;aAAM,IAAI,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC,EAAE,CAAC;YAC3C,qEAAqE;YACrE,uDAAuD;YACvD,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI;gBAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC3C,IAAI,CAAC,OAAO,EAAE,CAAC;QACjB,CAAC;aAAM,IAAI,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC;YACpD,IAAI,CAAC,KAAK,GAAG;gBACX,QAAQ,EAAE,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC;gBAC/C,SAAS,EAAE,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC;aACjD,CAAC;YACF,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;IAEO,IAAI,CAAC,IAAY;QACvB,IAAI,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC;YAAE,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC;aACpD,IAAI,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,EAAE,CAAC;YAC1D,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;IAEO,KAAK;QACX,qEAAqE;QACrE,kDAAkD;QAClD,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO;QAC5B,IAAI,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;YAC7D,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QAClC,CAAC;aAAM,IAAI,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAClE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YACtB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC;YAC3C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;YAClB,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;YAClC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACnB,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;YAC1C,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;gBACzC,MAAM,EAAE,GAAG,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC7C,MAAM,GAAG,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;gBACpC,IACE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;oBAChB,CAAC,CAAC,QAAQ,IAAI,CAAC,EAAE,IAAI,QAAQ,IAAI,EAAE,CAAC;oBACpC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EACjB,CAAC;oBACD,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;gBACzB,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;wBAChB,EAAE;wBACF,QAAQ;wBACR,SAAS,EAAE,GAAG;wBACd,OAAO,EAAE,IAAI,CAAC,OAAO;qBACtB,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;IAClB,CAAC;IAEO,IAAI,CAAC,IAAY;QACvB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAChD,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,EAAU;QACjC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;QACrC,IAAI,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,MAAM,CAAC,GAAG,MAAM,CAAC;QAC7C,IAAI,GAAG,KAAK,IAAI,CAAC,UAAU;YAAE,OAAO,IAAI,CAAC,OAAO,CAAC;QACjD,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;YACpB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAC9B,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,oBAAoB,EAAE,CAAC;gBAC9C,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;gBACjD,IAAI,MAAM,KAAK,SAAS;oBAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACzD,CAAC;QACH,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,MAAM,KAAK,CAAC;QAC3B,IAAI,CAAC,UAAU,GAAG,GAAG,CAAC;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAEO,KAAK,CAAC,KAAK;QACjB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACtC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,GAAG,IAAI,CAAC;QACrE,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,IAAI,CAAC,SAAS;gBAChB,MAAM,IAAI,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YACzD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACjD,IAAI,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,GAAG,WAAW,CAAC,CAAC,EAAE,CAAC;gBACtD,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAC/B,SAAS;YACX,CAAC;YACD,MAAM,MAAM,GACV,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;YAChE,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC,WAAW,IAAI,MAAM,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;gBACrE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;gBACvB,SAAS;YACX,CAAC;YACD,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC;YACjC,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;YACzB,MAAM,EAAE,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3D,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;YACxD,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC;YACf,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YACxB,MAAM,GAAG,GAAG,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;YACvC,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,KAAK,IAAI,IAAI,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC1D,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,GAAG,CAAC;YAC1B,CAAC;YACD,IAAI,IAAI,CAAC,OAAO,CAAC,EAAE,KAAK,IAAI,IAAI,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;gBACtD,IAAI,CAAC,OAAO,CAAC,EAAE,GAAG,GAAG,CAAC;YACxB,CAAC;QACH,CAAC;QACD,MAAM,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;IACpC,CAAC;CACF;AAOD,2EAA2E;AAC3E,4EAA4E;AAC5E,MAAM,iBAAiB,GAAG,MAAM,CAAC;AACjC,MAAM,aAAa,GAAG,KAAK,CAAC;AAC5B,MAAM,iBAAiB,GAAG,MAAM,CAAC;AACjC,MAAM,cAAc,GAAG,GAAG,CAAC;AAE3B;;;;GAIG;AACH,MAAM,OAAO,UAAU;IAIF;IACA;IACA;IALF,cAAc,CAAS;IAExC,YACmB,MAAoB,EACpB,SAAS,cAAc,EACvB,YAAY,iBAAiB;QAF7B,WAAM,GAAN,MAAM,CAAc;QACpB,WAAM,GAAN,MAAM,CAAiB;QACvB,cAAS,GAAT,SAAS,CAAoB;QAE9C,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,gBAAgB,CAAC;IAChD,CAAC;IAED,uEAAuE;IACvE,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,UAAU,EAAE,CAAC;QAClB,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,iBAAiB;YAAE,OAAO;QAC1D,MAAM,IAAI,CAAC,KAAK,CACd,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,aAAa,EAC/C,iCAAiC,CAClC,CAAC;IACJ,CAAC;IAED,oEAAoE;IACpE,KAAK,CAAC,OAAO;QACX,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC;QAC3C,MAAM,IAAI,CAAC,KAAK,CACd,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,IAAI,EACzE,qCAAqC,CACtC,CAAC;QACF,IAAI,CAAC,UAAU,EAAE,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,IAAmB,EAAE,OAAe;QACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC;QAC7C,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;YACf,IAAI,CAAC,UAAU,EAAE,CAAC;YAClB,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CACb,GAAG,OAAO,UAAU,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,cAAc;oBACjE,kEAAkE;oBAClE,iBAAiB,CACpB,CAAC;YACJ,CAAC;YACD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QACnE,CAAC;IACH,CAAC;IAEO,UAAU;QAChB,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,GAAG,IAAI,CAAC,cAAc,CAAC;QACnE,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CACb,sBAAsB,OAAO,sCAAsC;gBACjE,gEAAgE;gBAChE,8BAA8B,CACjC,CAAC;QACJ,CAAC;IACH,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAC/B,WAAsC,EACtC,OAAsB;IAEtB,OAAO,KAAK,EAAE,UAAU,EAAE,EAAE;QAC1B,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,WAAW,EAAE,CAAC;QAChD,MAAM,EAAE,GAAG,IAAI,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;QACvD,MAAM,KAAK,GAAG,QAAQ,CACpB,kBAAkB,EAClB,OAAO,CAAC,MAAM,CAAC,kBAAkB,EAAE,IAAI,EAAE,EAAE,CAAC,CAC7C,CAAC;QACF,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,IAAI,CACnC,2BAA2B,KAAK,iBAAiB,IAAI,eAAe,EAAE,IAAI;YACxE,0DAA0D,aAAa,KAAK;YAC5E,gCAAgC,CACnC,CAAC;QACF,OAAO,IAAI,GAAG,CACZ,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACzB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CACrD,CACF,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,WAAsC,EACtC,EAAU,EACV,SAAiB,EACjB,MAAM,GAAG,IAAI;IAEb,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IACxC,SAAS,CAAC;QACR,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,IAAI,CACnC,uBAAuB,cAAc,CAAC,gBAAgB,GAAG;YACvD,eAAe,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,yBAAyB;YAClE,iBAAiB,aAAa,GAAG,CACpC,CAAC;QACF,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC;YAAE,OAAO;QACpD,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CACb,+DAA+D;gBAC7D,+DAA+D;gBAC/D,aAAa,CAChB,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;IAC9D,CAAC;AACH,CAAC"}

package/dist/history-v1.d.ts

@@ -1,4 +1,5 @@
 import { QueryClient } from "./query-client.js";
+import { type ArchiveReader } from "./archive.js";
 interface HistoryOptions {
     startTime: Date;
     playbackRate: number;
@@ -15,7 +16,7 @@ interface Delta {
         }[];
     }[];
 }
-export declare function createHistoryProviderV1(queryClient: QueryClient, selfContext: string, debug: (msg: string) => void): {
+export declare function createHistoryProviderV1(queryClient: QueryClient, selfContext: string, debug: (msg: string) => void, archive?: ArchiveReader): {
     hasAnyData: (options: HistoryOptions, callback: (hasResults: boolean) => void) => void;
     streamHistory: (spark: {
         write: (data: unknown) => void;

package/dist/history-v1.js

@@ -1,5 +1,6 @@
 import { validateTimestamp } from "./query-client.js";
 import { isMissingKindColumn, isMissingSourceColumn } from "./schema-errors.js";
+import { ARCHIVE_TABLES, NO_ARCHIVE } from "./archive.js";
 // v1 replays raw deltas rather than aggregating, so all three value tables can
 // be read in one pass and interleaved by timestamp. Every branch below emits
 // the same unified shape — ts, path, context, valuetext, kind — which
@@ -18,10 +19,10 @@ import { isMissingKindColumn, isMissingSourceColumn } from "./schema-errors.js";
 function sourceCol(withSource) {
     return withSource ? `CAST(source AS STRING)` : `CAST(NULL AS STRING)`;
 }
-function numBranch(where, withSource, tail = "") {
+function numBranch(where, withSource, tail = "", table = "signalk") {
     return (`SELECT ts, path, context, ${sourceCol(withSource)} source, ` +
         `CAST(value AS STRING) valuetext, ` +
-        `'number' kind FROM signalk WHERE ${where}${tail}`);
+        `'number' kind FROM ${table} WHERE ${where}${tail}`);
 }
 // `withKind: false` degrades to the pre-migration shape. ensureTables() adds
 // value_kind at startup, but a read racing an unmigrated table (or an
@@ -36,16 +37,41 @@ function strBranch(where, withKind, withSource, tail = "") {
         `FROM signalk_str WHERE ${where}${tail}`);
 }
 // The track table has no path column, so the path is supplied as a literal.
-function posBranch(where, withSource, tail = "") {
+function posBranch(where, withSource, tail = "", table = "signalk_position") {
     return (`SELECT ts, 'navigation.position' path, context, ` +
         `${sourceCol(withSource)} source, ` +
         `concat(CAST(lat AS STRING), ',', CAST(lon AS STRING)) valuetext, ` +
-        `'position' kind FROM signalk_position WHERE ${where}${tail}`);
+        `'position' kind FROM ${table} WHERE ${where}${tail}`);
 }
-function unionValueRowsSQL(where, orderAndLimit, withKind = true, withSource = true) {
-    return (`${numBranch(where, withSource)} UNION ALL ` +
-        `${strBranch(where, withKind, withSource)} UNION ALL ` +
-        `${posBranch(where, withSource)} ${orderAndLimit}`);
+const NOT_ARCHIVED = { numeric: false, position: false };
+function unionValueRowsSQL(where, orderAndLimit, withKind = true, withSource = true, archived = NOT_ARCHIVED) {
+    const branches = [
+        numBranch(where, withSource),
+        strBranch(where, withKind, withSource),
+        posBranch(where, withSource),
+    ];
+    if (archived.numeric) {
+        branches.push(numBranch(where, withSource, "", ARCHIVE_TABLES.signalk));
+    }
+    if (archived.position) {
+        branches.push(posBranch(where, withSource, "", ARCHIVE_TABLES.signalk_position));
+    }
+    return `${branches.join(" UNION ALL ")} ${orderAndLimit}`;
+}
+/**
+ * Keep the newest row per path and context. A snapshot reading the archive
+ * as well as the base tables gets a LATEST ON row from each for a path both
+ * hold, and the older one must not end up applied after the newer.
+ */
+function newestPerPath(rows) {
+    const newest = new Map();
+    for (const row of rows) {
+        const key = `${String(row.path)}\u0000${String(row.context)}`;
+        const held = newest.get(key);
+        if (!held || String(row.ts) > String(held.ts))
+            newest.set(key, row);
+    }
+    return [...newest.values()];
 }
 // Turn a unified row back into the value a Signal K delta carries. Numbers
 // must not stay strings (consumers do arithmetic on them), positions must be
@@ -159,7 +185,16 @@ async function fetchLatestNames(queryClient, startTime) {
         return new Map();
     }
 }
-export function createHistoryProviderV1(queryClient, selfContext, debug) {
+export function createHistoryProviderV1(queryClient, selfContext, debug, 
+// Where retention kept the own vessel's older rows. A read includes it only
+// for a range it covers.
+archive = NO_ARCHIVE) {
+    function archivedIn(from, to) {
+        return {
+            numeric: archive.covers("signalk", from, to),
+            position: archive.covers("signalk_position", from, to),
+        };
+    }
     // Run a unified read, retrying without whichever optional column QuestDB
     // reports missing. value_kind and source are separate migrations, so
     // either — or both, on an external database ensureTables() has not
@@ -206,14 +241,21 @@ export function createHistoryProviderV1(queryClient, selfContext, debug) {
         // "Any data" must consider every value table: a vessel recording only
         // string/boolean channels (or only a position track) has history to play
         // back, and answering false here disables playback entirely.
+        const tables = ["signalk", "signalk_str", "signalk_position"];
+        const archived = archivedIn(startTime, new Date().toISOString());
+        if (archived.numeric)
+            tables.push(ARCHIVE_TABLES.signalk);
+        if (archived.position)
+            tables.push(ARCHIVE_TABLES.signalk_position);
         queryClient
             .exec(
         // Summed via a UNION ALL subquery: QuestDB rejects adding scalar
         // subqueries with `+` ("no matching operator").
         `SELECT sum(c) as cnt FROM (` +
-            `SELECT count() c FROM signalk WHERE ts >= '${startTime}'` +
-            ` UNION ALL SELECT count() c FROM signalk_str WHERE ts >= '${startTime}'` +
-            ` UNION ALL SELECT count() c FROM signalk_position WHERE ts >= '${startTime}')`)
+            tables
+                .map((table) => `SELECT count() c FROM ${table} WHERE ts >= '${startTime}'`)
+                .join(" UNION ALL ") +
+            `)`)
             .then((result) => {
             const count = result.dataset.length > 0 ? result.dataset[0][0] : 0;
             callback(count > 0);
@@ -270,7 +312,8 @@ export function createHistoryProviderV1(queryClient, selfContext, debug) {
             try {
                 const window = `ts >= '${from}' AND ts < '${to}'`;
                 const order = `ORDER BY ts LIMIT ${CHUNK_ROW_LIMIT}`;
-                const result = await execUnified((withKind, withSource) => unionValueRowsSQL(window, order, withKind, withSource), columnState);
+                const archived = archivedIn(from, to);
+                const result = await execUnified((withKind, withSource) => unionValueRowsSQL(window, order, withKind, withSource, archived), columnState);
                 if (result.dataset.length === 0) {
                     currentTime = chunkEnd;
                     if (!stopped) {
@@ -385,14 +428,29 @@ export function createHistoryProviderV1(queryClient, selfContext, debug) {
         const at = `ts <= '${ts}'`;
         const byPath = ` LATEST ON ts PARTITION BY path, context`;
         const byContext = ` LATEST ON ts PARTITION BY context`;
-        const snapshotSQL = (withKind, withSource) => `(${numBranch(at, withSource, byPath)})` +
-            ` UNION ALL ` +
-            `(${strBranch(at, withKind, withSource, byPath)})` +
-            ` UNION ALL ` +
-            `(${posBranch(at, withSource, byContext)})`;
+        // The archive answers for paths whose base rows retention has dropped,
+        // as one more per-table LATEST ON branch, never by wrapping the union.
+        const archived = archivedIn(new Date(0).toISOString(), ts);
+        const snapshotSQL = (withKind, withSource) => {
+            const branches = [
+                numBranch(at, withSource, byPath),
+                strBranch(at, withKind, withSource, byPath),
+                posBranch(at, withSource, byContext),
+            ];
+            if (archived.numeric) {
+                branches.push(numBranch(at, withSource, byPath, ARCHIVE_TABLES.signalk));
+            }
+            if (archived.position) {
+                branches.push(posBranch(at, withSource, byContext, ARCHIVE_TABLES.signalk_position));
+            }
+            return branches.map((branch) => `(${branch})`).join(" UNION ALL ");
+        };
         execUnified(snapshotSQL)
             .then((result) => {
-            const rows = queryClient.toObjects(result);
+            const objects = queryClient.toObjects(result);
+            const rows = archived.numeric || archived.position
+                ? newestPerPath(objects)
+                : objects;
             const deltas = groupRowsIntoDeltas(rows);
             callback(deltas);
         })