signalk-questdb 2.2.2 → 2.3.0

package/dist/index.js

@@ -7,6 +7,9 @@ import { unitsResolver } from "./history-units.js";
 import { createHistoryProviderV2 } from "./history-v2.js";
 import { createHistoryProviderV1 } from "./history-v1.js";
 import { startRetention } from "./retention.js";
+import { ARCHIVE_TABLES, ArchiveSpans, waitForApply } from "./archive.js";
+import { GpxImport, GpxImportCancelled, GpxParseError, IMPORT_SOURCE, WriterGate, recordedIntervals, waitForImportedRow, } from "./gpx-import.js";
+import { GpxImportConflict, GpxImportSession, GpxImports, SliceTooLarge, readSlice, } from "./gpx-import-session.js";
 import { RESTORE_SOURCE, restoreFromHistory } from "./restore.js";
 import { createConsoleProxy } from "./console-proxy.js";
 import { buildFullExportWhere } from "./full-export-range.js";
@@ -38,6 +41,8 @@ const FULL_EXPORT_TABLES = [
     "signalk",
     "signalk_str",
     "signalk_position",
+    ARCHIVE_TABLES.signalk,
+    ARCHIVE_TABLES.signalk_position,
 ];
 const FULL_EXPORT_TABLE_SET = new Set(FULL_EXPORT_TABLES);
 // Healthcheck for the QuestDB container. The `questdb/questdb` image ships no
@@ -126,7 +131,11 @@ export default (app) => {
     };
     let migrationCounter = 0;
     let queryClient = null;
-    let retentionTimer = null;
+    let retention = null;
+    // What the archive covers, for the history providers and the GPX import;
+    // null while the plugin is stopped.
+    let archiveSpans = null;
+    const gpxImports = new GpxImports();
     let schemaHealTimer = null;
     let currentConfig = null;
     // True when the `signalk` table exists with the wrong (ILP-auto-created)
@@ -846,6 +855,18 @@ export default (app) => {
         // the wrong (`timestamp`, not `ts`) schema; heal it before the writer can
         // ingest into the broken shape.
         await healSchemaTables();
+        // Reads add the archive only where it covers the asked range, so its span
+        // must be known before a provider answers anything. If it cannot be read
+        // now, reads leave the archive out until the heal heartbeat below reads
+        // it.
+        const archive = new ArchiveSpans(queryClient);
+        archiveSpans = archive;
+        try {
+            await archive.refresh();
+        }
+        catch (err) {
+            app.debug(`Archive span unknown: ${err instanceof Error ? err.message : String(err)}`);
+        }
         writer = new ILPWriter(ilpHost, ilpPort, (msg) => app.debug(msg), {
             // Surface a flapping ILP connection instead of leaving the status line
             // stuck on a cheerful "Recording" while every sample is dropped.
@@ -876,9 +897,9 @@ export default (app) => {
         const v2Provider = createHistoryProviderV2(queryClient, app.selfContext, config.historySourcePolicyAll ?? false, (msg) => app.debug(msg), unitsResolver(app), 
         // Twice the heartbeat: a path that updates at least once per heartbeat
         // gets a row within heartbeat + its own interval of the previous one.
-        config.unchangedHeartbeatSeconds * 2000);
+        config.unchangedHeartbeatSeconds * 2000, archive);
         app.registerHistoryApiProvider(v2Provider);
-        const v1Provider = createHistoryProviderV1(queryClient, app.selfContext, (msg) => app.debug(msg));
+        const v1Provider = createHistoryProviderV1(queryClient, app.selfContext, (msg) => app.debug(msg), archive);
         app.registerHistoryProvider(v1Provider);
         // Arm live-context tracking before the first delta can arrive, so the
         // restore sees every vessel that transmitted during startup.
@@ -1073,13 +1094,23 @@ export default (app) => {
         });
         unsubscribes.push(unsub);
         if (config.retentionDays && config.retentionDays > 0) {
-            retentionTimer = startRetention(queryClient, config.retentionDays, (msg) => app.debug(msg));
+            retention = startRetention(queryClient, {
+                retentionDays: config.retentionDays,
+                archiveOwnTrack: config.archiveOwnTrack,
+                archivePaths: new PathMatcher(config.archivePaths),
+                archiveResolutionSeconds: config.archiveResolutionSeconds,
+            }, (msg) => app.debug(msg), () => archive.refresh());
         }
         // Heartbeat: catch and repair a table that gets dropped (e.g. a manual
         // WAL recovery) and re-auto-created by ILP with the wrong schema, so
         // recording silently breaking heals itself instead of needing a restart.
         schemaHealTimer = setInterval(() => {
             void healSchemaTables();
+            if (!archive.known) {
+                archive.refresh().catch((err) => {
+                    app.debug(`Archive span still unknown: ${err instanceof Error ? err.message : String(err)}`);
+                });
+            }
         }, SCHEMA_HEAL_INTERVAL_MS);
         // Watch for WAL suspensions. Detection alone is not enough — a suspended
         // table keeps accepting ILP writes while committing nothing, and the only
@@ -1286,10 +1317,12 @@ export default (app) => {
                 }
             }
             unsubscribes.length = 0;
-            if (retentionTimer) {
-                clearInterval(retentionTimer);
-                retentionTimer = null;
+            if (retention) {
+                retention.stop();
+                retention = null;
             }
+            gpxImports.abortAll();
+            archiveSpans = null;
             if (schemaHealTimer) {
                 clearInterval(schemaHealTimer);
                 schemaHealTimer = null;
@@ -2154,10 +2187,12 @@ export default (app) => {
                             walMonitor = null;
                         }
                         clearWalAlertOnTeardown();
-                        if (retentionTimer) {
-                            clearInterval(retentionTimer);
-                            retentionTimer = null;
+                        if (retention) {
+                            retention.stop();
+                            retention = null;
                         }
+                        gpxImports.abortAll();
+                        archiveSpans = null;
                         if (writer) {
                             try {
                                 await writer.disconnect();
@@ -2453,6 +2488,181 @@ export default (app) => {
                     });
                 }
             });
+            // GPX import into the position archive (see gpx-import.ts). The file
+            // arrives as slices over several requests: POST announces it, each PUT
+            // carries the slice starting at `offset`, and POST …/finish ends it once
+            // every byte is in. Admin-only, like every route not registered through
+            // router.access(): an import writes history.
+            const importReply = (res, code, body) => {
+                res.status(code).json(body);
+            };
+            const importFailure = (res, err) => {
+                if (err instanceof GpxImportConflict) {
+                    importReply(res, 409, { error: err.message, import: err.status });
+                }
+                else if (err instanceof SliceTooLarge) {
+                    importReply(res, 413, { error: err.message });
+                }
+                else if (err instanceof GpxParseError) {
+                    importReply(res, 422, {
+                        error: err.message,
+                        import: gpxImports.latest(),
+                    });
+                }
+                else if (err instanceof GpxImportCancelled) {
+                    importReply(res, 409, {
+                        error: err.message,
+                        import: gpxImports.latest(),
+                    });
+                }
+                else {
+                    importReply(res, 500, {
+                        error: err instanceof Error ? err.message : String(err),
+                        import: gpxImports.latest(),
+                    });
+                }
+            };
+            router.get("/api/gpx-import", (_req, res) => {
+                importReply(res, 200, { import: gpxImports.latest() });
+            });
+            router.post("/api/gpx-import", async (req, res) => {
+                const body = (req.body ?? {});
+                const filename = typeof body.filename === "string" ? body.filename.slice(0, 255) : "";
+                const size = body.size;
+                if (typeof size !== "number" ||
+                    !Number.isSafeInteger(size) ||
+                    size <= 0) {
+                    importReply(res, 400, {
+                        error: "size must be the file's length in bytes",
+                    });
+                    return;
+                }
+                // Which tracks are the vessel's is the caller's call (see
+                // GpxImportOptions.tracks), so there is no default.
+                const tracks = body.tracks;
+                if (!Array.isArray(tracks) ||
+                    tracks.length === 0 ||
+                    tracks.length > 1000 ||
+                    !tracks.every((t) => typeof t === "string" && t.length <= 1024)) {
+                    importReply(res, 400, {
+                        error: 'tracks must list the names of the tracks to import ("" for one without a name)',
+                    });
+                    return;
+                }
+                const w = writer;
+                const client = queryClient;
+                const spans = archiveSpans;
+                const config = currentConfig;
+                if (!w || !client || !spans || !config) {
+                    importReply(res, 503, { error: "QuestDB writer not connected" });
+                    return;
+                }
+                // ILP would auto-create a missing archive with the wrong designated
+                // timestamp, and every read of it would come back empty.
+                let designated;
+                try {
+                    designated = await client.designatedTimestamp(ARCHIVE_TABLES.signalk_position);
+                }
+                catch (err) {
+                    importFailure(res, err);
+                    return;
+                }
+                if (designated !== "ts") {
+                    importReply(res, 503, {
+                        error: `${ARCHIVE_TABLES.signalk_position} is missing; restart the plugin to create it`,
+                    });
+                    return;
+                }
+                // From here to gpxImports.start() nothing awaits, so a migration
+                // cannot start between this check and the import taking the writer.
+                if (activeMigration && activeMigration.state === "running") {
+                    importReply(res, 409, {
+                        error: "An InfluxDB migration is running; import after it ends",
+                    });
+                    return;
+                }
+                // The rebuild carries over everything written during its copy, from
+                // the oldest line's time on: a GPX line from years back would make
+                // that years of rows.
+                if (legacyCleanupRunning) {
+                    importReply(res, 409, {
+                        error: "The string table is being rebuilt; import after it ends",
+                    });
+                    return;
+                }
+                try {
+                    const session = gpxImports.start(() => {
+                        const gate = new WriterGate(w);
+                        const importer = new GpxImport({
+                            tracks: new Set(tracks.map((t) => t.trim())),
+                            resolutionSeconds: config.archiveResolutionSeconds,
+                            write: (ms, latitude, longitude) => w.writePositionAtNanos("self", { latitude, longitude }, BigInt(ms) * 1000000n, IMPORT_SOURCE, ARCHIVE_TABLES.signalk_position),
+                            backpressure: () => gate.room(),
+                            ...(body.fillGapsOnly === false
+                                ? {}
+                                : { recorded: recordedIntervals(client, spans) }),
+                        });
+                        return new GpxImportSession(filename, size, importer, async () => {
+                            await gate.settled();
+                            await waitForApply(client, ARCHIVE_TABLES.signalk_position, 120_000);
+                            // QuestDB commits ILP on its own schedule, so a batch the
+                            // socket took may not be in the WAL yet; the last row written
+                            // reading back is what says the import is stored.
+                            const last = importer.lastWritten;
+                            if (last !== null)
+                                await waitForImportedRow(client, last, 120_000);
+                            await spans.refresh();
+                        });
+                    });
+                    importReply(res, 201, { import: session.status });
+                }
+                catch (err) {
+                    importFailure(res, err);
+                }
+            });
+            router.put("/api/gpx-import/:id", async (req, res) => {
+                const session = gpxImports.get(String(req.params.id));
+                if (!session) {
+                    importReply(res, 404, { error: "No such import" });
+                    return;
+                }
+                const offset = Number(req.query.offset);
+                if (!Number.isSafeInteger(offset) || offset < 0) {
+                    importReply(res, 400, { error: "offset must be a byte offset" });
+                    return;
+                }
+                try {
+                    const slice = await readSlice(req);
+                    importReply(res, 200, {
+                        import: await session.append(offset, slice),
+                    });
+                }
+                catch (err) {
+                    importFailure(res, err);
+                }
+            });
+            router.post("/api/gpx-import/:id/finish", async (req, res) => {
+                const session = gpxImports.get(String(req.params.id));
+                if (!session) {
+                    importReply(res, 404, { error: "No such import" });
+                    return;
+                }
+                try {
+                    importReply(res, 200, { import: await session.finish() });
+                }
+                catch (err) {
+                    importFailure(res, err);
+                }
+            });
+            router.delete("/api/gpx-import/:id", (req, res) => {
+                const session = gpxImports.get(String(req.params.id));
+                if (!session) {
+                    importReply(res, 404, { error: "No such import" });
+                    return;
+                }
+                session.abort();
+                importReply(res, 200, { import: session.status });
+            });
             router.post("/api/migration/start", async (req, res) => {
                 // One import at a time. Two concurrent runs would interleave on the
                 // single ILP writer and race each other for its buffer, and there is
@@ -2464,6 +2674,12 @@ export default (app) => {
                     });
                     return;
                 }
+                if (gpxImports.active) {
+                    res.status(409).json({
+                        error: "A GPX import is running; migrate after it ends",
+                    });
+                    return;
+                }
                 if (!writer) {
                     res.status(503).json({
                         error: "QuestDB writer not connected",
@@ -2698,6 +2914,12 @@ export default (app) => {
                     });
                     return;
                 }
+                if (gpxImports.active) {
+                    res.status(409).json({
+                        error: "A GPX import is running; migrate after it ends",
+                    });
+                    return;
+                }
                 if (!writer) {
                     res.status(503).json({
                         error: "QuestDB writer not connected",
@@ -2798,6 +3020,13 @@ export default (app) => {
                     });
                     return;
                 }
+                if (gpxImports.active) {
+                    res.status(409).json({
+                        error: "A GPX import is running; wait for it or abort it first",
+                        rows: 0,
+                    });
+                    return;
+                }
                 if (legacyCleanupRunning) {
                     res.status(409).json({
                         error: "The rows are already being removed",