signalk-questdb 2.1.4 → 2.1.5

package/dist/migration.d.ts

@@ -1,10 +1,11 @@
 import type { ILPWriter } from "./ilp-writer.js";
-import type { MigrationBucket, MigrationMeasurement, MigrationProgress, MigrationRunState } from "./api-contract.js";
+import type { CheckpointStore, ConfirmStored, MigrationCheckpoint } from "./migration-checkpoint.js";
+import type { MigrationBucket, MigrationMeasurement, MigrationProgress, MigrationResumePoint, MigrationRunState } from "./api-contract.js";
 /**
- * How much time one read covers. Small enough that a dense measurement does
- * not return a result set too large to hold, large enough that a sparse one
- * does not need thousands of round trips. A day is the natural unit for boat
- * data: it bounds a busy passage to a few hundred thousand points.
+ * How much time one query covers. Large enough that a sparse measurement does
+ * not need thousands of round trips, small enough that one query stays cheap
+ * for InfluxDB and progress moves in visible steps. It does not bound memory:
+ * a window is streamed in batches, however many points it holds.
  */
 export declare const DEFAULT_WINDOW_MS: number;
 export interface InfluxAuth {
@@ -25,8 +26,18 @@ export interface MigrationRequest {
     /** ISO instants bounding the import. */
     from: string;
     to: string;
-    /** Signal K context to write rows under. */
+    /** Signal K context to write the source's own vessel under. */
     context: string;
+    /**
+     * The `context` tag value the source uses for its own vessel. Rows tagged
+     * with it are written under `context`; rows tagged otherwise are other
+     * vessels and keep their tag (see `others`); untagged rows are the own
+     * vessel too. Absent: every row is the own vessel.
+     */
+    sourceSelfContext?: string;
+    /** What to do with rows of other vessels: keep them under their own
+     * context (the default), or leave them out. */
+    others?: "keep" | "skip";
     /** Restrict to these measurements; empty/absent means all of them. */
     measurements?: string[];
     /** `source` tag written on every imported row, so it is distinguishable. */
@@ -47,18 +58,70 @@ export declare function listMeasurements(req: {
     auth?: InfluxAuth;
 }, fetchImpl?: typeof fetch): Promise<MigrationMeasurement[]>;
 /**
- * Minimal annotated-CSV reader for Flux responses.
+ * The `context` tag values in a bucket/database: the vessels the source holds.
+ * Both Signal K InfluxDB writers tag every point with the delta's context, so
+ * this is how the source's own vessel is told from the AIS targets it recorded.
+ * A source with no such tag — a non-Signal K schema — returns none.
+ */
+export declare function listContexts(req: {
+    url: string;
+    type: string;
+    bucket: string;
+    auth?: InfluxAuth;
+}, fetchImpl?: typeof fetch): Promise<string[]>;
+/**
+ * Whether a string is a Signal K context this plugin will file history under:
+ * a vessel, aton, aircraft or SAR target with a well-formed identifier.
+ */
+export declare function isSignalKContext(context: string): boolean;
+/**
+ * Incremental annotated-CSV reader for Flux responses.
+ *
+ * Fed one physical line at a time, so a response is never held whole: the
+ * caller streams the body through it and sees each record as it completes.
  *
  * Flux emits one or more *tables*, each preceded by annotation lines starting
  * with `#` and then a header row. A response can therefore change its column
  * layout partway through, which is why the header is tracked per block rather
  * than read once at the top.
  */
+export declare class AnnotatedCsvReader {
+    private header;
+    private datatypes;
+    private groups;
+    private types;
+    private grouped;
+    private pending;
+    /**
+     * Feed one physical line, without its `\n`. Returns the record it
+     * completes, or null for annotations, headers, blanks and partial records.
+     *
+     * A quoted CSV value may contain a newline — verified against a live 2.9.1,
+     * which returns `"line1\r\nline2"` for a string value holding one. Splitting
+     * on newlines alone would tear that record in half and produce two garbage
+     * rows, so physical lines are joined until the quotes balance ("" inside a
+     * quoted value is an escaped quote and does not change the balance).
+     * \r is stripped per PHYSICAL line before joining: a quoted value spanning
+     * two CRLF lines would otherwise keep the \r of the first inside the joined
+     * value, so the imported string silently carries a stray carriage return.
+     */
+    push(physicalLine: string): AnnotatedRecord | null;
+    /** Call once after the last line. */
+    end(): AnnotatedRecord | null;
+    private logicalLine;
+}
+/** Parse a whole annotated-CSV response. For small results only. */
 export declare function parseAnnotatedCsv(text: string): AnnotatedRecord[];
-/** One Flux CSV record plus the `#datatype` of each of its columns. */
+/**
+ * One Flux CSV record, plus what the table's annotations say about its columns.
+ * `types` and `grouped` belong to the table and are shared by all its records.
+ */
 export interface AnnotatedRecord {
     values: Record<string, string>;
-    types: Record<string, string>;
+    /** `#datatype` per column. */
+    types: Readonly<Record<string, string>>;
+    /** Columns in the table's group key (`#group` true): tags, never fields. */
+    grouped: ReadonlySet<string>;
 }
 /**
  * Parse an RFC3339 instant into nanoseconds since the epoch, preserving digits
@@ -73,12 +136,27 @@ export declare function rfc3339ToNanos(value: string): bigint | null;
 /**
  * The Signal K path a measurement+field pair represents.
  *
- * signalk-to-influxdb writes the path as the measurement name and the value in
- * a field called `value`. Anything else (a measurement with several named
- * fields) is a non-Signal K schema, so the field name is appended to keep the
- * two apart instead of overwriting one with the other.
+ * The Signal K InfluxDB plugins write the path as the measurement name and the
+ * value in one of PATH_VALUE_FIELDS. Anything else (a measurement with several
+ * named fields) is a non-Signal K schema, so the field name is appended to keep
+ * the two apart instead of overwriting one with the other.
  */
 export declare function toSignalKPath(measurement: string, field: string): string;
+/**
+ * Decode `jsonValue` fields back into the values they encode.
+ *
+ * Left as the string it arrives as, a position would be recorded in
+ * `signalk_str` as text and `signalk_position` would stay empty. Decoded, it
+ * routes exactly like the live value did: a position to the position table,
+ * any other object to its scalar leaves.
+ *
+ * A value that does not parse cannot be represented, so it is dropped — and
+ * reported, so it lands in the run's skipped total.
+ */
+export declare function decodeJsonValueRows(rows: SourceRow[]): {
+    rows: SourceRow[];
+    dropped: number;
+};
 /** Coerce an InfluxDB field value (always text over CSV) to a JS value. */
 export declare function coerceValue(raw: string, dataType?: string): unknown;
 export interface MigrationRunHandle {
@@ -90,6 +168,7 @@ export interface MigrationRunHandle {
     finishedAt?: string;
     progress: MigrationProgress;
     error?: string;
+    resumedFrom?: MigrationResumePoint;
     cancel(): void;
 }
 /**
@@ -104,6 +183,7 @@ export declare class MigrationRun implements MigrationRunHandle {
     startedAt: string;
     finishedAt?: string;
     error?: string;
+    resumedFrom?: MigrationResumePoint;
     progress: MigrationProgress;
     private cancelled;
     constructor(id: string, url: string, bucket: string);
@@ -120,26 +200,63 @@ export declare class MigrationRun implements MigrationRunHandle {
  */
 export declare function runMigration(req: MigrationRequest, writer: Pick<ILPWriter, "writeAtNanos" | "writeStringAtNanos" | "writePositionAtNanos" | "pendingLines"> & {
     readonly droppedLineCount?: number;
+    readonly enqueuedLineCount?: number;
+    readonly settledLineCount?: number;
+    readonly atCapacity?: boolean;
 }, run: MigrationRun, deps?: {
     fetchImpl?: typeof fetch;
     debug?: (msg: string) => void;
     /** Injected so tests don't wait on real backpressure sleeps. */
     sleep?: (ms: number) => Promise<void>;
+    /** Injected so tests don't wait out a real stalled read. */
+    readIdleTimeoutMs?: number;
+    /** Where the import's position is kept, so a later run can resume it. */
+    checkpoints?: CheckpointStore;
+    /**
+     * A checkpoint to continue from. Only honoured if it belongs to this same
+     * import — anything else is ignored and the run starts from the beginning.
+     */
+    resumeFrom?: MigrationCheckpoint;
+    /** Injected so tests don't wait out the real checkpoint lag. */
+    checkpointLagMs?: number;
+    now?: () => number;
+    /**
+     * QuestDB's confirmation that rows can be read back, consulted before a
+     * position is saved. Without it a position rests on the writer's counters
+     * and the lag alone.
+     */
+    confirmStored?: ConfirmStored;
 }): Promise<void>;
+/**
+ * The Signal K context a row is written under, or null for a row that is left
+ * out — counted as skipped by the caller.
+ *
+ * The source's own vessel becomes `self`, whether the point was tagged with
+ * the source's own context, with the literal `vessels.self`, or not at all.
+ * Any other tag is another vessel: kept under its own context if it is a
+ * well-formed Signal K one, since that is what the history API is queried by,
+ * or left out when other vessels are not wanted. With no `sourceSelf` given,
+ * every row is the own vessel — what a single-vessel source amounts to.
+ */
+export declare function targetContext(rowContext: string | undefined, sourceSelf: string | undefined, self: string, others: "keep" | "skip"): string | null;
 /** One point read out of InfluxDB, normalised across the two dialects. */
 export interface SourceRow {
     tsNanos: bigint;
     field: string;
     value: unknown;
+    /** The source's `context` tag, when the point carried one. */
+    context?: string;
 }
 /**
  * Recombine latitude/longitude fields of a position measurement into one
  * object value at the same instant.
  *
- * The Signal K InfluxDB plugins store navigation.position as two fields
- * (`latitude`/`longitude`, or `lat`/`lon`). Left as-is they would import as
- * two numeric paths and the position history would be unusable — QuestDB keeps
- * positions in their own table.
+ * signalk-to-influxdb2 stores navigation.position as two fields (`lat`/`lon`),
+ * as does signalk-to-influxdb 1.x with `separateLatLon` on. Left as-is they
+ * would import as two numeric paths and the position history would be unusable
+ * — QuestDB keeps positions in their own table. The 1.x default encoding, a
+ * single `jsonValue`, is already whole by the time it gets here (see
+ * decodeJsonValueRows).
  */
 export declare function mergePositionRows(measurement: string, rows: SourceRow[]): {
     rows: SourceRow[];