signalk-questdb 1.8.3 → 1.9.0

package/.prettierignore

@@ -2,3 +2,4 @@ dist/
 node_modules/
 .mf/
 public/
+dist-panel-test/

package/README.md

@@ -34,7 +34,7 @@ The plugin embeds a React config panel in the Signal K Admin UI showing:
 - **Image Version** -- dropdown with latest, pre-releases, and last 3 stable releases
 - **Connection** -- managed container toggle, host/ports, PostgreSQL port for Grafana
 - **Recording** -- record self, record AIS targets, startup restore, console webapp, retention days
-- **Path filtering** (collapsible) -- exclude or include-only paths with glob patterns (e.g. exclude `navigation.position`)
+- **Path filtering** (collapsible) -- exclude or include-only paths with glob patterns
 - **Compression** (collapsible) -- LZ4/ZSTD codec selection for on-disk storage
 - **InfluxDB Migration** (collapsible) -- auto-detect with manual URL for remote instances
 - **Data Export** (collapsible) -- date range picker, Parquet/CSV format, download button
@@ -133,7 +133,7 @@ wants to slice it into kopia-dedup-friendly shards. Allowed tables:
 | QuestDB console webapp     | `true`       | Serve QuestDB's console in the Signal K admin UI, admin only (see Console webapp)                     |
 | Retention (days)           | `0`          | Auto-delete old partitions (0 = keep forever)                                                         |
 | Path filter mode           | `exclude`    | `exclude` matching paths, or `include` only matching paths                                            |
-| Path filter paths          | _(empty)_    | Glob patterns, one per line (e.g. `navigation.position`); empty = record everything                   |
+| Path filter paths          | _(empty)_    | Glob patterns, one per line (e.g. `notifications.*`); empty = record everything, which is the default |
 | Compression codec          | `lz4`        | On-disk WAL compression: `none`, `lz4`, or `zstd`                                                     |
 | Compression level          | `3`          | ZSTD level 1-22 (only when codec is zstd)                                                             |
 | Container network          | `sk-network` | Shared network for QuestDB (only applied when binding to 0.0.0.0)                                     |

package/dist/api-contract.d.ts

@@ -1,3 +1,4 @@
+import type { UlimitClamp } from "signalk-container-helper";
 import type { MaxMapCountStatus } from "./host-limits.js";
 import type { SkipPlan, SuspendedTable } from "./wal-monitor.js";
 /**
@@ -11,16 +12,15 @@ export interface ApiError {
     message?: string;
 }
 /**
- * Mirror of signalk-container's `UlimitClamp` event, redeclared (not
- * imported) for the same reason index.ts redeclares it: signalk-container is
- * an optional peer reached through globalThis, so there is no type to import.
+ * signalk-container's `UlimitClamp` event, re-exported under the name the
+ * API responses use.
+ *
+ * `import type` is load-bearing: this file is shared with the browser panel,
+ * and the helper's main entry carries ~19KB of Node-side runtime code. A
+ * value import would drag all of it into the federated bundle. The type-only
+ * form is erased at compile time, so nothing ships.
  */
-export interface UlimitClampStatus {
-    ulimit: string;
-    requested: number;
-    granted: number;
-    reason: string;
-}
+export type UlimitClampStatus = UlimitClamp;
 export type SuspendedTableStatus = SuspendedTable & {
     /** WAL monitor's verdict for the current stall point: "pending" | "failed". */
     autoResume: string | null;
@@ -51,6 +51,26 @@ export interface DbStatus extends ApiError {
      * display name gets an empty statusMessage and the count would be invisible.
      */
     restore?: RestoreStatus | null;
+    /**
+     * Paths seen carrying a value no table can hold — arrays, or objects nested
+     * deeper than the one level that gets flattened. Absent when nothing has
+     * been dropped. Surfaced because the previous behaviour was to drop them
+     * silently, which is how attitude went unrecorded unnoticed (issue #128).
+     */
+    unstorable?: UnstorableStatus | null;
+}
+/** Values encountered that cannot be represented in any table. */
+export interface UnstorableStatus {
+    /**
+     * Distinct paths dropped since the plugin started. A LOWER BOUND when
+     * `truncated` is set: tracking is capped, because Signal K paths embed
+     * instance and vessel identifiers and so are not a bounded vocabulary.
+     */
+    paths: number;
+    /** True once the cap was hit and further distinct paths stopped being kept. */
+    truncated: boolean;
+    /** A sample of those paths, for identifying them without reading logs. */
+    examples: string[];
 }
 /** Startup-restore outcome, shown on the config panel. */
 export interface RestoreStatus {

package/dist/configpanel/responses.d.ts

@@ -1,14 +1,4 @@
-import type { MigrationSource, QuestdbVersion } from "../api-contract.js";
-/**
- * GET /api/versions returns an array of releases. Anything else becomes an
- * empty list, and malformed members are dropped: the version dropdown is
- * then short or empty rather than the panel being gone.
- *
- * Members are validated, not just the array — `versions.filter(v =>
- * !v.prerelease)` runs during render, so a single null element is as fatal
- * as a non-array body.
- */
-export declare function toVersionList(body: unknown): QuestdbVersion[];
+import type { MigrationSource } from "../api-contract.js";
 /**
  * GET /api/migration/detect returns `{ sources: [...] }`. A body without it
  * means "nothing detected", which is also what an empty array means — so the

package/dist/configpanel/responses.js

@@ -9,25 +9,6 @@
 //
 // These helpers keep the "trust but verify" boundary in one testable place.
 const isRecord = (v) => typeof v === "object" && v !== null;
-/**
- * GET /api/versions returns an array of releases. Anything else becomes an
- * empty list, and malformed members are dropped: the version dropdown is
- * then short or empty rather than the panel being gone.
- *
- * Members are validated, not just the array — `versions.filter(v =>
- * !v.prerelease)` runs during render, so a single null element is as fatal
- * as a non-array body.
- */
-export function toVersionList(body) {
-    if (!Array.isArray(body))
-        return [];
-    return body.filter((v) => isRecord(v) &&
-        typeof v.tag === "string" &&
-        // Not merely cosmetic: the two filters below partition on this, so a
-        // missing or non-boolean value would list a pre-release as stable —
-        // and the stable list is what the version dropdown defaults to.
-        typeof v.prerelease === "boolean");
-}
 /**
  * GET /api/migration/detect returns `{ sources: [...] }`. A body without it
  * means "nothing detected", which is also what an empty array means — so the

package/dist/configpanel/responses.js.map

@@ -1 +1 @@
-{"version":3,"file":"responses.js","sourceRoot":"","sources":["../../src/configpanel/responses.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,EAAE;AACF,4EAA4E;AAC5E,6EAA6E;AAC7E,6EAA6E;AAC7E,wEAAwE;AACxE,4EAA4E;AAC5E,uEAAuE;AACvE,EAAE;AACF,4EAA4E;AAI5E,MAAM,QAAQ,GAAG,CAAC,CAAU,EAAgC,EAAE,CAC5D,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,CAAC;AAEtC;;;;;;;;GAQG;AACH,MAAM,UAAU,aAAa,CAAC,IAAa;IACzC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IACpC,OAAO,IAAI,CAAC,MAAM,CAChB,CAAC,CAAC,EAAuB,EAAE,CACzB,QAAQ,CAAC,CAAC,CAAC;QACX,OAAO,CAAC,CAAC,GAAG,KAAK,QAAQ;QACzB,qEAAqE;QACrE,oEAAoE;QACpE,gEAAgE;QAChE,OAAO,CAAC,CAAC,UAAU,KAAK,SAAS,CACpC,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAa;IAC9C,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;IACvC,OAAO,OAAO,CAAC,MAAM,CACnB,CAAC,CAAC,EAAwB,EAAE,CAC1B,QAAQ,CAAC,CAAC,CAAC;QACX,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ;QAC1B,OAAO,CAAC,CAAC,GAAG,KAAK,QAAQ;QACzB,iEAAiE;QACjE,kEAAkE;QAClE,iEAAiE;QACjE,sDAAsD;QACtD,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ;QAC5B,CAAC,CAAC,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ,CAAC,CAC7D,CAAC;AACJ,CAAC"}
+{"version":3,"file":"responses.js","sourceRoot":"","sources":["../../src/configpanel/responses.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,EAAE;AACF,4EAA4E;AAC5E,6EAA6E;AAC7E,6EAA6E;AAC7E,wEAAwE;AACxE,4EAA4E;AAC5E,uEAAuE;AACvE,EAAE;AACF,4EAA4E;AAI5E,MAAM,QAAQ,GAAG,CAAC,CAAU,EAAgC,EAAE,CAC5D,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,CAAC;AAEtC;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAa;IAC9C,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;IACvC,OAAO,OAAO,CAAC,MAAM,CACnB,CAAC,CAAC,EAAwB,EAAE,CAC1B,QAAQ,CAAC,CAAC,CAAC;QACX,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ;QAC1B,OAAO,CAAC,CAAC,GAAG,KAAK,QAAQ;QACzB,iEAAiE;QACjE,kEAAkE;QAClE,iEAAiE;QACjE,sDAAsD;QACtD,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ;QAC5B,CAAC,CAAC,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ,CAAC,CAC7D,CAAC;AACJ,CAAC"}

package/dist/delta-routing.d.ts

@@ -1,3 +1,57 @@
-export type DeltaRoute = "number" | "string" | "boolean" | "position" | null;
+export type DeltaRoute = "number" | "string" | "boolean" | "position" | "flatten" | null;
+/** One scalar leaf pulled out of an object value, ready to record. */
+export interface FlattenedLeaf {
+    /** Parent path plus the key, e.g. `navigation.attitude.roll`. */
+    path: string;
+    value: number | string | boolean;
+}
+/**
+ * Remembers which paths carried a value no table can hold, so the drop can be
+ * reported instead of happening in silence — the underlying bug behind #128.
+ *
+ * Capped, because the path vocabulary is NOT bounded: paths embed instance
+ * identifiers (`watermaker.0.*`) and notifications embed per-vessel ones, so
+ * an unstorable shape on a busy AIS stream would grow an uncapped set for the
+ * lifetime of the process. Past the cap the count still rises but no new path
+ * is retained, and `truncated` marks the count as a lower bound rather than
+ * letting it read as exact.
+ *
+ * Deliberately does no logging: `note()` runs per delta on a stream carrying
+ * 100+ values/sec, and building a message string there costs whether or not
+ * debug is enabled.
+ */
+export declare class UnstorableTracker {
+    private readonly cap;
+    private readonly paths;
+    private capped;
+    constructor(cap?: number);
+    /** Records a path. Cheap and idempotent; safe to call per delta. */
+    note(path: string): void;
+    get size(): number;
+    get truncated(): boolean;
+    examples(limit: number): string[];
+    clear(): void;
+}
 export declare function extractVesselName(path: string, value: unknown): string | null;
 export declare function routeDeltaValue(path: string, value: unknown): DeltaRoute;
+/**
+ * Scalar leaves of an object value, as dotted paths.
+ *
+ *   navigation.attitude {roll: 0.02, yaw: 1.57}
+ *     -> navigation.attitude.roll  0.02
+ *     -> navigation.attitude.yaw   1.57
+ *
+ * **One level deep, deliberately.** That covers attitude and effectively every
+ * real Signal K object, while a recursive walk would happily write out whole
+ * nested payloads (notifications, resource documents) that nobody asked to
+ * record. A nested object is therefore skipped, not descended into — and
+ * reported, so it is not another silent drop.
+ *
+ * The leaves are ordinary scalar paths, which is what makes this cheap: no
+ * schema change, and dedup, retention, sampling, path filtering and both
+ * history APIs apply to them unchanged.
+ */
+export declare function flattenObjectValue(path: string, value: unknown): {
+    leaves: FlattenedLeaf[];
+    skipped: string[];
+};

package/dist/delta-routing.js

@@ -1,3 +1,49 @@
+/**
+ * Remembers which paths carried a value no table can hold, so the drop can be
+ * reported instead of happening in silence — the underlying bug behind #128.
+ *
+ * Capped, because the path vocabulary is NOT bounded: paths embed instance
+ * identifiers (`watermaker.0.*`) and notifications embed per-vessel ones, so
+ * an unstorable shape on a busy AIS stream would grow an uncapped set for the
+ * lifetime of the process. Past the cap the count still rises but no new path
+ * is retained, and `truncated` marks the count as a lower bound rather than
+ * letting it read as exact.
+ *
+ * Deliberately does no logging: `note()` runs per delta on a stream carrying
+ * 100+ values/sec, and building a message string there costs whether or not
+ * debug is enabled.
+ */
+export class UnstorableTracker {
+    cap;
+    paths = new Set();
+    capped = false;
+    constructor(cap = 50) {
+        this.cap = cap;
+    }
+    /** Records a path. Cheap and idempotent; safe to call per delta. */
+    note(path) {
+        if (this.paths.has(path))
+            return;
+        if (this.paths.size >= this.cap) {
+            this.capped = true;
+            return;
+        }
+        this.paths.add(path);
+    }
+    get size() {
+        return this.paths.size;
+    }
+    get truncated() {
+        return this.capped;
+    }
+    examples(limit) {
+        return [...this.paths].slice(0, limit);
+    }
+    clear() {
+        this.paths.clear();
+        this.capped = false;
+    }
+}
 // Static vessel identity arrives as EMPTY-path object deltas —
 // `{path: "", value: {name: "..."}}` is how AIS static reports reach the
 // server, and the exact shape Freeboard reads names from. These never make
@@ -13,8 +59,15 @@ export function extractVesselName(path, value) {
     return typeof name === "string" && name.trim() !== "" ? name : null;
 }
 export function routeDeltaValue(path, value) {
+    // NaN and ±Infinity have no ILP representation. QuestDB does NOT reject
+    // them — verified against a live instance, `value=NaN` is accepted and
+    // stored — so an unguarded sensor fault silently poisons the numeric
+    // column, and every aggregate over that path afterwards. A source
+    // reporting a non-finite number is reporting "no reading", which is what
+    // recording nothing means. flattenObjectValue already applied this to
+    // leaves; the top-level path has to agree.
     if (typeof value === "number")
-        return "number";
+        return Number.isFinite(value) ? "number" : null;
     if (typeof value === "string")
         return "string";
     // Booleans are everywhere in Signal K — switch and relay states, pump and
@@ -34,6 +87,62 @@ export function routeDeltaValue(path, value) {
         Number.isFinite(value.latitude) &&
         Number.isFinite(value.longitude))
         return "position";
+    // Any other non-null, non-array object: record its scalar leaves
+    // individually. Arrays are excluded deliberately — their indices are not
+    // stable identities, so `foo.0` would silently mean a different thing from
+    // one delta to the next.
+    if (value !== null && typeof value === "object" && !Array.isArray(value))
+        return "flatten";
     return null;
 }
+/**
+ * Scalar leaves of an object value, as dotted paths.
+ *
+ *   navigation.attitude {roll: 0.02, yaw: 1.57}
+ *     -> navigation.attitude.roll  0.02
+ *     -> navigation.attitude.yaw   1.57
+ *
+ * **One level deep, deliberately.** That covers attitude and effectively every
+ * real Signal K object, while a recursive walk would happily write out whole
+ * nested payloads (notifications, resource documents) that nobody asked to
+ * record. A nested object is therefore skipped, not descended into — and
+ * reported, so it is not another silent drop.
+ *
+ * The leaves are ordinary scalar paths, which is what makes this cheap: no
+ * schema change, and dedup, retention, sampling, path filtering and both
+ * history APIs apply to them unchanged.
+ */
+export function flattenObjectValue(path, value) {
+    const leaves = [];
+    const skipped = [];
+    if (value === null || typeof value !== "object" || Array.isArray(value)) {
+        return { leaves, skipped };
+    }
+    // An empty parent would build ".name" — a leading-dot path that matches no
+    // Signal K path and no filter pattern. The recorder already returns before
+    // reaching here for path "" (that shape is the AIS static report, handled
+    // by extractVesselName), so this is belt-and-braces: the function must not
+    // depend on a caller's guard to avoid emitting a malformed path.
+    if (path === "")
+        return { leaves, skipped };
+    for (const [key, leaf] of Object.entries(value)) {
+        const leafPath = `${path}.${key}`;
+        if (typeof leaf === "number") {
+            // NaN and ±Infinity have no ILP representation and would poison the
+            // column; a sensor reporting them is reporting "no reading".
+            if (Number.isFinite(leaf))
+                leaves.push({ path: leafPath, value: leaf });
+            else
+                skipped.push(leafPath);
+        }
+        else if (typeof leaf === "string" || typeof leaf === "boolean") {
+            leaves.push({ path: leafPath, value: leaf });
+        }
+        else {
+            // Nested object, array, null or undefined.
+            skipped.push(leafPath);
+        }
+    }
+    return { leaves, skipped };
+}
 //# sourceMappingURL=delta-routing.js.map

package/dist/delta-routing.js.map

@@ -1 +1 @@
-{"version":3,"file":"delta-routing.js","sourceRoot":"","sources":["../src/delta-routing.ts"],"names":[],"mappings":"AAUA,+DAA+D;AAC/D,yEAAyE;AACzE,2EAA2E;AAC3E,sEAAsE;AACtE,wEAAwE;AACxE,uBAAuB;AACvB,MAAM,UAAU,iBAAiB,CAAC,IAAY,EAAE,KAAc;IAC5D,IAAI,IAAI,KAAK,EAAE;QAAE,OAAO,IAAI,CAAC;IAC7B,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC7D,MAAM,IAAI,GAAI,KAA4B,CAAC,IAAI,CAAC;IAChD,OAAO,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;AACtE,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAY,EAAE,KAAc;IAC1D,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAC/C,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAC/C,0EAA0E;IAC1E,yEAAyE;IACzE,iEAAiE;IACjE,0EAA0E;IAC1E,2EAA2E;IAC3E,yEAAyE;IACzE,qDAAqD;IACrD,IAAI,OAAO,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IACjD,IACE,IAAI,KAAK,qBAAqB;QAC9B,KAAK,KAAK,IAAI;QACd,OAAO,KAAK,KAAK,QAAQ;QACzB,UAAU,IAAI,KAAK;QACnB,WAAW,IAAI,KAAK;QACpB,MAAM,CAAC,QAAQ,CAAE,KAA+B,CAAC,QAAQ,CAAC;QAC1D,MAAM,CAAC,QAAQ,CAAE,KAAgC,CAAC,SAAS,CAAC;QAE5D,OAAO,UAAU,CAAC;IACpB,OAAO,IAAI,CAAC;AACd,CAAC"}
+{"version":3,"file":"delta-routing.js","sourceRoot":"","sources":["../src/delta-routing.ts"],"names":[],"mappings":"AAsBA;;;;;;;;;;;;;;GAcG;AACH,MAAM,OAAO,iBAAiB;IAIC;IAHZ,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,MAAM,GAAG,KAAK,CAAC;IAEvB,YAA6B,MAAM,EAAE;QAAR,QAAG,GAAH,GAAG,CAAK;IAAG,CAAC;IAEzC,oEAAoE;IACpE,IAAI,CAAC,IAAY;QACf,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO;QACjC,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YAChC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;YACnB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACvB,CAAC;IAED,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,QAAQ,CAAC,KAAa;QACpB,OAAO,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACzC,CAAC;IAED,KAAK;QACH,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;IACtB,CAAC;CACF;AAED,+DAA+D;AAC/D,yEAAyE;AACzE,2EAA2E;AAC3E,sEAAsE;AACtE,wEAAwE;AACxE,uBAAuB;AACvB,MAAM,UAAU,iBAAiB,CAAC,IAAY,EAAE,KAAc;IAC5D,IAAI,IAAI,KAAK,EAAE;QAAE,OAAO,IAAI,CAAC;IAC7B,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC7D,MAAM,IAAI,GAAI,KAA4B,CAAC,IAAI,CAAC;IAChD,OAAO,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;AACtE,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAY,EAAE,KAAc;IAC1D,wEAAwE;IACxE,uEAAuE;IACvE,qEAAqE;IACrE,kEAAkE;IAClE,yEAAyE;IACzE,sEAAsE;IACtE,2CAA2C;IAC3C,IAAI,OAAO,KAAK,KAAK,QAAQ;QAC3B,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC;IAClD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAC/C,0EAA0E;IAC1E,yEAAyE;IACzE,iEAAiE;IACjE,0EAA0E;IAC1E,2EAA2E;IAC3E,yEAAyE;IACzE,qDAAqD;IACrD,IAAI,OAAO,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IACjD,IACE,IAAI,KAAK,qBAAqB;QAC9B,KAAK,KAAK,IAAI;QACd,OAAO,KAAK,KAAK,QAAQ;QACzB,UAAU,IAAI,KAAK;QACnB,WAAW,IAAI,KAAK;QACpB,MAAM,CAAC,QAAQ,CAAE,KAA+B,CAAC,QAAQ,CAAC;QAC1D,MAAM,CAAC,QAAQ,CAAE,KAAgC,CAAC,SAAS,CAAC;QAE5D,OAAO,UAAU,CAAC;IACpB,iEAAiE;IACjE,yEAAyE;IACzE,2EAA2E;IAC3E,yBAAyB;IACzB,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QACtE,OAAO,SAAS,CAAC;IACnB,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,UAAU,kBAAkB,CAChC,IAAY,EACZ,KAAc;IAEd,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IAC7B,CAAC;IACD,2EAA2E;IAC3E,2EAA2E;IAC3E,0EAA0E;IAC1E,2EAA2E;IAC3E,iEAAiE;IACjE,IAAI,IAAI,KAAK,EAAE;QAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IAC5C,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAgC,CAAC,EAAE,CAAC;QAC3E,MAAM,QAAQ,GAAG,GAAG,IAAI,IAAI,GAAG,EAAE,CAAC;QAClC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC7B,oEAAoE;YACpE,6DAA6D;YAC7D,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAAE,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;;gBACnE,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC9B,CAAC;aAAM,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,OAAO,IAAI,KAAK,SAAS,EAAE,CAAC;YACjE,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/C,CAAC;aAAM,CAAC;YACN,2CAA2C;YAC3C,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;AAC7B,CAAC"}

package/dist/index.js

@@ -1,7 +1,8 @@
+import { waitForContainerManager } from "signalk-container-helper";
 import { ILPWriter } from "./ilp-writer.js";
 import { QueryClient, isReadOnlySQL } from "./query-client.js";
 import { ConfigSchema, normalizeConfig } from "./config/schema.js";
-import { extractVesselName, routeDeltaValue } from "./delta-routing.js";
+import { extractVesselName, flattenObjectValue, routeDeltaValue, UnstorableTracker, } from "./delta-routing.js";
 import { createHistoryProviderV2 } from "./history-v2.js";
 import { createHistoryProviderV1 } from "./history-v1.js";
 import { startRetention } from "./retention.js";
@@ -13,6 +14,12 @@ import { QUESTDB_INTERNAL_HTTP_PORT, QUESTDB_INTERNAL_ILP_PORT, QUESTDB_INTERNAL
 import { nofileClampSatisfied, readMaxMapCount } from "./host-limits.js";
 import { buildContainerEnv } from "./container-env.js";
 import { PathMatcher, RateMatcher, Throttle } from "./path-matcher.js";
+// signalk-container's API, imported rather than mirrored. The 156-line
+// hand-written copy that used to live here drifted by construction: it was
+// only ever updated when this plugin happened to need a new member, so
+// getContainerNofile sat unmirrored from 1.25.3 until a probe needed it.
+// signalk-container-helper carries the same types and pins them against
+// signalk-container/types in CI, so a drift fails there instead of here.
 // The managed QuestDB container's name. signalk-container prefixes it with
 // `sk-` for the actual container and its DNS name (e.g. `sk-signalk-questdb`).
 const QUESTDB_CONTAINER_NAME = "signalk-questdb";
@@ -184,6 +191,13 @@ export default (app) => {
     // without creating resources, so an aborted start can't resurrect a
     // just-purged container.
     let startAbort = null;
+    // How long a start waits for signalk-container to publish its API and for
+    // runtime detection to settle. Generous because both are one-off startup
+    // costs on a cold boot — signalk-container may still be pulling its own
+    // image — and the alternative to waiting is a spurious "plugin required"
+    // error on a system that has it. A purge aborts the wait, so a wedged start
+    // does not hold the lock for this long.
+    const CONTAINER_MANAGER_TIMEOUT_MS = 120000;
     // How long purge waits to acquire the lifecycle lock before proceeding
     // anyway. After this it forces its teardown through even if a hung start
     // never released the lock (see PURGE_LOCK_TIMEOUT_MS use below).
@@ -346,8 +360,22 @@ export default (app) => {
                 [`${QUESTDB_INTERNAL_ILP_PORT}/tcp`]: `${bind}:${ilpPort}`,
                 [`${QUESTDB_INTERNAL_PG_PORT}/tcp`]: `${bind}:${pgPort}`,
             };
+            // ensureNetwork is feature-detected: the helper types every
+            // version-gated member as optional so a plugin degrades on an older
+            // signalk-container rather than throwing. The previous hand-written
+            // mirror declared it required, which hid the need for this guard —
+            // and the connectToNetwork call further down was already guarding, so
+            // the two disagreed about the same API.
+            //
+            // networkMode is set REGARDLESS. Creating the network is the optional
+            // step; joining it is what the companion Grafana depends on, and on a
+            // signalk-container without ensureNetwork the network the operator
+            // named almost certainly exists already. Gating the assignment on the
+            // probe would silently drop the container off that network.
             if (config.networkName) {
-                await containers.ensureNetwork(config.networkName);
+                if (typeof containers.ensureNetwork === "function") {
+                    await containers.ensureNetwork(config.networkName);
+                }
                 containerConfig.networkMode = config.networkName;
             }
             // A 0.0.0.0-published port is reached on 127.0.0.1 or via the host
@@ -374,10 +402,17 @@ export default (app) => {
         // endpoint from whatever address signalk-container reports.
         containerConfig.signalkAccessiblePorts = QUESTDB_ACCESSIBLE_PORTS;
         return async () => {
+            // The two are feature-detected SEPARATELY. Requiring both would skip
+            // attachment entirely on a signalk-container that has connectToNetwork
+            // but not ensureNetwork -- and there the network almost certainly
+            // already exists, so attaching still works. Creating it is the
+            // optional step; joining it is the one Grafana depends on.
             if (config.networkName &&
                 typeof containers.connectToNetwork === "function") {
                 try {
-                    await containers.ensureNetwork(config.networkName);
+                    if (typeof containers.ensureNetwork === "function") {
+                        await containers.ensureNetwork(config.networkName);
+                    }
                     await containers.connectToNetwork(name, config.networkName);
                 }
                 catch (err) {
@@ -394,6 +429,18 @@ export default (app) => {
     // 100% CPU with an 89-pattern filter list.
     let pathFilterMatcher = null;
     let rateMatcher = null;
+    // Paths carrying a value with no representation in any table: arrays, and
+    // objects nested below the one level `flattenObjectValue` descends. These
+    // used to vanish without any trace at all — the whole reason
+    // navigation.attitude went unnoticed until a user asked (issue #128) — so
+    // record them and surface the count on /api/status.
+    //
+    // See UnstorableTracker: capped, and deliberately silent on the per-delta
+    // path. The diagnostic surfaces on /api/status, which is where a user will
+    // actually see it — this whole class of loss went unnoticed precisely
+    // because it was only ever visible in a log nobody had enabled (issue #128).
+    const unstorable = new UnstorableTracker();
+    const noteUnstorable = (path) => unstorable.note(path);
     // Takes only the mode: the patterns themselves live in pathFilterMatcher,
     // compiled from this same config at start. Accepting a `paths` array here
     // would imply it is consulted when it is not.
@@ -600,19 +647,34 @@ export default (app) => {
             },
         };
         if (config.managedContainer !== false) {
-            const containers = globalThis.__signalk_containerManager;
+            // Two phases in one call: poll for the manager global, then wait for
+            // runtime detection to settle. The previous code read the global ONCE,
+            // synchronously — which worked only because plugins start in
+            // alphabetical order and "signalk-questdb" sorts after
+            // "signalk-container". Any rename, or a slower container start, and the
+            // plugin reported "signalk-container required" on a system that has it.
+            //
+            // The signal is threaded through so a purge preempting a start stops
+            // the poll immediately rather than waiting out the full budget.
+            app.setPluginStatus("Waiting for signalk-container plugin...");
+            const { manager: containers, runtime } = await waitForContainerManager({
+                timeoutMs: CONTAINER_MANAGER_TIMEOUT_MS,
+                signal,
+                onWaiting: (phase) => {
+                    if (phase === "runtime")
+                        app.setPluginStatus("Waiting for container runtime detection...");
+                },
+            });
+            // Purge may have preempted us while the wait was pending; bail before
+            // touching the runtime so we don't recreate what purge is removing.
+            if (signal.aborted)
+                return;
             if (!containers) {
                 app.debug("containerManager not found");
                 publishError("signalk-container plugin required for managed mode. Install it or set managedContainer=false.");
                 return;
             }
-            app.setPluginStatus("Waiting for container runtime detection...");
-            await containers.whenReady();
-            // Purge may have preempted us while whenReady() was pending; bail before
-            // touching the runtime so we don't recreate what purge is removing.
-            if (signal.aborted)
-                return;
-            if (!containers.getRuntime()) {
+            if (!runtime) {
                 app.debug("container runtime not detected");
                 publishError("No container runtime detected. Check signalk-container plugin.");
                 return;
@@ -812,13 +874,32 @@ export default (app) => {
                 return;
             if (!isSelf && !config.recordOthers)
                 return;
-            if (!shouldRecord(path, config.pathFilter.mode))
-                return;
-            // Throttled per path AND context: the sampling rate bounds each
-            // vessel's stream, not the fleet's (issue #93). The stored context
-            // ("self" vs raw) is the key, matching what the rows carry.
-            if (isThrottled(path, isSelf ? "self" : context, config.defaultSamplingRate ?? 2000))
+            // Routed once, before the gates, because object values are gated on
+            // their leaf paths instead — see the "flatten" branch below. Applying
+            // the parent's gates to those would be wrong twice over: an
+            // include-filter listing only `navigation.attitude.roll` would drop the
+            // parent before any leaf was seen, and the parent would consume a
+            // throttle slot the leaves then have to wait out again, halving the
+            // effective sampling rate.
+            const route = routeDeltaValue(path, value);
+            // Recorded BEFORE the gates. A value nothing can store is worth
+            // reporting whether or not the user also filters or throttles that
+            // path — and gating it first would hide exactly the case this exists to
+            // surface, since a throttled array would return here and never be
+            // counted.
+            if (route === null) {
+                noteUnstorable(path);
                 return;
+            }
+            if (route !== "flatten") {
+                if (!shouldRecord(path, config.pathFilter.mode))
+                    return;
+                // Throttled per path AND context: the sampling rate bounds each
+                // vessel's stream, not the fleet's (issue #93). The stored context
+                // ("self" vs raw) is the key, matching what the rows carry.
+                if (isThrottled(path, isSelf ? "self" : context, config.defaultSamplingRate ?? 2000))
+                    return;
+            }
             // Rows are stamped with the server receive time, deliberately NOT the
             // delta's own timestamp: a boat is a set of independent clocks (GPS
             // time, RTC-less devices, gateway latencies), and storing per-source
@@ -832,7 +913,6 @@ export default (app) => {
             // even gets *more* accurate history. Re-sent ILP batches keep their
             // original stamps (baked at write() time), so replay-idempotency holds.
             const ctx = isSelf ? "self" : context;
-            const route = routeDeltaValue(path, value);
             if (route === "number") {
                 writer.write(path, ctx, value);
             }
@@ -848,6 +928,40 @@ export default (app) => {
             else if (route === "position") {
                 writer.writePosition(ctx, value);
             }
+            else if (route === "flatten") {
+                // Object values are recorded as their scalar leaves (issue #128).
+                const { leaves, skipped } = flattenObjectValue(path, value);
+                // Only the leaves are reported, never the parent as well. Each
+                // skipped leaf is already named here, so also noting the parent
+                // double-counts one problem and — worse — burns a second slot against
+                // the cap, hitting it sooner and hiding genuinely distinct paths.
+                //
+                // An object with NO storable leaf still gets reported: `skipped` then
+                // holds every key, so the drop is visible through them. The only
+                // silent case left is an empty object, which carries no data to lose.
+                for (const leafPath of skipped)
+                    noteUnstorable(leafPath);
+                for (const leaf of leaves) {
+                    // Filtered and throttled on the LEAF path, not the parent: the
+                    // leaves are what the history API exposes, so excluding
+                    // `navigation.attitude.yaw` has to actually exclude it. Throttling
+                    // per leaf also keeps one object's sampling budget from being
+                    // consumed by whichever key happened to be written first.
+                    if (!shouldRecord(leaf.path, config.pathFilter.mode))
+                        continue;
+                    if (isThrottled(leaf.path, ctx, config.defaultSamplingRate ?? 2000))
+                        continue;
+                    if (typeof leaf.value === "number") {
+                        writer.write(leaf.path, ctx, leaf.value);
+                    }
+                    else if (typeof leaf.value === "boolean") {
+                        writer.writeString(leaf.path, ctx, leaf.value ? "true" : "false", undefined, "boolean");
+                    }
+                    else {
+                        writer.writeString(leaf.path, ctx, leaf.value);
+                    }
+                }
+            }
         });
         unsubscribes.push(unsub);
         if (config.retentionDays && config.retentionDays > 0) {
@@ -1044,6 +1158,9 @@ export default (app) => {
             // restore count from the previous run.
             restoreSummary = null;
             restoreStatus = null;
+            // Same reason: a path that is unstorable under this config may not be
+            // under the next, so the count must not carry across a restart.
+            unstorable.clear();
             pluginErrorActive = false;
             for (const unsub of unsubscribes) {
                 try {
@@ -1245,6 +1362,17 @@ export default (app) => {
                             ? `${questdbEndpoints.http.host}:${questdbEndpoints.http.port}`
                             : null,
                         restore: restoreStatus,
+                        // Omitted entirely when nothing was dropped, so the panel can
+                        // treat presence as "there is something to report".
+                        unstorable: unstorable.size > 0
+                            ? {
+                                paths: unstorable.size,
+                                truncated: unstorable.truncated,
+                                // A handful is enough to identify the shape; the tracked
+                                // set is capped but still larger than a status body wants.
+                                examples: unstorable.examples(5),
+                            }
+                            : null,
                     });
                 }
                 catch (err) {