signalk-questdb 2.1.2 → 2.1.3

package/README.md

@@ -417,7 +417,7 @@ The plugin is optimized for Raspberry Pi and similar low-power devices:
 
 - **Default sampling rate** of 2000ms limits each path to 1 write per 2 seconds, keeping write volume modest on busy NMEA 2000 buses
 - **Resource caps** of 768 MB RAM and 1.5 CPU cores (cgroup limits via signalk-container) keep QuestDB from squeezing co-resident containers like Grafana, mayara, or signalk-backup. The JVM auto-sizes its heap to a fraction of the memory cap, so total footprint (heap + off-heap) is bounded
-- **QuestDB worker threads** reduced to 1 each (WAL, shared, ILP) to minimize CPU usage
+- **QuestDB worker tuning** cuts idle CPU on low-power hosts. QuestDB sizes its worker pools from the host core count and busy-polls before sleeping — sensible on a server, wasteful on a mostly idle boat database, where threads burn CPU waiting for work that never arrives. The plugin pins its worker pools down and quiets QuestDB's per-commit logging, keeping the messages that matter. One caveat worth knowing when upgrading the QuestDB image: an unrecognised setting is ignored silently, so a property renamed between releases looks exactly like one that worked
 - **ILP batching** commits every 5s (configurable via "Write batch interval"; a large backlog of buffered rows commits early). Each commit is one WAL transaction per table, and with deduplicated tables the apply cost of a transaction grows with partition size — frequent tiny commits eventually outpace what a Pi can apply and recording stalls. Bigger batches keep the WAL healthy; the cost is that at most one interval of buffered samples is lost on a hard crash
 - Per-path overrides allow faster rates for critical paths (e.g. `{ "environment.wind.*": 200 }`) while keeping slow-changing paths throttled
 - Set the memory limit to empty or CPU limit to `0` to disable the cap entirely on roomier hosts

package/dist/container-env.js

@@ -21,16 +21,82 @@
 // handful of fsyncs per commit interval (ILP batches at `ilpFlushIntervalMs`,
 // 5s by default), which at boat data rates is noise against losing a table.
 export const QUESTDB_COMMIT_MODE = "sync";
+// Poll cycles a worker spins before sleeping. QuestDB's default is 10000,
+// which keeps threads hot on a database that is idle most of the time. Low
+// enough to sleep promptly, high enough that a burst of ingest does not pay a
+// wake-up per batch.
+const WORKER_SLEEP_THRESHOLD = "100";
 export function buildContainerEnv(config) {
     return {
         QDB_TELEMETRY_ENABLED: "false",
         QDB_HTTP_ENABLED: "true",
         QDB_LINE_TCP_ENABLED: "true",
         QDB_CAIRO_COMMIT_MODE: QUESTDB_COMMIT_MODE,
-        // Reduce CPU usage on low-power devices (Pi, Cerbo)
-        QDB_CAIRO_WAL_APPLY_WORKER_COUNT: "1",
+        // ── Low-power tuning (Pi, Cerbo) ──────────────────────────────────────
+        //
+        // QuestDB sizes most worker pools from the host core count and defaults
+        // each pool to a busy-poll before it sleeps. That is right for a server
+        // and wrong for a boat: on a mostly idle database the threads burn CPU
+        // waiting for work that never comes, and the cost is a fixed floor rather
+        // than something that scales with the data.
+        //
+        // NOTE ON NAMING: QuestDB ignores an unrecognised QDB_* variable in
+        // silence — no warning, no startup failure — so a wrong name is
+        // indistinguishable from a setting that worked. Verify any change here
+        // against the running server rather than against the docs:
+        //
+        //   SELECT property_path, value, value_source FROM (SHOW PARAMETERS)
+        //     WHERE value_source = 'env'
+        //
+        // A key that reports `default` there was not applied, whatever was
+        // passed. (The startup log names each pool's `workers=N` too, but
+        // QDB_LOG_W_STDOUT_LEVEL below suppresses those lines, so the query is
+        // the reliable check.)
+        //
+        // `wal.apply.worker.count` has NO `cairo.` prefix, unlike its neighbours
+        // `cairo.wal.apply.*` — the one spelling in this block most likely to be
+        // guessed wrong.
+        QDB_WAL_APPLY_WORKER_COUNT: "1",
         QDB_SHARED_WORKER_COUNT: "1",
         QDB_LINE_TCP_WRITER_WORKER_COUNT: "1",
+        // Zero is this pool's default and does NOT mean "share the common pool"
+        // the way it does elsewhere — ILP keys per-worker state by worker id, so
+        // it gets a dedicated pool sized 2 regardless of the core count. Leaving
+        // it unset was our own gap: the writer pool was pinned to 1 while the io
+        // pool beside it still ran two threads.
+        QDB_LINE_TCP_IO_WORKER_COUNT: "1",
+        // Pools for features this plugin never uses. They are sized and polled
+        // regardless, which makes them precisely the ones nobody thinks to set.
+        QDB_VIEW_COMPILER_WORKER_COUNT: "1",
+        QDB_MAT_VIEW_REFRESH_WORKER_COUNT: "1",
+        QDB_LIVE_VIEW_REFRESH_WORKER_COUNT: "1",
+        QDB_EXPORT_WORKER_COUNT: "1",
+        // Poll cycles a worker spins before sleeping. There is no global default,
+        // so every pool needs its own — one left out keeps the full spin.
+        QDB_SHARED_WORKER_SLEEP_THRESHOLD: WORKER_SLEEP_THRESHOLD,
+        // The HTTP and pg-wire pools take their own threshold even though both
+        // currently draw workers from `shared-network` — QuestDB reports them at
+        // the 10000 default there rather than inheriting the shared value. Set
+        // one of the two `*.worker.count` keys and each becomes a dedicated pool
+        // where its own threshold is what governs, so leaving these unset is a
+        // gap that only opens when a default changes.
+        QDB_HTTP_WORKER_SLEEP_THRESHOLD: WORKER_SLEEP_THRESHOLD,
+        QDB_PG_WORKER_SLEEP_THRESHOLD: WORKER_SLEEP_THRESHOLD,
+        QDB_WAL_APPLY_WORKER_SLEEP_THRESHOLD: WORKER_SLEEP_THRESHOLD,
+        QDB_LINE_TCP_IO_WORKER_SLEEP_THRESHOLD: WORKER_SLEEP_THRESHOLD,
+        QDB_LINE_TCP_WRITER_WORKER_SLEEP_THRESHOLD: WORKER_SLEEP_THRESHOLD,
+        QDB_VIEW_COMPILER_WORKER_SLEEP_THRESHOLD: WORKER_SLEEP_THRESHOLD,
+        QDB_MAT_VIEW_REFRESH_WORKER_SLEEP_THRESHOLD: WORKER_SLEEP_THRESHOLD,
+        QDB_LIVE_VIEW_REFRESH_WORKER_SLEEP_THRESHOLD: WORKER_SLEEP_THRESHOLD,
+        QDB_EXPORT_WORKER_SLEEP_THRESHOLD: WORKER_SLEEP_THRESHOLD,
+        // QuestDB narrates every WAL commit at INFO — several lines a second of
+        // pure noise, written to the container log forever. ERROR is a floor, not
+        // an exact set, so CRITICAL and ADVISORY still come through: table
+        // suspension, ILP parse failures and the max_map_count warning all
+        // survive. What it drops is the INFO band — which also carries the WAL
+        // apply memory-pressure backoff that precedes a suspension, so raise this
+        // back to INFO when diagnosing a database that has stopped recording.
+        QDB_LOG_W_STDOUT_LEVEL: "ERROR",
         QDB_CAIRO_O3_COLUMN_MEMORY_SIZE: "262144",
         ...(config.compression && config.compression !== "none"
             ? {

package/dist/container-env.js.map

@@ -1 +1 @@
-{"version":3,"file":"container-env.js","sourceRoot":"","sources":["../src/container-env.ts"],"names":[],"mappings":"AAAA,qDAAqD;AACrD,EAAE;AACF,wEAAwE;AACxE,8EAA8E;AAC9E,6EAA6E;AAC7E,0EAA0E;AAC1E,6EAA6E;AAC7E,yBAAyB;AAIzB,kEAAkE;AAClE,6EAA6E;AAC7E,0EAA0E;AAC1E,0EAA0E;AAC1E,yEAAyE;AACzE,0EAA0E;AAC1E,6EAA6E;AAC7E,2EAA2E;AAC3E,0EAA0E;AAC1E,QAAQ;AACR,EAAE;AACF,4EAA4E;AAC5E,8EAA8E;AAC9E,4EAA4E;AAC5E,MAAM,CAAC,MAAM,mBAAmB,GAAG,MAAM,CAAC;AAE1C,MAAM,UAAU,iBAAiB,CAC/B,MAAuB;IAEvB,OAAO;QACL,qBAAqB,EAAE,OAAO;QAC9B,gBAAgB,EAAE,MAAM;QACxB,oBAAoB,EAAE,MAAM;QAC5B,qBAAqB,EAAE,mBAAmB;QAC1C,oDAAoD;QACpD,gCAAgC,EAAE,GAAG;QACrC,uBAAuB,EAAE,GAAG;QAC5B,gCAAgC,EAAE,GAAG;QACrC,+BAA+B,EAAE,QAAQ;QACzC,GAAG,CAAC,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,KAAK,MAAM;YACrD,CAAC,CAAC;gBACE,uCAAuC,EACrC,MAAM,CAAC,WAAW,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;gBAChD,GAAG,CAAC,MAAM,CAAC,WAAW,KAAK,MAAM,IAAI,MAAM,CAAC,gBAAgB;oBAC1D,CAAC,CAAC;wBACE,uCAAuC,EAAE,MAAM,CAC7C,MAAM,CAAC,gBAAgB,CACxB;qBACF;oBACH,CAAC,CAAC,EAAE,CAAC;aACR;YACH,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;AACJ,CAAC"}
+{"version":3,"file":"container-env.js","sourceRoot":"","sources":["../src/container-env.ts"],"names":[],"mappings":"AAAA,qDAAqD;AACrD,EAAE;AACF,wEAAwE;AACxE,8EAA8E;AAC9E,6EAA6E;AAC7E,0EAA0E;AAC1E,6EAA6E;AAC7E,yBAAyB;AAIzB,kEAAkE;AAClE,6EAA6E;AAC7E,0EAA0E;AAC1E,0EAA0E;AAC1E,yEAAyE;AACzE,0EAA0E;AAC1E,6EAA6E;AAC7E,2EAA2E;AAC3E,0EAA0E;AAC1E,QAAQ;AACR,EAAE;AACF,4EAA4E;AAC5E,8EAA8E;AAC9E,4EAA4E;AAC5E,MAAM,CAAC,MAAM,mBAAmB,GAAG,MAAM,CAAC;AAE1C,0EAA0E;AAC1E,2EAA2E;AAC3E,8EAA8E;AAC9E,qBAAqB;AACrB,MAAM,sBAAsB,GAAG,KAAK,CAAC;AAErC,MAAM,UAAU,iBAAiB,CAC/B,MAAuB;IAEvB,OAAO;QACL,qBAAqB,EAAE,OAAO;QAC9B,gBAAgB,EAAE,MAAM;QACxB,oBAAoB,EAAE,MAAM;QAC5B,qBAAqB,EAAE,mBAAmB;QAE1C,yEAAyE;QACzE,EAAE;QACF,wEAAwE;QACxE,wEAAwE;QACxE,uEAAuE;QACvE,0EAA0E;QAC1E,4CAA4C;QAC5C,EAAE;QACF,oEAAoE;QACpE,gEAAgE;QAChE,uEAAuE;QACvE,2DAA2D;QAC3D,EAAE;QACF,qEAAqE;QACrE,iCAAiC;QACjC,EAAE;QACF,mEAAmE;QACnE,kEAAkE;QAClE,uEAAuE;QACvE,uBAAuB;QACvB,EAAE;QACF,yEAAyE;QACzE,yEAAyE;QACzE,iBAAiB;QACjB,0BAA0B,EAAE,GAAG;QAC/B,uBAAuB,EAAE,GAAG;QAC5B,gCAAgC,EAAE,GAAG;QACrC,wEAAwE;QACxE,yEAAyE;QACzE,yEAAyE;QACzE,yEAAyE;QACzE,wCAAwC;QACxC,4BAA4B,EAAE,GAAG;QACjC,uEAAuE;QACvE,wEAAwE;QACxE,8BAA8B,EAAE,GAAG;QACnC,iCAAiC,EAAE,GAAG;QACtC,kCAAkC,EAAE,GAAG;QACvC,uBAAuB,EAAE,GAAG;QAE5B,0EAA0E;QAC1E,kEAAkE;QAClE,iCAAiC,EAAE,sBAAsB;QACzD,uEAAuE;QACvE,yEAAyE;QACzE,uEAAuE;QACvE,yEAAyE;QACzE,uEAAuE;QACvE,8CAA8C;QAC9C,+BAA+B,EAAE,sBAAsB;QACvD,6BAA6B,EAAE,sBAAsB;QACrD,oCAAoC,EAAE,sBAAsB;QAC5D,sCAAsC,EAAE,sBAAsB;QAC9D,0CAA0C,EAAE,sBAAsB;QAClE,wCAAwC,EAAE,sBAAsB;QAChE,2CAA2C,EAAE,sBAAsB;QACnE,4CAA4C,EAAE,sBAAsB;QACpE,iCAAiC,EAAE,sBAAsB;QAEzD,wEAAwE;QACxE,0EAA0E;QAC1E,mEAAmE;QACnE,mEAAmE;QACnE,uEAAuE;QACvE,0EAA0E;QAC1E,sEAAsE;QACtE,sBAAsB,EAAE,OAAO;QAE/B,+BAA+B,EAAE,QAAQ;QACzC,GAAG,CAAC,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,KAAK,MAAM;YACrD,CAAC,CAAC;gBACE,uCAAuC,EACrC,MAAM,CAAC,WAAW,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;gBAChD,GAAG,CAAC,MAAM,CAAC,WAAW,KAAK,MAAM,IAAI,MAAM,CAAC,gBAAgB;oBAC1D,CAAC,CAAC;wBACE,uCAAuC,EAAE,MAAM,CAC7C,MAAM,CAAC,gBAAgB,CACxB;qBACF;oBACH,CAAC,CAAC,EAAE,CAAC;aACR;YACH,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;AACJ,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "signalk-questdb",
-  "version": "2.1.2",
+  "version": "2.1.3",
   "type": "module",
   "description": "QuestDB history provider for Signal K — replaces signalk-to-influxdb",
   "keywords": [