signalk-questdb 1.5.2 → 1.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -0
- package/dist/history-v2.js +32 -2
- package/dist/history-v2.js.map +1 -1
- package/dist/host-limits.d.ts +9 -0
- package/dist/host-limits.js +58 -0
- package/dist/host-limits.js.map +1 -0
- package/dist/index.js +435 -49
- package/dist/index.js.map +1 -1
- package/dist/query-client.d.ts +1 -1
- package/dist/query-client.js +13 -2
- package/dist/query-client.js.map +1 -1
- package/dist/test/history-v2.test.js +93 -0
- package/dist/test/history-v2.test.js.map +1 -1
- package/dist/test/host-limits.test.d.ts +1 -0
- package/dist/test/host-limits.test.js +71 -0
- package/dist/test/host-limits.test.js.map +1 -0
- package/dist/test/query-client.test.js +72 -0
- package/dist/test/query-client.test.js.map +1 -1
- package/dist/test/wal-monitor.test.d.ts +1 -0
- package/dist/test/wal-monitor.test.js +333 -0
- package/dist/test/wal-monitor.test.js.map +1 -0
- package/dist/wal-monitor.d.ts +65 -0
- package/dist/wal-monitor.js +293 -0
- package/dist/wal-monitor.js.map +1 -0
- package/package.json +1 -1
- package/public/540.js +1 -1
- package/public/805.js +1 -1
- package/public/main.js +1 -1
- package/public/remoteEntry.js +1 -1
package/README.md
CHANGED
|
@@ -265,6 +265,35 @@ podman events --since 2h --stream=false --filter container=sk-signalk-questdb \
|
|
|
265
265
|
--filter event=oom --filter event=died
|
|
266
266
|
```
|
|
267
267
|
|
|
268
|
+
### History queries slow or timing out / "out-of-memory" errors with free RAM
|
|
269
|
+
|
|
270
|
+
QuestDB memory-maps every partition column file and every pending WAL segment
|
|
271
|
+
it touches. The Linux kernel caps how many memory mappings one process may
|
|
272
|
+
hold (`vm.max_map_count`), and the stock value on Debian, Ubuntu ≤ 22.04, and
|
|
273
|
+
RHEL (65530) is far below what QuestDB recommends (1048576). A fresh database
|
|
274
|
+
fits easily; months of daily partitions across the three tables — plus a
|
|
275
|
+
segment backlog if a table's WAL is suspended — can exhaust the limit. When
|
|
276
|
+
that happens `mmap` fails with out-of-memory errors (errno 12) **even though
|
|
277
|
+
plenty of RAM is free**: queries error out or crawl, and the WAL apply job can
|
|
278
|
+
suspend a table.
|
|
279
|
+
|
|
280
|
+
The QuestDB Web Console shows the same warning
|
|
281
|
+
(`vm.max_map_count limit is too low`), the plugin's config panel shows a
|
|
282
|
+
banner, and the server log gets a warning at plugin startup. Fix it in a
|
|
283
|
+
shell **on the host machine itself** — not inside the QuestDB container via
|
|
284
|
+
`podman exec`: the limit is kernel-global, a container cannot change it, and
|
|
285
|
+
the QuestDB image has neither `sudo` nor `/etc/sysctl.d`. Being kernel-global
|
|
286
|
+
also means this works no matter how Signal K itself is deployed:
|
|
287
|
+
|
|
288
|
+
```bash
|
|
289
|
+
echo 'vm.max_map_count=1048576' | sudo tee /etc/sysctl.d/99-questdb.conf
|
|
290
|
+
sudo sysctl --system
|
|
291
|
+
```
|
|
292
|
+
|
|
293
|
+
This takes effect immediately — no container or host restart needed. See
|
|
294
|
+
[QuestDB capacity planning](https://questdb.com/docs/getting-started/capacity-planning/#max-virtual-memory-areas-limit)
|
|
295
|
+
for background.
|
|
296
|
+
|
|
268
297
|
## History API Provider
|
|
269
298
|
|
|
270
299
|
QuestDB automatically registers as the **default** Signal K v2 History API provider. Any app or Grafana plugin that queries `/signalk/v2/api/history/` uses QuestDB.
|
package/dist/history-v2.js
CHANGED
|
@@ -82,9 +82,39 @@ function normalizeContext(context, selfContext) {
|
|
|
82
82
|
}
|
|
83
83
|
return context;
|
|
84
84
|
}
|
|
85
|
+
// Upper bound on SAMPLE BY buckets a single request may generate. The
|
|
86
|
+
// SAMPLE BY queries use FILL(NULL), which fabricates a row for EVERY bucket
|
|
87
|
+
// in the range whether data exists or not — a caller asking for weeks at 1s
|
|
88
|
+
// resolution would stream millions of rows through QuestDB, Node's JSON
|
|
89
|
+
// parser, and the HTTP response, wedging Pi-class servers for minutes. The
|
|
90
|
+
// well-behaved clients budget ~2000 points per request; a million covers any
|
|
91
|
+
// sane chart while keeping the worst case bounded.
|
|
92
|
+
const MAX_SAMPLE_BUCKETS = 1_000_000;
|
|
93
|
+
// Effective SAMPLE BY period: QuestDB rejects `SAMPLE BY 0s`, so fractional
|
|
94
|
+
// resolutions (e.g. 0.5) must clamp up to 1s instead of flooring to zero.
|
|
95
|
+
function effectiveResolution(resolution) {
|
|
96
|
+
return Math.max(1, Math.floor(resolution));
|
|
97
|
+
}
|
|
85
98
|
function createHistoryProviderV2(queryClient, selfContext) {
|
|
86
99
|
async function getValues(query) {
|
|
87
100
|
const range = (0, time_range_1.resolveTimeRange)(query);
|
|
101
|
+
// Only specs that actually run a SAMPLE BY query fabricate buckets:
|
|
102
|
+
// client-side aggregates (sma/ema/middle_index) read raw rows under a
|
|
103
|
+
// LIMIT and must not trip the cap. Each qualifying spec issues its own
|
|
104
|
+
// SAMPLE BY, so the fabricated total scales with their count.
|
|
105
|
+
const sampledSpecs = query.pathSpecs.filter((spec) => spec.path === "navigation.position" ||
|
|
106
|
+
!needsClientSideAggregation(spec.aggregate)).length;
|
|
107
|
+
if (sampledSpecs > 0 && query.resolution && query.resolution > 0) {
|
|
108
|
+
const rangeSec = (Date.parse(range.to) - Date.parse(range.from)) / 1000;
|
|
109
|
+
const bucketsPerSeries = Math.ceil(rangeSec / effectiveResolution(query.resolution));
|
|
110
|
+
const buckets = bucketsPerSeries * sampledSpecs;
|
|
111
|
+
if (buckets > MAX_SAMPLE_BUCKETS) {
|
|
112
|
+
throw new Error(`resolution ${query.resolution}s over this range produces ` +
|
|
113
|
+
`${buckets} sample buckets across ${sampledSpecs} paths ` +
|
|
114
|
+
`(max ${MAX_SAMPLE_BUCKETS}) — use a coarser resolution or ` +
|
|
115
|
+
`a shorter range`);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
88
118
|
const requestedContext = query.context ?? "vessels.self";
|
|
89
119
|
const storedContext = normalizeContext(requestedContext, selfContext);
|
|
90
120
|
const safeContext = (0, query_client_1.validateIdentifier)(storedContext);
|
|
@@ -104,7 +134,7 @@ function createHistoryProviderV2(queryClient, selfContext) {
|
|
|
104
134
|
const posAgg = spec.aggregate === "last" ? "last" : "first";
|
|
105
135
|
let sql;
|
|
106
136
|
if (query.resolution && query.resolution > 0) {
|
|
107
|
-
sql = `SELECT ts, ${posAgg}(lat) as lat, ${posAgg}(lon) as lon FROM ${table} WHERE ${where} SAMPLE BY ${
|
|
137
|
+
sql = `SELECT ts, ${posAgg}(lat) as lat, ${posAgg}(lon) as lon FROM ${table} WHERE ${where} SAMPLE BY ${effectiveResolution(query.resolution)}s FILL(NULL) ORDER BY ts`;
|
|
108
138
|
}
|
|
109
139
|
else {
|
|
110
140
|
sql = `SELECT ts, lat, lon FROM ${table} WHERE ${where} ORDER BY ts LIMIT 10000`;
|
|
@@ -148,7 +178,7 @@ function createHistoryProviderV2(queryClient, selfContext) {
|
|
|
148
178
|
const aggExpr = aggregateToSql(spec.aggregate);
|
|
149
179
|
let sql;
|
|
150
180
|
if (query.resolution && query.resolution > 0) {
|
|
151
|
-
sql = `SELECT ts, ${aggExpr} as agg_value FROM ${table} WHERE ${where} SAMPLE BY ${
|
|
181
|
+
sql = `SELECT ts, ${aggExpr} as agg_value FROM ${table} WHERE ${where} SAMPLE BY ${effectiveResolution(query.resolution)}s FILL(NULL) ORDER BY ts`;
|
|
152
182
|
}
|
|
153
183
|
else {
|
|
154
184
|
sql = `SELECT ts, value FROM ${table} WHERE ${where} ORDER BY ts LIMIT 10000`;
|
package/dist/history-v2.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"history-v2.js","sourceRoot":"","sources":["../src/history-v2.ts"],"names":[],"mappings":";;
|
|
1
|
+
{"version":3,"file":"history-v2.js","sourceRoot":"","sources":["../src/history-v2.ts"],"names":[],"mappings":";;AAgJA,0DAuLC;AAvUD,iDAIwB;AACxB,6CAA+D;AAoC/D,SAAS,cAAc,CAAC,MAAc;IACpC,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,SAAS;YACZ,OAAO,YAAY,CAAC;QACtB,KAAK,KAAK;YACR,OAAO,YAAY,CAAC;QACtB,KAAK,KAAK;YACR,OAAO,YAAY,CAAC;QACtB,KAAK,OAAO;YACV,OAAO,cAAc,CAAC;QACxB,KAAK,MAAM;YACT,OAAO,aAAa,CAAC;QACvB,KAAK,KAAK;YACR,OAAO,+BAA+B,CAAC;QACzC;YACE,OAAO,YAAY,CAAC;IACxB,CAAC;AACH,CAAC;AAED,SAAS,0BAA0B,CAAC,MAAc;IAChD,OAAO,MAAM,KAAK,cAAc,IAAI,MAAM,KAAK,KAAK,IAAI,MAAM,KAAK,KAAK,CAAC;AAC3E,CAAC;AAED,SAAS,UAAU,CAAC,MAAyB,EAAE,CAAS;IACtD,MAAM,MAAM,GAAsB,EAAE,CAAC;IACrC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClB,SAAS;QACX,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACf,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YAAE,MAAM,CAAC,KAAK,EAAE,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;IACjE,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,UAAU,CACjB,MAAyB,EACzB,KAAa;IAEb,MAAM,MAAM,GAAsB,EAAE,CAAC;IACrC,IAAI,IAAI,GAAkB,IAAI,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClB,SAAS;QACX,CAAC;QACD,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,IAAI,GAAG,CAAC,CAAC;QACX,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC;QACxC,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,eAAe,CAAC,KAAoB,EAAE,OAAgB;IAC7D,MAAM,IAAI,GAAG,IAAA,gCAAiB,EAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,EAAE,GAAG,IAAA,gCAAiB,EAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACvC,IAAI,KAAK,GAAG,UAAU,IAAI,gBAAgB,EAAE,GAAG,CAAC;IAChD,IAAI,OAAO,EAAE,CAAC;QACZ,KAAK,IAAI,mBAAmB,IAAA,iCAAkB,EAAC,OAAO,CAAC,GAAG,CAAC;IAC7D,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,gBAAgB,CAAC,OAAe,EAAE,WAAmB;IAC5D,IACE,OAAO,KAAK,MAAM;QAClB,OAAO,KAAK,cAAc;QAC1B,OAAO,KAAK,WAAW,EACvB,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,sEAAsE;AACtE,4EAA4E;AAC5E,4EAA4E;AAC5E,wEAAwE;AACxE,2EAA2E;AAC3E,6EAA6E;AAC7E,mDAAmD;AACnD,MAAM,kBAAkB,GAAG,SAAS,CAAC;AAErC,4EAA4E;AAC5E,0EAA0E;AAC1E,SAAS,mBAAmB,CAAC,UAAkB;IAC7C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;AAC7C,CAAC;AAED,SAAgB,uBAAuB,CACrC,WAAwB,EACxB,WAAmB;IAEnB,KAAK,UAAU,SAAS,CAAC,KAAoB;QAC3C,MAAM,KAAK,GAAG,IAAA,6BAAgB,EAAC,KAAY,CAAC,CAAC;QAE7C,oEAAoE;QACpE,sEAAsE;QACtE,uEAAuE;QACvE,8DAA8D;QAC9D,MAAM,YAAY,GAAG,KAAK,CAAC,SAAS,CAAC,MAAM,CACzC,CAAC,IAAI,EAAE,EAAE,CACP,IAAI,CAAC,IAAI,KAAK,qBAAqB;YACnC,CAAC,0BAA0B,CAAC,IAAI,CAAC,SAAS,CAAC,CAC9C,CAAC,MAAM,CAAC;QAET,IAAI,YAAY,GAAG,CAAC,IAAI,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;YACjE,MAAM,QAAQ,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC;YACxE,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAChC,QAAQ,GAAG,mBAAmB,CAAC,KAAK,CAAC,UAAU,CAAC,CACjD,CAAC;YACF,MAAM,OAAO,GAAG,gBAAgB,GAAG,YAAY,CAAC;YAChD,IAAI,OAAO,GAAG,kBAAkB,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,CACb,cAAc,KAAK,CAAC,UAAU,6BAA6B;oBACzD,GAAG,OAAO,0BAA0B,YAAY,SAAS;oBACzD,QAAQ,kBAAkB,kCAAkC;oBAC5D,iBAAiB,CACpB,CAAC;YACJ,CAAC;QACH,CAAC;QAED,MAAM,gBAAgB,GAAG,KAAK,CAAC,OAAO,IAAI,cAAc,CAAC;QACzD,MAAM,aAAa,GAAG,gBAAgB,CAAC,gBAAgB,EAAE,WAAW,CAAC,CAAC;QACtE,MAAM,WAAW,GAAG,IAAA,iCAAkB,EAAC,aAAa,CAAC,CAAC;QAEtD,MAAM,UAAU,GAAuC,EAAE,CAAC;QAC1D,MAAM,UAAU,GAAqC,IAAI,GAAG,EAAE,CAAC;QAE/D,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,IAAA,iCAAkB,EAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/C,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;YAE7D,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,KAAK,qBAAqB,CAAC;YACvD,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,SAAS,CAAC;YAE1D,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,KAAK,GAAG,eAAe,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;gBAClD,oEAAoE;gBACpE,+DAA+D;gBAC/D,oEAAoE;gBACpE,+DAA+D;gBAC/D,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;gBAC5D,IAAI,GAAW,CAAC;gBAChB,IAAI,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;oBAC7C,GAAG,GAAG,cAAc,MAAM,iBAAiB,MAAM,qBAAqB,KAAK,UAAU,KAAK,cAAc,mBAAmB,CAAC,KAAK,CAAC,UAAU,CAAC,0BAA0B,CAAC;gBAC1K,CAAC;qBAAM,CAAC;oBACN,GAAG,GAAG,4BAA4B,KAAK,UAAU,KAAK,0BAA0B,CAAC;gBACnF,CAAC;gBACD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC3C,MAAM,IAAI,GAAwB,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC;oBAC5D,GAAG,CAAC,CAAC,CAAW;oBAChB,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI;wBAChC,CAAC,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE;wBACzC,CAAC,CAAC,IAAI;iBACT,CAAC,CAAC;gBACH,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;gBAChC,SAAS;YACX,CAAC;YAED,MAAM,KAAK,GAAG,GAAG,eAAe,CAAC,KAAK,EAAE,WAAW,CAAC,gBAAgB,QAAQ,GAAG,CAAC;YAEhF,IAAI,0BAA0B,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC/C,MAAM,GAAG,GAAG,yBAAyB,KAAK,UAAU,KAAK,0BAA0B,CAAC;gBACpF,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC3C,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAW,CAAC,CAAC;gBAC7D,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAkB,CAAC,CAAC;gBAEnE,IAAI,QAA2B,CAAC;gBAChC,IAAI,IAAI,CAAC,SAAS,KAAK,KAAK,EAAE,CAAC;oBAC7B,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;oBACjD,QAAQ,GAAG,UAAU,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;gBACtC,CAAC;qBAAM,IAAI,IAAI,CAAC,SAAS,KAAK,KAAK,EAAE,CAAC;oBACpC,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC;oBACrD,QAAQ,GAAG,UAAU,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;gBAC1C,CAAC;qBAAM,CAAC;oBACN,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBAC7C,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;gBACxE,CAAC;gBAED,MAAM,IAAI,GAAwB,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC1D,EAAE;oBACF,QAAQ,CAAC,CAAC,CAAC;iBACZ,CAAC,CAAC;gBACH,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;gBAChC,SAAS;YACX,CAAC;YAED,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC/C,IAAI,GAAW,CAAC;YAChB,IAAI,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;gBAC7C,GAAG,GAAG,cAAc,OAAO,sBAAsB,KAAK,UAAU,KAAK,cAAc,mBAAmB,CAAC,KAAK,CAAC,UAAU,CAAC,0BAA0B,CAAC;YACrJ,CAAC;iBAAM,CAAC;gBACN,GAAG,GAAG,yBAAyB,KAAK,UAAU,KAAK,0BAA0B,CAAC;YAChF,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC3C,MAAM,IAAI,GAAwB,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC;gBAC5D,GAAG,CAAC,CAAC,CAAW;gBAChB,GAAG,CAAC,CAAC,CAAC;aACP,CAAC,CAAC;YACH,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAClC,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;QACxC,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YACvC,KAAK,MAAM,CAAC,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;gBACxB,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QACD,MAAM,gBAAgB,GAAG,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,IAAI,EAAE,CAAC;QAE1D,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACrD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAgC,CAAC;QAC1D,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,UAAU,EAAE,CAAC;YACtC,MAAM,CAAC,GAAG,IAAI,GAAG,EAAmB,CAAC;YACrC,KAAK,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;gBAC7B,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;YACjB,CAAC;YACD,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACzB,CAAC;QAED,MAAM,IAAI,GAA6B,gBAAgB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;YACjE,MAAM,GAAG,GAA2B,CAAC,EAAE,CAAC,CAAC;YACzC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;gBAC7B,MAAM,CAAC,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAC9B,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC;YAC/B,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,gBAAgB;YACzB,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE;YACzC,MAAM,EAAE,UAAU;YAClB,IAAI;SACL,CAAC;IACJ,CAAC;IAED,KAAK,UAAU,QAAQ,CAAC,KAAmB;QACzC,MAAM,KAAK,GAAG,IAAA,6BAAgB,EAAC,KAAY,CAAC,CAAC;QAC7C,MAAM,KAAK,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAErC,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,IAAI,CACnC,2CAA2C,KAAK;;qDAED,KAAK;qBACrC,CAChB,CAAC;QAEF,OAAO,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAW,CAAC,CAAC;IACvD,CAAC;IAED,KAAK,UAAU,WAAW,CAAC,KAAsB;QAC/C,MAAM,KAAK,GAAG,IAAA,6BAAgB,EAAC,KAAY,CAAC,CAAC;QAC7C,MAAM,KAAK,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAErC,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,IAAI,CACnC,8CAA8C,KAAK;;wDAED,KAAK;wBACrC,CACnB,CAAC;QAEF,oEAAoE;QACpE,OAAO,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YAChC,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,CAAW,CAAC;YAC7B,OAAO,GAAG,KAAK,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC;QAC/C,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;AAC9C,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export declare const RECOMMENDED_MAX_MAP_COUNT = 1048576;
|
|
2
|
+
export interface MaxMapCountStatus {
|
|
3
|
+
current: number;
|
|
4
|
+
recommended: number;
|
|
5
|
+
tooLow: boolean;
|
|
6
|
+
}
|
|
7
|
+
export declare function parseMaxMapCount(content: string): number | null;
|
|
8
|
+
export declare function evaluateMaxMapCount(current: number): MaxMapCountStatus;
|
|
9
|
+
export declare function readMaxMapCount(path?: string): Promise<MaxMapCountStatus | null>;
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// Host kernel limits that QuestDB depends on but cannot raise from inside its
|
|
3
|
+
// container.
|
|
4
|
+
//
|
|
5
|
+
// QuestDB memory-maps every column file of every partition it touches, plus
|
|
6
|
+
// every pending WAL segment. `vm.max_map_count` caps how many mappings one
|
|
7
|
+
// process may hold; when the cap is hit, mmap fails with ENOMEM (errno=12) —
|
|
8
|
+
// queries error out and the WAL apply job can suspend a table, even though
|
|
9
|
+
// plenty of RAM is free. The stock Debian/Ubuntu/RHEL value (65530) is fine
|
|
10
|
+
// for a fresh database but months of daily partitions × three tables — plus
|
|
11
|
+
// a segment backlog once a table IS suspended — can exhaust it. QuestDB's own
|
|
12
|
+
// web console warns below 1048576; we mirror exactly that check so both
|
|
13
|
+
// surfaces agree.
|
|
14
|
+
//
|
|
15
|
+
// The sysctl is kernel-global (not namespaced), so reading it from the Signal
|
|
16
|
+
// K process gives the value that governs the QuestDB container whenever both
|
|
17
|
+
// share the kernel — i.e. in managed-container mode, regardless of whether
|
|
18
|
+
// Signal K itself runs bare-metal or containerized. In external mode QuestDB
|
|
19
|
+
// may live on another machine, so the local value proves nothing and the
|
|
20
|
+
// probe should not be used.
|
|
21
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
22
|
+
exports.RECOMMENDED_MAX_MAP_COUNT = void 0;
|
|
23
|
+
exports.parseMaxMapCount = parseMaxMapCount;
|
|
24
|
+
exports.evaluateMaxMapCount = evaluateMaxMapCount;
|
|
25
|
+
exports.readMaxMapCount = readMaxMapCount;
|
|
26
|
+
const promises_1 = require("fs/promises");
|
|
27
|
+
exports.RECOMMENDED_MAX_MAP_COUNT = 1048576;
|
|
28
|
+
const MAX_MAP_COUNT_PATH = "/proc/sys/vm/max_map_count";
|
|
29
|
+
// The proc file holds a single decimal number (plus a trailing newline), but
|
|
30
|
+
// parse defensively: a garbled read must degrade to "unknown", never to a
|
|
31
|
+
// bogus warning.
|
|
32
|
+
function parseMaxMapCount(content) {
|
|
33
|
+
const trimmed = content.trim();
|
|
34
|
+
if (!/^\d+$/.test(trimmed))
|
|
35
|
+
return null;
|
|
36
|
+
const value = Number(trimmed);
|
|
37
|
+
return Number.isSafeInteger(value) ? value : null;
|
|
38
|
+
}
|
|
39
|
+
function evaluateMaxMapCount(current) {
|
|
40
|
+
return {
|
|
41
|
+
current,
|
|
42
|
+
recommended: exports.RECOMMENDED_MAX_MAP_COUNT,
|
|
43
|
+
tooLow: current < exports.RECOMMENDED_MAX_MAP_COUNT,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
// Null on non-Linux, unreadable proc, or unparseable content — callers treat
|
|
47
|
+
// null as "unknown, say nothing" rather than warning on speculation.
|
|
48
|
+
async function readMaxMapCount(path = MAX_MAP_COUNT_PATH) {
|
|
49
|
+
try {
|
|
50
|
+
const content = await (0, promises_1.readFile)(path, "utf8");
|
|
51
|
+
const value = parseMaxMapCount(content);
|
|
52
|
+
return value === null ? null : evaluateMaxMapCount(value);
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
return null;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
//# sourceMappingURL=host-limits.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"host-limits.js","sourceRoot":"","sources":["../src/host-limits.ts"],"names":[],"mappings":";AAAA,8EAA8E;AAC9E,aAAa;AACb,EAAE;AACF,4EAA4E;AAC5E,2EAA2E;AAC3E,6EAA6E;AAC7E,2EAA2E;AAC3E,4EAA4E;AAC5E,4EAA4E;AAC5E,8EAA8E;AAC9E,wEAAwE;AACxE,kBAAkB;AAClB,EAAE;AACF,8EAA8E;AAC9E,6EAA6E;AAC7E,2EAA2E;AAC3E,6EAA6E;AAC7E,yEAAyE;AACzE,4BAA4B;;;AAiB5B,4CAKC;AAED,kDAMC;AAID,0CAUC;AA1CD,0CAAuC;AAE1B,QAAA,yBAAyB,GAAG,OAAO,CAAC;AAEjD,MAAM,kBAAkB,GAAG,4BAA4B,CAAC;AAQxD,6EAA6E;AAC7E,0EAA0E;AAC1E,iBAAiB;AACjB,SAAgB,gBAAgB,CAAC,OAAe;IAC9C,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAC/B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IACxC,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;IAC9B,OAAO,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;AACpD,CAAC;AAED,SAAgB,mBAAmB,CAAC,OAAe;IACjD,OAAO;QACL,OAAO;QACP,WAAW,EAAE,iCAAyB;QACtC,MAAM,EAAE,OAAO,GAAG,iCAAyB;KAC5C,CAAC;AACJ,CAAC;AAED,6EAA6E;AAC7E,qEAAqE;AAC9D,KAAK,UAAU,eAAe,CACnC,OAAe,kBAAkB;IAEjC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,IAAA,mBAAQ,EAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC7C,MAAM,KAAK,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;QACxC,OAAO,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAC5D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
|