sailkick-boat 0.23.0 → 0.23.3

package/README.md

@@ -52,6 +52,17 @@ files to send later — nothing is lost in the gap.
 
 - Network errors, `429` and `5xx` are retried with backoff (1 s → 60 s); the data stays
   on disk.
+- **All cloud traffic uses core `https`, not `fetch`** — telemetry sync, the offline
+  mirror, the cache-manifest poller, the contract check, the backfill and the Sync page
+  share one connection pool (`lib/net.js`), so one reset clears every subsystem. Twice in one afternoon this boat's
+  Signal K process stopped being able to open *any* outbound HTTPS connection — zero
+  sockets to `:443`, while a second process in the same container reached the same host
+  in under a second — and it never recovered on its own. Starlink sits behind CGNAT,
+  which drops idle NAT mappings without an RST, so a pooled keep-alive socket looks alive
+  to the client and is dead on the wire. `fetch` offers no supported way to reset its
+  pool from a plugin. Owning an agent means the pool can be rebuilt after repeated
+  transport failures (and it is, automatically, after five), and it means the log names
+  `ECONNRESET` or `ETIMEDOUT` instead of `fetch`'s uniformly useless "fetch failed".
 - A malformed batch (`4xx` other than the three below) is quarantined to `spool/dead/`
   rather than retried forever, because it would otherwise wedge the queue behind it.
 - **`404`, `401` and `403` are held, not quarantined.** A missing bucket or a rejected

package/lib/backfill/index.js

@@ -24,6 +24,7 @@ const fs = require('fs')
 const path = require('path')
 const { writeLines } = require('../sync/influxWrite')
 const { csvToLineProtocol } = require('./lineproto')
+const { request } = require('../net')
 
 const HOUR_MS = 3600000
 const DEFAULTS = {
@@ -121,13 +122,13 @@ function createBackfill (app, options) {
     const url = `${conn.url.replace(/\/+$/, '')}/api/v2/query?org=${encodeURIComponent(conn.org)}`
     let resp
     try {
-      resp = await fetch(url, {
+      resp = await request(url, {
         method: 'POST',
         headers: { Authorization: `Token ${conn.token}`, 'Content-Type': 'application/json', Accept: 'application/csv' },
         body: JSON.stringify({ type: 'flux', query: body, dialect: { header: true, annotations: ['datatype'] } }),
-        signal: AbortSignal.timeout(cfg.queryTimeoutMs)
+        timeoutMs: cfg.queryTimeoutMs
       })
-    } catch (e) { return { ok: false, message: e.message } }
+    } catch (e) { return { ok: false, message: `${e.message}${e.code ? ` (${e.code})` : ''}` } }
     if (!resp.ok) {
       const t = await resp.text().catch(() => '')
       return { ok: false, status: resp.status, message: `HTTP ${resp.status}: ${t.slice(0, 160)}` }

package/lib/cloud/index.js

@@ -27,6 +27,8 @@
 const fs = require('fs')
 const fsp = fs.promises
 const path = require('path')
+// Named to avoid shadowing this module's own request() below.
+const { request: httpRequest } = require('../net')
 
 const COOKIE_NAME = 'sk_session'
 const MAX_BODY_BYTES = 8 * 1024 * 1024 // a polar CSV is tiny; this is just a sane ceiling
@@ -105,11 +107,11 @@ function createCloud (app, options = {}) {
 
   async function login (slug, password) {
     if (!upstream) return { ok: false, code: 'no-upstream', message: 'no cloud host configured' }
-    const got = await attempt(() => fetch(`${upstream}/api/auth/login`, {
+    const got = await attempt(() => httpRequest(`${upstream}/api/auth/login`, {
       method: 'POST',
       headers: { 'Content-Type': 'application/json' },
       body: JSON.stringify({ slug, password }),
-      signal: AbortSignal.timeout(timeoutMs)
+      timeoutMs
     }))
     if (!got.ok) {
       warn(`login could not reach ${upstream} after 3 attempts — ${why(got.error)}`)
@@ -151,15 +153,15 @@ function createCloud (app, options = {}) {
   // handler and a boat is offline most of the time.
   async function request (apiPath, { method = 'GET', body = null } = {}) {
     if (!session) return { ok: false, status: 401, code: 'logged-out', message: 'not logged in to the cloud' }
-    const got = await attempt(() => fetch(upstream + apiPath, {
+    const got = await attempt(() => httpRequest(upstream + apiPath, {
       method,
       headers: {
         Cookie: session.cookie,
         Accept: 'application/json',
         ...(body ? { 'Content-Type': 'application/json' } : {})
       },
-      body: body || undefined,
-      signal: AbortSignal.timeout(timeoutMs)
+      body: body || null,
+      timeoutMs
     }))
     if (!got.ok) {
       return { ok: false, status: 0, code: 'offline', message: `cannot reach the cloud — ${why(got.error)}. The boat's link may have dropped; try again.` }

package/lib/history/index.js

@@ -51,6 +51,7 @@ function createHistory (app, options) {
       const persistFile = options.ringPersist !== false ? path.join(ringDir, 'history-ring.jsonl') : null
       provider = new RingHistoryProvider({
         source: options.ringSource,
+        perfSource: options.perfSource || null, // lib/perf — the computed polar %
         windowSec: options.ringWindowSec,
         sampleSec: options.ringSampleSec,
         persistFile

package/lib/net.js

@@ -0,0 +1,149 @@
+'use strict'
+
+// The one outbound HTTP client for everything that talks to the cloud.
+//
+// WHY NOT fetch()
+//
+// Twice in one afternoon this boat's Signal K process stopped being able to open ANY
+// outbound HTTPS connection: zero sockets to :443, while live connections to the
+// Starlink dish and the local database stayed up, file descriptors at 50 of 524288, and
+// a second node process in the SAME container reached the same host in 979 ms. It never
+// recovered — 33 minutes the second time. Restarting bought about twenty minutes.
+//
+// Starlink is behind CGNAT, which drops an idle NAT mapping WITHOUT sending an RST. The
+// pooled keep-alive socket then looks alive to the client and is dead on the wire.
+// fetch() (undici) keeps such sockets, and a plugin has no supported way to reset that
+// pool: `undici` is not requirable on the boat (Node bundles it internally),
+// `Connection: close` is a forbidden header that fetch strips, and reaching for the
+// global-dispatcher symbol is version-specific guesswork.
+//
+// Core http/https gives what is actually needed:
+//   - an agent we own, so a poisoned pool can be thrown away (resetTransport)
+//   - a short keep-alive, so an idle socket is dropped by US before CGNAT drops it
+//   - the REAL error code. fetch reports every transport failure as the uniformly
+//     useless string "fetch failed" and hides the reason in e.cause, so a wedged process
+//     and a boat genuinely at sea produced identical logs. Here a caller sees
+//     ECONNRESET / ETIMEDOUT / ENOTFOUND / ECONNREFUSED directly.
+//
+// ONE pool for the whole plugin, so one reset clears every subsystem at once — during
+// the incident sync, the mirror, the manifest poller and the contract check were all
+// wedged together, because they share the process, not because they share code.
+//
+// The response shape mirrors the parts of fetch() the callers actually used, so the call
+// sites read the same: { ok, status, headers.get(), headers.forEach(), headers
+// .getSetCookie(), buffer, text(), json() }. Transport failures THROW, as fetch does —
+// with `.code` set, which fetch never gave us.
+
+const http = require('http')
+const https = require('https')
+const { URL } = require('url')
+
+// Short keep-alive: long enough that a steady stream reuses a socket, short enough that
+// an idle one is closed by us well before a CGNAT mapping expires.
+const AGENT_OPTS = { keepAlive: true, keepAliveMsecs: 5000, timeout: 15000, maxSockets: 8 }
+const MAX_BODY_BYTES = 64 * 1024 * 1024 // Cesium.js is ~6 MB; this is a sanity ceiling
+
+let agents = null
+let generation = 0
+
+function pool () {
+  if (!agents) agents = { http: new http.Agent(AGENT_OPTS), https: new https.Agent(AGENT_OPTS) }
+  return agents
+}
+
+// Throw the pools away. The next request builds fresh sockets — the only thing that
+// recovers a pool whose sockets are dead but look open. destroy() closes idle sockets
+// only; anything in flight finishes normally.
+function resetTransport () {
+  if (agents) {
+    try { agents.http.destroy() } catch {}
+    try { agents.https.destroy() } catch {}
+  }
+  agents = null
+  return ++generation
+}
+
+function headersView (raw) {
+  const lower = {}
+  for (const [k, v] of Object.entries(raw || {})) lower[k.toLowerCase()] = v
+  return {
+    get (name) {
+      const v = lower[String(name).toLowerCase()]
+      return v == null ? null : (Array.isArray(v) ? v.join(', ') : String(v))
+    },
+    forEach (fn) {
+      for (const [k, v] of Object.entries(lower)) fn(Array.isArray(v) ? v.join(', ') : String(v), k)
+    },
+    // Node keeps set-cookie as an array already — the one header that must not be joined.
+    getSetCookie () {
+      const v = lower['set-cookie']
+      return v == null ? [] : (Array.isArray(v) ? v : [String(v)])
+    }
+  }
+}
+
+// fetch-shaped, minus the parts nothing here uses. Throws on transport failure with
+// `.code` populated; an HTTP error status resolves normally (check `ok`/`status`).
+function request (url, { method = 'GET', headers = {}, body = null, timeoutMs = 20000 } = {}) {
+  return new Promise((resolve, reject) => {
+    const settled = [] // run when the socket returns to the pool — see the note below
+    let u
+    try { u = new URL(url) } catch (e) { return reject(Object.assign(new Error(`bad url: ${url}`), { code: 'ERR_INVALID_URL' })) }
+    const lib = u.protocol === 'https:' ? https : http
+    const agent = u.protocol === 'https:' ? pool().https : pool().http
+
+    const hdrs = { ...headers }
+    if (body != null && hdrs['Content-Length'] == null && hdrs['content-length'] == null) {
+      hdrs['Content-Length'] = Buffer.byteLength(body)
+    }
+
+    const req = lib.request({
+      protocol: u.protocol,
+      hostname: u.hostname,
+      port: u.port || undefined,
+      path: u.pathname + u.search,
+      method,
+      agent,
+      headers: hdrs
+    }, (res) => {
+      const chunks = []
+      let size = 0
+      res.on('data', (c) => {
+        size += c.length
+        if (size > MAX_BODY_BYTES) { req.destroy(Object.assign(new Error('response too large'), { code: 'EMSGSIZE' })); return }
+        chunks.push(c)
+      })
+      res.on('error', (e) => { settled.forEach((f) => f()); reject(Object.assign(e, { code: e.code || 'ERR_STREAM' })) })
+      res.on('end', () => {
+        settled.forEach((f) => f())
+        const buffer = Buffer.concat(chunks)
+        resolve({
+          ok: res.statusCode >= 200 && res.statusCode < 300,
+          status: res.statusCode,
+          headers: headersView(res.headers),
+          buffer,
+          text: async () => buffer.toString('utf8'),
+          json: async () => JSON.parse(buffer.toString('utf8')),
+          arrayBuffer: async () => buffer
+        })
+      })
+    })
+
+    if (timeoutMs) {
+      req.setTimeout(timeoutMs, () => req.destroy(Object.assign(new Error(`timed out after ${timeoutMs}ms`), { code: 'ETIMEDOUT' })))
+    }
+    // An IDLE pooled socket must never be the reason the host process cannot exit — Signal
+    // K has to be able to shut down and a test runner has to finish. But an IN-FLIGHT one
+    // must hold the loop, or Node exits mid-request and the caller never resolves (which
+    // is exactly what a first attempt at this did). So: ref while the request is running,
+    // unref once it is back in the pool.
+    let sock = null
+    const idle = () => { try { if (sock && sock.unref) sock.unref() } catch {} }
+    req.on('socket', (s) => { sock = s; try { if (s.ref) s.ref() } catch {} })
+    settled.push(idle)
+    req.on('error', (e) => { settled.forEach((f) => f()); reject(Object.assign(e, { code: e.code || 'ERR_REQUEST' })) })
+    req.end(body == null ? undefined : body)
+  })
+}
+
+module.exports = { request, resetTransport, _agents: () => agents, _generation: () => generation }

package/lib/proxy/cache.js

@@ -14,6 +14,7 @@
 
 const fs = require('fs')
 const fsp = fs.promises
+const { request } = require('../net') // owned connection pool + real error codes; see lib/net.js
 const path = require('path')
 const crypto = require('crypto')
 
@@ -47,7 +48,7 @@ async function readFromDisk (file, meta) {
 async function fetchAndStore (url, file, meta, timeoutMs) {
   let resp
   try {
-    resp = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) })
+    resp = await request(url, { timeoutMs })
   } catch (e) {
     const err = new Error(`offline / unreachable: ${e.message}`); err.offline = true; throw err
   }

package/lib/proxy/index.js

@@ -8,6 +8,7 @@ const { createManifest } = require('./manifest')
 const { createContractCheck } = require('../telemetry/contract')
 const { createSeeder } = require('./seed')
 const { countBboxTiles, bboxTiles, boxAround } = require('./tiles')
+const { request } = require('../net')
 
 // Caching proxy module. The standalone server (origin root, `proxyPort`) is what
 // the browser points at. It routes by path:
@@ -181,7 +182,7 @@ function createProxy (app, options) {
     if (zoomCache) return zoomCache
     const out = { ...FALLBACK_MAX_ZOOM }
     try {
-      const r = await fetch(cfg.upstream + '/api/assets', { signal: AbortSignal.timeout(cfg.timeoutMs) })
+      const r = await request(cfg.upstream + '/api/assets', { timeoutMs: cfg.timeoutMs })
       if (r.ok) {
         const j = await r.json()
         const raster = j && j.tiles && j.tiles.manifest && j.tiles.manifest.maxZoom
@@ -338,7 +339,7 @@ function createProxy (app, options) {
     }
     let r
     try {
-      r = await fetch(target, { method: req.method, headers: fwd, body: chunks.length ? Buffer.concat(chunks) : undefined })
+      r = await request(target, { method: req.method, headers: fwd, body: chunks.length ? Buffer.concat(chunks) : undefined, timeoutMs: cfg.timeoutMs })
     } catch (e) { res.statusCode = 502; res.end('upstream unreachable'); return }
     res.statusCode = r.status
     r.headers.forEach((v, k) => {

package/lib/proxy/manifest.js

@@ -18,6 +18,7 @@
 const fs = require('fs')
 const fsp = fs.promises
 const path = require('path')
+const { request } = require('../net')
 
 function createManifest (app, options) {
   const log = (m) => (app.debug ? app.debug('[manifest] ' + m) : console.log('[sailkick-boat:manifest]', m))
@@ -61,7 +62,7 @@ function createManifest (app, options) {
     if (!cfg) return
     let m
     try {
-      const resp = await fetch(cfg.upstream + cfg.path, { signal: AbortSignal.timeout(cfg.timeoutMs) })
+      const resp = await request(cfg.upstream + cfg.path, { timeoutMs: cfg.timeoutMs })
       if (!resp.ok) return
       m = await resp.json()
     } catch { return } // offline / bad JSON → no-op, nothing invalidated

package/lib/sync/index.js

@@ -8,9 +8,15 @@
 const fs = require('fs')
 const path = require('path')
 const { Spool, DEFAULT_MAX_BYTES } = require('./spool')
-const { writeLines } = require('./influxWrite')
+const { writeLines, resetTransport } = require('./influxWrite')
 const { deltaToLines } = require('./lineprotocol')
 
+// Rebuild the connection pool after this many consecutive TRANSPORT failures. Low enough
+// that a wedged process recovers in seconds instead of needing a restart, high enough
+// that an ordinary offline stretch does not churn agents.
+const RESET_AFTER_FAILURES = 5
+const WARN_REPEAT_MS = 300000 // re-state an ongoing outage every 5 min, not every retry
+
 function createSync (app, options) {
   const log = (m) => (app.debug ? app.debug('[sync] ' + m) : console.log('[sailkick-boat:sync]', m))
   // Anything that stops telemetry reaching the cloud goes to `warn`, which lands in the
@@ -18,7 +24,6 @@ function createSync (app, options) {
   // an outage must never be visible ONLY in the status line, which is what made a
   // day-long silent failure possible.
   const warn = (m) => (app.error ? app.error('[sailkick-boat:sync] ' + m) : console.error('[sailkick-boat:sync]', m))
-  const WARN_REPEAT_MS = 300000 // re-state an ongoing outage every 5 min, not every retry
   let state = null
 
   function start () {
@@ -55,11 +60,26 @@ function createSync (app, options) {
     // never reach its endpoint used to produce no log output at all.
     const noteFailure = (res) => {
       state.failCount++
+      // A poisoned connection pool cannot heal itself: the sockets look open and are dead
+      // on the wire (Starlink is behind CGNAT, which drops idle mappings without an RST),
+      // so every retry writes into the same corpse. Twice this left the process unable to
+      // open ANY outbound HTTPS for over half an hour while a second process in the same
+      // container reached the host in under a second. Throwing the pool away is the one
+      // thing that recovers it. Only for TRANSPORT failures — an HTTP status means the
+      // connection worked fine.
+      if (res && res.networkError && state.failCount % RESET_AFTER_FAILURES === 0) {
+        const gen = resetTransport()
+        warn(`${state.failCount} consecutive transport failures — rebuilt the connection pool (generation ${gen}); if the link is up this recovers on the next attempt`)
+      }
       const now = Date.now()
       if (state.failing && now - state.lastWarnAt < WARN_REPEAT_MS) return
       state.failing = true
       state.lastWarnAt = now
-      const why = res && res.status ? `HTTP ${res.status}` : `unreachable${res && res.error ? ` (${res.error})` : ''}`
+      // Name the REASON. "fetch failed" told us nothing through two incidents; ECONNRESET
+      // and ETIMEDOUT are different problems with different fixes.
+      const why = res && res.status
+        ? `HTTP ${res.status}`
+        : `unreachable${res && (res.code || res.error) ? ` (${res.code || res.error})` : ''}`
       warn(`cannot write to ${cfg.influxUrl} — ${why}; ${state.failCount} failed attempt(s), telemetry is buffering on disk`)
     }
     const noteSuccess = () => {

package/lib/sync/influxWrite.js

@@ -4,10 +4,10 @@
 //
 // Return shape:
 //   { ok: true, status: 204 }
-//   { ok: false, retryable: true,  networkError: true }        connection failed
-//   { ok: false, retryable: true,  status }                     429 / 5xx (transient)
-//   { ok: false, configError: true, status, body }              404 / 401 / 403
-//   { ok: false, retryable: false, status, body }               other 4xx (bad data)
+//   { ok: false, retryable: true,  networkError: true, code, error }  connection failed
+//   { ok: false, retryable: true,  status }                            429 / 5xx (transient)
+//   { ok: false, configError: true, status, body }                     404 / 401 / 403
+//   { ok: false, retryable: false, status, body }                      other 4xx (bad data)
 //
 // Most 4xx is non-retryable: retrying a malformed batch forever would wedge the queue,
 // so the caller quarantines it instead.
@@ -21,7 +21,11 @@
 // and keeps retrying slowly, so correcting the config recovers on its own.
 const CONFIG_ERROR_STATUS = new Set([401, 403, 404])
 
+// The transport (an owned connection pool, real error codes) lives in lib/net.js —
+// see the long note there on why fetch() is unusable on a CGNAT satellite link.
+
 const zlib = require('zlib')
+const { request, resetTransport } = require('../net')
 
 async function writeLines (cfg, body) {
   const base = cfg.influxUrl.replace(/\/+$/, '')
@@ -31,10 +35,9 @@ async function writeLines (cfg, body) {
     '&precision=ns'
 
   const gz = zlib.gzipSync(Buffer.from(body, 'utf8'))
-
-  let res
+  let r
   try {
-    res = await fetch(url, {
+    r = await request(url, {
       method: 'POST',
       headers: {
         Authorization: `Token ${cfg.token}`,
@@ -42,21 +45,20 @@ async function writeLines (cfg, body) {
         'Content-Encoding': 'gzip'
       },
       body: gz,
-      signal: cfg.timeoutMs ? AbortSignal.timeout(cfg.timeoutMs) : undefined
+      timeoutMs: cfg.timeoutMs
     })
   } catch (e) {
-    return { ok: false, retryable: true, networkError: true, error: e.message }
+    // The REAL code, not fetch's "fetch failed" — see lib/net.js.
+    return { ok: false, retryable: true, networkError: true, code: e.code || null, error: e.message }
   }
+  const status = r.status
 
-  if (res.status === 204) return { ok: true, status: 204 }
-
-  let text = ''
-  try { text = await res.text() } catch {}
-  if (CONFIG_ERROR_STATUS.has(res.status)) {
-    return { ok: false, configError: true, retryable: false, status: res.status, body: text }
+  if (status === 204) return { ok: true, status: 204 }
+  if (CONFIG_ERROR_STATUS.has(status)) {
+    return { ok: false, configError: true, retryable: false, status, body: await r.text() }
   }
-  const retryable = res.status === 429 || res.status >= 500
-  return { ok: false, retryable, status: res.status, body: text }
+  const retryable = status === 429 || status >= 500
+  return { ok: false, retryable, status, body: await r.text() }
 }
 
-module.exports = { writeLines }
+module.exports = { writeLines, resetTransport }

package/lib/telemetry/contract.js

@@ -22,6 +22,8 @@
 //     sha256sum <sailkick>/public/engine/signalk-map.js | cut -c1-12
 
 // sha256(app public/engine/signalk-map.js)[0..12] as ported in v0.18.6
+const { request } = require('../net') // owned connection pool + real error codes
+
 const PINNED_APP_HASH = '2015ae986cd8'
 
 function createContractCheck (app, options = {}) {
@@ -37,7 +39,7 @@ function createContractCheck (app, options = {}) {
     if (!upstream) return drifted
     let remote
     try {
-      const r = await fetch(String(upstream).replace(/\/+$/, '') + '/health', { signal: AbortSignal.timeout(timeoutMs) })
+      const r = await request(String(upstream).replace(/\/+$/, '') + '/health', { timeoutMs })
       if (!r.ok) return drifted
       const body = await r.json()
       remote = body && body.contracts && body.contracts.signalkMap

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sailkick-boat",
-  "version": "0.23.0",
+  "version": "0.23.3",
   "description": "Run the sailkick app on board with no internet: charts, weather, climatology, trends and AIS all served from the boat itself. With a sailkick account it also syncs your metrics to the cloud in real time. Alpha, invite-only \u2014 info@sailkick.io",
   "main": "index.js",
   "scripts": {