sailkick-boat 0.21.1 → 0.21.3

package/README.md

@@ -242,6 +242,15 @@ so the boat's own app opens with no password, fully offline. Everything else in
 config passes through untouched. (Hand-edit `proxy.openAccess: false` to keep the cloud
 login gate — there is no toggle, since the gate cannot complete over the mirror anyway.)
 
+It also fills in **`boat.perfKey`**, which the cloud only sends to a logged-in session.
+Without it the app has no identity and the **performance data cloud** — the recorded
+(TWA, STW) samples behind the polar plot, baked to `/perf/<perfKey>/estimate.json` — never
+loads, even though the bake itself is cached and reachable. The key is derived rather than
+configured: the app server takes a boat's Influx bucket and its perf directory from the
+same identity, so the bucket minus its `_raw` suffix *is* the perf key, for a UUID account
+and a grandfathered slug one alike. An unpaired boat has no bucket, so `boat` is left
+exactly as the cloud sent it.
+
 ## Routes, polars and settings — stored on the boat
 The app reads and writes these through `/api/profile/*`. On the cloud that router is
 session-gated, and the mirror can never satisfy it: the caching GET path forwards no
@@ -316,6 +325,15 @@ past a second the spool was never empty for an instant and the backfill stood do
 ever — silently, since that path logs nothing. It now yields at a real backlog, which an
 outage produces within a minute, and logs both the stand-down and the resume.
 
+**It restarts itself.** A run stops after a streak of failed writes, so a boat that has
+gone offline never marks a window falsely done — but it then schedules a fresh walk (1 min,
+backing off to 30 min, reset by any successful window) instead of waiting for a human. It
+used to end with "resumes on restart", and nothing restarted it: one boat sat idle for 8.6
+hours at 96% complete after seven link drops in 13 minutes, while live sync rode out the
+same drops with a single retry. A rejected destination (a renamed bucket, a bad token) is
+retried the same way, since those are settings and settings get corrected. Genuinely
+malformed data still stops for good — retrying cannot fix it.
+
 It needs a **cloud read+write token**, not the write token from signup. Every hour it
 uploads is verified by counting the destination, and a write-only token cannot read. A
 `204` means InfluxDB accepted the bytes, not that every point landed — without the count

package/index.js

@@ -266,9 +266,17 @@ module.exports = function (app) {
       if (upstream.ignored) {
         ;(app.error || console.error)(`[sailkick-boat] ignoring proxy.sailkickUrl "${upstream.ignored}" left over from an older config — mirroring ${upstream.url}. Set proxy.selfHosted:true to keep your own server.`)
       }
+      // Boat identity for the app, derived — never guessed. The cloud only fills
+      // config.boat for a logged-in session and the mirror forwards no cookie, so the app
+      // ran with no identity and public/engine/polar-cloud.js could not find its data
+      // cloud (it keys on boat.perfKey). The app server defaults `bucket` and `perfKey`
+      // from the same identity (server/auth/registry.js), so the bucket minus its _raw
+      // suffix IS the perf key — for a UUID account and a grandfathered slug one alike.
+      const perfKey = b && b.bucket ? String(b.bucket).replace(/_raw$/, '') : null
       const pOpts = {
         sailkickUrl: upstream.url,
         storeDir: store,
+        boat: perfKey ? { perfKey, slug: (b && b.slug) || perfKey } : null,
         proxyPort: (proxyPort = p.proxyPort == null ? 8080 : p.proxyPort),
         localSignalkUrl: p.localSignalkUrl || 'http://127.0.0.1:3000',
         localPaths: (p.localPaths && p.localPaths.length) ? p.localPaths : PROXY_TUNING.localPaths,

package/lib/backfill/index.js

@@ -50,9 +50,21 @@ const DEFAULTS = {
   // comparison: an outage puts hundreds of files on disk within a minute.
   backlogFiles: 5,
   maxErrorStreak: 5,
+  // A run used to END on a streak of failures — "it resumes on restart", except nothing
+  // restarts it, so a boat 96% through its archive sat idle for 8.6 hours after seven
+  // Starlink drops in 13 minutes. Live sync shrugged the same drops off in one retry.
+  // Transient exits now schedule a fresh walk instead: short at first, backing off if the
+  // condition persists, reset as soon as a window succeeds.
+  retryRunMinMs: 60000,
+  retryRunMaxMs: 1800000,
   queryTimeoutMs: 120000
 }
 
+// Returned by run() when it stopped for something that may clear on its own — an
+// unreachable source, a rejected destination, a streak of failed writes. Anything else
+// (complete, misconfigured, stopped) is final and schedules nothing.
+const TRANSIENT = 'transient'
+
 const iso = (ms) => new Date(ms).toISOString()
 const hourFloor = (ms) => Math.floor(ms / HOUR_MS) * HOUR_MS
 const sleep = (ms) => new Promise((r) => { const t = setTimeout(r, ms); if (t.unref) t.unref() })
@@ -72,6 +84,8 @@ function createBackfill (app, options) {
   let stopped = false
   let statusLine = 'backfill: off'
   let runPromise = null
+  let retryTimer = null
+  let retryDelay = 0 // set from cfg on first use; grows while a transient condition holds
 
   // --- persisted manifest -------------------------------------------------------
   function load () {
@@ -234,7 +248,15 @@ function createBackfill (app, options) {
       const body = lines.slice(i, i + cfg.batchSize).join('\n') + '\n'
       const res = await writeLines({ influxUrl: cfg.dst.url, org: cfg.dst.org, bucket: cfg.dst.bucket, token: cfg.dst.token, timeoutMs: cfg.queryTimeoutMs }, body)
       if (!res.ok) {
-        // 4xx is fatal: bad credentials or malformed data, and retrying cannot fix it.
+        // A wrong bucket or token is a SETTING, not bad data — it will keep rejecting
+        // until someone changes the config, and then start working with no code change.
+        // Worth retrying slowly rather than stopping for good. (See the same split in
+        // lib/sync/influxWrite.js.)
+        if (res.configError) {
+          warn(`write to ${cfg.dst.bucket} rejected (HTTP ${res.status}) — ${res.status === 404 ? 'that bucket does not exist; check "Data bucket" in the plugin config' : 'check the cloud read+write token'}`)
+          return 'config'
+        }
+        // Other 4xx: malformed data. Retrying cannot fix it.
         if (!res.retryable) {
           // Name the destination: after a bucket rename this is the first thing to check.
           warn(`write to ${cfg.dst.bucket} REJECTED (HTTP ${res.status}) — ${res.body ? String(res.body).slice(0, 160) : 'no detail'}`)
@@ -255,7 +277,7 @@ function createBackfill (app, options) {
     try {
       // Validate the source and decide the filter before walking 15k windows.
       const contexts = await sourceContexts()
-      if (contexts == null) { warn(`could not read contexts from ${cfg.src.bucket} — is the source reachable?`); statusLine = 'backfill: source unreachable'; return }
+      if (contexts == null) { warn(`could not read contexts from ${cfg.src.bucket} — is the source reachable?`); statusLine = 'backfill: source unreachable'; return TRANSIENT }
       if (!contexts.length) {
         warn(`source bucket "${cfg.src.bucket}" (org "${cfg.src.org}") holds no data at all — check the org and bucket names`)
         statusLine = `backfill: source ${cfg.src.org}/${cfg.src.bucket} is empty — check the names`
@@ -265,7 +287,7 @@ function createBackfill (app, options) {
 
       if (state.earliest == null) {
         const e = await earliestPoint()
-        if (e == null) { warn('could not read the oldest point from the source — is it reachable?'); statusLine = 'backfill: source unreachable'; return }
+        if (e == null) { warn('could not read the oldest point from the source — is it reachable?'); statusLine = 'backfill: source unreachable'; return TRANSIENT }
         state.earliest = e
         log(`oldest point in ${cfg.src.bucket}: ${iso(e)}`)
         save()
@@ -320,8 +342,9 @@ function createBackfill (app, options) {
         const r = await doWindow(startMs, stopMs, tally)
         if (r === 'stopped') break
         if (r === 'fatal') { statusLine = 'backfill: stopped — write rejected, see the log'; return }
+        if (r === 'config') { statusLine = 'backfill: waiting — destination rejected the write, see the log'; return TRANSIENT }
         if (r === 'retry') {
-          if (++errStreak >= cfg.maxErrorStreak) { warn(`${errStreak} consecutive failures — stopping this run, it resumes on restart`); statusLine = 'backfill: paused after repeated errors'; return }
+          if (++errStreak >= cfg.maxErrorStreak) { warn(`${errStreak} consecutive failures — pausing this run`); statusLine = 'backfill: paused after repeated errors'; return TRANSIENT }
           continue
         }
 
@@ -334,17 +357,18 @@ function createBackfill (app, options) {
           const dstCount = await count(cfg.dst, cfg.dst.bucket, startMs, stopMs)
           if (dstCount == null) {
             warn(`could not verify ${key} — leaving it for the next run`)
-            if (++errStreak >= cfg.maxErrorStreak) { statusLine = 'backfill: paused after repeated errors'; return }
+            if (++errStreak >= cfg.maxErrorStreak) { statusLine = 'backfill: paused after repeated errors'; return TRANSIENT }
             continue
           }
           if (dstCount < tally.written) {
             warn(`${key} MISMATCH: wrote ${tally.written}, destination has ${dstCount} — not marking done`)
-            if (++errStreak >= cfg.maxErrorStreak) { statusLine = 'backfill: paused after repeated errors'; return }
+            if (++errStreak >= cfg.maxErrorStreak) { statusLine = 'backfill: paused after repeated errors'; return TRANSIENT }
             continue
           }
           state.points += tally.written
         }
         errStreak = 0
+        retryDelay = cfg.retryRunMinMs // real progress: a later stall starts from the short delay again
         state.done[key] = r === 'empty' ? 'empty' : 'ok'
         didWork++
         save()
@@ -375,20 +399,49 @@ function createBackfill (app, options) {
     }
   }
 
+  // Re-enter the walk after a transient stop. The walk itself is resumable — every
+  // completed window is in the manifest — so a fresh run simply skips them and carries on
+  // from where it left off.
+  function scheduleRerun () {
+    if (stopped) return
+    clearTimeout(retryTimer)
+    if (!retryDelay) retryDelay = cfg.retryRunMinMs
+    const delay = retryDelay
+    retryDelay = Math.min(retryDelay * 2, cfg.retryRunMaxMs)
+    log(`paused; retrying in ${Math.round(delay / 1000)}s`)
+    statusLine += `; retry in ${Math.round(delay / 1000)}s`
+    retryTimer = setTimeout(() => { if (!stopped) begin() }, delay)
+    if (retryTimer.unref) retryTimer.unref()
+  }
+
+  function begin () {
+    if (running) return runPromise
+    state = load()
+    if (state.complete) { statusLine = `backfill: complete — ${state.points} point(s)`; return null }
+    runPromise = run()
+      .then((r) => { if (r === TRANSIENT) scheduleRerun() })
+      .catch((e) => {
+        warn('run failed: ' + e.message)
+        statusLine = 'backfill: error — ' + e.message
+        scheduleRerun() // an unexpected throw is exactly the case worth retrying
+      })
+    return runPromise
+  }
+
   function start () {
     if (running) return runPromise
     stopped = false
+    retryDelay = cfg.retryRunMinMs
     state = load()
     if (state.complete) { statusLine = `backfill: complete — ${state.points} point(s)`; return null }
     if (!cfg.src.token || !cfg.src.bucket || !cfg.src.url) { statusLine = 'backfill: not configured (source)'; return null }
     if (!cfg.dst.token || !cfg.dst.bucket) { statusLine = 'backfill: not configured (cloud token)'; return null }
     log(`${cfg.src.url} ${cfg.src.org}/${cfg.src.bucket} -> ${cfg.dst.url} ${cfg.dst.org}/${cfg.dst.bucket}`)
     statusLine = 'backfill: starting'
-    runPromise = run().catch((e) => { warn('run failed: ' + e.message); statusLine = 'backfill: error — ' + e.message })
-    return runPromise
+    return begin()
   }
 
-  function stop () { stopped = true }
+  function stop () { stopped = true; clearTimeout(retryTimer); retryTimer = null }
   function status () { return statusLine }
 
   return { start, stop, status, _state: () => state, _doWindow: doWindow, _wait: () => runPromise }

package/lib/proxy/index.js

@@ -93,9 +93,11 @@ function createProxy (app, options) {
       history: options.history || null,
       aisTargets: options.aisTargets || null,
       profile: options.profile || null,
+      boat: options.boat || null, // { perfKey, slug } — patched into /api/config, see serveConfig
       openAccess: options.openAccess !== false
     }
     log(`mirroring ${cfg.upstream}; local SignalK ${cfg.localSignalk}; store ${cfg.storeDir}`)
+    if (cfg.boat && cfg.boat.perfKey) log(`boat identity for the app: perfKey=${cfg.boat.perfKey} (performance data cloud at /perf/${cfg.boat.perfKey}/)`)
 
     // Cache-manifest poller: auto-refresh a dataset lazily when the cloud
     // announces a new bake. Tiles are otherwise pinned (no time-based expiry).
@@ -293,6 +295,26 @@ function createProxy (app, options) {
         if (j && typeof j === 'object') {
           j.auth = { ...(j.auth || {}), required: false } // single-tenant boat: no cloud login
           if (cfg.history && cfg.history.available()) j.historyAvailable = true // served locally (InfluxDB or ring)
+          // The cloud fills `boat` only for a logged-in session, and the mirror forwards no
+          // cookie — so it always arrived as null and the app had no identity. Harmless for
+          // most of the UI, but public/engine/polar-cloud.js keys the performance data cloud
+          // on boat.perfKey and throws 'no boat identity' without it, so the polar cloud was
+          // simply absent on the boat while the bake sat cached and reachable.
+          //
+          // perfKey is not guessed: server/auth/registry.js defaults `bucket` and `perfKey`
+          // from the same identity, so the bucket minus its _raw suffix IS the perf key —
+          // for UUID accounts and for grandfathered slug ones alike.
+          if (cfg.boat && cfg.boat.perfKey) {
+            j.boat = {
+              ...(j.boat || {}),
+              perfKey: cfg.boat.perfKey,
+              slug: cfg.boat.slug || cfg.boat.perfKey,
+              name: (j.boat && j.boat.name) || cfg.boat.slug || cfg.boat.perfKey,
+              // NEVER readOnly: that flag exists for public visitor sessions, and it hides
+              // the editing affordances. The owner at the chart table has full control.
+              readOnly: false
+            }
+          }
           out = Buffer.from(JSON.stringify(j))
           ct = 'application/json'
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sailkick-boat",
-  "version": "0.21.1",
+  "version": "0.21.3",
   "description": "Run the sailkick app on board with no internet: charts, weather, climatology, trends and AIS all served from the boat itself. With a sailkick account it also syncs your metrics to the cloud in real time. Alpha, invite-only \u2014 info@sailkick.io",
   "main": "index.js",
   "scripts": {