sailkick-boat 0.21.0 → 0.21.2

package/README.md

@@ -307,9 +307,24 @@ arriving now, and the spool only replays what it captured itself while offline.
 
 Fill the **Copy older history to the cloud** section and save. It walks backwards in
 one-hour windows (newest first, so recent history lands first), resumes after a restart
-from a manifest, and stands aside whenever live telemetry has a backlog — the data-
+from a manifest, and stands aside whenever live telemetry has a **backlog** — the data-
 critical path is never starved by a bulk upload. Progress shows in the status line.
 
+"Backlog" means several files waiting, not merely an upload in flight. Requiring an empty
+spool was a race: live sync flushes every second, so once the round trip to the cloud grew
+past a second the spool was never empty for an instant and the backfill stood down for
+ever — silently, since that path logs nothing. It now yields at a real backlog, which an
+outage produces within a minute, and logs both the stand-down and the resume.
+
+**It restarts itself.** A run stops after a streak of failed writes, so a boat that has
+gone offline never marks a window falsely done — but it then schedules a fresh walk (1 min,
+backing off to 30 min, reset by any successful window) instead of waiting for a human. It
+used to end with "resumes on restart", and nothing restarted it: one boat sat idle for 8.6
+hours at 96% complete after seven link drops in 13 minutes, while live sync rode out the
+same drops with a single retry. A rejected destination (a renamed bucket, a bad token) is
+retried the same way, since those are settings and settings get corrected. Genuinely
+malformed data still stops for good — retrying cannot fix it.
+
 It needs a **cloud read+write token**, not the write token from signup. Every hour it
 uploads is verified by counting the destination, and a write-only token cannot read. A
 `204` means InfluxDB accepted the bytes, not that every point landed — without the count

package/lib/backfill/index.js

@@ -41,10 +41,30 @@ const DEFAULTS = {
   minWindowMs: 60000, // never subdivide below a minute
   idleMs: 250, // pending() is the real backpressure; this is just politeness
   backlogWaitMs: 15000, // how long to stand down when live sync has a backlog
+  // Stand down at a genuine BACKLOG, not at any activity. Requiring an EMPTY spool was a
+  // race rather than a design: live sync flushes every second, and a file exists for as
+  // long as its upload is in flight. Once the round trip to the cloud grew past the flush
+  // interval — which it did — the spool stopped being empty even for an instant (measured
+  // on the boat: non-empty in 30 of 30 one-second samples) and the backfill parked for
+  // ever, silently, because that path logs nothing. A real backlog is unmistakable by
+  // comparison: an outage puts hundreds of files on disk within a minute.
+  backlogFiles: 5,
   maxErrorStreak: 5,
+  // A run used to END on a streak of failures — "it resumes on restart", except nothing
+  // restarts it, so a boat 96% through its archive sat idle for 8.6 hours after seven
+  // Starlink drops in 13 minutes. Live sync shrugged the same drops off in one retry.
+  // Transient exits now schedule a fresh walk instead: short at first, backing off if the
+  // condition persists, reset as soon as a window succeeds.
+  retryRunMinMs: 60000,
+  retryRunMaxMs: 1800000,
   queryTimeoutMs: 120000
 }
 
+// Returned by run() when it stopped for something that may clear on its own — an
+// unreachable source, a rejected destination, a streak of failed writes. Anything else
+// (complete, misconfigured, stopped) is final and schedules nothing.
+const TRANSIENT = 'transient'
+
 const iso = (ms) => new Date(ms).toISOString()
 const hourFloor = (ms) => Math.floor(ms / HOUR_MS) * HOUR_MS
 const sleep = (ms) => new Promise((r) => { const t = setTimeout(r, ms); if (t.unref) t.unref() })
@@ -64,6 +84,8 @@ function createBackfill (app, options) {
   let stopped = false
   let statusLine = 'backfill: off'
   let runPromise = null
+  let retryTimer = null
+  let retryDelay = 0 // set from cfg on first use; grows while a transient condition holds
 
   // --- persisted manifest -------------------------------------------------------
   function load () {
@@ -226,7 +248,15 @@ function createBackfill (app, options) {
       const body = lines.slice(i, i + cfg.batchSize).join('\n') + '\n'
       const res = await writeLines({ influxUrl: cfg.dst.url, org: cfg.dst.org, bucket: cfg.dst.bucket, token: cfg.dst.token, timeoutMs: cfg.queryTimeoutMs }, body)
       if (!res.ok) {
-        // 4xx is fatal: bad credentials or malformed data, and retrying cannot fix it.
+        // A wrong bucket or token is a SETTING, not bad data — it will keep rejecting
+        // until someone changes the config, and then start working with no code change.
+        // Worth retrying slowly rather than stopping for good. (See the same split in
+        // lib/sync/influxWrite.js.)
+        if (res.configError) {
+          warn(`write to ${cfg.dst.bucket} rejected (HTTP ${res.status}) — ${res.status === 404 ? 'that bucket does not exist; check "Data bucket" in the plugin config' : 'check the cloud read+write token'}`)
+          return 'config'
+        }
+        // Other 4xx: malformed data. Retrying cannot fix it.
         if (!res.retryable) {
           // Name the destination: after a bucket rename this is the first thing to check.
           warn(`write to ${cfg.dst.bucket} REJECTED (HTTP ${res.status}) — ${res.body ? String(res.body).slice(0, 160) : 'no detail'}`)
@@ -247,7 +277,7 @@ function createBackfill (app, options) {
     try {
       // Validate the source and decide the filter before walking 15k windows.
       const contexts = await sourceContexts()
-      if (contexts == null) { warn(`could not read contexts from ${cfg.src.bucket} — is the source reachable?`); statusLine = 'backfill: source unreachable'; return }
+      if (contexts == null) { warn(`could not read contexts from ${cfg.src.bucket} — is the source reachable?`); statusLine = 'backfill: source unreachable'; return TRANSIENT }
       if (!contexts.length) {
         warn(`source bucket "${cfg.src.bucket}" (org "${cfg.src.org}") holds no data at all — check the org and bucket names`)
         statusLine = `backfill: source ${cfg.src.org}/${cfg.src.bucket} is empty — check the names`
@@ -257,7 +287,7 @@ function createBackfill (app, options) {
 
       if (state.earliest == null) {
         const e = await earliestPoint()
-        if (e == null) { warn('could not read the oldest point from the source — is it reachable?'); statusLine = 'backfill: source unreachable'; return }
+        if (e == null) { warn('could not read the oldest point from the source — is it reachable?'); statusLine = 'backfill: source unreachable'; return TRANSIENT }
         state.earliest = e
         log(`oldest point in ${cfg.src.bucket}: ${iso(e)}`)
         save()
@@ -294,22 +324,27 @@ function createBackfill (app, options) {
         const key = iso(startMs)
         if (state.done[key]) continue
 
-        // Live telemetry is data-critical; this is not. Stand down while it is behind.
+        // Live telemetry is data-critical; this is not. Stand down while it is BEHIND —
+        // which means a backlog, not merely an upload in flight. See cfg.backlogFiles.
+        let waited = false
         while (!stopped && cfg.pending) {
           let depth = 0
           try { depth = (await cfg.pending()).count || 0 } catch {}
-          if (!depth) break
+          if (depth < cfg.backlogFiles) break
+          if (!waited) { waited = true; log(`standing down — live sync is behind (${depth} file(s)); telemetry has the link`) }
           statusLine = `backfill: paused — live sync backlog (${depth} file(s))`
           await sleep(cfg.backlogWaitMs)
         }
+        if (waited && !stopped) log('live sync caught up — resuming')
         if (stopped) break
 
         const tally = { written: 0 }
         const r = await doWindow(startMs, stopMs, tally)
         if (r === 'stopped') break
         if (r === 'fatal') { statusLine = 'backfill: stopped — write rejected, see the log'; return }
+        if (r === 'config') { statusLine = 'backfill: waiting — destination rejected the write, see the log'; return TRANSIENT }
         if (r === 'retry') {
-          if (++errStreak >= cfg.maxErrorStreak) { warn(`${errStreak} consecutive failures — stopping this run, it resumes on restart`); statusLine = 'backfill: paused after repeated errors'; return }
+          if (++errStreak >= cfg.maxErrorStreak) { warn(`${errStreak} consecutive failures — pausing this run`); statusLine = 'backfill: paused after repeated errors'; return TRANSIENT }
           continue
         }
 
@@ -322,17 +357,18 @@ function createBackfill (app, options) {
           const dstCount = await count(cfg.dst, cfg.dst.bucket, startMs, stopMs)
           if (dstCount == null) {
             warn(`could not verify ${key} — leaving it for the next run`)
-            if (++errStreak >= cfg.maxErrorStreak) { statusLine = 'backfill: paused after repeated errors'; return }
+            if (++errStreak >= cfg.maxErrorStreak) { statusLine = 'backfill: paused after repeated errors'; return TRANSIENT }
             continue
           }
           if (dstCount < tally.written) {
             warn(`${key} MISMATCH: wrote ${tally.written}, destination has ${dstCount} — not marking done`)
-            if (++errStreak >= cfg.maxErrorStreak) { statusLine = 'backfill: paused after repeated errors'; return }
+            if (++errStreak >= cfg.maxErrorStreak) { statusLine = 'backfill: paused after repeated errors'; return TRANSIENT }
             continue
           }
           state.points += tally.written
         }
         errStreak = 0
+        retryDelay = cfg.retryRunMinMs // real progress: a later stall starts from the short delay again
         state.done[key] = r === 'empty' ? 'empty' : 'ok'
         didWork++
         save()
@@ -363,20 +399,49 @@ function createBackfill (app, options) {
     }
   }
 
+  // Re-enter the walk after a transient stop. The walk itself is resumable — every
+  // completed window is in the manifest — so a fresh run simply skips them and carries on
+  // from where it left off.
+  function scheduleRerun () {
+    if (stopped) return
+    clearTimeout(retryTimer)
+    if (!retryDelay) retryDelay = cfg.retryRunMinMs
+    const delay = retryDelay
+    retryDelay = Math.min(retryDelay * 2, cfg.retryRunMaxMs)
+    log(`paused; retrying in ${Math.round(delay / 1000)}s`)
+    statusLine += `; retry in ${Math.round(delay / 1000)}s`
+    retryTimer = setTimeout(() => { if (!stopped) begin() }, delay)
+    if (retryTimer.unref) retryTimer.unref()
+  }
+
+  function begin () {
+    if (running) return runPromise
+    state = load()
+    if (state.complete) { statusLine = `backfill: complete — ${state.points} point(s)`; return null }
+    runPromise = run()
+      .then((r) => { if (r === TRANSIENT) scheduleRerun() })
+      .catch((e) => {
+        warn('run failed: ' + e.message)
+        statusLine = 'backfill: error — ' + e.message
+        scheduleRerun() // an unexpected throw is exactly the case worth retrying
+      })
+    return runPromise
+  }
+
   function start () {
     if (running) return runPromise
     stopped = false
+    retryDelay = cfg.retryRunMinMs
     state = load()
     if (state.complete) { statusLine = `backfill: complete — ${state.points} point(s)`; return null }
     if (!cfg.src.token || !cfg.src.bucket || !cfg.src.url) { statusLine = 'backfill: not configured (source)'; return null }
     if (!cfg.dst.token || !cfg.dst.bucket) { statusLine = 'backfill: not configured (cloud token)'; return null }
     log(`${cfg.src.url} ${cfg.src.org}/${cfg.src.bucket} -> ${cfg.dst.url} ${cfg.dst.org}/${cfg.dst.bucket}`)
     statusLine = 'backfill: starting'
-    runPromise = run().catch((e) => { warn('run failed: ' + e.message); statusLine = 'backfill: error — ' + e.message })
-    return runPromise
+    return begin()
   }
 
-  function stop () { stopped = true }
+  function stop () { stopped = true; clearTimeout(retryTimer); retryTimer = null }
   function status () { return statusLine }
 
   return { start, stop, status, _state: () => state, _doWindow: doWindow, _wait: () => runPromise }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sailkick-boat",
-  "version": "0.21.0",
+  "version": "0.21.2",
   "description": "Run the sailkick app on board with no internet: charts, weather, climatology, trends and AIS all served from the boat itself. With a sailkick account it also syncs your metrics to the cloud in real time. Alpha, invite-only \u2014 info@sailkick.io",
   "main": "index.js",
   "scripts": {