sailkick-boat 0.17.0 → 0.17.2

package/README.md

@@ -201,9 +201,30 @@ backfill runs: **revoke it afterwards**, live sync is unaffected.
 
 Safe to re-run. Points are keyed by (measurement, tagset, nanosecond timestamp), so an
 identical point overwrites rather than duplicating — an interrupted migration is simply
-run again. Everything in the bucket is copied, including AIS contexts; hand-edit
-`backfill.selfOnly: true` to restrict it to your own vessel if cloud series cardinality
-becomes a problem.
+run again.
+
+**Only this boat's data is copied**, and that is not configurable. The cloud's history
+queries assume your bucket holds one vessel, so uploading an archive's AIS would put
+other ships into your own SOG and heading charts. If the source holds several contexts
+the plugin copies yours and logs which it skipped. If it holds exactly **one** context it
+is copied whatever identity string it uses — a bucket with one vessel cannot be an AIS
+collection, and this is what lets an archive recorded under an older Signal K UUID (or an
+MMSI URN) still migrate. Hand-edit `backfill.context` to force a specific one.
+
+A run that copies **zero** points is reported as a problem, not as success: that almost
+always means the org or bucket is wrong rather than that the archive is empty.
+
+**It starts below what live sync already covers.** The destination's own oldest point is
+the moment cloud sync began, so the walk begins there rather than at *now*. Without that,
+a source archive that is still being written — a `signalk-to-influxdb-v2` bucket still
+recording — makes the first windows re-upload today's data. The timestamps are correct,
+but it is data the cloud already has, and a lot of wasted uplink.
+
+**Dense archives are subdivided.** A window is read whole and converted in memory, and a
+busy boat can produce millions of points an hour (54M/day was measured on a real boat —
+about 400 MB of CSV per hour). When a window holds more than `maxRowsPerChunk` points it
+is halved until it fits, down to a one-minute floor. The count is already known before
+the read, so this costs nothing extra.
 
 **True wind comes from your instruments.** If the boat publishes
 `environment.wind.speedTrue` / `directionTrue`, those are stored verbatim — a wind

package/index.js

@@ -373,7 +373,10 @@ module.exports = function (app) {
           backfill = createBackfill(app, {
             src,
             dst,
-            selfOnly: bf.selfOnly === true,
+            // Never a config option: the plugin must not upload data that is not this
+            // boat's, and the cloud's history queries depend on that holding.
+            selfContext: app.selfContext || ('vessels.' + (app.selfId || 'self')),
+            context: String(bf.context || '').trim() || null, // hand-edit escape hatch
             startBound: bf.startBound,
             stateFile: path.join((app.getDataDirPath && app.getDataDirPath()) || '.', 'backfill.json'),
             pending: sync ? sync.pending : null

package/lib/backfill/index.js

@@ -29,7 +29,13 @@ const HOUR_MS = 3600000
 const DEFAULTS = {
   windowMs: HOUR_MS,
   batchSize: 10000,
-  idleMs: 2000, // breathing room between windows so a slow link isn't monopolised
+  // A window is read whole and converted in memory. A busy archive can hold millions of
+  // points per hour (54M/day was measured on a real boat = ~2.25M/hour ~ 400 MB of CSV),
+  // which would exhaust a Raspberry Pi. When a window is denser than this it is halved
+  // until it fits — the count is already known before the read, so this costs nothing.
+  maxRowsPerChunk: 100000,
+  minWindowMs: 60000, // never subdivide below a minute
+  idleMs: 250, // pending() is the real backpressure; this is just politeness
   backlogWaitMs: 15000, // how long to stand down when live sync has a backlog
   maxErrorStreak: 5,
   queryTimeoutMs: 120000
@@ -59,9 +65,9 @@ function createBackfill (app, options) {
   function load () {
     try {
       const j = JSON.parse(fs.readFileSync(cfg.stateFile, 'utf8'))
-      if (j && typeof j === 'object') return { done: j.done || {}, earliest: j.earliest || null, points: j.points || 0, complete: !!j.complete }
+      if (j && typeof j === 'object') return { done: j.done || {}, earliest: j.earliest || null, ceiling: j.ceiling || null, points: j.points || 0, complete: !!j.complete }
     } catch {}
-    return { done: {}, earliest: null, points: 0, complete: false }
+    return { done: {}, earliest: null, ceiling: null, points: 0, complete: false }
   }
   function save () {
     try {
@@ -106,12 +112,12 @@ function createBackfill (app, options) {
     return 0
   }
 
-  async function earliestPoint () {
+  async function earliestPoint (conn = cfg.src, bucket = cfg.src.bucket) {
     // `first()` reduces each series before anything is merged, then _time is isolated
     // BEFORE group(). Both matter on a real bucket: grouping the raw stream fails with
     // "schema collision: cannot group boolean and integer types together" the moment the
     // database holds more than one field type, which any real boat's does.
-    const r = await flux(cfg.src, `from(bucket:"${cfg.src.bucket}")|>range(start:0)|>first()|>keep(columns:["_time"])|>group()|>min(column:"_time")`)
+    const r = await flux(conn, `from(bucket:"${bucket}")|>range(start:0)|>first()|>keep(columns:["_time"])|>group()|>min(column:"_time")`)
     if (!r.ok) return null
     for (const line of r.text.split('\n')) {
       if (!line || line.startsWith('#') || line.includes('_time')) continue
@@ -128,18 +134,70 @@ function createBackfill (app, options) {
     return null
   }
 
+  // Distinct contexts in the source bucket. Returns null on any failure — never an
+  // empty list, so an unreachable database is not mistaken for an empty archive.
+  async function sourceContexts () {
+    const r = await flux(cfg.src, `import "influxdata/influxdb/schema"\nschema.tagValues(bucket:"${cfg.src.bucket}", tag:"context")`)
+    if (!r.ok) return null
+    const out = []
+    for (const line of r.text.split('\n')) {
+      if (!line || line.startsWith('#')) continue
+      const cells = line.trim().split(',')
+      const v = cells[cells.length - 1]
+      if (v && v !== '_value') out.push(v)
+    }
+    return out
+  }
+
+  // The plugin must never upload data that is not this boat's. Live sync guarantees
+  // that by subscribing to vessels.self; the backfill is the only thing that could
+  // break it, and the cloud's history queries depend on it holding. So this is decided
+  // in code, not exposed as an option someone can get wrong.
+  //
+  // The wrinkle: an imported archive may carry a DIFFERENT context than the boat's
+  // current identity — a UUID from a since-reinstalled Signal K, or an MMSI URN. A
+  // strict match would then copy nothing, silently. Hence the single-context rule: a
+  // bucket holding exactly one context contains one vessel by definition and cannot be
+  // an AIS collection, so it is copied whatever its identity string says.
+  function contextFilterFor (contexts) {
+    const selfCtx = cfg.selfContext
+    if (cfg.context) {
+      log(`copying only context ${cfg.context} (explicit override)`)
+      return `|>filter(fn:(r)=>r.context=="${cfg.context}")`
+    }
+    if (contexts.length <= 1) {
+      log(`source holds a single context (${contexts[0] || 'none'}) — copying all of it`)
+      return ''
+    }
+    const others = contexts.filter((c) => c !== selfCtx)
+    warn(`source holds ${contexts.length} contexts — copying only this boat (${selfCtx}) and skipping ${others.length} other(s), e.g. ${others.slice(0, 3).join(', ')}`)
+    return `|>filter(fn:(r)=>r.self=="true" or r.context=="${selfCtx}")`
+  }
+
   // --- one window ---------------------------------------------------------------
-  // Returns 'done' | 'empty' | 'retry' | 'stopped'.
+  // Returns 'done' | 'empty' | 'retry' | 'fatal' | 'stopped'. Subdivides itself when a
+  // window holds more points than can be held in memory at once.
   async function doWindow (startMs, stopMs) {
     const srcCount = await count(cfg.src, cfg.src.bucket, startMs, stopMs)
     if (srcCount == null) return 'retry'
     if (srcCount === 0) return 'empty'
 
-    const filter = cfg.selfOnly ? '|>filter(fn:(r)=>r.self=="true")' : ''
+    if (srcCount > cfg.maxRowsPerChunk && (stopMs - startMs) > cfg.minWindowMs) {
+      // Too dense to read whole. Halve it — newest half first, keeping the run's
+      // newest-first order. Both halves must succeed for the caller to mark the window
+      // done, so a failure part-way is simply retried next time.
+      const mid = startMs + Math.floor((stopMs - startMs) / 2)
+      log(`${iso(startMs)} holds ${srcCount} points — splitting`)
+      const newer = await doWindow(mid, stopMs)
+      if (newer !== 'done' && newer !== 'empty') return newer
+      return doWindow(startMs, mid)
+    }
+
+    const filter = cfg._filter || ''
     const r = await flux(cfg.src, `from(bucket:"${cfg.src.bucket}")|>range(start:${iso(startMs)},stop:${iso(stopMs)})${filter}|>drop(columns:["_start","_stop"])`)
     if (!r.ok) { warn(`read ${iso(startMs)} failed — ${r.message}`); return 'retry' }
 
-    const { lines, skipped } = csvToLineProtocol(r.text, { selfOnly: cfg.selfOnly })
+    const { lines, skipped } = csvToLineProtocol(r.text)
     if (skipped) log(`${iso(startMs)}: skipped ${skipped} unconvertible row(s)`)
     if (!lines.length) return 'empty'
 
@@ -171,6 +229,16 @@ function createBackfill (app, options) {
   async function run () {
     running = true
     try {
+      // Validate the source and decide the filter before walking 15k windows.
+      const contexts = await sourceContexts()
+      if (contexts == null) { warn(`could not read contexts from ${cfg.src.bucket} — is the source reachable?`); statusLine = 'backfill: source unreachable'; return }
+      if (!contexts.length) {
+        warn(`source bucket "${cfg.src.bucket}" (org "${cfg.src.org}") holds no data at all — check the org and bucket names`)
+        statusLine = `backfill: source ${cfg.src.org}/${cfg.src.bucket} is empty — check the names`
+        return
+      }
+      cfg._filter = contextFilterFor(contexts)
+
       if (state.earliest == null) {
         const e = await earliestPoint()
         if (e == null) { warn('could not read the oldest point from the source — is it reachable?'); statusLine = 'backfill: source unreachable'; return }
@@ -186,7 +254,20 @@ function createBackfill (app, options) {
       const rawFloor = cfg.startBound ? Math.max(state.earliest, Date.parse(cfg.startBound)) : state.earliest
       const floor = hourFloor(rawFloor)
 
-      let cursor = hourFloor(Date.now()) // newest-first: recent history lands first
+      // Start below where live sync has already delivered, not at "now". A still-live
+      // source archive otherwise makes the first windows re-upload today's data —
+      // correct timestamps, but data the cloud already has, and a lot of wasted uplink.
+      // The destination's own oldest point IS the moment cloud sync began, and the
+      // read+write token can see it.
+      if (state.ceiling == null) {
+        const dstOldest = await earliestPoint(cfg.dst, cfg.dst.bucket)
+        state.ceiling = dstOldest != null ? hourFloor(dstOldest) : hourFloor(Date.now())
+        log(dstOldest != null
+          ? `live sync covers everything from ${iso(state.ceiling)} — backfilling only what is older`
+          : 'destination is empty — backfilling everything up to now')
+        save()
+      }
+      let cursor = state.ceiling // newest-first: recent history lands first
       let errStreak = 0
       let didWork = 0
 
@@ -223,6 +304,16 @@ function createBackfill (app, options) {
       }
 
       if (!stopped && cursor < floor) {
+        if (state.points === 0) {
+          // A walk that finishes having copied nothing is far more likely to be a wrong
+          // org/bucket, or a filter that matched no rows, than a genuinely empty
+          // archive. Marking it complete would dress a silent no-op as success and stop
+          // it ever retrying.
+          save()
+          warn(`walked ${Object.keys(state.done).length} window(s) and copied ZERO points — check that org "${cfg.src.org}" / bucket "${cfg.src.bucket}" is right, and that its data belongs to this boat. NOT marking complete.`)
+          statusLine = `backfill: finished with 0 points — check ${cfg.src.org}/${cfg.src.bucket}`
+          return
+        }
         state.complete = true
         save()
         statusLine = `backfill: complete — ${state.points} point(s) from ${iso(floor).slice(0, 10)}`
@@ -242,7 +333,7 @@ function createBackfill (app, options) {
     if (state.complete) { statusLine = `backfill: complete — ${state.points} point(s)`; return null }
     if (!cfg.src.token || !cfg.src.bucket || !cfg.src.url) { statusLine = 'backfill: not configured (source)'; return null }
     if (!cfg.dst.token || !cfg.dst.bucket) { statusLine = 'backfill: not configured (cloud token)'; return null }
-    log(`${cfg.src.url} ${cfg.src.org}/${cfg.src.bucket} -> ${cfg.dst.url} ${cfg.dst.org}/${cfg.dst.bucket}${cfg.selfOnly ? ' [self only]' : ''}`)
+    log(`${cfg.src.url} ${cfg.src.org}/${cfg.src.bucket} -> ${cfg.dst.url} ${cfg.dst.org}/${cfg.dst.bucket}`)
     statusLine = 'backfill: starting'
     runPromise = run().catch((e) => { warn('run failed: ' + e.message); statusLine = 'backfill: error — ' + e.message })
     return runPromise

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sailkick-boat",
-  "version": "0.17.0",
+  "version": "0.17.2",
   "description": "EARLY ALPHA — cloud telemetry + offline maps for sailkick boats (www.sailkick.io; register on the web, paste the write token). Gapless boat→cloud telemetry sync to InfluxDB, and a local proxy that keeps the sailkick app and its charts/maps working fully offline on board.",
   "main": "index.js",
   "scripts": {