sailkick-boat 0.18.0 → 0.18.2

package/README.md

@@ -236,6 +236,22 @@ a source archive that is still being written — a `signalk-to-influxdb-v2` buck
 recording — makes the first windows re-upload today's data. The timestamps are correct,
 but it is data the cloud already has, and a lot of wasted uplink.
 
+**Latency is the cost, not bandwidth.** Measured on a real archive, about 3 s of every
+4.2 s chunk was cloud round trips while all the boat-side work (query, parse, convert)
+took ~1.2 s. So batches carry up to 50k lines rather than 10k — one or two round trips
+per chunk instead of ten — and each **hour** is verified once rather than each of its
+~32 chunks. The verification itself is unchanged in kind: a `204` means the bytes were
+accepted, not that every point landed, so the destination is still counted before an
+hour is marked done. On a mismatch the hour is left unmarked and simply redone, which is
+safe because writes are idempotent.
+
+**Field types come from the source, not from guessing.** Queries ask InfluxDB for the
+`#datatype` annotation explicitly. Without it the response is unannotated and types have
+to be inferred from the text — which fails hard on a string field whose values sometimes
+look numeric (`"8"` emitted bare as a float, `"1.2.3.4"` quoted as a string), producing
+`422 field type conflict` and aborting the run. Integers also keep their type instead of
+silently becoming floats.
+
 **Dense archives are subdivided.** A window is read whole and converted in memory, and a
 busy boat can produce millions of points an hour (54M/day was measured on a real boat —
 about 400 MB of CSV per hour). When a window holds more than `maxRowsPerChunk` points it

package/lib/backfill/index.js

@@ -28,7 +28,11 @@ const { csvToLineProtocol } = require('./lineproto')
 const HOUR_MS = 3600000
 const DEFAULTS = {
   windowMs: HOUR_MS,
-  batchSize: 10000,
+  // Latency dominates, not bandwidth: measured on a real run, ~3s of every ~4.2s chunk
+  // was cloud round trips while all the Pi-side work took ~1.2s. At 10k a 100k chunk
+  // cost TEN sequential TLS round trips. 50k halves the peak memory of the gzip buffer
+  // versus one giant batch while still cutting the round trips fivefold.
+  batchSize: 50000,
   // A window is read whole and converted in memory. A busy archive can hold millions of
   // points per hour (54M/day was measured on a real boat = ~2.25M/hour ~ 400 MB of CSV),
   // which would exhaust a Raspberry Pi. When a window is denser than this it is halved
@@ -79,14 +83,26 @@ function createBackfill (app, options) {
   }
 
   // --- InfluxDB reads -----------------------------------------------------------
+  // Always ask for the #datatype annotation. A raw-Flux body (Content-Type
+  // application/vnd.flux) cannot carry a dialect, and InfluxDB then returns UNANNOTATED
+  // CSV — so field types have to be guessed from the text, and guessing is wrong in a
+  // way that breaks the write outright.
+  //
+  // The case that bit: a STRING field whose values sometimes look numeric. "8" is
+  // emitted bare (a float) while "1.2.3.4" is quoted (a string), so one batch declares
+  // two types for one field and InfluxDB rejects the lot with
+  //   422 field type conflict: input field "value" on measurement "network..."
+  // With the annotation the source declares `string` and every value is quoted
+  // consistently. Integers keep their `i` suffix too, instead of silently becoming
+  // floats.
   async function flux (conn, body) {
     const url = `${conn.url.replace(/\/+$/, '')}/api/v2/query?org=${encodeURIComponent(conn.org)}`
     let resp
     try {
       resp = await fetch(url, {
         method: 'POST',
-        headers: { Authorization: `Token ${conn.token}`, 'Content-Type': 'application/vnd.flux', Accept: 'application/csv' },
-        body,
+        headers: { Authorization: `Token ${conn.token}`, 'Content-Type': 'application/json', Accept: 'application/csv' },
+        body: JSON.stringify({ type: 'flux', query: body, dialect: { header: true, annotations: ['datatype'] } }),
         signal: AbortSignal.timeout(cfg.queryTimeoutMs)
       })
     } catch (e) { return { ok: false, message: e.message } }
@@ -176,8 +192,10 @@ function createBackfill (app, options) {
 
   // --- one window ---------------------------------------------------------------
   // Returns 'done' | 'empty' | 'retry' | 'fatal' | 'stopped'. Subdivides itself when a
-  // window holds more points than can be held in memory at once.
-  async function doWindow (startMs, stopMs) {
+  // window holds more points than can be held in memory at once. Points written are
+  // added to `tally` so the CALLER can verify once per hour rather than once per chunk —
+  // an hour is ~32 chunks, and each verification is a cloud round trip.
+  async function doWindow (startMs, stopMs, tally) {
     const srcCount = await count(cfg.src, cfg.src.bucket, startMs, stopMs)
     if (srcCount == null) return 'retry'
     if (srcCount === 0) return 'empty'
@@ -188,9 +206,11 @@ function createBackfill (app, options) {
       // done, so a failure part-way is simply retried next time.
       const mid = startMs + Math.floor((stopMs - startMs) / 2)
       log(`${iso(startMs)} holds ${srcCount} points — splitting`)
-      const newer = await doWindow(mid, stopMs)
+      const newer = await doWindow(mid, stopMs, tally)
       if (newer !== 'done' && newer !== 'empty') return newer
-      return doWindow(startMs, mid)
+      const older = await doWindow(startMs, mid, tally)
+      if (older !== 'done' && older !== 'empty') return older
+      return (newer === 'done' || older === 'done') ? 'done' : 'empty'
     }
 
     const filter = cfg._filter || ''
@@ -213,15 +233,7 @@ function createBackfill (app, options) {
       }
     }
 
-    // Verify: a 204 says the bytes were accepted, not that every point landed. This is
-    // the check the write-only sync token could never perform.
-    const dstCount = await count(cfg.dst, cfg.dst.bucket, startMs, stopMs)
-    if (dstCount == null) { warn(`could not verify ${iso(startMs)} — leaving it for the next run`); return 'retry' }
-    if (dstCount < lines.length) {
-      warn(`${iso(startMs)} MISMATCH: wrote ${lines.length}, destination has ${dstCount} — not marking done`)
-      return 'retry'
-    }
-    if (state) state.points += lines.length
+    if (tally) tally.written += lines.length
     return 'done'
   }
 
@@ -288,13 +300,34 @@ function createBackfill (app, options) {
         }
         if (stopped) break
 
-        const r = await doWindow(startMs, stopMs)
+        const tally = { written: 0 }
+        const r = await doWindow(startMs, stopMs, tally)
         if (r === 'stopped') break
         if (r === 'fatal') { statusLine = 'backfill: stopped — write rejected, see the log'; return }
         if (r === 'retry') {
           if (++errStreak >= cfg.maxErrorStreak) { warn(`${errStreak} consecutive failures — stopping this run, it resumes on restart`); statusLine = 'backfill: paused after repeated errors'; return }
           continue
         }
+
+        // Verify ONCE for the whole hour. A 204 says the bytes were accepted, not that
+        // every point landed, so this check still has to happen — but at the hour rather
+        // than the chunk it costs one cloud round trip instead of ~32. On a mismatch the
+        // hour is left unmarked and simply redone, which is safe because writes are
+        // idempotent. (>= not ==: the boundary hour also holds live-sync data.)
+        if (tally.written > 0) {
+          const dstCount = await count(cfg.dst, cfg.dst.bucket, startMs, stopMs)
+          if (dstCount == null) {
+            warn(`could not verify ${key} — leaving it for the next run`)
+            if (++errStreak >= cfg.maxErrorStreak) { statusLine = 'backfill: paused after repeated errors'; return }
+            continue
+          }
+          if (dstCount < tally.written) {
+            warn(`${key} MISMATCH: wrote ${tally.written}, destination has ${dstCount} — not marking done`)
+            if (++errStreak >= cfg.maxErrorStreak) { statusLine = 'backfill: paused after repeated errors'; return }
+            continue
+          }
+          state.points += tally.written
+        }
         errStreak = 0
         state.done[key] = r === 'empty' ? 'empty' : 'ok'
         didWork++

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sailkick-boat",
-  "version": "0.18.0",
+  "version": "0.18.2",
   "description": "EARLY ALPHA — cloud telemetry + offline maps for sailkick boats (www.sailkick.io; register on the web, paste the write token). Gapless boat→cloud telemetry sync to InfluxDB, and a local proxy that keeps the sailkick app and its charts/maps working fully offline on board.",
   "main": "index.js",
   "scripts": {