sailkick-boat 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -3
- package/index.js +22 -1
- package/lib/ais/targets.js +171 -0
- package/lib/backfill/index.js +101 -10
- package/lib/proxy/index.js +10 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -155,6 +155,22 @@ A local InfluxDB is not a competitor here anyway. The app never requests finer t
|
|
|
155
155
|
`ringSampleSec: 5` and it matches anything the UI can draw, from live state. What an old
|
|
156
156
|
database *is* good for is its contents ending up **in the cloud** — see below.
|
|
157
157
|
|
|
158
|
+
## AIS on the boat's own chart
|
|
159
|
+
The app draws other vessels from `GET /api/ais`. The cloud serves that by polling a
|
|
160
|
+
SignalK server over the LAN and gates it behind a boat session — neither of which can
|
|
161
|
+
work from a boat on a mobile link, and the mirror forwards no cookies, so proxying it
|
|
162
|
+
returns 401 whatever you do.
|
|
163
|
+
|
|
164
|
+
The plugin therefore serves `/api/ais` **from the boat's own SignalK**, in the same
|
|
165
|
+
envelope the app already consumes: position, SOG, COG, heading, rate of turn, name,
|
|
166
|
+
dimensions and ship type, plus a ~1 h trail per vessel. Anchored ships stay a single dot
|
|
167
|
+
— a trail point is added only once a vessel has moved more than 30 m — and a target
|
|
168
|
+
unheard for 15 min is dropped. A failed poll keeps the last snapshot rather than blanking
|
|
169
|
+
the chart.
|
|
170
|
+
|
|
171
|
+
This works **with no uplink at all**, which is when other vessels on your chart matter
|
|
172
|
+
most. Turn it off by hand-editing `proxy.serveAis: false`.
|
|
173
|
+
|
|
158
174
|
## Uploading AIS targets
|
|
159
175
|
The cloud app already draws other vessels, but its AIS source polls a SignalK server over
|
|
160
176
|
the LAN and keeps everything in memory — which cannot work once a boat is on a mobile
|
|
@@ -201,9 +217,30 @@ backfill runs: **revoke it afterwards**, live sync is unaffected.
|
|
|
201
217
|
|
|
202
218
|
Safe to re-run. Points are keyed by (measurement, tagset, nanosecond timestamp), so an
|
|
203
219
|
identical point overwrites rather than duplicating — an interrupted migration is simply
|
|
204
|
-
run again.
|
|
205
|
-
|
|
206
|
-
|
|
220
|
+
run again.
|
|
221
|
+
|
|
222
|
+
**Only this boat's data is copied**, and that is not configurable. The cloud's history
|
|
223
|
+
queries assume your bucket holds one vessel, so uploading an archive's AIS would put
|
|
224
|
+
other ships into your own SOG and heading charts. If the source holds several contexts
|
|
225
|
+
the plugin copies yours and logs which it skipped. If it holds exactly **one** context it
|
|
226
|
+
is copied whatever identity string it uses — a bucket with one vessel cannot be an AIS
|
|
227
|
+
collection, and this is what lets an archive recorded under an older Signal K UUID (or an
|
|
228
|
+
MMSI URN) still migrate. Hand-edit `backfill.context` to force a specific one.
|
|
229
|
+
|
|
230
|
+
A run that copies **zero** points is reported as a problem, not as success: that almost
|
|
231
|
+
always means the org or bucket is wrong rather than that the archive is empty.
|
|
232
|
+
|
|
233
|
+
**It starts below what live sync already covers.** The destination's own oldest point is
|
|
234
|
+
the moment cloud sync began, so the walk begins there rather than at *now*. Without that,
|
|
235
|
+
a source archive that is still being written — a `signalk-to-influxdb-v2` bucket still
|
|
236
|
+
recording — makes the first windows re-upload today's data. The timestamps are correct,
|
|
237
|
+
but it is data the cloud already has, and a lot of wasted uplink.
|
|
238
|
+
|
|
239
|
+
**Dense archives are subdivided.** A window is read whole and converted in memory, and a
|
|
240
|
+
busy boat can produce millions of points an hour (54M/day was measured on a real boat —
|
|
241
|
+
about 400 MB of CSV per hour). When a window holds more than `maxRowsPerChunk` points it
|
|
242
|
+
is halved until it fits, down to a one-minute floor. The count is already known before
|
|
243
|
+
the read, so this costs nothing extra.
|
|
207
244
|
|
|
208
245
|
**True wind comes from your instruments.** If the boat publishes
|
|
209
246
|
`environment.wind.speedTrue` / `directionTrue`, those are stored verbatim — a wind
|
package/index.js
CHANGED
|
@@ -8,6 +8,7 @@ const { createTelemetry } = require('./lib/telemetry')
|
|
|
8
8
|
const { createHistory } = require('./lib/history')
|
|
9
9
|
const { createBackfill } = require('./lib/backfill')
|
|
10
10
|
const { createAis } = require('./lib/ais')
|
|
11
|
+
const { createAisTargets } = require('./lib/ais/targets')
|
|
11
12
|
const { resolveAccountConfig } = require('./lib/account')
|
|
12
13
|
|
|
13
14
|
// sailkick-boat: one Signal K plugin, two independently-toggleable modules —
|
|
@@ -94,6 +95,7 @@ module.exports = function (app) {
|
|
|
94
95
|
let history = null
|
|
95
96
|
let backfill = null
|
|
96
97
|
let ais = null
|
|
98
|
+
let aisTargets = null
|
|
97
99
|
let statusTimer = null
|
|
98
100
|
let accountStatus = null
|
|
99
101
|
let syncWarning = null
|
|
@@ -291,6 +293,19 @@ module.exports = function (app) {
|
|
|
291
293
|
}
|
|
292
294
|
}
|
|
293
295
|
|
|
296
|
+
// AIS targets for the boat's own browser, from local SignalK. Independent of the
|
|
297
|
+
// AIS upload above: this one works with no uplink, which is when it matters most.
|
|
298
|
+
if (p.serveAis !== false) {
|
|
299
|
+
try {
|
|
300
|
+
aisTargets = createAisTargets(app, { localSignalkUrl: pOpts.localSignalkUrl })
|
|
301
|
+
aisTargets.start()
|
|
302
|
+
pOpts.aisTargets = aisTargets
|
|
303
|
+
} catch (e) {
|
|
304
|
+
(app.error || console.error)('[sailkick-boat] AIS targets start failed: ' + e.message)
|
|
305
|
+
aisTargets = null
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
|
|
294
309
|
if (p.serveTelemetry !== false) {
|
|
295
310
|
try {
|
|
296
311
|
telemetry = createTelemetry(app, {})
|
|
@@ -373,7 +388,10 @@ module.exports = function (app) {
|
|
|
373
388
|
backfill = createBackfill(app, {
|
|
374
389
|
src,
|
|
375
390
|
dst,
|
|
376
|
-
|
|
391
|
+
// Never a config option: the plugin must not upload data that is not this
|
|
392
|
+
// boat's, and the cloud's history queries depend on that holding.
|
|
393
|
+
selfContext: app.selfContext || ('vessels.' + (app.selfId || 'self')),
|
|
394
|
+
context: String(bf.context || '').trim() || null, // hand-edit escape hatch
|
|
377
395
|
startBound: bf.startBound,
|
|
378
396
|
stateFile: path.join((app.getDataDirPath && app.getDataDirPath()) || '.', 'backfill.json'),
|
|
379
397
|
pending: sync ? sync.pending : null
|
|
@@ -399,6 +417,7 @@ module.exports = function (app) {
|
|
|
399
417
|
if (proxy) parts.push(proxy.status())
|
|
400
418
|
if (telemetry) parts.push(telemetry.status())
|
|
401
419
|
if (history) parts.push(history.status())
|
|
420
|
+
if (aisTargets) parts.push(aisTargets.status())
|
|
402
421
|
if (ais) parts.push(ais.status())
|
|
403
422
|
if (backfill) parts.push(backfill.status())
|
|
404
423
|
try { app.setPluginStatus(parts.join(' | ') || 'idle (both features off)') } catch {}
|
|
@@ -410,6 +429,7 @@ module.exports = function (app) {
|
|
|
410
429
|
try { if (sync) sync.stop() } catch {}
|
|
411
430
|
try { if (telemetry) telemetry.stop() } catch {}
|
|
412
431
|
try { if (history) history.stop() } catch {}
|
|
432
|
+
try { if (aisTargets) aisTargets.stop() } catch {}
|
|
413
433
|
try { if (ais) ais.stop() } catch {}
|
|
414
434
|
try { if (backfill) backfill.stop() } catch {}
|
|
415
435
|
try { if (proxy) proxy.stop() } catch {}
|
|
@@ -418,6 +438,7 @@ module.exports = function (app) {
|
|
|
418
438
|
history = null
|
|
419
439
|
backfill = null
|
|
420
440
|
ais = null
|
|
441
|
+
aisTargets = null
|
|
421
442
|
proxy = null
|
|
422
443
|
accountStatus = null
|
|
423
444
|
syncWarning = null
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
// Serve the app's GET /api/ais from the boat's OWN SignalK, so other vessels appear on
|
|
4
|
+
// the chart with no uplink at all.
|
|
5
|
+
//
|
|
6
|
+
// Ported from the cloud's server/ais/service.js — same polling shape, same trail rules,
|
|
7
|
+
// same JSON envelope — so public/viewer/ais.js cannot tell the difference. Keep the
|
|
8
|
+
// field set and the trail thresholds in step with that copy.
|
|
9
|
+
//
|
|
10
|
+
// This is the counterpart of lib/ais/index.js, not a duplicate of it: that one pushes
|
|
11
|
+
// AIS to the cloud so the boat's surroundings can be seen from shore; this one answers
|
|
12
|
+
// the browser sitting on the boat. Offshore, only this one can work — and it is the more
|
|
13
|
+
// valuable of the two, since AIS targets on your own chart matter most exactly when
|
|
14
|
+
// there is no connectivity.
|
|
15
|
+
//
|
|
16
|
+
// Why not reuse the upload module's delta stream? It subscribes only to the paths it
|
|
17
|
+
// forwards, and a vessel's NAME arrives as a vessel-level delta (empty path) which that
|
|
18
|
+
// path list deliberately ignores. Polling the REST tree gets identity and dimensions
|
|
19
|
+
// without a second subscription, exactly as the cloud does.
|
|
20
|
+
|
|
21
|
+
const MS_TO_KT = 1.94384
|
|
22
|
+
const R2D = 180 / Math.PI
|
|
23
|
+
const num = (x) => (Number.isFinite(Number(x)) ? Number(x) : null)
|
|
24
|
+
const wrap360 = (d) => ((d % 360) + 360) % 360
|
|
25
|
+
|
|
26
|
+
// ws/wss/http, with or without a /signalk path → REST origin.
|
|
27
|
+
function restOrigin (url) {
|
|
28
|
+
return String(url || '').trim()
|
|
29
|
+
.replace(/\/+$/, '')
|
|
30
|
+
.replace(/\/signalk\/.*$/i, '')
|
|
31
|
+
.replace(/^ws(s?):\/\//i, 'http$1://')
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function haversineM (lat1, lon1, lat2, lon2) {
|
|
35
|
+
const toR = Math.PI / 180
|
|
36
|
+
const R = 6371000
|
|
37
|
+
const dLat = (lat2 - lat1) * toR
|
|
38
|
+
const dLon = (lon2 - lon1) * toR
|
|
39
|
+
const a = Math.sin(dLat / 2) ** 2 + Math.cos(lat1 * toR) * Math.cos(lat2 * toR) * Math.sin(dLon / 2) ** 2
|
|
40
|
+
return 2 * R * Math.asin(Math.min(1, Math.sqrt(a)))
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function createAisTargets (app, options = {}) {
|
|
44
|
+
const log = (m) => (app.debug ? app.debug('[ais-targets] ' + m) : console.log('[sailkick-boat:ais-targets]', m))
|
|
45
|
+
const cfg = {
|
|
46
|
+
url: options.localSignalkUrl || 'http://127.0.0.1:3000',
|
|
47
|
+
pollMs: options.pollMs || 10000,
|
|
48
|
+
trailMs: options.trailMs || 3600000, // ~1 h of trail
|
|
49
|
+
trailMinMoveM: options.trailMinMoveM || 30, // a point only after moving this far
|
|
50
|
+
trailMaxPoints: options.trailMaxPoints || 240,
|
|
51
|
+
staleMs: options.staleMs || 15 * 60000 // drop a vessel unseen for this long
|
|
52
|
+
}
|
|
53
|
+
const base = restOrigin(cfg.url)
|
|
54
|
+
const state = new Map() // mmsi -> record
|
|
55
|
+
let selfMmsi = null
|
|
56
|
+
let lastPollAt = null
|
|
57
|
+
let lastError = null
|
|
58
|
+
let stopped = false
|
|
59
|
+
let timer = null
|
|
60
|
+
|
|
61
|
+
async function fetchSelf () {
|
|
62
|
+
try {
|
|
63
|
+
const r = await fetch(`${base}/signalk/v1/api/self`, { signal: AbortSignal.timeout(5000) })
|
|
64
|
+
if (r.ok) selfMmsi = String(await r.json()).split(':').pop() // "vessels.urn:…:mmsi:<n>"
|
|
65
|
+
} catch { /* retry next poll */ }
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function ingest (vessels) {
|
|
69
|
+
const now = Date.now()
|
|
70
|
+
for (const [key, v] of Object.entries(vessels || {})) {
|
|
71
|
+
const mmsi = String(key).split(':').pop()
|
|
72
|
+
if (!mmsi || mmsi === selfMmsi) continue
|
|
73
|
+
const p = v && v.navigation && v.navigation.position && v.navigation.position.value
|
|
74
|
+
if (!p || !Number.isFinite(p.latitude) || !Number.isFinite(p.longitude)) continue
|
|
75
|
+
// The AIS message's own time, beside .value — the poll cadence is not the age of
|
|
76
|
+
// the report.
|
|
77
|
+
const posTs = Date.parse(v.navigation.position.timestamp) || now
|
|
78
|
+
|
|
79
|
+
const cog = num(v.navigation.courseOverGroundTrue && v.navigation.courseOverGroundTrue.value)
|
|
80
|
+
const hT = num(v.navigation.headingTrue && v.navigation.headingTrue.value)
|
|
81
|
+
const hM = num(v.navigation.headingMagnetic && v.navigation.headingMagnetic.value)
|
|
82
|
+
const varn = num(v.navigation.magneticVariation && v.navigation.magneticVariation.value)
|
|
83
|
+
const headingDeg = hT != null ? wrap360(hT * R2D) : hM != null ? wrap360((hM + (varn || 0)) * R2D) : null
|
|
84
|
+
|
|
85
|
+
const rec = state.get(mmsi) || { mmsi, trail: [] }
|
|
86
|
+
// SignalK exposes `name` as a bare string, not the usual {value} wrapper.
|
|
87
|
+
rec.name = (typeof v.name === 'string' ? v.name : v.name && v.name.value) || rec.name || null
|
|
88
|
+
rec.lat = p.latitude
|
|
89
|
+
rec.lon = p.longitude
|
|
90
|
+
const sog = num(v.navigation.speedOverGround && v.navigation.speedOverGround.value)
|
|
91
|
+
rec.sogKt = sog != null ? +(sog * MS_TO_KT).toFixed(1) : null
|
|
92
|
+
rec.cogDeg = cog != null ? wrap360(cog * R2D) : null
|
|
93
|
+
rec.headingDeg = headingDeg
|
|
94
|
+
const len = v.design && v.design.length && v.design.length.value
|
|
95
|
+
rec.loaM = num(len && (len.overall != null ? len.overall : len.hull != null ? len.hull : len))
|
|
96
|
+
rec.beamM = num(v.design && v.design.beam && v.design.beam.value)
|
|
97
|
+
const at = v.design && v.design.aisShipType && v.design.aisShipType.value // { id, name } | id
|
|
98
|
+
rec.shipType = (at && typeof at === 'object' ? at.name : null) || rec.shipType || null
|
|
99
|
+
rec.aisType = (typeof at === 'number' ? at : num(at && at.id)) || rec.aisType || null
|
|
100
|
+
rec.rotRadS = num(v.navigation.rateOfTurn && v.navigation.rateOfTurn.value)
|
|
101
|
+
rec.posTs = posTs
|
|
102
|
+
rec.updatedAt = now
|
|
103
|
+
|
|
104
|
+
// Trail: a point only once the vessel has actually moved, so anchored ships stay a
|
|
105
|
+
// single dot instead of a jittering cloud.
|
|
106
|
+
const last = rec.trail[rec.trail.length - 1]
|
|
107
|
+
if (!last || haversineM(last[0], last[1], p.latitude, p.longitude) > cfg.trailMinMoveM) {
|
|
108
|
+
rec.trail.push([p.latitude, p.longitude, now])
|
|
109
|
+
}
|
|
110
|
+
const cutoff = now - cfg.trailMs
|
|
111
|
+
while (rec.trail.length && rec.trail[0][2] < cutoff) rec.trail.shift()
|
|
112
|
+
if (rec.trail.length > cfg.trailMaxPoints) rec.trail.splice(0, rec.trail.length - cfg.trailMaxPoints)
|
|
113
|
+
|
|
114
|
+
state.set(mmsi, rec)
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
async function poll () {
|
|
119
|
+
if (stopped) return
|
|
120
|
+
try {
|
|
121
|
+
if (!selfMmsi) await fetchSelf()
|
|
122
|
+
const r = await fetch(`${base}/signalk/v1/api/vessels`, { signal: AbortSignal.timeout(8000) })
|
|
123
|
+
if (!r.ok) throw new Error(`vessels ${r.status}`)
|
|
124
|
+
ingest(await r.json())
|
|
125
|
+
lastPollAt = Date.now()
|
|
126
|
+
lastError = null
|
|
127
|
+
} catch (e) {
|
|
128
|
+
lastError = e.message // a failed poll keeps the last snapshot; never throws
|
|
129
|
+
} finally {
|
|
130
|
+
if (!stopped) { timer = setTimeout(poll, cfg.pollMs); if (timer.unref) timer.unref() }
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function getVessels () {
|
|
135
|
+
const cutoff = Date.now() - cfg.staleMs
|
|
136
|
+
const vessels = []
|
|
137
|
+
for (const rec of state.values()) {
|
|
138
|
+
if (rec.updatedAt < cutoff) { state.delete(rec.mmsi); continue }
|
|
139
|
+
vessels.push({
|
|
140
|
+
mmsi: rec.mmsi, name: rec.name, lat: rec.lat, lon: rec.lon,
|
|
141
|
+
sogKt: rec.sogKt, cogDeg: rec.cogDeg, headingDeg: rec.headingDeg, rotRadS: rec.rotRadS,
|
|
142
|
+
loaM: rec.loaM, beamM: rec.beamM, shipType: rec.shipType, aisType: rec.aisType,
|
|
143
|
+
posTs: rec.posTs, updatedAt: rec.updatedAt,
|
|
144
|
+
trail: rec.trail.map((q) => [q[0], q[1]])
|
|
145
|
+
})
|
|
146
|
+
}
|
|
147
|
+
return { polledAt: lastPollAt, error: lastError, count: vessels.length, vessels }
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function start () { log(`polling ${base}/signalk/v1/api/vessels every ${Math.round(cfg.pollMs / 1000)}s`); poll() }
|
|
151
|
+
function stop () { stopped = true; if (timer) clearTimeout(timer) }
|
|
152
|
+
function available () { return true } // SignalK is local; a failed poll just serves the last snapshot
|
|
153
|
+
|
|
154
|
+
// GET /api/ais — byte-for-byte the envelope the cloud returns.
|
|
155
|
+
function handleAis (req, res) {
|
|
156
|
+
const body = JSON.stringify({ available: true, ...getVessels() })
|
|
157
|
+
res.statusCode = 200
|
|
158
|
+
res.setHeader('Content-Type', 'application/json')
|
|
159
|
+
res.end(req.method === 'HEAD' ? undefined : body)
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
function status () {
|
|
163
|
+
const n = state.size
|
|
164
|
+
const age = lastPollAt ? Math.round((Date.now() - lastPollAt) / 1000) + 's ago' : 'never'
|
|
165
|
+
return `ais targets: ${n} vessel(s), polled ${age}${lastError ? ' (' + lastError + ')' : ''}`
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
return { start, stop, status, available, handleAis, getVessels, _ingest: ingest, _state: () => state }
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
module.exports = { createAisTargets, restOrigin, haversineM }
|
package/lib/backfill/index.js
CHANGED
|
@@ -29,7 +29,13 @@ const HOUR_MS = 3600000
|
|
|
29
29
|
const DEFAULTS = {
|
|
30
30
|
windowMs: HOUR_MS,
|
|
31
31
|
batchSize: 10000,
|
|
32
|
-
|
|
32
|
+
// A window is read whole and converted in memory. A busy archive can hold millions of
|
|
33
|
+
// points per hour (54M/day was measured on a real boat = ~2.25M/hour ~ 400 MB of CSV),
|
|
34
|
+
// which would exhaust a Raspberry Pi. When a window is denser than this it is halved
|
|
35
|
+
// until it fits — the count is already known before the read, so this costs nothing.
|
|
36
|
+
maxRowsPerChunk: 100000,
|
|
37
|
+
minWindowMs: 60000, // never subdivide below a minute
|
|
38
|
+
idleMs: 250, // pending() is the real backpressure; this is just politeness
|
|
33
39
|
backlogWaitMs: 15000, // how long to stand down when live sync has a backlog
|
|
34
40
|
maxErrorStreak: 5,
|
|
35
41
|
queryTimeoutMs: 120000
|
|
@@ -59,9 +65,9 @@ function createBackfill (app, options) {
|
|
|
59
65
|
function load () {
|
|
60
66
|
try {
|
|
61
67
|
const j = JSON.parse(fs.readFileSync(cfg.stateFile, 'utf8'))
|
|
62
|
-
if (j && typeof j === 'object') return { done: j.done || {}, earliest: j.earliest || null, points: j.points || 0, complete: !!j.complete }
|
|
68
|
+
if (j && typeof j === 'object') return { done: j.done || {}, earliest: j.earliest || null, ceiling: j.ceiling || null, points: j.points || 0, complete: !!j.complete }
|
|
63
69
|
} catch {}
|
|
64
|
-
return { done: {}, earliest: null, points: 0, complete: false }
|
|
70
|
+
return { done: {}, earliest: null, ceiling: null, points: 0, complete: false }
|
|
65
71
|
}
|
|
66
72
|
function save () {
|
|
67
73
|
try {
|
|
@@ -106,12 +112,12 @@ function createBackfill (app, options) {
|
|
|
106
112
|
return 0
|
|
107
113
|
}
|
|
108
114
|
|
|
109
|
-
async function earliestPoint () {
|
|
115
|
+
async function earliestPoint (conn = cfg.src, bucket = cfg.src.bucket) {
|
|
110
116
|
// `first()` reduces each series before anything is merged, then _time is isolated
|
|
111
117
|
// BEFORE group(). Both matter on a real bucket: grouping the raw stream fails with
|
|
112
118
|
// "schema collision: cannot group boolean and integer types together" the moment the
|
|
113
119
|
// database holds more than one field type, which any real boat's does.
|
|
114
|
-
const r = await flux(
|
|
120
|
+
const r = await flux(conn, `from(bucket:"${bucket}")|>range(start:0)|>first()|>keep(columns:["_time"])|>group()|>min(column:"_time")`)
|
|
115
121
|
if (!r.ok) return null
|
|
116
122
|
for (const line of r.text.split('\n')) {
|
|
117
123
|
if (!line || line.startsWith('#') || line.includes('_time')) continue
|
|
@@ -128,18 +134,70 @@ function createBackfill (app, options) {
|
|
|
128
134
|
return null
|
|
129
135
|
}
|
|
130
136
|
|
|
137
|
+
// Distinct contexts in the source bucket. Returns null on any failure — never an
|
|
138
|
+
// empty list, so an unreachable database is not mistaken for an empty archive.
|
|
139
|
+
async function sourceContexts () {
|
|
140
|
+
const r = await flux(cfg.src, `import "influxdata/influxdb/schema"\nschema.tagValues(bucket:"${cfg.src.bucket}", tag:"context")`)
|
|
141
|
+
if (!r.ok) return null
|
|
142
|
+
const out = []
|
|
143
|
+
for (const line of r.text.split('\n')) {
|
|
144
|
+
if (!line || line.startsWith('#')) continue
|
|
145
|
+
const cells = line.trim().split(',')
|
|
146
|
+
const v = cells[cells.length - 1]
|
|
147
|
+
if (v && v !== '_value') out.push(v)
|
|
148
|
+
}
|
|
149
|
+
return out
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// The plugin must never upload data that is not this boat's. Live sync guarantees
|
|
153
|
+
// that by subscribing to vessels.self; the backfill is the only thing that could
|
|
154
|
+
// break it, and the cloud's history queries depend on it holding. So this is decided
|
|
155
|
+
// in code, not exposed as an option someone can get wrong.
|
|
156
|
+
//
|
|
157
|
+
// The wrinkle: an imported archive may carry a DIFFERENT context than the boat's
|
|
158
|
+
// current identity — a UUID from a since-reinstalled Signal K, or an MMSI URN. A
|
|
159
|
+
// strict match would then copy nothing, silently. Hence the single-context rule: a
|
|
160
|
+
// bucket holding exactly one context contains one vessel by definition and cannot be
|
|
161
|
+
// an AIS collection, so it is copied whatever its identity string says.
|
|
162
|
+
function contextFilterFor (contexts) {
|
|
163
|
+
const selfCtx = cfg.selfContext
|
|
164
|
+
if (cfg.context) {
|
|
165
|
+
log(`copying only context ${cfg.context} (explicit override)`)
|
|
166
|
+
return `|>filter(fn:(r)=>r.context=="${cfg.context}")`
|
|
167
|
+
}
|
|
168
|
+
if (contexts.length <= 1) {
|
|
169
|
+
log(`source holds a single context (${contexts[0] || 'none'}) — copying all of it`)
|
|
170
|
+
return ''
|
|
171
|
+
}
|
|
172
|
+
const others = contexts.filter((c) => c !== selfCtx)
|
|
173
|
+
warn(`source holds ${contexts.length} contexts — copying only this boat (${selfCtx}) and skipping ${others.length} other(s), e.g. ${others.slice(0, 3).join(', ')}`)
|
|
174
|
+
return `|>filter(fn:(r)=>r.self=="true" or r.context=="${selfCtx}")`
|
|
175
|
+
}
|
|
176
|
+
|
|
131
177
|
// --- one window ---------------------------------------------------------------
|
|
132
|
-
// Returns 'done' | 'empty' | 'retry' | 'stopped'.
|
|
178
|
+
// Returns 'done' | 'empty' | 'retry' | 'fatal' | 'stopped'. Subdivides itself when a
|
|
179
|
+
// window holds more points than can be held in memory at once.
|
|
133
180
|
async function doWindow (startMs, stopMs) {
|
|
134
181
|
const srcCount = await count(cfg.src, cfg.src.bucket, startMs, stopMs)
|
|
135
182
|
if (srcCount == null) return 'retry'
|
|
136
183
|
if (srcCount === 0) return 'empty'
|
|
137
184
|
|
|
138
|
-
|
|
185
|
+
if (srcCount > cfg.maxRowsPerChunk && (stopMs - startMs) > cfg.minWindowMs) {
|
|
186
|
+
// Too dense to read whole. Halve it — newest half first, keeping the run's
|
|
187
|
+
// newest-first order. Both halves must succeed for the caller to mark the window
|
|
188
|
+
// done, so a failure part-way is simply retried next time.
|
|
189
|
+
const mid = startMs + Math.floor((stopMs - startMs) / 2)
|
|
190
|
+
log(`${iso(startMs)} holds ${srcCount} points — splitting`)
|
|
191
|
+
const newer = await doWindow(mid, stopMs)
|
|
192
|
+
if (newer !== 'done' && newer !== 'empty') return newer
|
|
193
|
+
return doWindow(startMs, mid)
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
const filter = cfg._filter || ''
|
|
139
197
|
const r = await flux(cfg.src, `from(bucket:"${cfg.src.bucket}")|>range(start:${iso(startMs)},stop:${iso(stopMs)})${filter}|>drop(columns:["_start","_stop"])`)
|
|
140
198
|
if (!r.ok) { warn(`read ${iso(startMs)} failed — ${r.message}`); return 'retry' }
|
|
141
199
|
|
|
142
|
-
const { lines, skipped } = csvToLineProtocol(r.text
|
|
200
|
+
const { lines, skipped } = csvToLineProtocol(r.text)
|
|
143
201
|
if (skipped) log(`${iso(startMs)}: skipped ${skipped} unconvertible row(s)`)
|
|
144
202
|
if (!lines.length) return 'empty'
|
|
145
203
|
|
|
@@ -171,6 +229,16 @@ function createBackfill (app, options) {
|
|
|
171
229
|
async function run () {
|
|
172
230
|
running = true
|
|
173
231
|
try {
|
|
232
|
+
// Validate the source and decide the filter before walking 15k windows.
|
|
233
|
+
const contexts = await sourceContexts()
|
|
234
|
+
if (contexts == null) { warn(`could not read contexts from ${cfg.src.bucket} — is the source reachable?`); statusLine = 'backfill: source unreachable'; return }
|
|
235
|
+
if (!contexts.length) {
|
|
236
|
+
warn(`source bucket "${cfg.src.bucket}" (org "${cfg.src.org}") holds no data at all — check the org and bucket names`)
|
|
237
|
+
statusLine = `backfill: source ${cfg.src.org}/${cfg.src.bucket} is empty — check the names`
|
|
238
|
+
return
|
|
239
|
+
}
|
|
240
|
+
cfg._filter = contextFilterFor(contexts)
|
|
241
|
+
|
|
174
242
|
if (state.earliest == null) {
|
|
175
243
|
const e = await earliestPoint()
|
|
176
244
|
if (e == null) { warn('could not read the oldest point from the source — is it reachable?'); statusLine = 'backfill: source unreachable'; return }
|
|
@@ -186,7 +254,20 @@ function createBackfill (app, options) {
|
|
|
186
254
|
const rawFloor = cfg.startBound ? Math.max(state.earliest, Date.parse(cfg.startBound)) : state.earliest
|
|
187
255
|
const floor = hourFloor(rawFloor)
|
|
188
256
|
|
|
189
|
-
|
|
257
|
+
// Start below where live sync has already delivered, not at "now". A still-live
|
|
258
|
+
// source archive otherwise makes the first windows re-upload today's data —
|
|
259
|
+
// correct timestamps, but data the cloud already has, and a lot of wasted uplink.
|
|
260
|
+
// The destination's own oldest point IS the moment cloud sync began, and the
|
|
261
|
+
// read+write token can see it.
|
|
262
|
+
if (state.ceiling == null) {
|
|
263
|
+
const dstOldest = await earliestPoint(cfg.dst, cfg.dst.bucket)
|
|
264
|
+
state.ceiling = dstOldest != null ? hourFloor(dstOldest) : hourFloor(Date.now())
|
|
265
|
+
log(dstOldest != null
|
|
266
|
+
? `live sync covers everything from ${iso(state.ceiling)} — backfilling only what is older`
|
|
267
|
+
: 'destination is empty — backfilling everything up to now')
|
|
268
|
+
save()
|
|
269
|
+
}
|
|
270
|
+
let cursor = state.ceiling // newest-first: recent history lands first
|
|
190
271
|
let errStreak = 0
|
|
191
272
|
let didWork = 0
|
|
192
273
|
|
|
@@ -223,6 +304,16 @@ function createBackfill (app, options) {
|
|
|
223
304
|
}
|
|
224
305
|
|
|
225
306
|
if (!stopped && cursor < floor) {
|
|
307
|
+
if (state.points === 0) {
|
|
308
|
+
// A walk that finishes having copied nothing is far more likely to be a wrong
|
|
309
|
+
// org/bucket, or a filter that matched no rows, than a genuinely empty
|
|
310
|
+
// archive. Marking it complete would dress a silent no-op as success and stop
|
|
311
|
+
// it ever retrying.
|
|
312
|
+
save()
|
|
313
|
+
warn(`walked ${Object.keys(state.done).length} window(s) and copied ZERO points — check that org "${cfg.src.org}" / bucket "${cfg.src.bucket}" is right, and that its data belongs to this boat. NOT marking complete.`)
|
|
314
|
+
statusLine = `backfill: finished with 0 points — check ${cfg.src.org}/${cfg.src.bucket}`
|
|
315
|
+
return
|
|
316
|
+
}
|
|
226
317
|
state.complete = true
|
|
227
318
|
save()
|
|
228
319
|
statusLine = `backfill: complete — ${state.points} point(s) from ${iso(floor).slice(0, 10)}`
|
|
@@ -242,7 +333,7 @@ function createBackfill (app, options) {
|
|
|
242
333
|
if (state.complete) { statusLine = `backfill: complete — ${state.points} point(s)`; return null }
|
|
243
334
|
if (!cfg.src.token || !cfg.src.bucket || !cfg.src.url) { statusLine = 'backfill: not configured (source)'; return null }
|
|
244
335
|
if (!cfg.dst.token || !cfg.dst.bucket) { statusLine = 'backfill: not configured (cloud token)'; return null }
|
|
245
|
-
log(`${cfg.src.url} ${cfg.src.org}/${cfg.src.bucket} -> ${cfg.dst.url} ${cfg.dst.org}/${cfg.dst.bucket}
|
|
336
|
+
log(`${cfg.src.url} ${cfg.src.org}/${cfg.src.bucket} -> ${cfg.dst.url} ${cfg.dst.org}/${cfg.dst.bucket}`)
|
|
246
337
|
statusLine = 'backfill: starting'
|
|
247
338
|
runPromise = run().catch((e) => { warn('run failed: ' + e.message); statusLine = 'backfill: error — ' + e.message })
|
|
248
339
|
return runPromise
|
package/lib/proxy/index.js
CHANGED
|
@@ -57,6 +57,7 @@ function createProxy (app, options) {
|
|
|
57
57
|
localPaths: (options.localPaths && options.localPaths.length) ? options.localPaths : ['/signalk'],
|
|
58
58
|
telemetryPath: options.telemetryPath || '/ws/telemetry',
|
|
59
59
|
history: options.history || null,
|
|
60
|
+
aisTargets: options.aisTargets || null,
|
|
60
61
|
openAccess: options.openAccess !== false
|
|
61
62
|
}
|
|
62
63
|
log(`mirroring ${cfg.upstream}; local SignalK ${cfg.localSignalk}; store ${cfg.storeDir}`)
|
|
@@ -140,6 +141,15 @@ function createProxy (app, options) {
|
|
|
140
141
|
if (pathname === '/api/history/series') return cfg.history.handleSeries(req, res)
|
|
141
142
|
if (pathname === '/api/history/track') return cfg.history.handleTrack(req, res)
|
|
142
143
|
}
|
|
144
|
+
// AIS targets, from the boat's own SignalK. The cloud's /api/ais is gated behind a
|
|
145
|
+
// boat session the mirror can never hold (it forwards no cookies), and its poller
|
|
146
|
+
// reads a LAN address it cannot reach once the boat is on a mobile link — so
|
|
147
|
+
// proxying it returns 401 whatever we do. Served locally it also works with no
|
|
148
|
+
// uplink at all, which is when other vessels on the chart matter most.
|
|
149
|
+
if (req.method === 'GET' && cfg.aisTargets && cfg.aisTargets.available() &&
|
|
150
|
+
req.url.split('?')[0] === '/api/ais') {
|
|
151
|
+
return cfg.aisTargets.handleAis(req, res)
|
|
152
|
+
}
|
|
143
153
|
// /api/config drives the app's login gate + Trends toggle. On the single-tenant
|
|
144
154
|
// boat we serve it with auth turned off (no cloud login over the offline HTTP
|
|
145
155
|
// mirror) and history forced on when we serve it locally. All other config passes
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "sailkick-boat",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.18.0",
|
|
4
4
|
"description": "EARLY ALPHA — cloud telemetry + offline maps for sailkick boats (www.sailkick.io; register on the web, paste the write token). Gapless boat→cloud telemetry sync to InfluxDB, and a local proxy that keeps the sailkick app and its charts/maps working fully offline on board.",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"scripts": {
|