signalk-edge-link 2.3.0 → 2.4.1

package/lib/pipeline-v2-server.js

@@ -56,6 +56,8 @@ const msgpack = __importStar(require("@msgpack/msgpack"));
 const crypto_1 = require("./crypto");
 const pathDictionary_1 = require("./pathDictionary");
 const delta_sanitizer_1 = require("./delta-sanitizer");
+const source_dispatch_1 = require("./source-dispatch");
+const source_snapshot_1 = require("./source-snapshot");
 const packet_1 = require("./packet");
 const sequence_1 = require("./sequence");
 const metrics_publisher_1 = require("./metrics-publisher");
@@ -154,6 +156,8 @@ function createPipelineV2Server(app, state, metricsApi) {
             // packet can still be processed for this request without polluting state.
             return {
                 key,
+                sourceClientInstanceId: null,
+                clientId: null,
                 address: rinfo.address,
                 port: rinfo.port,
                 sequenceTracker: new sequence_1.SequenceTracker({
@@ -175,11 +179,15 @@ function createPipelineV2Server(app, state, metricsApi) {
                 rateLimitWindowStart: Date.now(),
                 metaRequested: false,
                 lastMetaEnvSeq: null,
-                seenMetaChunkIdx: new Set()
+                seenMetaChunkIdx: new Set(),
+                lastSourceEnvSeq: null,
+                seenSourceChunkIdx: new Set()
             };
         }
         const session = {
             key,
+            sourceClientInstanceId: null,
+            clientId: null,
             address: rinfo.address,
             port: rinfo.port,
             sequenceTracker: new sequence_1.SequenceTracker({
@@ -206,7 +214,9 @@ function createPipelineV2Server(app, state, metricsApi) {
             metaRequested: false,
             // Stale-envelope rejection for METADATA packets
             lastMetaEnvSeq: null,
-            seenMetaChunkIdx: new Set()
+            seenMetaChunkIdx: new Set(),
+            lastSourceEnvSeq: null,
+            seenSourceChunkIdx: new Set()
         };
         clientSessions.set(key, session);
         app.debug(`[v2-server] new client session: ${key}`);
@@ -252,6 +262,8 @@ function createPipelineV2Server(app, state, metricsApi) {
     let lastMetricsTime = Date.now();
     let lastBytesReceived = 0;
     let lastPacketsReceived = 0;
+    let previousSourceMissingIdentity = 0;
+    let previousSourceConflicts = 0;
     // Rate-limit operator-visible warnings for protocol-version mismatches so a
     // persistently misconfigured peer is noticeable in logs without flooding them.
     const PROTOCOL_VERSION_MISMATCH_WARN_INTERVAL_MS = 60000;
@@ -450,6 +462,61 @@ function createPipelineV2Server(app, state, metricsApi) {
             ackTimer = null;
         }
     }
+    function shouldDropEnvelopeBySeq(session, env, channel) {
+        if (!session || typeof env.seq !== "number" || !Number.isFinite(env.seq)) {
+            return false;
+        }
+        const envSeq = env.seq >>> 0;
+        const envIdx = typeof env.idx === "number" && Number.isFinite(env.idx) ? env.idx >>> 0 : 0;
+        const isSource = channel === "source snapshot";
+        const seenChunkIdx = isSource ? session.seenSourceChunkIdx : session.seenMetaChunkIdx;
+        let lastEnvSeq = isSource ? session.lastSourceEnvSeq : session.lastMetaEnvSeq;
+        const setLastEnvSeq = (value) => {
+            if (isSource) {
+                session.lastSourceEnvSeq = value;
+            }
+            else {
+                session.lastMetaEnvSeq = value;
+            }
+            lastEnvSeq = value;
+        };
+        // Sender-restart detection: the client's envelope sequence counter is
+        // initialised to 0 at process start, so an incoming envSeq of 0 with a
+        // sufficiently-advanced previous seq is a strong signal that the peer
+        // restarted. The threshold guards against first-packet replays.
+        if (lastEnvSeq !== null && envSeq === 0 && lastEnvSeq >= META_RESTART_THRESHOLD) {
+            app.debug(`[v2-server] ${channel} sender restart detected for ${session.key} ` +
+                `(last seq was ${lastEnvSeq}); resetting ${channel} state`);
+            setLastEnvSeq(null);
+            seenChunkIdx.clear();
+            if (!isSource) {
+                session.metaRequested = false;
+            }
+        }
+        if (lastEnvSeq !== null) {
+            const distance = (envSeq - lastEnvSeq) >>> 0;
+            if (distance !== 0 && distance >= 0x80000000) {
+                metrics.duplicatePackets = (metrics.duplicatePackets || 0) + 1;
+                app.debug(`[v2-server] stale ${channel} envelope seq=${envSeq} from ${session.key} (last=${lastEnvSeq}), dropping`);
+                return true;
+            }
+            if (distance !== 0) {
+                setLastEnvSeq(envSeq);
+                seenChunkIdx.clear();
+            }
+            else if (seenChunkIdx.has(envIdx)) {
+                metrics.duplicatePackets = (metrics.duplicatePackets || 0) + 1;
+                app.debug(`[v2-server] duplicate ${channel} chunk seq=${envSeq} idx=${envIdx} from ${session.key}, dropping`);
+                return true;
+            }
+        }
+        else {
+            setLastEnvSeq(envSeq);
+            seenChunkIdx.clear();
+        }
+        seenChunkIdx.add(envIdx);
+        return false;
+    }
     /**
      * Decrypt and dispatch a METADATA (0x06) packet.
      *
@@ -496,12 +563,20 @@ function createPipelineV2Server(app, state, metricsApi) {
                 return;
             }
             const env = content;
-            if (!Array.isArray(env.entries) || env.entries.length === 0) {
+            const hasSourceSnapshot = env.kind === "sources" &&
+                env.sources !== null &&
+                typeof env.sources === "object" &&
+                !Array.isArray(env.sources);
+            const entries = Array.isArray(env.entries) ? env.entries : [];
+            if (!hasSourceSnapshot && entries.length === 0) {
                 metrics.malformedPackets = (metrics.malformedPackets || 0) + 1;
                 app.debug("v2 META envelope has no entries, dropping");
                 recordError("general", "v2 META envelope has no entries");
                 return;
             }
+            if (hasSourceSnapshot && shouldDropEnvelopeBySeq(session, env, "source snapshot")) {
+                return;
+            }
             // Drop stale/duplicate envelopes that UDP reordered or replayed. The
             // inner envelope `seq` identifies a batch (shared across all chunks of
             // a multi-chunk snapshot/diff); the inner `idx` identifies a specific
@@ -510,7 +585,10 @@ function createPipelineV2Server(app, state, metricsApi) {
             //   - within the current batch, reject any (seq, idx) pair already
             //     processed ("exact replay"); other idx values for the same seq
             //     remain accepted so multi-chunk batches still apply in full.
-            if (session && typeof env.seq === "number" && Number.isFinite(env.seq)) {
+            if (!hasSourceSnapshot &&
+                session &&
+                typeof env.seq === "number" &&
+                Number.isFinite(env.seq)) {
                 const envSeq = env.seq >>> 0;
                 const envIdx = typeof env.idx === "number" && Number.isFinite(env.idx) ? env.idx >>> 0 : 0;
                 // Sender-restart detection: the client's meta sequence counter is
@@ -558,12 +636,17 @@ function createPipelineV2Server(app, state, metricsApi) {
                 }
                 session.seenMetaChunkIdx.add(envIdx);
             }
+            if (hasSourceSnapshot) {
+                const added = (0, source_snapshot_1.mergeSourceSnapshot)(app, env.sources);
+                app.debug(`v2 source snapshot received: sources=${Object.keys(env.sources || {}).length}, added=${added}, envSeq=${env.seq ?? "?"}`);
+                return;
+            }
             // Group entries by context so the local Signal K server sees one delta
             // per context rather than one per path. Reduces app.handleMessage
             // overhead on big snapshots without changing semantics.
             const nowIso = new Date().toISOString();
             const byContext = new Map();
-            for (const rawEntry of env.entries) {
+            for (const rawEntry of entries) {
                 if (!rawEntry || typeof rawEntry.meta !== "object" || !rawEntry.meta) {
                     continue;
                 }
@@ -605,7 +688,7 @@ function createPipelineV2Server(app, state, metricsApi) {
                 };
                 app.handleMessage("", deltaMessage);
             }
-            app.debug(`v2 meta received: kind=${env.kind ?? "?"}, entries=${env.entries.length}, contexts=${byContext.size}, envSeq=${env.v ?? "?"}`);
+            app.debug(`v2 meta received: kind=${env.kind ?? "?"}, entries=${entries.length}, contexts=${byContext.size}, envSeq=${env.seq ?? "?"}`);
         }
         catch (err) {
             const msg = err instanceof Error ? err.message : String(err);
@@ -712,6 +795,16 @@ function createPipelineV2Server(app, state, metricsApi) {
                 try {
                     const info = JSON.parse(parsed.payload.toString());
                     app.debug(`v2 hello from client: ${JSON.stringify(info)}`);
+                    if (session && info && typeof info === "object") {
+                        const helloClientId = typeof info.clientId === "string" && info.clientId.trim()
+                            ? info.clientId.trim()
+                            : null;
+                        const helloInstanceId = typeof info.instanceId === "string" && info.instanceId.trim()
+                            ? info.instanceId.trim()
+                            : null;
+                        session.clientId = helloClientId;
+                        session.sourceClientInstanceId = helloInstanceId || helloClientId;
+                    }
                 }
                 catch (parseErr) {
                     app.error(`v2 failed to parse HELLO payload: ${parseErr instanceof Error ? parseErr.message : String(parseErr)}`);
@@ -879,12 +972,25 @@ function createPipelineV2Server(app, state, metricsApi) {
                     continue;
                 }
                 deltaMessage = sanitizedDelta;
+                deltaMessage = (0, source_dispatch_1.normalizeDeltaSourceRefs)(deltaMessage);
                 _ingestRemoteTelemetry(deltaMessage);
                 if (!Array.isArray(deltaMessage.updates) || deltaMessage.updates.length === 0) {
                     continue;
                 }
+                if (state.sourceRegistry && typeof state.sourceRegistry.upsertFromDelta === "function") {
+                    const deltaRecord = deltaMessage;
+                    const deltaSourceInstanceId = deltaMessage &&
+                        typeof deltaMessage === "object" &&
+                        typeof deltaRecord.sourceClientInstanceId === "string"
+                        ? deltaRecord.sourceClientInstanceId || null
+                        : null;
+                    const stableSourceClientId = (session && (session.sourceClientInstanceId || session.clientId)) ||
+                        deltaSourceInstanceId ||
+                        "unknown";
+                    state.sourceRegistry.upsertFromDelta(deltaMessage, stableSourceClientId);
+                }
                 trackPathStats(deltaMessage, decompressed.length / deltas.length);
-                app.handleMessage("", deltaMessage);
+                (0, source_dispatch_1.handleMessageBySource)(app, deltaMessage);
                 metrics.deltasReceived++;
             }
             app.debug(`v2 received: seq=${parsed.sequence}, ${deltaCount} deltas, ${packet.length} bytes`);
@@ -943,7 +1049,8 @@ function createPipelineV2Server(app, state, metricsApi) {
             sessions,
             totalSessions: clientSessions.size,
             acksSent: metrics.acksSent,
-            naksSent: metrics.naksSent
+            naksSent: metrics.naksSent,
+            sourceReplication: state.sourceRegistry ? state.sourceRegistry.getMetrics() : null
         };
     }
     /**
@@ -1027,6 +1134,16 @@ function createPipelineV2Server(app, state, metricsApi) {
         const effectiveQueueDepth = hasRemoteTelemetry ? remote.queueDepth || 0 : 0;
         const effectiveRetransmitRate = hasRemoteTelemetry ? remote.retransmitRate || 0 : 0;
         const effectiveActiveLink = hasRemoteTelemetry ? remote.activeLink || "primary" : "primary";
+        const sourceReplicationMetrics = state.sourceRegistry
+            ? state.sourceRegistry.getMetrics()
+            : { upserts: 0, noops: 0, missingIdentity: 0, conflicts: 0 };
+        const deltaMissing = sourceReplicationMetrics.missingIdentity - previousSourceMissingIdentity;
+        const deltaConflicts = sourceReplicationMetrics.conflicts - previousSourceConflicts;
+        if (deltaMissing > 0 || deltaConflicts > 0) {
+            app.debug(`[source-replication] +missingIdentity=${deltaMissing} +conflicts=${deltaConflicts} totalMissingIdentity=${sourceReplicationMetrics.missingIdentity} totalConflicts=${sourceReplicationMetrics.conflicts} size=${state.sourceRegistry.snapshot().size}`);
+        }
+        previousSourceMissingIdentity = sourceReplicationMetrics.missingIdentity;
+        previousSourceConflicts = sourceReplicationMetrics.conflicts;
         // Publish to Signal K
         metricsPublisher.publish({
             rtt: effectiveRtt,

package/lib/pipeline.js

@@ -36,6 +36,7 @@ const msgpack = __importStar(require("@msgpack/msgpack"));
 const crypto_1 = require("./crypto");
 const pathDictionary_1 = require("./pathDictionary");
 const delta_sanitizer_1 = require("./delta-sanitizer");
+const source_dispatch_1 = require("./source-dispatch");
 const pipeline_utils_1 = require("./pipeline-utils");
 const constants_1 = require("./constants");
 const metadata_1 = require("./metadata");
@@ -294,9 +295,10 @@ function createPipeline(app, state, metricsApi) {
                     app.debug(`Skipping delta with no valid Signal K values at index ${i}`);
                     continue;
                 }
+                deltaMessage = (0, source_dispatch_1.normalizeDeltaSourceRefs)(deltaMessage);
                 // Track path stats for server-side analytics
                 trackPathStats(deltaMessage, decompressed.length / deltas.length);
-                app.handleMessage("", deltaMessage);
+                (0, source_dispatch_1.handleMessageBySource)(app, deltaMessage);
                 // Log a compact summary only — never log full delta values which may
                 // contain sensitive data (position, fuel, MMSI) in plaintext logs.
                 app.debug(`delta ctx=${deltaMessage.context ?? "?"} updates=${Array.isArray(deltaMessage.updates) ? deltaMessage.updates.length : 0}`);

package/lib/routes/metrics.js

@@ -9,7 +9,7 @@ const prometheus_1 = require("../prometheus");
  * @param ctx - Shared route context (helpers, middleware, registry)
  */
 function register(router, ctx) {
-    const { rateLimitMiddleware, instanceRegistry, getFirstBundle, getEffectiveNetworkQuality, getActiveMetricsPublisher, buildFullMetricsResponse } = ctx;
+    const { rateLimitMiddleware, instanceRegistry, getFirstBundle, getEffectiveNetworkQuality, getActiveMetricsPublisher, buildFullMetricsResponse, managementAuthMiddleware } = ctx;
     router.get("/metrics", rateLimitMiddleware, (req, res) => {
         try {
             const bundle = getFirstBundle();
@@ -124,4 +124,28 @@ function register(router, ctx) {
             res.status(500).json({ error: err instanceof Error ? err.message : String(err) });
         }
     });
+    router.get("/sources", rateLimitMiddleware, managementAuthMiddleware("sources.read"), (req, res) => {
+        try {
+            const serverBundle = instanceRegistry
+                .getAll()
+                .find((bundle) => bundle.state && bundle.state.isServerMode && bundle.state.sourceRegistry) || null;
+            const bundle = serverBundle || getFirstBundle();
+            if (!bundle) {
+                return res.status(503).json({ error: "Plugin not started" });
+            }
+            const { state } = bundle;
+            if (!state.sourceRegistry) {
+                return res.json({
+                    schemaVersion: 1,
+                    size: 0,
+                    sources: [],
+                    legacy: { byLabel: {}, bySourceRef: {} }
+                });
+            }
+            return res.json(state.sourceRegistry.snapshot());
+        }
+        catch (err) {
+            return res.status(500).json({ error: err instanceof Error ? err.message : String(err) });
+        }
+    });
 }

package/lib/routes.js

@@ -442,6 +442,12 @@ function createRoutes(app, instanceRegistry, pluginRef) {
                     timestamp: metrics.lastErrorTime,
                     timeAgo: metrics.lastErrorTime ? Date.now() - metrics.lastErrorTime : null
                 }
+                : null,
+            sourceReplication: state.sourceRegistry
+                ? {
+                    metrics: state.sourceRegistry.getMetrics(),
+                    registry: null
+                }
                 : null
         };
         return metricsData;

package/lib/shared/connection-schema.js

@@ -15,7 +15,7 @@
  * results to `RJSFSchema` at call sites.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.alertThresholdsProperty = exports.enableNotificationsProperty = exports.bondingProperty = exports.congestionControlProperty = exports.serverReliabilityProperty = exports.clientReliabilityProperty = exports.clientTransportProperties = exports.commonConnectionProperties = void 0;
+exports.alertThresholdsProperty = exports.skipOwnDataProperty = exports.enableNotificationsProperty = exports.bondingProperty = exports.congestionControlProperty = exports.serverReliabilityProperty = exports.clientReliabilityProperty = exports.clientTransportProperties = exports.commonConnectionProperties = void 0;
 exports.buildConnectionItemSchema = buildConnectionItemSchema;
 exports.buildWebappConnectionSchema = buildWebappConnectionSchema;
 const crypto_constants_1 = require("./crypto-constants");
@@ -424,6 +424,13 @@ exports.enableNotificationsProperty = {
     description: "Emit Signal K notifications for alerts and failover events.",
     default: false
 };
+// ── Client-only: skip forwarding plugin-generated data ────────────────────────
+exports.skipOwnDataProperty = {
+    type: "boolean",
+    title: "Skip Plugin's Own Data",
+    description: "Do not forward data this plugin publishes locally over the link. Strips entries under 'networking.edgeLink.*' and the v1 RTT path 'networking.modem.rtt' / 'networking.modem.<id>.rtt'; other 'networking.modem.*' paths from external providers are left intact. Also suppresses the v2/v3 client telemetry packet that mirrors local link metrics to the receiver.",
+    default: false
+};
 // ── v2/v3 monitoring alert thresholds (client) ────────────────────────────────
 exports.alertThresholdsProperty = {
     type: "object",
@@ -501,6 +508,7 @@ function buildConnectionItemSchema() {
                             congestionControl: exports.congestionControlProperty,
                             bonding: exports.bondingProperty,
                             enableNotifications: exports.enableNotificationsProperty,
+                            skipOwnData: exports.skipOwnDataProperty,
                             alertThresholds: exports.alertThresholdsProperty
                         },
                         required: ["udpAddress", "testAddress", "testPort"]
@@ -524,6 +532,7 @@ function buildWebappConnectionSchema(isClient, protocolVersion) {
     if (isClient) {
         Object.assign(props, exports.clientTransportProperties);
         props.enableNotifications = exports.enableNotificationsProperty;
+        props.skipOwnData = exports.skipOwnDataProperty;
         required.push("udpAddress", "testAddress", "testPort");
         if (isReliableProtocol) {
             props.reliability = exports.clientReliabilityProperty;

package/lib/source-dispatch.js

@@ -0,0 +1,98 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalizeDeltaSourceRefs = normalizeDeltaSourceRefs;
+exports.handleMessageBySource = handleMessageBySource;
+function isRecord(value) {
+    return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+function getSourceLabel(update) {
+    const source = isRecord(update.source) ? update.source : null;
+    const label = source && typeof source.label === "string" ? source.label.trim() : "";
+    return label.length > 0 ? label : "";
+}
+function hasStaleEdgeLinkSourceRef(update) {
+    const sourceLabel = getSourceLabel(update);
+    const sourceRef = typeof update.$source === "string" ? update.$source.trim() : "";
+    if (!sourceLabel || !sourceRef || sourceLabel === "signalk-edge-link") {
+        return false;
+    }
+    return (sourceRef === "signalk-edge-link" ||
+        sourceRef.startsWith("signalk-edge-link.") ||
+        sourceRef.startsWith("signalk-edge-link:"));
+}
+function normalizeUpdateSourceRef(update) {
+    if (!hasStaleEdgeLinkSourceRef(update)) {
+        return update;
+    }
+    const cloned = { ...update };
+    delete cloned.$source;
+    return cloned;
+}
+function cloneUpdate(update) {
+    const normalized = normalizeUpdateSourceRef(update);
+    const cloned = {
+        ...normalized,
+        source: isRecord(normalized.source)
+            ? { ...normalized.source }
+            : normalized.source,
+        values: Array.isArray(normalized.values)
+            ? normalized.values.map((value) => ({ ...value }))
+            : normalized.values,
+        meta: Array.isArray(normalized.meta)
+            ? normalized.meta.map((entry) => ({ ...entry }))
+            : normalized.meta
+    };
+    return cloned;
+}
+function normalizeDeltaSourceRefs(delta) {
+    if (!delta || !Array.isArray(delta.updates)) {
+        return delta;
+    }
+    let changed = false;
+    const updates = delta.updates.map((update) => {
+        const normalized = normalizeUpdateSourceRef(update);
+        if (normalized !== update) {
+            changed = true;
+        }
+        return normalized;
+    });
+    return changed ? { ...delta, updates } : delta;
+}
+/**
+ * Signal K's app.handleMessage(providerId, delta) rewrites update.source.label
+ * to providerId before applying the delta. Remote updates can contain several
+ * original source labels, so dispatch them under their original label. Stale
+ * edge-link `$source` values are removed separately before dispatch so Signal K
+ * can recompute them from the structured source object.
+ */
+function handleMessageBySource(app, delta) {
+    if (!delta || !Array.isArray(delta.updates) || delta.updates.length === 0) {
+        return;
+    }
+    const grouped = new Map();
+    let hasOriginalSourceLabel = false;
+    for (const update of delta.updates) {
+        const sourceLabel = getSourceLabel(update);
+        if (sourceLabel) {
+            hasOriginalSourceLabel = true;
+        }
+        const providerId = sourceLabel || "";
+        const updates = grouped.get(providerId);
+        if (updates) {
+            updates.push(update);
+        }
+        else {
+            grouped.set(providerId, [update]);
+        }
+    }
+    if (!hasOriginalSourceLabel) {
+        app.handleMessage("", delta);
+        return;
+    }
+    for (const [providerId, updates] of grouped) {
+        app.handleMessage(providerId, {
+            ...delta,
+            updates: updates.map(cloneUpdate)
+        });
+    }
+}