triflux 10.44.0 → 10.45.0

package/hub/router.mjs

@@ -376,6 +376,9 @@ export function createRouter(store) {
   const queuesByAgent = new Map();
   const liveMessages = new Map();
   const roleStates = new Map();
+  const hubInstanceId = uuidv7();
+  const storeFingerprint =
+    store.fingerprint || `${store.type || "unknown"}:unknown`;
   const updateRegistrationIdentity = store.db?.prepare
     ? store.db.prepare(`
         UPDATE agents SET
@@ -392,6 +395,70 @@ export function createRouter(store) {
   let latencyIdx = 0;
   const deliveryLatencies = new Array(MAX_LATENCY_SAMPLES).fill(0);
 
+  function hubIdentity() {
+    return {
+      hub_instance_id: hubInstanceId,
+      store_fingerprint: storeFingerprint,
+    };
+  }
+
+  function validateCallerIdentity(input = {}) {
+    if (
+      input.hub_instance_id != null &&
+      input.hub_instance_id !== hubInstanceId
+    ) {
+      return {
+        ok: false,
+        error: {
+          code: "STALE_HUB_INSTANCE",
+          message: "caller is bound to a previous hub instance",
+        },
+        data: hubIdentity(),
+      };
+    }
+    if (
+      input.store_fingerprint != null &&
+      input.store_fingerprint !== storeFingerprint
+    ) {
+      return {
+        ok: false,
+        error: {
+          code: "STORE_FINGERPRINT_MISMATCH",
+          message: "caller store fingerprint does not match this hub",
+        },
+        data: hubIdentity(),
+      };
+    }
+    return null;
+  }
+
+  function isControlPlaneRegistration(args = {}) {
+    const metadata = args.metadata || {};
+    const values = [
+      ...(Array.isArray(args.capabilities) ? args.capabilities : []),
+      ...(Array.isArray(args.topics) ? args.topics : []),
+      metadata.role,
+      ...(Array.isArray(metadata.roles) ? metadata.roles : []),
+    ]
+      .flat()
+      .map((value) => String(value || "").toLowerCase());
+    return (
+      metadata.control_plane === true ||
+      values.some((value) => value === "cto" || value === "role:cto")
+    );
+  }
+
+  function staleOwnerFailure(agentId, result) {
+    return {
+      ok: false,
+      error: {
+        code: "STALE_PRESENCE_OWNER",
+        message: `stale presence owner: ${agentId}`,
+      },
+      data: { ...result, ...hubIdentity() },
+    };
+  }
+
   function ensureAgentQueue(agentId) {
     let queue = queuesByAgent.get(agentId);
     if (!queue) {
@@ -1120,6 +1187,18 @@ export function createRouter(store) {
     deliveryEmitter,
 
     registerAgent(args) {
+      const identityFailure = validateCallerIdentity(args);
+      if (identityFailure) return identityFailure;
+      if (isControlPlaneRegistration(args) && store.type !== "sqlite") {
+        return {
+          ok: false,
+          error: {
+            code: "CONTROL_PLANE_STORE_DEGRADED",
+            message: "control-plane presence requires durable SQLite storage",
+          },
+          data: { ...hubIdentity(), store_type: store.type || "unknown" },
+        };
+      }
       const identity = normalizeRegistrationIdentity(args);
       const result = store.registerAgent(args);
       if (!registerPresenceIsEffective(result)) {
@@ -1139,11 +1218,23 @@ export function createRouter(store) {
           transferBacklog: true,
         });
       }
-      return { ok: true, data: { ...result, hub_pid: process.pid } };
+      return {
+        ok: true,
+        data: { ...result, hub_pid: process.pid, ...hubIdentity() },
+      };
     },
 
-    refreshAgentLease(agentId, ttlMs = 30000) {
-      const result = store.refreshLease(agentId, ttlMs);
+    refreshAgentLease(
+      agentId,
+      ttlMs = 30000,
+      presenceGeneration,
+      callerIdentity = {},
+    ) {
+      const identityFailure = validateCallerIdentity(callerIdentity);
+      if (identityFailure) return identityFailure;
+      const result = store.refreshLease(agentId, ttlMs, presenceGeneration);
+      if (result?.effective?.updated === false)
+        return staleOwnerFailure(agentId, result);
       refreshRoleCandidateForAgent(agentId);
       for (const roleName of ROLE_TOPICS) {
         ensureRoleLeader(roleName, {
@@ -1151,7 +1242,7 @@ export function createRouter(store) {
           transferBacklog: true,
         });
       }
-      return result;
+      return { ok: true, data: result };
     },
 
     subscribeAgent(agentId, topics, { replace = false } = {}) {
@@ -1170,11 +1261,15 @@ export function createRouter(store) {
       return listRuntimeTopics(agentId);
     },
 
-    updateAgentStatus(agentId, status) {
+    updateAgentStatus(agentId, status, presenceGeneration) {
       if (status === "offline") {
         runtimeTopics.delete(agentId);
       }
-      const updated = store.updateAgentStatus(agentId, status);
+      const updated = store.updateAgentStatus(
+        agentId,
+        status,
+        presenceGeneration,
+      );
       refreshRoleCandidateForAgent(agentId);
       for (const roleName of ROLE_TOPICS) {
         ensureRoleLeader(roleName, {
@@ -1185,6 +1280,39 @@ export function createRouter(store) {
       return updated;
     },
 
+    transitionAgentStatus(
+      agentId,
+      status,
+      presenceGeneration,
+      callerIdentity = {},
+    ) {
+      const identityFailure = validateCallerIdentity(callerIdentity);
+      if (identityFailure) return identityFailure;
+      const updated = store.updateAgentStatus(
+        agentId,
+        status,
+        presenceGeneration,
+      );
+      if (!updated) return staleOwnerFailure(agentId, { status });
+      if (status === "offline") runtimeTopics.delete(agentId);
+      refreshRoleCandidateForAgent(agentId);
+      for (const roleName of ROLE_TOPICS) {
+        ensureRoleLeader(roleName, {
+          reason: `status:${status}`,
+          transferBacklog: true,
+        });
+      }
+      return {
+        ok: true,
+        data: {
+          agent_id: agentId,
+          status,
+          presence_generation: presenceGeneration,
+          ...hubIdentity(),
+        },
+      };
+    },
+
     route(msg) {
       const recipients = uniqueStrings(resolveRecipients(msg));
       if (!recipients.length) {
@@ -1369,7 +1497,14 @@ export function createRouter(store) {
       agent_id,
       reason = "manual",
       requested_by = "manual",
+      hub_instance_id,
+      store_fingerprint,
     } = {}) {
+      const identityFailure = validateCallerIdentity({
+        hub_instance_id,
+        store_fingerprint,
+      });
+      if (identityFailure) return identityFailure;
       const roleName = normalizeRoleName(role);
       if (!roleName) {
         return {
@@ -1440,6 +1575,7 @@ export function createRouter(store) {
           leader_epoch: snapshot.leader_epoch,
           ...transfer,
           status: snapshot,
+          ...hubIdentity(),
         },
       };
     },
@@ -1647,18 +1783,15 @@ export function createRouter(store) {
       return scheduleAssignRetry(job, reason, job.error, requested_by);
     },
 
-    sweepExpired() {
-      const now = Date.now();
-      let expired = 0;
-      const deadLettered = [];
+    sweepExpired({ now_ms = Date.now(), source = "manual" } = {}) {
+      const result = store.sweepExpired({ now_ms });
       for (const [messageId, record] of Array.from(liveMessages.entries())) {
-        if (record.message.expires_at_ms > now) continue;
-        store.moveToDeadLetter(messageId, "ttl_expired", null);
-        deadLettered.push(record.message);
+        if (record.message.expires_at_ms > now_ms) continue;
         removeMessage(messageId);
-        expired += 1;
       }
-      for (const message of deadLettered) {
+      for (const transition of result.transitions) {
+        if (!transition.dlq_inserted) continue;
+        const { message } = transition;
         if (
           !["handoff", "request"].includes(message.type) ||
           message.payload?.kind === "assign.job"
@@ -1683,12 +1816,15 @@ export function createRouter(store) {
             original_to: message.to_agent,
             reason: "ttl_expired",
             task: message.payload?.task ?? null,
+            expired_at_ms: message.expires_at_ms,
+            failed_at_ms: transition.failed_at_ms,
+            recovery_source: source,
           },
           trace_id: message.trace_id,
           correlation_id: message.correlation_id,
         });
       }
-      return { messages: expired };
+      return result;
     },
 
     sweepTimedOutAssigns() {
@@ -1709,19 +1845,49 @@ export function createRouter(store) {
       return { timed_out, retried };
     },
 
-    startSweeper() {
+    startSweeper({
+      run_immediately = false,
+      interval_ms = 10000,
+      onSweepError,
+    } = {}) {
       if (sweepTimer) return;
-      sweepTimer = setInterval(() => {
+      const reportSweepError = (source, error, failures) => {
+        if (typeof onSweepError === "function") {
+          onSweepError({ source, error, consecutive_failures: failures });
+        }
+      };
+      let sweepFailures = 0;
+      let assignFailures = 0;
+      const runSweep = () => {
+        try {
+          router.sweepExpired({ source: "periodic" });
+          sweepFailures = 0;
+        } catch (error) {
+          sweepFailures += 1;
+          reportSweepError("periodic", error, sweepFailures);
+        }
         try {
-          router.sweepExpired();
           router.sweepTimedOutAssigns();
-        } catch {}
-      }, 10000);
+          assignFailures = 0;
+        } catch (error) {
+          assignFailures += 1;
+          reportSweepError("assign_timeout", error, assignFailures);
+        }
+      };
+      if (run_immediately) runSweep();
+      sweepTimer = setInterval(() => {
+        runSweep();
+      }, interval_ms);
+      let staleFailures = 0;
       staleTimer = setInterval(() => {
         try {
           store.sweepStaleAgents();
           router.reelectStaleRoles();
-        } catch {}
+          staleFailures = 0;
+        } catch (error) {
+          staleFailures += 1;
+          reportSweepError("stale_agents", error, staleFailures);
+        }
       }, 120000);
       sweepTimer.unref();
       staleTimer.unref();
@@ -1777,8 +1943,19 @@ export function createRouter(store) {
 
     getStatus(
       scope = "hub",
-      { agent_id, trace_id, include_metrics = true } = {},
+      {
+        agent_id,
+        trace_id,
+        include_metrics = true,
+        hub_instance_id,
+        store_fingerprint,
+      } = {},
     ) {
+      const identityFailure = validateCallerIdentity({
+        hub_instance_id,
+        store_fingerprint,
+      });
+      if (identityFailure) return identityFailure;
       const data = {};
 
       if (scope === "hub" || scope === "queue") {
@@ -1787,6 +1964,7 @@ export function createRouter(store) {
           uptime_ms: (process.uptime() * 1000) | 0,
           realtime_transport: "named-pipe",
           audit_store: store.type || "sqlite",
+          ...hubIdentity(),
         };
         data.roles = {
           cto: getRoleSnapshot("cto"),
@@ -1840,7 +2018,7 @@ export function createRouter(store) {
         data.trace = store.getMessagesByTrace(trace_id);
       }
 
-      return { ok: true, data };
+      return { ok: true, data: { ...data, ...hubIdentity() } };
     },
   };
 

package/hub/schema.sql

@@ -10,6 +10,7 @@ CREATE TABLE IF NOT EXISTS agents (
   topics_json TEXT NOT NULL DEFAULT '[]',
   last_seen_ms INTEGER NOT NULL,
   lease_expires_ms INTEGER NOT NULL,
+  presence_generation INTEGER NOT NULL DEFAULT 0,
   status TEXT NOT NULL CHECK (status IN ('online','stale','offline')),
   metadata_json TEXT NOT NULL DEFAULT '{}',
   project_id TEXT,