triflux 10.43.0 → 10.45.0

package/hub/schema.sql

@@ -10,8 +10,13 @@ CREATE TABLE IF NOT EXISTS agents (
   topics_json TEXT NOT NULL DEFAULT '[]',
   last_seen_ms INTEGER NOT NULL,
   lease_expires_ms INTEGER NOT NULL,
+  presence_generation INTEGER NOT NULL DEFAULT 0,
   status TEXT NOT NULL CHECK (status IN ('online','stale','offline')),
-  metadata_json TEXT NOT NULL DEFAULT '{}'
+  metadata_json TEXT NOT NULL DEFAULT '{}',
+  project_id TEXT,
+  session_id TEXT,
+  host_id TEXT,
+  transport_locators_json TEXT NOT NULL DEFAULT '[]'
 );
 
 -- 메시지 테이블
@@ -28,7 +33,61 @@ CREATE TABLE IF NOT EXISTS messages (
   correlation_id TEXT NOT NULL,
   trace_id TEXT NOT NULL,
   payload_json TEXT NOT NULL DEFAULT '{}',
-  status TEXT NOT NULL CHECK (status IN ('queued','delivered','acked','expired','dead_letter'))
+  status TEXT NOT NULL CHECK (status IN ('queued','delivered','acked','expired','dead_letter')),
+  role_key_wire TEXT
+);
+
+-- 동적 역할 레지스트리 (schema v6)
+CREATE TABLE IF NOT EXISTS role_registry (
+  role_key_wire TEXT PRIMARY KEY,
+  project_id TEXT NOT NULL,
+  role_kind TEXT NOT NULL CHECK (role_kind IN ('cto','lead')),
+  scope_id TEXT NOT NULL DEFAULT '',
+  state TEXT NOT NULL CHECK (
+    state IN (
+      'electable',
+      'elected-probing',
+      'active',
+      'unreachable',
+      'quarantined'
+    )
+  ),
+  holder_agent_id TEXT,
+  previous_holder_agent_id TEXT,
+  epoch INTEGER NOT NULL DEFAULT 0 CHECK (epoch >= 0),
+  activation_seq INTEGER NOT NULL DEFAULT 0 CHECK (activation_seq >= 0),
+  activation_id TEXT,
+  activation_deadline_ms INTEGER,
+  holder_lease_expires_ms INTEGER,
+  charter_version TEXT,
+  charter_acked_epoch INTEGER,
+  retry_count INTEGER NOT NULL DEFAULT 0,
+  next_probe_ms INTEGER,
+  blocked_reason TEXT,
+  last_transition_ms INTEGER NOT NULL,
+  last_reason TEXT NOT NULL,
+  version INTEGER NOT NULL DEFAULT 0,
+  CHECK (
+    (role_kind = 'cto' AND scope_id = '') OR
+    (role_kind = 'lead' AND scope_id <> '')
+  ),
+  UNIQUE(project_id, role_kind, scope_id)
+);
+
+CREATE TABLE IF NOT EXISTS role_candidates (
+  role_key_wire TEXT NOT NULL,
+  agent_id TEXT NOT NULL,
+  source TEXT NOT NULL CHECK (source IN ('grant','standup','operator')),
+  priority INTEGER NOT NULL DEFAULT 0,
+  transport_locators_json TEXT NOT NULL DEFAULT '[]',
+  consecutive_probe_failures INTEGER NOT NULL DEFAULT 0,
+  excluded_until_ms INTEGER,
+  last_probe_ms INTEGER,
+  last_probe_code TEXT,
+  updated_at_ms INTEGER NOT NULL,
+  PRIMARY KEY(role_key_wire, agent_id),
+  FOREIGN KEY(role_key_wire)
+    REFERENCES role_registry(role_key_wire) ON DELETE CASCADE
 );
 
 -- 메시지 수신함 (배달 추적)
@@ -73,6 +132,7 @@ CREATE INDEX IF NOT EXISTS idx_messages_correlation ON messages(correlation_id);
 CREATE INDEX IF NOT EXISTS idx_messages_trace ON messages(trace_id);
 CREATE INDEX IF NOT EXISTS idx_messages_expires ON messages(expires_at_ms);
 CREATE INDEX IF NOT EXISTS idx_messages_priority ON messages(priority DESC, created_at_ms ASC);
+CREATE INDEX IF NOT EXISTS idx_messages_role_key ON messages(role_key_wire, status, expires_at_ms);
 CREATE INDEX IF NOT EXISTS idx_inbox_agent ON message_inbox(agent_id, delivered_at_ms);
 CREATE INDEX IF NOT EXISTS idx_inbox_message ON message_inbox(message_id);
 CREATE INDEX IF NOT EXISTS idx_human_requests_state ON human_requests(state);

package/hub/server.mjs

@@ -961,6 +961,7 @@ function servePublicFile(res, path) {
  * @param {string} [opts.host]
  * @param {string|number} [opts.sessionId]
  * @param {(options: { cwd: string }) => object} [opts.createDelegatorWorker]
+ * @param {(dbPath: string) => object|Promise<object>} [opts.createStoreAdapterFn] test seam
  */
 export async function startHub({
   port: portOpt,
@@ -968,6 +969,7 @@ export async function startHub({
   host = "127.0.0.1",
   sessionId = process.pid,
   createDelegatorWorker = createDelegatorMcpWorker,
+  createStoreAdapterFn = createStoreAdapter,
 } = {}) {
   const resolvedPort = parseHubPort(portOpt);
   const portSpecified = Number.isFinite(resolvedPort);
@@ -1062,7 +1064,7 @@ export async function startHub({
     cleanupOwnedTokenFile({ token: HUB_TOKEN, port });
   }
 
-  const store = await createStoreAdapter(dbPath);
+  const store = await createStoreAdapterFn(dbPath);
   const router = createRouter(store);
   const fingerprintService = createAdaptiveFingerprintService({ store });
 
@@ -1558,6 +1560,38 @@ export async function startHub({
         try {
           const body = await parseBody(req);
           const { sessionId } = body || {};
+          if (body?.agent_id && body?.cli) {
+            const registered = router.registerAgent({
+              agent_id: body.agent_id,
+              cli: body.cli,
+              capabilities: Array.isArray(body.capabilities)
+                ? body.capabilities
+                : [],
+              topics: Array.isArray(body.topics) ? body.topics : [],
+              heartbeat_ttl_ms: body.heartbeat_ttl_ms ?? 300000,
+              metadata:
+                body.metadata && typeof body.metadata === "object"
+                  ? body.metadata
+                  : { source: "session_start" },
+              ...(Object.hasOwn(body, "project_id")
+                ? { project_id: body.project_id }
+                : {}),
+              ...(Object.hasOwn(body, "session_id")
+                ? { session_id: body.session_id }
+                : {}),
+              ...(Object.hasOwn(body, "host_id")
+                ? { host_id: body.host_id }
+                : {}),
+              ...(Object.hasOwn(body, "transport_locators_json")
+                ? { transport_locators_json: body.transport_locators_json }
+                : {}),
+            });
+            if (!registered?.ok) {
+              throw new Error(
+                registered?.error?.message || "hub agent register failed",
+              );
+            }
+          }
           const result = synapseRegistry.register(sessionId, body);
           if (!result?.ok) {
             throw new Error(result?.reason || "register failed");
@@ -1688,6 +1722,8 @@ export async function startHub({
               agent_id,
               reason = "manual",
               requested_by = "http",
+              hub_instance_id,
+              store_fingerprint,
             } = body;
             if (!role || !agent_id) {
               return writeJson(res, 400, {
@@ -1700,6 +1736,8 @@ export async function startHub({
               agent_id,
               reason,
               requested_by,
+              hub_instance_id,
+              store_fingerprint,
             });
             return writeJson(res, result.ok ? 200 : 409, result);
           }
@@ -1712,6 +1750,12 @@ export async function startHub({
               topics = [],
               capabilities = [],
               metadata = {},
+              project_id,
+              session_id,
+              host_id,
+              transport_locators_json,
+              hub_instance_id,
+              store_fingerprint,
             } = body;
             if (!agent_id || !cli) {
               return writeJson(res, 400, {
@@ -1740,6 +1784,14 @@ export async function startHub({
               topics,
               heartbeat_ttl_ms,
               metadata,
+              ...(project_id === undefined ? {} : { project_id }),
+              ...(session_id === undefined ? {} : { session_id }),
+              ...(host_id === undefined ? {} : { host_id }),
+              ...(transport_locators_json === undefined
+                ? {}
+                : { transport_locators_json }),
+              ...(hub_instance_id === undefined ? {} : { hub_instance_id }),
+              ...(store_fingerprint === undefined ? {} : { store_fingerprint }),
             });
             if (result.ok) {
               workerSpans.set(agent_id, {
@@ -1774,7 +1826,13 @@ export async function startHub({
           }
 
           if (path === "/bridge/heartbeat" && req.method === "POST") {
-            const { agent_id, ttl_ms } = body;
+            const {
+              agent_id,
+              ttl_ms,
+              presence_generation,
+              hub_instance_id,
+              store_fingerprint,
+            } = body;
             if (!agent_id)
               return writeJson(res, 400, { ok: false, error: "agent_id 필수" });
             const result = await pipe.executeCommand("heartbeat", {
@@ -1783,6 +1841,9 @@ export async function startHub({
                 ttl_ms,
                 resolveWorkerLeaseTtlMs(),
               ),
+              presence_generation,
+              hub_instance_id,
+              store_fingerprint,
             });
             return writeJson(res, result.ok ? 200 : 404, result);
           }
@@ -1863,6 +1924,9 @@ export async function startHub({
               status,
               include_metrics = true,
               trace_id,
+              presence_generation,
+              hub_instance_id,
+              store_fingerprint,
             } = body;
 
             if (agent_id && status) {
@@ -1883,7 +1947,15 @@ export async function startHub({
               ]).has(normalizedStatus)
                 ? normalizedStatus
                 : "online";
-              router.updateAgentStatus(normalizedAgentId, statusForStore);
+              const transition = router.transitionAgentStatus(
+                normalizedAgentId,
+                statusForStore,
+                presence_generation,
+                { hub_instance_id, store_fingerprint },
+              );
+              if (!transition.ok) {
+                return writeJson(res, 409, transition);
+              }
               const snapshot = await pipe.executeQuery("status", {
                 scope: "agent",
                 agent_id: normalizedAgentId,
@@ -1897,6 +1969,8 @@ export async function startHub({
                   reported_status: normalizedStatus,
                   reported_at_ms: Date.now(),
                   snapshot: snapshot?.data?.agent || null,
+                  hub_instance_id: transition.data.hub_instance_id,
+                  store_fingerprint: transition.data.store_fingerprint,
                 },
               });
             }
@@ -1906,6 +1980,8 @@ export async function startHub({
               agent_id,
               include_metrics,
               trace_id,
+              hub_instance_id,
+              store_fingerprint,
             });
             return writeJson(res, 200, result);
           }
@@ -2125,12 +2201,20 @@ export async function startHub({
           }
 
           if (path === "/bridge/deregister" && req.method === "POST") {
-            const { agent_id } = body;
+            const {
+              agent_id,
+              presence_generation,
+              hub_instance_id,
+              store_fingerprint,
+            } = body;
             if (!agent_id) {
               return writeJson(res, 400, { ok: false, error: "agent_id 필수" });
             }
             const result = await pipe.executeCommand("deregister", {
               agent_id,
+              presence_generation,
+              hub_instance_id,
+              store_fingerprint,
             });
             if (result.ok) {
               finishWorkerTrace(agent_id, "deregister");
@@ -2274,102 +2358,107 @@ export async function startHub({
   httpServer.requestTimeout = 30000;
   httpServer.headersTimeout = 10000;
 
-  router.startSweeper();
+  let hitlTimer = null;
+  let sessionTimer = null;
+  let synapsePruneTimer = null;
+  let orphanCleanupTimer = null;
+  let rateLimitTimer = null;
 
-  const hitlTimer = setInterval(() => {
-    try {
-      hitl.checkTimeouts();
-    } catch (err) {
-      hubLog.warn({ err }, "hitl.timeout_check_failed");
-    }
-  }, 10000);
-  hitlTimer.unref();
-
-  // MCP session TTL: sessions idle for SESSION_TTL_MS are closed automatically.
-  // Configurable via SESSION_TTL_MS (default 30 minutes). The sweep runs every 60 s.
-  const SESSION_TTL_MS =
-    parseInt(process.env.TFX_SESSION_TTL_MS || "", 10) || 30 * 60 * 1000;
-  const sessionTimer = setInterval(() => {
-    const now = Date.now();
-    for (const [sid, session] of transports) {
-      if (now - (session.transport._lastActivity || 0) <= SESSION_TTL_MS)
-        continue;
-      void closeMcpTransportSession(sid, session, "session_ttl");
-    }
-  }, 60000);
-  sessionTimer.unref();
+  const startBackgroundTimers = () => {
+    hitlTimer = setInterval(() => {
+      try {
+        hitl.checkTimeouts();
+      } catch (err) {
+        hubLog.warn({ err }, "hitl.timeout_check_failed");
+      }
+    }, 10000);
+    hitlTimer.unref();
+
+    // MCP session TTL: sessions idle for SESSION_TTL_MS are closed automatically.
+    // Configurable via SESSION_TTL_MS (default 30 minutes). The sweep runs every 60 s.
+    const SESSION_TTL_MS =
+      parseInt(process.env.TFX_SESSION_TTL_MS || "", 10) || 30 * 60 * 1000;
+    sessionTimer = setInterval(() => {
+      const now = Date.now();
+      for (const [sid, session] of transports) {
+        if (now - (session.transport._lastActivity || 0) <= SESSION_TTL_MS)
+          continue;
+        void closeMcpTransportSession(sid, session, "session_ttl");
+      }
+    }, 60000);
+    sessionTimer.unref();
 
-  const synapsePruneTimer = setInterval(() => {
-    try {
-      const { count } = synapseRegistry.pruneExpired();
-      if (count > 0) hubLog.info({ count }, "synapse.prune");
-    } catch (err) {
-      hubLog.warn({ err }, "synapse.prune_failed");
-    }
-  }, 60000);
-  synapsePruneTimer.unref();
-
-  // 고아 node.exe 프로세스 + stale spawn 세션 주기적 정리 (5분마다)
-  // TFX_DISABLE_ORPHAN_CLEANUP=1 로 비활성 (active SSH/swarm 세션 보호 회피용 hotfix gate)
-  const orphanCleanupTimer = setInterval(
-    () => {
-      if (process.env.TFX_DISABLE_ORPHAN_CLEANUP === "1") return;
+    synapsePruneTimer = setInterval(() => {
       try {
-        const { killed, killedProcesses = [] } = cleanupOrphanNodeProcesses();
-        const {
-          killed: runtimeKilled,
-          killedProcesses: runtimeKilledProcesses = [],
-        } = cleanupOrphanRuntimeProcesses();
-        const totalKilled = killed + runtimeKilled;
-        if (totalKilled > 0) {
-          hubLog.info(
-            {
-              killed: totalKilled,
-              processes: [...killedProcesses, ...runtimeKilledProcesses],
-              caller: "timer",
-            },
-            "hub.orphan_cleanup",
-          );
-        }
+        const { count } = synapseRegistry.pruneExpired();
+        if (count > 0) hubLog.info({ count }, "synapse.prune");
+      } catch (err) {
+        hubLog.warn({ err }, "synapse.prune_failed");
+      }
+    }, 60000);
+    synapsePruneTimer.unref();
+
+    // 고아 node.exe 프로세스 + stale spawn 세션 주기적 정리 (5분마다)
+    // TFX_DISABLE_ORPHAN_CLEANUP=1 로 비활성 (active SSH/swarm 세션 보호 회피용 hotfix gate)
+    orphanCleanupTimer = setInterval(
+      () => {
+        if (process.env.TFX_DISABLE_ORPHAN_CLEANUP === "1") return;
+        try {
+          const { killed, killedProcesses = [] } = cleanupOrphanNodeProcesses();
+          const {
+            killed: runtimeKilled,
+            killedProcesses: runtimeKilledProcesses = [],
+          } = cleanupOrphanRuntimeProcesses();
+          const totalKilled = killed + runtimeKilled;
+          if (totalKilled > 0) {
+            hubLog.info(
+              {
+                killed: totalKilled,
+                processes: [...killedProcesses, ...runtimeKilledProcesses],
+                caller: "timer",
+              },
+              "hub.orphan_cleanup",
+            );
+          }
 
-        const { killed: fsmonitorKilled, stale } = cleanupStaleFsmonitorDaemons(
-          {
-            minAgeMs: 24 * 60 * 60 * 1000,
-          },
-        );
-        if (fsmonitorKilled > 0) {
-          hubLog.info(
-            { killed: fsmonitorKilled, stale: stale.length },
-            "hub.fsmonitor_cleanup",
-          );
-        }
-      } catch {}
+          const { killed: fsmonitorKilled, stale } =
+            cleanupStaleFsmonitorDaemons({
+              minAgeMs: 24 * 60 * 60 * 1000,
+            });
+          if (fsmonitorKilled > 0) {
+            hubLog.info(
+              { killed: fsmonitorKilled, stale: stale.length },
+              "hub.fsmonitor_cleanup",
+            );
+          }
+        } catch {}
 
-      // stale tfx-spawn-* psmux 세션 정리 (30분 이상 idle)
-      try {
-        const staleKilled = cleanupStaleSpawnSessions(hubLog);
-        if (staleKilled > 0) {
-          hubLog.info({ killed: staleKilled }, "hub.stale_spawn_cleanup");
+        // stale tfx-spawn-* psmux 세션 정리 (30분 이상 idle)
+        try {
+          const staleKilled = cleanupStaleSpawnSessions(hubLog);
+          if (staleKilled > 0) {
+            hubLog.info({ killed: staleKilled }, "hub.stale_spawn_cleanup");
+          }
+        } catch {}
+      },
+      5 * 60 * 1000,
+    );
+    orphanCleanupTimer.unref();
+
+    // Evict stale rate-limit buckets once per minute to bound memory usage.
+    rateLimitTimer = setInterval(() => {
+      const cutoff = Date.now() - RATE_LIMIT_WINDOW_MS;
+      for (const [ip, timestamps] of rateLimitMap) {
+        const fresh = timestamps.filter((t) => t >= cutoff);
+        if (fresh.length === 0) {
+          rateLimitMap.delete(ip);
+        } else {
+          rateLimitMap.set(ip, fresh);
         }
-      } catch {}
-    },
-    5 * 60 * 1000,
-  );
-  orphanCleanupTimer.unref();
-
-  // Evict stale rate-limit buckets once per minute to bound memory usage.
-  const rateLimitTimer = setInterval(() => {
-    const cutoff = Date.now() - RATE_LIMIT_WINDOW_MS;
-    for (const [ip, timestamps] of rateLimitMap) {
-      const fresh = timestamps.filter((t) => t >= cutoff);
-      if (fresh.length === 0) {
-        rateLimitMap.delete(ip);
-      } else {
-        rateLimitMap.set(ip, fresh);
       }
-    }
-  }, RATE_LIMIT_WINDOW_MS);
-  rateLimitTimer.unref();
+    }, RATE_LIMIT_WINDOW_MS);
+    rateLimitTimer.unref();
+  };
 
   mkdirSync(PID_DIR, { recursive: true });
 
@@ -2377,6 +2466,19 @@ export async function startHub({
     try {
       router.stopSweeper();
     } catch {}
+    for (const timer of [
+      hitlTimer,
+      sessionTimer,
+      synapsePruneTimer,
+      orphanCleanupTimer,
+      rateLimitTimer,
+    ]) {
+      if (timer) clearInterval(timer);
+    }
+    for (const [sid, session] of Array.from(transports)) {
+      await closeMcpTransportSession(sid, session, "hub.startup_failure");
+    }
+    transports.clear();
     try {
       await pipe.stop();
     } catch {}
@@ -2386,6 +2488,9 @@ export async function startHub({
     try {
       await delegatorWorker.stop();
     } catch {}
+    try {
+      synapseRegistry.destroy();
+    } catch {}
     try {
       store.close();
     } catch {}
@@ -2396,6 +2501,22 @@ export async function startHub({
   };
 
   try {
+    router.sweepExpired({ now_ms: Date.now(), source: "startup" });
+    router.startSweeper({
+      run_immediately: false,
+      interval_ms: 10000,
+      onSweepError: ({ source, error, consecutive_failures }) => {
+        hubLog.warn(
+          {
+            source,
+            code: error?.code || "UNKNOWN",
+            consecutive_failures,
+          },
+          "hub.sweep_failed",
+        );
+      },
+    });
+    startBackgroundTimers();
     await pipe.start();
     await assignCallbacks.start();
   } catch (error) {