target-supabase-sdk 0.2.13 → 0.2.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/dist/browser.js +1 -1
  2. package/dist/chunks/{enable-DqnfLPsQ.js → enable-C7lFooTG.js} +2 -2
  3. package/dist/chunks/{enable-DqnfLPsQ.js.map → enable-C7lFooTG.js.map} +1 -1
  4. package/dist/chunks/{task-queue-6X3aglJh.js → task-queue-qSnsfiIg.js} +5 -3
  5. package/dist/chunks/task-queue-qSnsfiIg.js.map +1 -0
  6. package/dist/node/env/index.d.ts +1 -1
  7. package/dist/node/env/index.d.ts.map +1 -1
  8. package/dist/node/node-runtime.base.d.ts +14 -1
  9. package/dist/node/node-runtime.base.d.ts.map +1 -1
  10. package/dist/node/node.api.d.ts.map +1 -1
  11. package/dist/node/node.constant.d.ts +3 -0
  12. package/dist/node/node.constant.d.ts.map +1 -0
  13. package/dist/node/node.utils.d.ts +0 -2
  14. package/dist/node/node.utils.d.ts.map +1 -1
  15. package/dist/node/readiness/index.d.ts +2 -0
  16. package/dist/node/readiness/index.d.ts.map +1 -1
  17. package/dist/node/{service-guard → readiness}/run-readiness-gate.d.ts +1 -1
  18. package/dist/node/readiness/run-readiness-gate.d.ts.map +1 -0
  19. package/dist/node/readiness/service-ready-gate.d.ts.map +1 -1
  20. package/dist/node/runtime-state/create-service-runtime-state-store.d.ts.map +1 -1
  21. package/dist/node/runtime-state/service-runtime-state.types.d.ts +16 -0
  22. package/dist/node/runtime-state/service-runtime-state.types.d.ts.map +1 -1
  23. package/dist/node/service-guard/index.d.ts +4 -4
  24. package/dist/node/service-guard/index.d.ts.map +1 -1
  25. package/dist/node/service-guard/register-service-guard-runner.d.ts +1 -2
  26. package/dist/node/service-guard/register-service-guard-runner.d.ts.map +1 -1
  27. package/dist/node/service-guard/registry-slot-guard-step.d.ts.map +1 -0
  28. package/dist/node/service-guard/run-service-guard-tick.d.ts.map +1 -1
  29. package/dist/node/service-guard/service-guard-node.d.ts +25 -2
  30. package/dist/node/service-guard/service-guard-node.d.ts.map +1 -1
  31. package/dist/node/service-guard/service-guard.interface.d.ts +25 -10
  32. package/dist/node/service-guard/service-guard.interface.d.ts.map +1 -1
  33. package/dist/node/service-host/claim-registry-slot.d.ts +15 -0
  34. package/dist/node/service-host/claim-registry-slot.d.ts.map +1 -0
  35. package/dist/node/service-host/create-l3-child-launcher.d.ts +38 -0
  36. package/dist/node/service-host/create-l3-child-launcher.d.ts.map +1 -0
  37. package/dist/node/service-host/index.d.ts +2 -5
  38. package/dist/node/service-host/index.d.ts.map +1 -1
  39. package/dist/node/service-host/service-host.d.ts +3 -3
  40. package/dist/node/service-host/service-host.d.ts.map +1 -1
  41. package/dist/node/service-host/single-process-service.d.ts +3 -3
  42. package/dist/node/service-host/single-process-service.d.ts.map +1 -1
  43. package/dist/node.d.ts +8 -6
  44. package/dist/node.d.ts.map +1 -1
  45. package/dist/node.js +627 -264
  46. package/dist/node.js.map +1 -1
  47. package/dist/preload.js +1 -1
  48. package/package.json +1 -1
  49. package/src/node/env/index.ts +0 -4
  50. package/src/node/node-runtime.base.ts +33 -5
  51. package/src/node/node.api.ts +1 -1
  52. package/src/node/node.constant.ts +2 -0
  53. package/src/node/node.utils.ts +2 -2
  54. package/src/node/readiness/index.ts +2 -0
  55. package/src/node/{service-guard → readiness}/run-readiness-gate.ts +2 -2
  56. package/src/node/readiness/service-ready-gate.ts +27 -23
  57. package/src/node/runtime-state/create-service-runtime-state-store.ts +11 -0
  58. package/src/node/runtime-state/service-runtime-state.types.ts +12 -0
  59. package/src/node/service-guard/index.ts +7 -8
  60. package/src/node/service-guard/register-service-guard-runner.ts +0 -2
  61. package/src/node/service-guard/run-service-guard-tick.ts +5 -2
  62. package/src/node/service-guard/service-guard-node.ts +229 -6
  63. package/src/node/service-guard/service-guard.interface.ts +35 -10
  64. package/src/node/service-host/claim-registry-slot.ts +33 -0
  65. package/src/node/service-host/create-l3-child-launcher.ts +157 -0
  66. package/src/node/service-host/index.ts +5 -24
  67. package/src/node/service-host/service-host.ts +11 -18
  68. package/src/node/service-host/single-process-service.ts +10 -20
  69. package/src/node.ts +21 -18
  70. package/dist/chunks/task-queue-6X3aglJh.js.map +0 -1
  71. package/dist/node/service-guard/run-readiness-gate.d.ts.map +0 -1
  72. package/dist/node/service-host/registry-slot-guard-step.d.ts.map +0 -1
  73. /package/dist/node/{service-host → service-guard}/registry-slot-guard-step.d.ts +0 -0
  74. /package/src/node/{service-host → service-guard}/registry-slot-guard-step.ts +0 -0
package/dist/preload.js CHANGED
@@ -1,6 +1,6 @@
1
1
  import { resolve } from 'node:path';
2
2
  import { w as logManager, H as resolveLogMinLevel } from './chunks/list.api-BzbTJBsM.js';
3
- import { r as resolveProjectRootByPackageName, p as pinEnvProfileFromArgv, l as loadEnvFiles, v as validateLogSpoolPreloadEnv, a as logSpoolEnabledFromEnv, m as mergeLogSpoolExtraProcessRolesEnv } from './chunks/enable-DqnfLPsQ.js';
3
+ import { r as resolveProjectRootByPackageName, p as pinEnvProfileFromArgv, l as loadEnvFiles, v as validateLogSpoolPreloadEnv, a as logSpoolEnabledFromEnv, m as mergeLogSpoolExtraProcessRolesEnv } from './chunks/enable-C7lFooTG.js';
4
4
  import 'zod';
5
5
  import 'lodash-es';
6
6
  import '@supabase/supabase-js';
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "target-supabase-sdk",
3
- "version": "0.2.13",
3
+ "version": "0.2.14",
4
4
  "description": "Supabase client initializer and data access APIs",
5
5
  "license": "MIT",
6
6
  "author": "KyLoc20",
@@ -7,10 +7,6 @@ export {
7
7
  CLI_PROD_FLAG,
8
8
  envProfileFromProcess,
9
9
  loadEnvFiles,
10
- parseEnvFile,
11
- parseEnvFileContent,
12
- parseEnvLine,
13
- pinEnvProfileFromArgv,
14
10
  resolveDefaultEnvFiles,
15
11
  SERVICE_ENV_PROFILE_ENV_KEY,
16
12
  SERVICE_ENV_PROFILE_PROD,
@@ -12,9 +12,12 @@ import {
12
12
  } from "../shared/log";
13
13
  import { getErrorMessage } from "../shared/utils/error.utils";
14
14
  import { patchChangeNodeStatus, patchNodeHeartBeat, patchStopNode, postRegisterNode } from "./node.api";
15
+ import { LOG_TOPIC_NODE } from "./node.constant";
15
16
  import { NodeStatus } from "./node.interface";
16
17
  import { formatHeartbeat, getRandomInterval } from "./node.utils";
17
18
 
19
+ export { LOG_TOPIC_NODE };
20
+
18
21
  /** 主循环单轮运行时上下文 — 由 start() 创建并向下传递 */
19
22
  export interface NodeLoopContext {
20
23
  loopTraceId: string;
@@ -31,7 +34,6 @@ export interface BaseNodeRuntimeOptions {
31
34
  beforeProcessExit?: () => void | Promise<void>;
32
35
  }
33
36
 
34
- export const LOG_TOPIC_NODE = "node";
35
37
  const LOG_TOPIC_COMMAND = "command";
36
38
  const LOG_TOPIC_PROCESS = "process";
37
39
 
@@ -118,6 +120,25 @@ abstract class BaseNodeRuntime {
118
120
  return getRandomInterval();
119
121
  }
120
122
 
123
+ /**
124
+ * When true, the main loop skips commands and {@link runLoopSteps} (heartbeat only).
125
+ * Used by the L3 guard process while waiting for connectivity to recover.
126
+ */
127
+ protected isSilenced(): boolean {
128
+ return false;
129
+ }
130
+
131
+ /**
132
+ * Consecutive heartbeat failures reached {@link HEARTBEAT_FAILURE_THRESHOLD}.
133
+ * Default: shut down the node. Override to wait for recovery instead of exiting.
134
+ */
135
+ protected async onHealthCheckFailed(_ctx: NodeLoopContext): Promise<void> {
136
+ this.requestShutdown("heartbeat:consecutive-failures");
137
+ }
138
+
139
+ /** Heartbeat succeeded while {@link isSilenced} is true. Default: no-op. */
140
+ protected async onHeartbeatRecovered(_ctx: NodeLoopContext): Promise<void> {}
141
+
121
142
  private getLogNodeId(): string {
122
143
  return this.localNodeId ?? BaseNodeRuntime.NODE_ID_PENDING_ASSIGNMENT;
123
144
  }
@@ -284,7 +305,9 @@ abstract class BaseNodeRuntime {
284
305
  data: { nodeId, loopCount: this.loopCount },
285
306
  });
286
307
  try {
287
- await this.batchCommand(loopCtx);
308
+ if (!this.isSilenced()) {
309
+ await this.batchCommand(loopCtx);
310
+ }
288
311
  if (this.isShuttingDown()) {
289
312
  iterLogger.debug("節點關閉中,跳過本輪剩餘步驟", { topic: LOG_TOPIC_NODE });
290
313
  break;
@@ -294,7 +317,9 @@ abstract class BaseNodeRuntime {
294
317
  iterLogger.debug("節點關閉中,跳過本輪剩餘步驟", { topic: LOG_TOPIC_NODE });
295
318
  break;
296
319
  }
297
- await this.runLoopSteps(loopCtx, heartbeatOk);
320
+ if (!this.isSilenced()) {
321
+ await this.runLoopSteps(loopCtx, heartbeatOk);
322
+ }
298
323
  } catch (error) {
299
324
  iterLogger.critical(`主循環第 ${this.loopCount} 輪遇到未知錯誤`, {
300
325
  topic: LOG_TOPIC_NODE,
@@ -410,11 +435,11 @@ abstract class BaseNodeRuntime {
410
435
  threshold: BaseNodeRuntime.HEARTBEAT_FAILURE_THRESHOLD,
411
436
  };
412
437
  if (failureCount >= BaseNodeRuntime.HEARTBEAT_FAILURE_THRESHOLD) {
413
- logger.critical("連續心跳失敗達上限,準備關閉節點", {
438
+ logger.critical("連續心跳失敗達上限", {
414
439
  topic: LOG_TOPIC_NODE,
415
440
  data: logData,
416
441
  });
417
- this.requestShutdown("heartbeat:consecutive-failures");
442
+ await this.onHealthCheckFailed(ctx);
418
443
  } else {
419
444
  logger.warn("心跳更新失敗", { topic: LOG_TOPIC_NODE, data: logData });
420
445
  }
@@ -422,6 +447,9 @@ abstract class BaseNodeRuntime {
422
447
  }
423
448
 
424
449
  this.consecutiveHeartbeatFailures = 0;
450
+ if (this.isSilenced()) {
451
+ await this.onHeartbeatRecovered(ctx);
452
+ }
425
453
  // Success is the steady state every loop — do not INFO-spam; failures already warn/critical.
426
454
  logger.debug("心跳更新成功", {
427
455
  topic: LOG_TOPIC_NODE,
@@ -2,10 +2,10 @@ import { z } from "zod";
2
2
  import { createTarget, type QueryFilter, updateTargetDetails, validateWithSchema } from "../core.api";
3
3
  import { generateResponse } from "../core.interface";
4
4
  import { createLogger } from "../shared/log";
5
+ import { LOG_TOPIC_NODE } from "./node.constant";
5
6
  import { CategoryNode, type Node, type NodeDetails, NodeStatus } from "./node.interface";
6
7
 
7
8
  const NODE_STATUS_FIELD = "details->>status" as const;
8
- const LOG_TOPIC_NODE = "node";
9
9
 
10
10
  const nodeIdSchema = z.string().trim().min(1);
11
11
  const traceIdSchema = z.string().trim().min(1).optional();
@@ -0,0 +1,2 @@
1
+ /** Shared log topic for Node domain APIs and {@link BaseNodeRuntime}. */
2
+ export const LOG_TOPIC_NODE = "node";
@@ -1,4 +1,4 @@
1
- export const formatTimestamp = (timestamp: number): string => {
1
+ const formatTimestamp = (timestamp: number): string => {
2
2
  return new Intl.DateTimeFormat("zh-CN", {
3
3
  year: "numeric",
4
4
  month: "2-digit",
@@ -11,7 +11,7 @@ export const formatTimestamp = (timestamp: number): string => {
11
11
  }).format(timestamp);
12
12
  };
13
13
 
14
- export const formatRelativeTime = (timestamp: number): string => {
14
+ const formatRelativeTime = (timestamp: number): string => {
15
15
  const now = Date.now();
16
16
  const diff = now - timestamp;
17
17
 
@@ -12,6 +12,8 @@ export {
12
12
  createSupabaseReachableCheck,
13
13
  } from "./readiness-checks";
14
14
  export { runReadinessChecks } from "./run-readiness-checks";
15
+ export type { RunReadinessGateInput } from "./run-readiness-gate";
16
+ export { runReadinessGate } from "./run-readiness-gate";
15
17
  export type {
16
18
  ServiceReadyGate,
17
19
  ServiceReadySnapshot,
@@ -1,6 +1,6 @@
1
1
  import type { LoggerWithScope } from "../../shared/log";
2
- import type { ReadinessCheck, ReadinessReport } from "../readiness/readiness.types";
3
- import { runReadinessChecks } from "../readiness/run-readiness-checks";
2
+ import type { ReadinessCheck, ReadinessReport } from "./readiness.types";
3
+ import { runReadinessChecks } from "./run-readiness-checks";
4
4
 
5
5
  export interface RunReadinessGateInput {
6
6
  checks: ReadinessCheck[];
@@ -1,8 +1,5 @@
1
1
  import type { LoggerWithScope } from "../../shared/log";
2
-
3
- function sleep(ms: number): Promise<void> {
4
- return new Promise((resolve) => setTimeout(resolve, ms));
5
- }
2
+ import { pollUntil } from "./poll-until";
6
3
 
7
4
  export interface ServiceReadySnapshot {
8
5
  failed: boolean;
@@ -36,30 +33,37 @@ export async function waitForServiceReady(
36
33
  ): Promise<ServiceReadySnapshot> {
37
34
  const pollMs = options.pollMs ?? 500;
38
35
  const defaultFailureMessage = options.failureMessage ?? "Readiness checks failed";
39
- const deadline = Date.now() + options.timeoutMs;
40
36
  let last: ServiceReadySnapshot = { failed: false, ready: false };
41
37
 
42
- while (Date.now() < deadline) {
43
- last = await gate.read();
44
-
38
+ try {
39
+ await pollUntil({
40
+ timeoutMs: options.timeoutMs,
41
+ intervalMs: pollMs,
42
+ until: async () => {
43
+ last = await gate.read();
44
+ if (last.failed) {
45
+ throw new Error(last.failureMessage ?? defaultFailureMessage);
46
+ }
47
+ return last.ready;
48
+ },
49
+ });
50
+ } catch (error) {
45
51
  if (last.failed) {
46
- throw new Error(last.failureMessage ?? defaultFailureMessage);
52
+ throw error instanceof Error ? error : new Error(defaultFailureMessage);
47
53
  }
48
-
49
- if (last.ready) {
50
- options.logger?.info("service ready", {
51
- topic: "readiness",
52
- data: last.readyDetail ?? {},
53
- });
54
- return last;
54
+ const isPollTimeout = error instanceof Error && error.message.startsWith("pollUntil timeout after");
55
+ if (!isPollTimeout) {
56
+ throw error;
55
57
  }
56
-
57
- await sleep(pollMs);
58
+ const message =
59
+ options.formatTimeoutError?.(last, options.timeoutMs) ??
60
+ `Service ready timeout after ${options.timeoutMs}ms (failed=${last.failed}, ready=${last.ready})`;
61
+ throw new Error(message);
58
62
  }
59
63
 
60
- const message =
61
- options.formatTimeoutError?.(last, options.timeoutMs) ??
62
- `Service ready timeout after ${options.timeoutMs}ms (failed=${last.failed}, ready=${last.ready})`;
63
-
64
- throw new Error(message);
64
+ options.logger?.info("service ready", {
65
+ topic: "readiness",
66
+ data: last.readyDetail ?? {},
67
+ });
68
+ return last;
65
69
  }
@@ -58,11 +58,22 @@ function createDefaultServiceRuntimeState<TExtraSlices extends Record<string, ob
58
58
  lastDecision: null,
59
59
  lastSpawnAt: null,
60
60
  spawnCount: 0,
61
+ mode: "healthy",
62
+ silentEnteredAt: null,
63
+ silentLastHeartbeatAt: null,
64
+ silentBackoffMs: 0,
65
+ silentConsecutiveFailures: 0,
66
+ silentRecoveryAttempt: 0,
67
+ silentEvents: [],
61
68
  };
62
69
  const scheduler: SchedulerRuntimeSlice = {
63
70
  lastTickAt: null,
64
71
  lastPostedCount: 0,
65
72
  lastFired: {},
73
+ pid: null,
74
+ spawnedAt: null,
75
+ ready: false,
76
+ readyAt: null,
66
77
  };
67
78
  const worker: WorkerRuntimeSlice = {
68
79
  pid: null,
@@ -15,12 +15,24 @@ export interface GuardRuntimeSlice {
15
15
  lastDecision: string | null;
16
16
  lastSpawnAt: string | null;
17
17
  spawnCount: number;
18
+ /** `healthy` when omitted (legacy shards). Silent: heartbeat-only wait. Recovering: respawn business nodes. */
19
+ mode: "healthy" | "silent" | "recovering";
20
+ silentEnteredAt: string | null;
21
+ silentLastHeartbeatAt: string | null;
22
+ silentBackoffMs: number;
23
+ silentConsecutiveFailures: number;
24
+ silentRecoveryAttempt: number;
25
+ silentEvents: { at: string; type: string; detail?: string }[];
18
26
  }
19
27
 
20
28
  export interface SchedulerRuntimeSlice {
21
29
  lastTickAt: string | null;
22
30
  lastPostedCount: number;
23
31
  lastFired: Record<string, string>;
32
+ pid: number | null;
33
+ spawnedAt: string | null;
34
+ ready: boolean;
35
+ readyAt: string | null;
24
36
  }
25
37
 
26
38
  export interface WorkerRuntimeSlice {
@@ -1,6 +1,9 @@
1
1
  export { registerServiceGuardRunner } from "./register-service-guard-runner";
2
- export type { RunReadinessGateInput } from "./run-readiness-gate";
3
- export { runReadinessGate } from "./run-readiness-gate";
2
+ export type {
3
+ ApplyRegistrySlotGuardInput,
4
+ ApplyRegistrySlotGuardResult,
5
+ } from "./registry-slot-guard-step";
6
+ export { applyRegistrySlotGuardStep } from "./registry-slot-guard-step";
4
7
  export { runServiceGuardTick } from "./run-service-guard-tick";
5
8
  export type {
6
9
  RegisterServiceGuardRunnerOptions,
@@ -8,10 +11,6 @@ export type {
8
11
  ServiceGuardTickInput,
9
12
  ServiceGuardTickResult,
10
13
  } from "./service-guard.interface";
11
- export { SERVICE_GUARD_RUNNER_KEY } from "./service-guard.interface";
14
+ export { guardRetryAfterSec, isGuardAvailable, SERVICE_GUARD_RUNNER_KEY } from "./service-guard.interface";
12
15
  export { ServiceGuardNode } from "./service-guard-node";
13
- export {
14
- getWorkerSpawnCooldownLastAt,
15
- markWorkerSpawned,
16
- resetWorkerSpawnCooldownForTests,
17
- } from "./worker-spawn-cooldown";
16
+ export { markWorkerSpawned } from "./worker-spawn-cooldown";
@@ -60,5 +60,3 @@ export function registerServiceGuardRunner(options: RegisterServiceGuardRunnerOp
60
60
  fn: (ctx) => runServiceGuardRunner(ctx, options),
61
61
  });
62
62
  }
63
-
64
- export { SERVICE_GUARD_RUNNER_KEY };
@@ -1,5 +1,8 @@
1
- import { CategoryNode, evaluateBusyNodeLiveness, type Node, patchServiceRuntime, scanTargetList } from "../../browser";
2
- import { applyRegistrySlotGuardStep } from "../service-host/registry-slot-guard-step";
1
+ import { scanTargetList } from "../../core.api";
2
+ import { patchServiceRuntime } from "../../service/registry.service";
3
+ import { CategoryNode, type Node } from "../node.interface";
4
+ import { evaluateBusyNodeLiveness } from "../node-liveness";
5
+ import { applyRegistrySlotGuardStep } from "./registry-slot-guard-step";
3
6
  import type { ServiceGuardTickInput, ServiceGuardTickResult } from "./service-guard.interface";
4
7
  import { getWorkerSpawnCooldownLastAt, markWorkerSpawned } from "./worker-spawn-cooldown";
5
8
 
@@ -1,18 +1,35 @@
1
- import type { LoggerWithScope } from "../../shared/log";
1
+ import { createLogger, type LoggerWithScope } from "../../shared/log";
2
2
  import { registerCollectLogRunner } from "../../shared/log/spool/register-collect-log-runner";
3
+ import { getErrorMessage } from "../../shared/utils/error.utils";
3
4
  import { TriggerNode } from "../../trigger/trigger-node";
5
+ import type { NodeLoopContext } from "../node-runtime.base";
6
+ import { pollUntil } from "../readiness/poll-until";
7
+ import { runReadinessGate } from "../readiness/run-readiness-gate";
8
+ import type { GuardRuntimeSlice } from "../runtime-state/service-runtime-state.types";
4
9
  import { registerServiceGuardRunner } from "./register-service-guard-runner";
5
- import { runReadinessGate } from "./run-readiness-gate";
6
10
  import type { ServiceGuardNodeOptions } from "./service-guard.interface";
11
+ import { markWorkerSpawned } from "./worker-spawn-cooldown";
12
+
13
+ const SILENT_BACKOFF_MS = [15_000, 30_000, 60_000, 120_000, 240_000, 300_000] as const;
14
+ const SILENT_EVENT_LIMIT = 32;
15
+ const DEFAULT_BUSINESS_READY_TIMEOUT_MS = 180_000;
7
16
 
8
17
  /**
9
- * L3 guard process TriggerNode: readiness gate, TaskNode worker spawn owner,
18
+ * L3 guard process TriggerNode: readiness gate, business-node spawn owner,
10
19
  * and {@link registerServiceGuardRunner} tick (registry slot + liveness + runtime).
11
20
  *
21
+ * Consecutive heartbeat failure enters silent mode (no process exit). Recovery
22
+ * respawns business nodes; leaving silent requires heartbeat + all nodes ready.
23
+ *
12
24
  * Use {@link ServiceGuardNode.create} so the guard runner is registered before `start()`.
13
25
  */
14
26
  class ServiceGuardNode extends TriggerNode {
15
27
  private readonly options: ServiceGuardNodeOptions;
28
+ private readonly businessReadyTimeoutMs: number;
29
+ private mode: GuardRuntimeSlice["mode"] = "healthy";
30
+ private silentBackoffIndex = 0;
31
+ private silentEvents: GuardRuntimeSlice["silentEvents"] = [];
32
+ private recoverPromise: Promise<void> | null = null;
16
33
 
17
34
  private constructor(options: ServiceGuardNodeOptions) {
18
35
  super({
@@ -20,14 +37,16 @@ class ServiceGuardNode extends TriggerNode {
20
37
  beforeProcessExit: options.beforeProcessExit,
21
38
  });
22
39
  this.options = options;
40
+ this.businessReadyTimeoutMs = options.businessReadyTimeoutMs ?? DEFAULT_BUSINESS_READY_TIMEOUT_MS;
23
41
  }
24
42
 
25
43
  /** Register the guard runner and return a ready-to-start node. */
26
44
  static create(options: ServiceGuardNodeOptions): ServiceGuardNode {
27
45
  registerServiceGuardRunner({
46
+ ...options.guardRunner,
28
47
  serviceValue: options.serviceValue,
29
48
  logTopic: options.logTopic,
30
- ...options.guardRunner,
49
+ spawnWorker: options.spawnBusinessNodes,
31
50
  });
32
51
  registerCollectLogRunner({
33
52
  serviceValue: options.serviceValue,
@@ -36,6 +55,38 @@ class ServiceGuardNode extends TriggerNode {
36
55
  return new ServiceGuardNode(options);
37
56
  }
38
57
 
58
+ protected isSilenced(): boolean {
59
+ return this.mode !== "healthy";
60
+ }
61
+
62
+ protected getLoopIntervalMs(): number {
63
+ if (this.mode === "silent") {
64
+ return SILENT_BACKOFF_MS[this.silentBackoffIndex] ?? 300_000;
65
+ }
66
+ return super.getLoopIntervalMs();
67
+ }
68
+
69
+ protected async onHealthCheckFailed(_ctx: NodeLoopContext): Promise<void> {
70
+ if (this.mode !== "healthy") {
71
+ this.bumpSilentBackoff();
72
+ await this.persistGuard({
73
+ lastDecision: "silent_heartbeat_fail",
74
+ silentConsecutiveFailures: this.silentConsecutiveFailures(),
75
+ silentBackoffMs: this.getLoopIntervalMs(),
76
+ silentLastHeartbeatAt: new Date().toISOString(),
77
+ });
78
+ return;
79
+ }
80
+ await this.enterSilent("heartbeat:consecutive-failures");
81
+ }
82
+
83
+ protected async onHeartbeatRecovered(ctx: NodeLoopContext): Promise<void> {
84
+ if (this.mode !== "silent") {
85
+ return;
86
+ }
87
+ await this.recoverFromSilent(ctx);
88
+ }
89
+
39
90
  protected async onBeforeRegisterNode(nodeLogger: LoggerWithScope): Promise<void> {
40
91
  await runReadinessGate({
41
92
  checks: this.options.readinessChecks,
@@ -46,11 +97,183 @@ class ServiceGuardNode extends TriggerNode {
46
97
 
47
98
  await super.onBeforeRegisterNode(nodeLogger);
48
99
 
49
- nodeLogger.info("Spawning TaskNode worker (single spawn owner)", {
100
+ nodeLogger.info("Spawning business nodes", {
50
101
  topic: this.options.logTopic ?? "guard",
51
102
  });
52
- await this.options.spawnWorker("guard:readiness-passed");
103
+ await this.options.spawnBusinessNodes("guard:readiness-passed");
104
+ markWorkerSpawned();
53
105
  this.options.onWorkerSpawned?.();
106
+
107
+ await this.waitUntilBusinessReady();
108
+ await this.persistGuard({
109
+ mode: "healthy",
110
+ lastDecision: "bootstrap_ready",
111
+ });
112
+ }
113
+
114
+ protected async runLoopSteps(ctx: NodeLoopContext, heartbeatOk: boolean): Promise<void> {
115
+ await super.runLoopSteps(ctx, heartbeatOk);
116
+ if (this.mode !== "healthy" || !heartbeatOk) {
117
+ return;
118
+ }
119
+ // Idempotent — respawn exited children without entering silent (worker stale is the runner's job).
120
+ try {
121
+ await this.options.spawnBusinessNodes("guard:ensure-business");
122
+ } catch (error) {
123
+ this.createGuardLogger().warn("ensure-business spawn failed", {
124
+ topic: this.options.logTopic ?? "guard",
125
+ data: { error: getErrorMessage(error) },
126
+ });
127
+ }
128
+ }
129
+
130
+ private silentConsecutiveFailures(): number {
131
+ return this.silentBackoffIndex + 1;
132
+ }
133
+
134
+ private bumpSilentBackoff(): void {
135
+ if (this.silentBackoffIndex < SILENT_BACKOFF_MS.length - 1) {
136
+ this.silentBackoffIndex += 1;
137
+ }
138
+ }
139
+
140
+ private async enterSilent(reason: string): Promise<void> {
141
+ this.mode = "silent";
142
+ this.silentBackoffIndex = 0;
143
+ const enteredAt = new Date().toISOString();
144
+ this.pushEvent("enter", reason);
145
+ await this.persistGuard({
146
+ mode: "silent",
147
+ lastDecision: `silent:${reason}`,
148
+ silentEnteredAt: enteredAt,
149
+ silentLastHeartbeatAt: enteredAt,
150
+ silentBackoffMs: SILENT_BACKOFF_MS[0],
151
+ silentConsecutiveFailures: 1,
152
+ silentRecoveryAttempt: 0,
153
+ silentEvents: this.silentEvents,
154
+ });
155
+
156
+ const logger = this.createGuardLogger();
157
+ logger.warn("Entering silent mode — service unavailable until heartbeat + business nodes recover", {
158
+ topic: this.options.logTopic ?? "guard",
159
+ data: { reason },
160
+ });
161
+
162
+ try {
163
+ await this.options.stopBusinessNodes();
164
+ } catch (error) {
165
+ logger.warn("stopBusinessNodes failed (best-effort)", {
166
+ topic: this.options.logTopic ?? "guard",
167
+ data: { error: getErrorMessage(error) },
168
+ });
169
+ }
170
+ }
171
+
172
+ private async recoverFromSilent(ctx: NodeLoopContext): Promise<void> {
173
+ if (this.recoverPromise != null) {
174
+ return this.recoverPromise;
175
+ }
176
+ this.recoverPromise = this.runRecover(ctx).finally(() => {
177
+ this.recoverPromise = null;
178
+ });
179
+ return this.recoverPromise;
180
+ }
181
+
182
+ private async runRecover(ctx: NodeLoopContext): Promise<void> {
183
+ this.mode = "recovering";
184
+ this.pushEvent("recover_start", ctx.loopTraceId);
185
+ await this.persistGuard({
186
+ mode: "recovering",
187
+ lastDecision: "recovering",
188
+ silentLastHeartbeatAt: new Date().toISOString(),
189
+ silentRecoveryAttempt: this.silentEvents.filter((event) => event.type === "recover_start").length,
190
+ silentEvents: this.silentEvents,
191
+ });
192
+
193
+ const logger = this.createGuardLogger();
194
+ logger.info("Heartbeat restored — recovering business nodes", {
195
+ topic: this.options.logTopic ?? "guard",
196
+ });
197
+
198
+ try {
199
+ await this.options.stopBusinessNodes();
200
+ } catch (error) {
201
+ logger.warn("stopBusinessNodes failed before respawn", {
202
+ topic: this.options.logTopic ?? "guard",
203
+ data: { error: getErrorMessage(error) },
204
+ });
205
+ }
206
+
207
+ try {
208
+ await this.options.spawnBusinessNodes("guard:silent-recover");
209
+ markWorkerSpawned();
210
+ await this.waitUntilBusinessReady();
211
+ } catch (error) {
212
+ this.pushEvent("recover_fail", getErrorMessage(error));
213
+ this.mode = "silent";
214
+ this.bumpSilentBackoff();
215
+ await this.persistGuard({
216
+ mode: "silent",
217
+ lastDecision: "recover_fail",
218
+ silentBackoffMs: this.getLoopIntervalMs(),
219
+ silentEvents: this.silentEvents,
220
+ });
221
+ logger.warn("Recovery failed — returning to silent", {
222
+ topic: this.options.logTopic ?? "guard",
223
+ data: { error: getErrorMessage(error) },
224
+ });
225
+ try {
226
+ await this.options.stopBusinessNodes();
227
+ } catch {
228
+ // best-effort — leftover children are stopped again on the next recover
229
+ }
230
+ return;
231
+ }
232
+
233
+ this.mode = "healthy";
234
+ this.silentBackoffIndex = 0;
235
+ this.pushEvent("recover_ok");
236
+ await this.persistGuard({
237
+ mode: "healthy",
238
+ lastDecision: "recover_ok",
239
+ silentEnteredAt: null,
240
+ silentBackoffMs: 0,
241
+ silentConsecutiveFailures: 0,
242
+ silentEvents: this.silentEvents,
243
+ });
244
+ logger.success("Left silent mode — business nodes ready", {
245
+ topic: this.options.logTopic ?? "guard",
246
+ });
247
+ }
248
+
249
+ private async waitUntilBusinessReady(): Promise<void> {
250
+ try {
251
+ await pollUntil({
252
+ timeoutMs: this.businessReadyTimeoutMs,
253
+ intervalMs: 500,
254
+ until: () => this.options.isBusinessReady(),
255
+ });
256
+ } catch {
257
+ throw new Error(`Business nodes not ready within ${this.businessReadyTimeoutMs}ms`);
258
+ }
259
+ }
260
+
261
+ private pushEvent(type: string, detail?: string): void {
262
+ this.silentEvents = [...this.silentEvents, { at: new Date().toISOString(), type, detail }].slice(
263
+ -SILENT_EVENT_LIMIT,
264
+ );
265
+ }
266
+
267
+ private async persistGuard(patch: Partial<GuardRuntimeSlice>): Promise<void> {
268
+ try {
269
+ await this.options.guardRunner.onGuardPatch(patch);
270
+ } catch {
271
+ // best-effort — disk/runtime state must not take down the guard loop
272
+ }
273
+ }
274
+
275
+ private createGuardLogger(): LoggerWithScope {
276
+ return createLogger({ module: "service-guard" });
54
277
  }
55
278
  }
56
279