target-supabase-sdk 0.2.13 → 0.2.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/dist/browser.js +1 -1
  2. package/dist/chunks/{enable-DqnfLPsQ.js → enable-C7lFooTG.js} +2 -2
  3. package/dist/chunks/{enable-DqnfLPsQ.js.map → enable-C7lFooTG.js.map} +1 -1
  4. package/dist/chunks/{task-queue-6X3aglJh.js → task-queue-qSnsfiIg.js} +5 -3
  5. package/dist/chunks/task-queue-qSnsfiIg.js.map +1 -0
  6. package/dist/node/env/index.d.ts +1 -1
  7. package/dist/node/env/index.d.ts.map +1 -1
  8. package/dist/node/node-runtime.base.d.ts +14 -1
  9. package/dist/node/node-runtime.base.d.ts.map +1 -1
  10. package/dist/node/node.api.d.ts.map +1 -1
  11. package/dist/node/node.constant.d.ts +3 -0
  12. package/dist/node/node.constant.d.ts.map +1 -0
  13. package/dist/node/node.utils.d.ts +0 -2
  14. package/dist/node/node.utils.d.ts.map +1 -1
  15. package/dist/node/readiness/index.d.ts +2 -0
  16. package/dist/node/readiness/index.d.ts.map +1 -1
  17. package/dist/node/{service-guard → readiness}/run-readiness-gate.d.ts +1 -1
  18. package/dist/node/readiness/run-readiness-gate.d.ts.map +1 -0
  19. package/dist/node/readiness/service-ready-gate.d.ts.map +1 -1
  20. package/dist/node/runtime-state/create-service-runtime-state-store.d.ts.map +1 -1
  21. package/dist/node/runtime-state/service-runtime-state.types.d.ts +16 -0
  22. package/dist/node/runtime-state/service-runtime-state.types.d.ts.map +1 -1
  23. package/dist/node/service-guard/index.d.ts +4 -4
  24. package/dist/node/service-guard/index.d.ts.map +1 -1
  25. package/dist/node/service-guard/register-service-guard-runner.d.ts +1 -2
  26. package/dist/node/service-guard/register-service-guard-runner.d.ts.map +1 -1
  27. package/dist/node/service-guard/registry-slot-guard-step.d.ts.map +1 -0
  28. package/dist/node/service-guard/run-service-guard-tick.d.ts.map +1 -1
  29. package/dist/node/service-guard/service-guard-node.d.ts +25 -2
  30. package/dist/node/service-guard/service-guard-node.d.ts.map +1 -1
  31. package/dist/node/service-guard/service-guard.interface.d.ts +25 -10
  32. package/dist/node/service-guard/service-guard.interface.d.ts.map +1 -1
  33. package/dist/node/service-host/claim-registry-slot.d.ts +15 -0
  34. package/dist/node/service-host/claim-registry-slot.d.ts.map +1 -0
  35. package/dist/node/service-host/create-l3-child-launcher.d.ts +38 -0
  36. package/dist/node/service-host/create-l3-child-launcher.d.ts.map +1 -0
  37. package/dist/node/service-host/index.d.ts +2 -5
  38. package/dist/node/service-host/index.d.ts.map +1 -1
  39. package/dist/node/service-host/service-host.d.ts +3 -3
  40. package/dist/node/service-host/service-host.d.ts.map +1 -1
  41. package/dist/node/service-host/single-process-service.d.ts +3 -3
  42. package/dist/node/service-host/single-process-service.d.ts.map +1 -1
  43. package/dist/node.d.ts +8 -6
  44. package/dist/node.d.ts.map +1 -1
  45. package/dist/node.js +627 -264
  46. package/dist/node.js.map +1 -1
  47. package/dist/preload.js +1 -1
  48. package/package.json +1 -1
  49. package/src/node/env/index.ts +0 -4
  50. package/src/node/node-runtime.base.ts +33 -5
  51. package/src/node/node.api.ts +1 -1
  52. package/src/node/node.constant.ts +2 -0
  53. package/src/node/node.utils.ts +2 -2
  54. package/src/node/readiness/index.ts +2 -0
  55. package/src/node/{service-guard → readiness}/run-readiness-gate.ts +2 -2
  56. package/src/node/readiness/service-ready-gate.ts +27 -23
  57. package/src/node/runtime-state/create-service-runtime-state-store.ts +11 -0
  58. package/src/node/runtime-state/service-runtime-state.types.ts +12 -0
  59. package/src/node/service-guard/index.ts +7 -8
  60. package/src/node/service-guard/register-service-guard-runner.ts +0 -2
  61. package/src/node/service-guard/run-service-guard-tick.ts +5 -2
  62. package/src/node/service-guard/service-guard-node.ts +229 -6
  63. package/src/node/service-guard/service-guard.interface.ts +35 -10
  64. package/src/node/service-host/claim-registry-slot.ts +33 -0
  65. package/src/node/service-host/create-l3-child-launcher.ts +157 -0
  66. package/src/node/service-host/index.ts +5 -24
  67. package/src/node/service-host/service-host.ts +11 -18
  68. package/src/node/service-host/single-process-service.ts +10 -20
  69. package/src/node.ts +21 -18
  70. package/dist/chunks/task-queue-6X3aglJh.js.map +0 -1
  71. package/dist/node/service-guard/run-readiness-gate.d.ts.map +0 -1
  72. package/dist/node/service-host/registry-slot-guard-step.d.ts.map +0 -1
  73. /package/dist/node/{service-host → service-guard}/registry-slot-guard-step.d.ts +0 -0
  74. /package/src/node/{service-host → service-guard}/registry-slot-guard-step.ts +0 -0
package/dist/node.js CHANGED
@@ -1,13 +1,13 @@
1
1
  import { z } from 'zod';
2
2
  import { M as MAX_POLL_TARGET_LIST_SIZE, V as validateWithSchema, g as createLogger, z as pollTargetList, k as generateResponse, h as createTarget, K as supabase, w as logManager, X as createScope, d as LogLevel, Y as patchScope, J as scanTargetList, Z as withModule, l as getPossibleTarget, t as isOptimisticLockResponse, R as updateTargetDetails } from './chunks/list.api-BzbTJBsM.js';
3
3
  export { B as BaseValidator, C as CREATE_TARGET_ALREADY_EXISTS_MESSAGE, a as CategoryList, L as LOG_BATCH_LOADER_KEY, b as LOG_MIN_LEVEL_ENV_KEY, c as LOG_PERSIST_LOADER_KEY, e as MAX_TARGET_LIST_PAGE_SIZE, O as OPTIMISTIC_LOCK_ERROR_CODE, f as OPTIMISTIC_LOCK_FAILED_MESSAGE, P as PostListValidator, S as StatusCode, U as UPDATE_TARGET_NOT_FOUND_MESSAGE, i as deleteTarget, j as formatZodError, m as getTarget, n as getTargetList, o as getTargetTotalCount, p as handleSupabaseError, q as isCreateTargetAlreadyExistsError, r as isDevEnvironment, s as isOptimisticLockError, u as logDev, v as logDevGroup, x as logMinLevelFromEnv, y as patchTarget, A as postListCreate, D as postLogListCreate, E as postTarget, F as postTargetPayloadSchema, G as queryFilterSchema, H as resolveLogMinLevel, I as safeParseWithSchema, N as targetDraftSchema, Q as updateTarget, T as validateWith } from './chunks/list.api-BzbTJBsM.js';
4
- import { l as loadEnvFiles, b as readEnv, c as requireEnv, d as resolveLogSpoolSpawnEnvForLabel, e as resolveLogSpoolConfig, f as resolveLogSpoolRoot, g as resolveAllLogSpoolProcessRoles, h as logSpoolProcessDir, s as spoolLogger, i as parseLogSpoolBatchFilename, j as computeLogBatchIdempotencyKey, k as buildLogListDraft, n as postLogBatch, a as logSpoolEnabledFromEnv, o as logSpoolConfigFromEnv, L as LOG_SPOOL_SERVICE_ID_ENV, q as ensureLogSpoolFromEnv, t as shutdownLogSpoolFromEnv, u as getLogSpoolStats } from './chunks/enable-DqnfLPsQ.js';
5
- export { C as CLI_PROD_FLAG, S as SERVICE_ENV_PROFILE_ENV_KEY, w as SERVICE_ENV_PROFILE_PROD, x as buildLogSpoolSpawnEnv, y as envBool, z as envInt, A as envMs, B as envNumber, D as envPort, E as envProfileFromProcess, F as parseEnvFile, G as parseEnvLine, p as pinEnvProfileFromArgv, H as resolveDefaultEnvFiles, r as resolveProjectRootByPackageName, I as resolveProjectRootFromModule, v as validateLogSpoolPreloadEnv } from './chunks/enable-DqnfLPsQ.js';
4
+ import { l as loadEnvFiles, b as readEnv, c as requireEnv, d as resolveLogSpoolSpawnEnvForLabel, e as resolveLogSpoolConfig, f as resolveLogSpoolRoot, g as resolveAllLogSpoolProcessRoles, h as logSpoolProcessDir, s as spoolLogger, i as parseLogSpoolBatchFilename, j as computeLogBatchIdempotencyKey, k as buildLogListDraft, n as postLogBatch, a as logSpoolEnabledFromEnv, o as logSpoolConfigFromEnv, L as LOG_SPOOL_SERVICE_ID_ENV, q as ensureLogSpoolFromEnv, t as shutdownLogSpoolFromEnv, u as getLogSpoolStats } from './chunks/enable-C7lFooTG.js';
5
+ export { C as CLI_PROD_FLAG, S as SERVICE_ENV_PROFILE_ENV_KEY, w as SERVICE_ENV_PROFILE_PROD, x as buildLogSpoolSpawnEnv, y as envBool, z as envInt, A as envMs, B as envNumber, D as envPort, E as envProfileFromProcess, F as resolveDefaultEnvFiles, r as resolveProjectRootByPackageName, G as resolveProjectRootFromModule, v as validateLogSpoolPreloadEnv } from './chunks/enable-C7lFooTG.js';
6
6
  import { randomBytes, createHash } from 'node:crypto';
7
7
  import { readFile, unlink, mkdir, writeFile, rename, access, stat, readdir, constants } from 'node:fs/promises';
8
8
  import { dirname, resolve, join, isAbsolute } from 'node:path';
9
- import { bh as sha256Hex, bm as getErrorMessage, aL as postRegisterNode, ap as patchChangeNodeStatus, N as NodeStatus, aA as patchStopNode, ax as patchNodeHeartBeat, E as EMPTY_REGISTRY_SLOT_RUNTIME_STATE, ag as isLogEntry, b7 as runRegistrySlotGuardCheck, aZ as registrySlotRuntimePatchFromGuardResult, c as CategoryNode, Q as evaluateBusyNodeLiveness, az as patchServiceRuntime, k as ServiceRegistryError, x as claimServiceRegistrySlot, e as CategoryRepo, R as ResultCode, bn as LOG_TOPIC_TASK, m as TASK_REPO_USAGE, a6 as getScanRemoteRepoValues, at as patchClaimTask, bo as CRITICAL_UNKNOWN_TASK_TRACE_ID, ar as patchChangeTaskStatus, o as TaskStatusAction, bp as toError, aT as postTaskSchema, bq as createTaskRow, n as TaskStatus } from './chunks/task-queue-6X3aglJh.js';
10
- export { A as ApiMethod, C as CategoryConfig, a as CategoryExtraction, b as CategoryLink, d as CategoryParcel, f as CategoryService, g as CategoryTask, D as DEFAULT_SYSTEM_REGISTRY_SEED_SLOTS, L as LINK_MANIFEST_VERSION, h as LIST_MANIFEST_VERSION, P as POST_TASK_DEFAULT_MAX_ATTEMPTS, i as ParcelManager, j as PostLinkValidator, S as ServiceLifecycleStatus, l as ServiceSlotStatus, T as TARGET_SYSTEM_REGISTRY_KEY, p as apiDetailsSchema, q as appendSystemRegistrySlots, r as assertRegistrySlotAvailable, s as assertRegistrySlotOwner, t as buildEmptyServiceSlots, u as buildLinkTargetDraft, v as buildListTargetDraft, w as buildSystemRegistryConfigDetails, y as classifyNetworkError, z as countTasksByType, B as createActiveServiceLifecycle, F as createClaimedRegistrySlotRuntimeState, G as createStorageProviderRegistry, H as createUploadTracker, I as decodeLogBatch, J as decodeLogBatchList, K as defaultL3ServiceDetails, M as deleteParcel, O as deleteParcelSchema, U as fetchBinaryWithRetry, V as fetchWithRetry, W as fieldDefinitionSchema, X as flattenLogBatch, Y as flattenLogBatches, Z as formatNetworkError, _ as getApi, $ as getApiSchema, a0 as getConfig, a1 as getConfigSchema, a2 as getExtraction, a3 as getExtractionSchema, a4 as getParcel, a5 as getParcelSchema, a7 as getScanRemoteRepoValuesSchema, a8 as getService, a9 as getServiceSchema, aa as getTargetSystemRegistry, ab as getValueAtPath, ac as installChunkFetchRegistry, ad as isHttpUrl, ae as isLocalFilesystemPath, af as isLogBatchList, ah as isOpaqueChunkUrl, ai as isRetryableHttpStatus, aj as linkTargetDraftBuildInputSchema, ak as parcelDetailsSchema, al as parseLogBatchMeta, am as parseProviderPrefixedUrl, an as parseServiceSlot, ao as parseServiceSlots, aq as patchChangeNodeStatusSchema, as as patchChangeTaskStatusSchema, au as patchClaimTaskSchema, av as patchExtractionObjects, aw as patchExtractionObjectsSchema, ay as patchNodeHeartBeatSchema, aB as patchStopNodeSchema, aC as patchTaskProgress, aD as patchTaskProgressSchema, aE as postApi, aF as postApiSchema, aG as postExtraction, aH as postExtractionSchema, aI as postLinkCreate, aJ as postParcel, aK as postParcelSchema, aM as postRegisterNodeSchema, aN as postService, aO as postServiceInstance, aP as postServiceSchema, aQ as postSystemRegistryConfig, aR as postSystemRegistryConfigSchema, aS as postTask, aU as postTaskWithRetry, aV as publishParcel, aW as registerProviderChunkResolver, aX as registerService, aY as registerServiceAtStartup, a_ as releaseSystemRegistrySlots, a$ as releaseSystemRegistrySlotsByServiceId, b0 as requestHasBody, b1 as resetSystemRegistryConfig, b2 as resetSystemRegistryConfigSchema, b3 as resolveActiveRegistryServiceId, b4 as resolveFetchUrl, b5 as restoreParcel, b6 as restoreParcelById, b8 as sanitizeFileName, b9 as sanitizeParcelFileName, ba as scanExtractionList, bb as scanExtractionListSchema, bc as schemaDefinitionSchema, bd as serviceDetailsSchema, be as serviceLifecycleSchema, bf as serviceNodeSnapshotSchema, bg as serviceRuntimeSchema, bi as summarizeTaskQueue, bj as systemRegistrySeedSlotSchema, bk as unregisterService, bl as unregisterServiceAtShutdown } from './chunks/task-queue-6X3aglJh.js';
9
+ import { bh as sha256Hex, bm as getErrorMessage, bn as LOG_TOPIC_NODE, aL as postRegisterNode, ap as patchChangeNodeStatus, N as NodeStatus, aA as patchStopNode, ax as patchNodeHeartBeat, E as EMPTY_REGISTRY_SLOT_RUNTIME_STATE, b7 as runRegistrySlotGuardCheck, aZ as registrySlotRuntimePatchFromGuardResult, c as CategoryNode, Q as evaluateBusyNodeLiveness, az as patchServiceRuntime, ag as isLogEntry, x as claimServiceRegistrySlot, k as ServiceRegistryError, e as CategoryRepo, R as ResultCode, bo as LOG_TOPIC_TASK, m as TASK_REPO_USAGE, a6 as getScanRemoteRepoValues, at as patchClaimTask, bp as CRITICAL_UNKNOWN_TASK_TRACE_ID, ar as patchChangeTaskStatus, o as TaskStatusAction, bq as toError, aT as postTaskSchema, br as createTaskRow, n as TaskStatus } from './chunks/task-queue-qSnsfiIg.js';
10
+ export { A as ApiMethod, C as CategoryConfig, a as CategoryExtraction, b as CategoryLink, d as CategoryParcel, f as CategoryService, g as CategoryTask, D as DEFAULT_SYSTEM_REGISTRY_SEED_SLOTS, L as LINK_MANIFEST_VERSION, h as LIST_MANIFEST_VERSION, P as POST_TASK_DEFAULT_MAX_ATTEMPTS, i as ParcelManager, j as PostLinkValidator, S as ServiceLifecycleStatus, l as ServiceSlotStatus, T as TARGET_SYSTEM_REGISTRY_KEY, p as apiDetailsSchema, q as appendSystemRegistrySlots, r as assertRegistrySlotAvailable, s as assertRegistrySlotOwner, t as buildEmptyServiceSlots, u as buildLinkTargetDraft, v as buildListTargetDraft, w as buildSystemRegistryConfigDetails, y as classifyNetworkError, z as countTasksByType, B as createActiveServiceLifecycle, F as createClaimedRegistrySlotRuntimeState, G as createStorageProviderRegistry, H as createUploadTracker, I as decodeLogBatch, J as decodeLogBatchList, K as defaultL3ServiceDetails, M as deleteParcel, O as deleteParcelSchema, U as fetchBinaryWithRetry, V as fetchWithRetry, W as fieldDefinitionSchema, X as flattenLogBatch, Y as flattenLogBatches, Z as formatNetworkError, _ as getApi, $ as getApiSchema, a0 as getConfig, a1 as getConfigSchema, a2 as getExtraction, a3 as getExtractionSchema, a4 as getParcel, a5 as getParcelSchema, a7 as getScanRemoteRepoValuesSchema, a8 as getService, a9 as getServiceSchema, aa as getTargetSystemRegistry, ab as getValueAtPath, ac as installChunkFetchRegistry, ad as isHttpUrl, ae as isLocalFilesystemPath, af as isLogBatchList, ah as isOpaqueChunkUrl, ai as isRetryableHttpStatus, aj as linkTargetDraftBuildInputSchema, ak as parcelDetailsSchema, al as parseLogBatchMeta, am as parseProviderPrefixedUrl, an as parseServiceSlot, ao as parseServiceSlots, aq as patchChangeNodeStatusSchema, as as patchChangeTaskStatusSchema, au as patchClaimTaskSchema, av as patchExtractionObjects, aw as patchExtractionObjectsSchema, ay as patchNodeHeartBeatSchema, aB as patchStopNodeSchema, aC as patchTaskProgress, aD as patchTaskProgressSchema, aE as postApi, aF as postApiSchema, aG as postExtraction, aH as postExtractionSchema, aI as postLinkCreate, aJ as postParcel, aK as postParcelSchema, aM as postRegisterNodeSchema, aN as postService, aO as postServiceInstance, aP as postServiceSchema, aQ as postSystemRegistryConfig, aR as postSystemRegistryConfigSchema, aS as postTask, aU as postTaskWithRetry, aV as publishParcel, aW as registerProviderChunkResolver, aX as registerService, aY as registerServiceAtStartup, a_ as releaseSystemRegistrySlots, a$ as releaseSystemRegistrySlotsByServiceId, b0 as requestHasBody, b1 as resetSystemRegistryConfig, b2 as resetSystemRegistryConfigSchema, b3 as resolveActiveRegistryServiceId, b4 as resolveFetchUrl, b5 as restoreParcel, b6 as restoreParcelById, b8 as sanitizeFileName, b9 as sanitizeParcelFileName, ba as scanExtractionList, bb as scanExtractionListSchema, bc as schemaDefinitionSchema, bd as serviceDetailsSchema, be as serviceLifecycleSchema, bf as serviceNodeSnapshotSchema, bg as serviceRuntimeSchema, bi as summarizeTaskQueue, bj as systemRegistrySeedSlotSchema, bk as unregisterService, bl as unregisterServiceAtShutdown } from './chunks/task-queue-qSnsfiIg.js';
11
11
  import { spawn } from 'node:child_process';
12
12
  import { tmpdir } from 'node:os';
13
13
  import { pathToFileURL } from 'node:url';
@@ -288,7 +288,6 @@ const formatHeartbeat = (timestamp) => {
288
288
  return `${absolute} (${relative})`;
289
289
  };
290
290
 
291
- const LOG_TOPIC_NODE = "node";
292
291
  const LOG_TOPIC_COMMAND = "command";
293
292
  const LOG_TOPIC_PROCESS = "process";
294
293
  /**
@@ -357,6 +356,22 @@ class BaseNodeRuntime {
357
356
  getLoopIntervalMs() {
358
357
  return getRandomInterval();
359
358
  }
359
+ /**
360
+ * When true, the main loop skips commands and {@link runLoopSteps} (heartbeat only).
361
+ * Used by the L3 guard process while waiting for connectivity to recover.
362
+ */
363
+ isSilenced() {
364
+ return false;
365
+ }
366
+ /**
367
+ * Consecutive heartbeat failures reached {@link HEARTBEAT_FAILURE_THRESHOLD}.
368
+ * Default: shut down the node. Override to wait for recovery instead of exiting.
369
+ */
370
+ async onHealthCheckFailed(_ctx) {
371
+ this.requestShutdown("heartbeat:consecutive-failures");
372
+ }
373
+ /** Heartbeat succeeded while {@link isSilenced} is true. Default: no-op. */
374
+ async onHeartbeatRecovered(_ctx) { }
360
375
  getLogNodeId() {
361
376
  return this.localNodeId ?? BaseNodeRuntime.NODE_ID_PENDING_ASSIGNMENT;
362
377
  }
@@ -513,7 +528,9 @@ class BaseNodeRuntime {
513
528
  data: { nodeId, loopCount: this.loopCount },
514
529
  });
515
530
  try {
516
- await this.batchCommand(loopCtx);
531
+ if (!this.isSilenced()) {
532
+ await this.batchCommand(loopCtx);
533
+ }
517
534
  if (this.isShuttingDown()) {
518
535
  iterLogger.debug("節點關閉中,跳過本輪剩餘步驟", { topic: LOG_TOPIC_NODE });
519
536
  break;
@@ -523,7 +540,9 @@ class BaseNodeRuntime {
523
540
  iterLogger.debug("節點關閉中,跳過本輪剩餘步驟", { topic: LOG_TOPIC_NODE });
524
541
  break;
525
542
  }
526
- await this.runLoopSteps(loopCtx, heartbeatOk);
543
+ if (!this.isSilenced()) {
544
+ await this.runLoopSteps(loopCtx, heartbeatOk);
545
+ }
527
546
  }
528
547
  catch (error) {
529
548
  iterLogger.critical(`主循環第 ${this.loopCount} 輪遇到未知錯誤`, {
@@ -635,11 +654,11 @@ class BaseNodeRuntime {
635
654
  threshold: BaseNodeRuntime.HEARTBEAT_FAILURE_THRESHOLD,
636
655
  };
637
656
  if (failureCount >= BaseNodeRuntime.HEARTBEAT_FAILURE_THRESHOLD) {
638
- logger.critical("連續心跳失敗達上限,準備關閉節點", {
657
+ logger.critical("連續心跳失敗達上限", {
639
658
  topic: LOG_TOPIC_NODE,
640
659
  data: logData,
641
660
  });
642
- this.requestShutdown("heartbeat:consecutive-failures");
661
+ await this.onHealthCheckFailed(ctx);
643
662
  }
644
663
  else {
645
664
  logger.warn("心跳更新失敗", { topic: LOG_TOPIC_NODE, data: logData });
@@ -647,6 +666,9 @@ class BaseNodeRuntime {
647
666
  return false;
648
667
  }
649
668
  this.consecutiveHeartbeatFailures = 0;
669
+ if (this.isSilenced()) {
670
+ await this.onHeartbeatRecovered(ctx);
671
+ }
650
672
  // Success is the steady state every loop — do not INFO-spam; failures already warn/critical.
651
673
  logger.debug("心跳更新成功", {
652
674
  topic: LOG_TOPIC_NODE,
@@ -901,7 +923,7 @@ function createManagedChildProcesses(options) {
901
923
  });
902
924
  }
903
925
 
904
- function sleep$1(ms) {
926
+ function sleep(ms) {
905
927
  return new Promise((resolve) => setTimeout(resolve, ms));
906
928
  }
907
929
  /**
@@ -915,7 +937,7 @@ async function pollUntil(options) {
915
937
  if (await options.until()) {
916
938
  return;
917
939
  }
918
- await sleep$1(intervalMs);
940
+ await sleep(intervalMs);
919
941
  }
920
942
  throw new Error(`pollUntil timeout after ${options.timeoutMs}ms`);
921
943
  }
@@ -989,9 +1011,26 @@ async function runReadinessChecks(checks) {
989
1011
  };
990
1012
  }
991
1013
 
992
- function sleep(ms) {
993
- return new Promise((resolve) => setTimeout(resolve, ms));
1014
+ /** Run readiness checks, persist report via callback, fail-fast when not ok. */
1015
+ async function runReadinessGate(input) {
1016
+ const { checks, onReport, logger, logTopic = "guard" } = input;
1017
+ logger.info("Running readiness checks", { topic: logTopic });
1018
+ const report = await runReadinessChecks(checks);
1019
+ await onReport(report);
1020
+ if (!report.ok) {
1021
+ logger.error("Readiness checks failed", {
1022
+ topic: logTopic,
1023
+ data: { message: report.message, checks: report.checks },
1024
+ });
1025
+ throw new Error(report.message ?? "Readiness checks failed");
1026
+ }
1027
+ logger.debug("Readiness checks passed", {
1028
+ topic: logTopic,
1029
+ data: { checks: report.checks.map((check) => check.name) },
1030
+ });
1031
+ return report;
994
1032
  }
1033
+
995
1034
  /**
996
1035
  * Poll {@link ServiceReadyGate} until ready, fail fast on `failed`, or timeout.
997
1036
  * Returns the final ready snapshot.
@@ -999,25 +1038,37 @@ function sleep(ms) {
999
1038
  async function waitForServiceReady(gate, options) {
1000
1039
  const pollMs = options.pollMs ?? 500;
1001
1040
  const defaultFailureMessage = options.failureMessage ?? "Readiness checks failed";
1002
- const deadline = Date.now() + options.timeoutMs;
1003
1041
  let last = { failed: false, ready: false };
1004
- while (Date.now() < deadline) {
1005
- last = await gate.read();
1042
+ try {
1043
+ await pollUntil({
1044
+ timeoutMs: options.timeoutMs,
1045
+ intervalMs: pollMs,
1046
+ until: async () => {
1047
+ last = await gate.read();
1048
+ if (last.failed) {
1049
+ throw new Error(last.failureMessage ?? defaultFailureMessage);
1050
+ }
1051
+ return last.ready;
1052
+ },
1053
+ });
1054
+ }
1055
+ catch (error) {
1006
1056
  if (last.failed) {
1007
- throw new Error(last.failureMessage ?? defaultFailureMessage);
1057
+ throw error instanceof Error ? error : new Error(defaultFailureMessage);
1008
1058
  }
1009
- if (last.ready) {
1010
- options.logger?.info("service ready", {
1011
- topic: "readiness",
1012
- data: last.readyDetail ?? {},
1013
- });
1014
- return last;
1059
+ const isPollTimeout = error instanceof Error && error.message.startsWith("pollUntil timeout after");
1060
+ if (!isPollTimeout) {
1061
+ throw error;
1015
1062
  }
1016
- await sleep(pollMs);
1063
+ const message = options.formatTimeoutError?.(last, options.timeoutMs) ??
1064
+ `Service ready timeout after ${options.timeoutMs}ms (failed=${last.failed}, ready=${last.ready})`;
1065
+ throw new Error(message);
1017
1066
  }
1018
- const message = options.formatTimeoutError?.(last, options.timeoutMs) ??
1019
- `Service ready timeout after ${options.timeoutMs}ms (failed=${last.failed}, ready=${last.ready})`;
1020
- throw new Error(message);
1067
+ options.logger?.info("service ready", {
1068
+ topic: "readiness",
1069
+ data: last.readyDetail ?? {},
1070
+ });
1071
+ return last;
1021
1072
  }
1022
1073
 
1023
1074
  const RUNTIME_STATE_SHARD_DIR = "runtime-state";
@@ -1181,11 +1232,22 @@ function createDefaultServiceRuntimeState(options) {
1181
1232
  lastDecision: null,
1182
1233
  lastSpawnAt: null,
1183
1234
  spawnCount: 0,
1235
+ mode: "healthy",
1236
+ silentEnteredAt: null,
1237
+ silentLastHeartbeatAt: null,
1238
+ silentBackoffMs: 0,
1239
+ silentConsecutiveFailures: 0,
1240
+ silentRecoveryAttempt: 0,
1241
+ silentEvents: [],
1184
1242
  };
1185
1243
  const scheduler = {
1186
1244
  lastTickAt: null,
1187
1245
  lastPostedCount: 0,
1188
1246
  lastFired: {},
1247
+ pid: null,
1248
+ spawnedAt: null,
1249
+ ready: false,
1250
+ readyAt: null,
1189
1251
  };
1190
1252
  const worker = {
1191
1253
  pid: null,
@@ -1477,6 +1539,193 @@ async function runRunner(ctx, runner) {
1477
1539
  return succeeded;
1478
1540
  }
1479
1541
 
1542
+ /**
1543
+ * Run registry slot ownership check and optionally SIGTERM the parent main process.
1544
+ * Used by guard / supervisor TriggerNode runners across L3 services.
1545
+ */
1546
+ async function applyRegistrySlotGuardStep(input) {
1547
+ const result = await runRegistrySlotGuardCheck({
1548
+ serviceValue: input.serviceValue,
1549
+ serviceId: input.serviceId,
1550
+ });
1551
+ const registryPatch = registrySlotRuntimePatchFromGuardResult(result, input.checkedAt);
1552
+ if (result.shouldShutdown) {
1553
+ input.logger.critical("Registry slot lost — shutting down service", {
1554
+ topic: input.logTopic ?? "guard",
1555
+ data: {
1556
+ traceId: input.traceId,
1557
+ serviceId: input.serviceId,
1558
+ serviceValue: input.serviceValue,
1559
+ },
1560
+ });
1561
+ process.kill(process.ppid, "SIGTERM");
1562
+ return { continueTick: false, registryPatch };
1563
+ }
1564
+ if (result.error != null) {
1565
+ input.logger.warn("Registry slot check failed (transient) — skipping enforcement this tick", {
1566
+ topic: input.logTopic ?? "guard",
1567
+ data: {
1568
+ traceId: input.traceId,
1569
+ serviceId: input.serviceId,
1570
+ message: result.error,
1571
+ },
1572
+ });
1573
+ }
1574
+ return { continueTick: true, registryPatch };
1575
+ }
1576
+
1577
+ let lastWorkerSpawnAt = 0;
1578
+ function getWorkerSpawnCooldownLastAt() {
1579
+ return lastWorkerSpawnAt;
1580
+ }
1581
+ function markWorkerSpawned(at = Date.now()) {
1582
+ lastWorkerSpawnAt = at;
1583
+ }
1584
+
1585
+ /**
1586
+ * One guard-runner tick: registry slot check, TaskNode liveness, optional worker
1587
+ * respawn, Service runtime heartbeat rollup.
1588
+ */
1589
+ async function runServiceGuardTick(input) {
1590
+ const { serviceValue, serviceId, nodeId, loopTraceId, logger, logTopic = "guard", checkedAt, nowMs, taskNodeStaleMs, workerSpawnCooldownMs, maxNodes = 200, spawnWorker, getLastWorkerSpawnAt = getWorkerSpawnCooldownLastAt, markWorkerSpawned: markWorkerSpawnedAt = markWorkerSpawned, } = input;
1591
+ const continueTick = true;
1592
+ let registryPatch;
1593
+ let decision = "idle";
1594
+ if (serviceId != null) {
1595
+ const guard = await applyRegistrySlotGuardStep({
1596
+ serviceValue,
1597
+ serviceId,
1598
+ checkedAt,
1599
+ logger,
1600
+ logTopic,
1601
+ traceId: loopTraceId,
1602
+ });
1603
+ registryPatch = guard.registryPatch;
1604
+ if (!guard.continueTick) {
1605
+ return { continueTick: false, decision: "slot_lost", nodes: [], registryPatch };
1606
+ }
1607
+ decision = "slot_ok";
1608
+ }
1609
+ const { data: nodes = [] } = await scanTargetList({
1610
+ category: CategoryNode.NODE,
1611
+ maxRows: maxNodes,
1612
+ });
1613
+ const liveness = evaluateBusyNodeLiveness(nodes, {
1614
+ staleMs: taskNodeStaleMs,
1615
+ now: nowMs,
1616
+ excludeNodeId: nodeId,
1617
+ onlyFreshCandidates: true,
1618
+ });
1619
+ const ageMs = liveness.freshest?.ageMs ?? Number.POSITIVE_INFINITY;
1620
+ if (liveness.healthy) {
1621
+ decision = "healthy";
1622
+ }
1623
+ else if (nowMs - getLastWorkerSpawnAt() < workerSpawnCooldownMs) {
1624
+ decision = "spawn_cooldown";
1625
+ logger.warn("TaskNode unavailable but spawn cooldown active", {
1626
+ topic: logTopic,
1627
+ data: {
1628
+ traceId: loopTraceId,
1629
+ ageMs,
1630
+ staleMs: taskNodeStaleMs,
1631
+ cooldownMs: workerSpawnCooldownMs,
1632
+ },
1633
+ });
1634
+ }
1635
+ else {
1636
+ decision = "spawn_worker";
1637
+ logger.warn("TaskNode stale — restarting worker", {
1638
+ topic: logTopic,
1639
+ data: {
1640
+ traceId: loopTraceId,
1641
+ peerWorkerCount: liveness.busyWorkerCount,
1642
+ freshPeerCount: liveness.freshWorkerCount,
1643
+ staleMs: taskNodeStaleMs,
1644
+ },
1645
+ });
1646
+ await spawnWorker("guard:restart-unhealthy-tasknode");
1647
+ markWorkerSpawnedAt(nowMs);
1648
+ }
1649
+ try {
1650
+ await patchServiceRuntime({
1651
+ serviceValue,
1652
+ serviceId: serviceId ?? undefined,
1653
+ nodes,
1654
+ lastHeartBeat: nowMs,
1655
+ });
1656
+ }
1657
+ catch (error) {
1658
+ const message = error instanceof Error ? error.message : String(error);
1659
+ logger.warn("Failed to patch Service runtime rollup", {
1660
+ topic: logTopic,
1661
+ data: { traceId: loopTraceId, message },
1662
+ });
1663
+ }
1664
+ return { continueTick, decision, nodes, registryPatch };
1665
+ }
1666
+
1667
+ /** Default TriggerManager runner key for {@link registerServiceGuardRunner}. */
1668
+ const SERVICE_GUARD_RUNNER_KEY = "service-guard";
1669
+ /** True when the guard slice is serving traffic (`mode` omitted = legacy healthy). */
1670
+ function isGuardAvailable(guard) {
1671
+ return guard.mode == null || guard.mode === "healthy";
1672
+ }
1673
+ /**
1674
+ * Seconds for `Retry-After` when the guard is not serving traffic.
1675
+ * `null` when {@link isGuardAvailable} is true.
1676
+ */
1677
+ function guardRetryAfterSec(guard) {
1678
+ if (isGuardAvailable(guard)) {
1679
+ return null;
1680
+ }
1681
+ const backoffMs = guard.silentBackoffMs > 0 ? guard.silentBackoffMs : 15_000;
1682
+ return Math.max(1, Math.ceil(backoffMs / 1000));
1683
+ }
1684
+
1685
+ async function runServiceGuardRunner(ctx, options) {
1686
+ const { loopTraceId, nodeId, logger } = ctx;
1687
+ const nowMs = options.now?.() ?? Date.now();
1688
+ const checkedAt = new Date(nowMs).toISOString();
1689
+ const serviceId = await options.getServiceId();
1690
+ const result = await runServiceGuardTick({
1691
+ serviceValue: options.serviceValue,
1692
+ serviceId,
1693
+ nodeId,
1694
+ loopTraceId,
1695
+ logger,
1696
+ logTopic: options.logTopic,
1697
+ checkedAt,
1698
+ nowMs,
1699
+ taskNodeStaleMs: options.taskNodeStaleMs,
1700
+ workerSpawnCooldownMs: options.workerSpawnCooldownMs,
1701
+ maxNodes: options.maxNodes,
1702
+ spawnWorker: options.spawnWorker,
1703
+ });
1704
+ if (result.registryPatch != null) {
1705
+ await options.onRegistryPatch(result.registryPatch);
1706
+ }
1707
+ if (!result.continueTick) {
1708
+ return;
1709
+ }
1710
+ await options.onGuardPatch({
1711
+ nodeId,
1712
+ lastCheckAt: checkedAt,
1713
+ lastDecision: result.decision,
1714
+ });
1715
+ if (options.onDecision != null) {
1716
+ await options.onDecision(result.decision);
1717
+ }
1718
+ }
1719
+ /** Register the standard L3 guard runner (registry slot + TaskNode + Service runtime). */
1720
+ function registerServiceGuardRunner(options) {
1721
+ TriggerManager.registerRunner({
1722
+ key: options.runnerKey ?? SERVICE_GUARD_RUNNER_KEY,
1723
+ intervalMs: options.intervalMs,
1724
+ initialDelayMs: options.initialDelayMs,
1725
+ fn: (ctx) => runServiceGuardRunner(ctx, options),
1726
+ });
1727
+ }
1728
+
1480
1729
  const UPLOAD_LANE = "slow";
1481
1730
  /** Per serviceId — guard collect-log tick runs GC at most once per `gc.intervalMs`. */
1482
1731
  const lastGcAtMsByServiceId = new Map();
@@ -1786,205 +2035,13 @@ function registerCollectLogRunner(options) {
1786
2035
  }
1787
2036
 
1788
2037
  /**
1789
- * Run registry slot ownership check and optionally SIGTERM the parent main process.
1790
- * Used by guard / supervisor TriggerNode runners across L3 services.
1791
- */
1792
- async function applyRegistrySlotGuardStep(input) {
1793
- const result = await runRegistrySlotGuardCheck({
1794
- serviceValue: input.serviceValue,
1795
- serviceId: input.serviceId,
1796
- });
1797
- const registryPatch = registrySlotRuntimePatchFromGuardResult(result, input.checkedAt);
1798
- if (result.shouldShutdown) {
1799
- input.logger.critical("Registry slot lost — shutting down service", {
1800
- topic: input.logTopic ?? "guard",
1801
- data: {
1802
- traceId: input.traceId,
1803
- serviceId: input.serviceId,
1804
- serviceValue: input.serviceValue,
1805
- },
1806
- });
1807
- process.kill(process.ppid, "SIGTERM");
1808
- return { continueTick: false, registryPatch };
1809
- }
1810
- if (result.error != null) {
1811
- input.logger.warn("Registry slot check failed (transient) — skipping enforcement this tick", {
1812
- topic: input.logTopic ?? "guard",
1813
- data: {
1814
- traceId: input.traceId,
1815
- serviceId: input.serviceId,
1816
- message: result.error,
1817
- },
1818
- });
1819
- }
1820
- return { continueTick: true, registryPatch };
1821
- }
1822
-
1823
- let lastWorkerSpawnAt = 0;
1824
- function getWorkerSpawnCooldownLastAt() {
1825
- return lastWorkerSpawnAt;
1826
- }
1827
- function markWorkerSpawned(at = Date.now()) {
1828
- lastWorkerSpawnAt = at;
1829
- }
1830
-
1831
- /**
1832
- * One guard-runner tick: registry slot check, TaskNode liveness, optional worker
1833
- * respawn, Service runtime heartbeat rollup.
1834
- */
1835
- async function runServiceGuardTick(input) {
1836
- const { serviceValue, serviceId, nodeId, loopTraceId, logger, logTopic = "guard", checkedAt, nowMs, taskNodeStaleMs, workerSpawnCooldownMs, maxNodes = 200, spawnWorker, getLastWorkerSpawnAt = getWorkerSpawnCooldownLastAt, markWorkerSpawned: markWorkerSpawnedAt = markWorkerSpawned, } = input;
1837
- const continueTick = true;
1838
- let registryPatch;
1839
- let decision = "idle";
1840
- if (serviceId != null) {
1841
- const guard = await applyRegistrySlotGuardStep({
1842
- serviceValue,
1843
- serviceId,
1844
- checkedAt,
1845
- logger,
1846
- logTopic,
1847
- traceId: loopTraceId,
1848
- });
1849
- registryPatch = guard.registryPatch;
1850
- if (!guard.continueTick) {
1851
- return { continueTick: false, decision: "slot_lost", nodes: [], registryPatch };
1852
- }
1853
- decision = "slot_ok";
1854
- }
1855
- const { data: nodes = [] } = await scanTargetList({
1856
- category: CategoryNode.NODE,
1857
- maxRows: maxNodes,
1858
- });
1859
- const liveness = evaluateBusyNodeLiveness(nodes, {
1860
- staleMs: taskNodeStaleMs,
1861
- now: nowMs,
1862
- excludeNodeId: nodeId,
1863
- onlyFreshCandidates: true,
1864
- });
1865
- const ageMs = liveness.freshest?.ageMs ?? Number.POSITIVE_INFINITY;
1866
- if (liveness.healthy) {
1867
- decision = "healthy";
1868
- }
1869
- else if (nowMs - getLastWorkerSpawnAt() < workerSpawnCooldownMs) {
1870
- decision = "spawn_cooldown";
1871
- logger.warn("TaskNode unavailable but spawn cooldown active", {
1872
- topic: logTopic,
1873
- data: {
1874
- traceId: loopTraceId,
1875
- ageMs,
1876
- staleMs: taskNodeStaleMs,
1877
- cooldownMs: workerSpawnCooldownMs,
1878
- },
1879
- });
1880
- }
1881
- else {
1882
- decision = "spawn_worker";
1883
- logger.warn("TaskNode stale — restarting worker", {
1884
- topic: logTopic,
1885
- data: {
1886
- traceId: loopTraceId,
1887
- peerWorkerCount: liveness.busyWorkerCount,
1888
- freshPeerCount: liveness.freshWorkerCount,
1889
- staleMs: taskNodeStaleMs,
1890
- },
1891
- });
1892
- await spawnWorker("guard:restart-unhealthy-tasknode");
1893
- markWorkerSpawnedAt(nowMs);
1894
- }
1895
- try {
1896
- await patchServiceRuntime({
1897
- serviceValue,
1898
- serviceId: serviceId ?? undefined,
1899
- nodes,
1900
- lastHeartBeat: nowMs,
1901
- });
1902
- }
1903
- catch (error) {
1904
- const message = error instanceof Error ? error.message : String(error);
1905
- logger.warn("Failed to patch Service runtime rollup", {
1906
- topic: logTopic,
1907
- data: { traceId: loopTraceId, message },
1908
- });
1909
- }
1910
- return { continueTick, decision, nodes, registryPatch };
1911
- }
1912
-
1913
- /** Default TriggerManager runner key for {@link registerServiceGuardRunner}. */
1914
- const SERVICE_GUARD_RUNNER_KEY = "service-guard";
1915
-
1916
- async function runServiceGuardRunner(ctx, options) {
1917
- const { loopTraceId, nodeId, logger } = ctx;
1918
- const nowMs = options.now?.() ?? Date.now();
1919
- const checkedAt = new Date(nowMs).toISOString();
1920
- const serviceId = await options.getServiceId();
1921
- const result = await runServiceGuardTick({
1922
- serviceValue: options.serviceValue,
1923
- serviceId,
1924
- nodeId,
1925
- loopTraceId,
1926
- logger,
1927
- logTopic: options.logTopic,
1928
- checkedAt,
1929
- nowMs,
1930
- taskNodeStaleMs: options.taskNodeStaleMs,
1931
- workerSpawnCooldownMs: options.workerSpawnCooldownMs,
1932
- maxNodes: options.maxNodes,
1933
- spawnWorker: options.spawnWorker,
1934
- });
1935
- if (result.registryPatch != null) {
1936
- await options.onRegistryPatch(result.registryPatch);
1937
- }
1938
- if (!result.continueTick) {
1939
- return;
1940
- }
1941
- await options.onGuardPatch({
1942
- nodeId,
1943
- lastCheckAt: checkedAt,
1944
- lastDecision: result.decision,
1945
- });
1946
- if (options.onDecision != null) {
1947
- await options.onDecision(result.decision);
1948
- }
1949
- }
1950
- /** Register the standard L3 guard runner (registry slot + TaskNode + Service runtime). */
1951
- function registerServiceGuardRunner(options) {
1952
- TriggerManager.registerRunner({
1953
- key: options.runnerKey ?? SERVICE_GUARD_RUNNER_KEY,
1954
- intervalMs: options.intervalMs,
1955
- initialDelayMs: options.initialDelayMs,
1956
- fn: (ctx) => runServiceGuardRunner(ctx, options),
1957
- });
1958
- }
1959
-
1960
- /** Run readiness checks, persist report via callback, fail-fast when not ok. */
1961
- async function runReadinessGate(input) {
1962
- const { checks, onReport, logger, logTopic = "guard" } = input;
1963
- logger.info("Running readiness checks", { topic: logTopic });
1964
- const report = await runReadinessChecks(checks);
1965
- await onReport(report);
1966
- if (!report.ok) {
1967
- logger.error("Readiness checks failed", {
1968
- topic: logTopic,
1969
- data: { message: report.message, checks: report.checks },
1970
- });
1971
- throw new Error(report.message ?? "Readiness checks failed");
1972
- }
1973
- logger.debug("Readiness checks passed", {
1974
- topic: logTopic,
1975
- data: { checks: report.checks.map((check) => check.name) },
1976
- });
1977
- return report;
1978
- }
1979
-
1980
- /**
1981
- * Local interval scheduler node.
1982
- * Main loop: commands → heartbeat → {@link TriggerManager.tick}.
1983
- *
1984
- * Register runners via {@link TriggerManager.registerRunner} before {@link start}.
1985
- * Does not scan Supabase trigger rows — see `trigger.api.ts` for remote admin APIs.
1986
- *
1987
- * Deploy **one** TriggerNode per logical scheduler — multiple processes duplicate runner work.
2038
+ * Local interval scheduler node.
2039
+ * Main loop: commands → heartbeat → {@link TriggerManager.tick}.
2040
+ *
2041
+ * Register runners via {@link TriggerManager.registerRunner} before {@link start}.
2042
+ * Does not scan Supabase trigger rows — see `trigger.api.ts` for remote admin APIs.
2043
+ *
2044
+ * Deploy **one** TriggerNode per logical scheduler — multiple processes duplicate runner work.
1988
2045
  */
1989
2046
  class TriggerNode extends BaseNodeRuntime {
1990
2047
  requireRunners;
@@ -2047,27 +2104,39 @@ class TriggerNode extends BaseNodeRuntime {
2047
2104
  }
2048
2105
  }
2049
2106
 
2107
+ const SILENT_BACKOFF_MS = [15_000, 30_000, 60_000, 120_000, 240_000, 300_000];
2108
+ const DEFAULT_BUSINESS_READY_TIMEOUT_MS = 180_000;
2050
2109
  /**
2051
- * L3 guard process TriggerNode: readiness gate, TaskNode worker spawn owner,
2110
+ * L3 guard process TriggerNode: readiness gate, business-node spawn owner,
2052
2111
  * and {@link registerServiceGuardRunner} tick (registry slot + liveness + runtime).
2053
2112
  *
2113
+ * Consecutive heartbeat failure enters silent mode (no process exit). Recovery
2114
+ * respawns business nodes; leaving silent requires heartbeat + all nodes ready.
2115
+ *
2054
2116
  * Use {@link ServiceGuardNode.create} so the guard runner is registered before `start()`.
2055
2117
  */
2056
2118
  class ServiceGuardNode extends TriggerNode {
2057
2119
  options;
2120
+ businessReadyTimeoutMs;
2121
+ mode = "healthy";
2122
+ silentBackoffIndex = 0;
2123
+ silentEvents = [];
2124
+ recoverPromise = null;
2058
2125
  constructor(options) {
2059
2126
  super({
2060
2127
  requireRunners: true,
2061
2128
  beforeProcessExit: options.beforeProcessExit,
2062
2129
  });
2063
2130
  this.options = options;
2131
+ this.businessReadyTimeoutMs = options.businessReadyTimeoutMs ?? DEFAULT_BUSINESS_READY_TIMEOUT_MS;
2064
2132
  }
2065
2133
  /** Register the guard runner and return a ready-to-start node. */
2066
2134
  static create(options) {
2067
2135
  registerServiceGuardRunner({
2136
+ ...options.guardRunner,
2068
2137
  serviceValue: options.serviceValue,
2069
2138
  logTopic: options.logTopic,
2070
- ...options.guardRunner,
2139
+ spawnWorker: options.spawnBusinessNodes,
2071
2140
  });
2072
2141
  registerCollectLogRunner({
2073
2142
  serviceValue: options.serviceValue,
@@ -2075,6 +2144,34 @@ class ServiceGuardNode extends TriggerNode {
2075
2144
  });
2076
2145
  return new ServiceGuardNode(options);
2077
2146
  }
2147
+ isSilenced() {
2148
+ return this.mode !== "healthy";
2149
+ }
2150
+ getLoopIntervalMs() {
2151
+ if (this.mode === "silent") {
2152
+ return SILENT_BACKOFF_MS[this.silentBackoffIndex] ?? 300_000;
2153
+ }
2154
+ return super.getLoopIntervalMs();
2155
+ }
2156
+ async onHealthCheckFailed(_ctx) {
2157
+ if (this.mode !== "healthy") {
2158
+ this.bumpSilentBackoff();
2159
+ await this.persistGuard({
2160
+ lastDecision: "silent_heartbeat_fail",
2161
+ silentConsecutiveFailures: this.silentConsecutiveFailures(),
2162
+ silentBackoffMs: this.getLoopIntervalMs(),
2163
+ silentLastHeartbeatAt: new Date().toISOString(),
2164
+ });
2165
+ return;
2166
+ }
2167
+ await this.enterSilent("heartbeat:consecutive-failures");
2168
+ }
2169
+ async onHeartbeatRecovered(ctx) {
2170
+ if (this.mode !== "silent") {
2171
+ return;
2172
+ }
2173
+ await this.recoverFromSilent(ctx);
2174
+ }
2078
2175
  async onBeforeRegisterNode(nodeLogger) {
2079
2176
  await runReadinessGate({
2080
2177
  checks: this.options.readinessChecks,
@@ -2083,11 +2180,298 @@ class ServiceGuardNode extends TriggerNode {
2083
2180
  logTopic: this.options.logTopic,
2084
2181
  });
2085
2182
  await super.onBeforeRegisterNode(nodeLogger);
2086
- nodeLogger.info("Spawning TaskNode worker (single spawn owner)", {
2183
+ nodeLogger.info("Spawning business nodes", {
2087
2184
  topic: this.options.logTopic ?? "guard",
2088
2185
  });
2089
- await this.options.spawnWorker("guard:readiness-passed");
2186
+ await this.options.spawnBusinessNodes("guard:readiness-passed");
2187
+ markWorkerSpawned();
2090
2188
  this.options.onWorkerSpawned?.();
2189
+ await this.waitUntilBusinessReady();
2190
+ await this.persistGuard({
2191
+ mode: "healthy",
2192
+ lastDecision: "bootstrap_ready",
2193
+ });
2194
+ }
2195
+ async runLoopSteps(ctx, heartbeatOk) {
2196
+ await super.runLoopSteps(ctx, heartbeatOk);
2197
+ if (this.mode !== "healthy" || !heartbeatOk) {
2198
+ return;
2199
+ }
2200
+ // Idempotent — respawn exited children without entering silent (worker stale is the runner's job).
2201
+ try {
2202
+ await this.options.spawnBusinessNodes("guard:ensure-business");
2203
+ }
2204
+ catch (error) {
2205
+ this.createGuardLogger().warn("ensure-business spawn failed", {
2206
+ topic: this.options.logTopic ?? "guard",
2207
+ data: { error: getErrorMessage(error) },
2208
+ });
2209
+ }
2210
+ }
2211
+ silentConsecutiveFailures() {
2212
+ return this.silentBackoffIndex + 1;
2213
+ }
2214
+ bumpSilentBackoff() {
2215
+ if (this.silentBackoffIndex < SILENT_BACKOFF_MS.length - 1) {
2216
+ this.silentBackoffIndex += 1;
2217
+ }
2218
+ }
2219
+ async enterSilent(reason) {
2220
+ this.mode = "silent";
2221
+ this.silentBackoffIndex = 0;
2222
+ const enteredAt = new Date().toISOString();
2223
+ this.pushEvent("enter", reason);
2224
+ await this.persistGuard({
2225
+ mode: "silent",
2226
+ lastDecision: `silent:${reason}`,
2227
+ silentEnteredAt: enteredAt,
2228
+ silentLastHeartbeatAt: enteredAt,
2229
+ silentBackoffMs: SILENT_BACKOFF_MS[0],
2230
+ silentConsecutiveFailures: 1,
2231
+ silentRecoveryAttempt: 0,
2232
+ silentEvents: this.silentEvents,
2233
+ });
2234
+ const logger = this.createGuardLogger();
2235
+ logger.warn("Entering silent mode — service unavailable until heartbeat + business nodes recover", {
2236
+ topic: this.options.logTopic ?? "guard",
2237
+ data: { reason },
2238
+ });
2239
+ try {
2240
+ await this.options.stopBusinessNodes();
2241
+ }
2242
+ catch (error) {
2243
+ logger.warn("stopBusinessNodes failed (best-effort)", {
2244
+ topic: this.options.logTopic ?? "guard",
2245
+ data: { error: getErrorMessage(error) },
2246
+ });
2247
+ }
2248
+ }
2249
+ async recoverFromSilent(ctx) {
2250
+ if (this.recoverPromise != null) {
2251
+ return this.recoverPromise;
2252
+ }
2253
+ this.recoverPromise = this.runRecover(ctx).finally(() => {
2254
+ this.recoverPromise = null;
2255
+ });
2256
+ return this.recoverPromise;
2257
+ }
2258
+ async runRecover(ctx) {
2259
+ this.mode = "recovering";
2260
+ this.pushEvent("recover_start", ctx.loopTraceId);
2261
+ await this.persistGuard({
2262
+ mode: "recovering",
2263
+ lastDecision: "recovering",
2264
+ silentLastHeartbeatAt: new Date().toISOString(),
2265
+ silentRecoveryAttempt: this.silentEvents.filter((event) => event.type === "recover_start").length,
2266
+ silentEvents: this.silentEvents,
2267
+ });
2268
+ const logger = this.createGuardLogger();
2269
+ logger.info("Heartbeat restored — recovering business nodes", {
2270
+ topic: this.options.logTopic ?? "guard",
2271
+ });
2272
+ try {
2273
+ await this.options.stopBusinessNodes();
2274
+ }
2275
+ catch (error) {
2276
+ logger.warn("stopBusinessNodes failed before respawn", {
2277
+ topic: this.options.logTopic ?? "guard",
2278
+ data: { error: getErrorMessage(error) },
2279
+ });
2280
+ }
2281
+ try {
2282
+ await this.options.spawnBusinessNodes("guard:silent-recover");
2283
+ markWorkerSpawned();
2284
+ await this.waitUntilBusinessReady();
2285
+ }
2286
+ catch (error) {
2287
+ this.pushEvent("recover_fail", getErrorMessage(error));
2288
+ this.mode = "silent";
2289
+ this.bumpSilentBackoff();
2290
+ await this.persistGuard({
2291
+ mode: "silent",
2292
+ lastDecision: "recover_fail",
2293
+ silentBackoffMs: this.getLoopIntervalMs(),
2294
+ silentEvents: this.silentEvents,
2295
+ });
2296
+ logger.warn("Recovery failed — returning to silent", {
2297
+ topic: this.options.logTopic ?? "guard",
2298
+ data: { error: getErrorMessage(error) },
2299
+ });
2300
+ try {
2301
+ await this.options.stopBusinessNodes();
2302
+ }
2303
+ catch {
2304
+ // best-effort — leftover children are stopped again on the next recover
2305
+ }
2306
+ return;
2307
+ }
2308
+ this.mode = "healthy";
2309
+ this.silentBackoffIndex = 0;
2310
+ this.pushEvent("recover_ok");
2311
+ await this.persistGuard({
2312
+ mode: "healthy",
2313
+ lastDecision: "recover_ok",
2314
+ silentEnteredAt: null,
2315
+ silentBackoffMs: 0,
2316
+ silentConsecutiveFailures: 0,
2317
+ silentEvents: this.silentEvents,
2318
+ });
2319
+ logger.success("Left silent mode — business nodes ready", {
2320
+ topic: this.options.logTopic ?? "guard",
2321
+ });
2322
+ }
2323
+ async waitUntilBusinessReady() {
2324
+ try {
2325
+ await pollUntil({
2326
+ timeoutMs: this.businessReadyTimeoutMs,
2327
+ intervalMs: 500,
2328
+ until: () => this.options.isBusinessReady(),
2329
+ });
2330
+ }
2331
+ catch {
2332
+ throw new Error(`Business nodes not ready within ${this.businessReadyTimeoutMs}ms`);
2333
+ }
2334
+ }
2335
+ pushEvent(type, detail) {
2336
+ this.silentEvents = [...this.silentEvents, { at: new Date().toISOString(), type, detail }].slice(-32);
2337
+ }
2338
+ async persistGuard(patch) {
2339
+ try {
2340
+ await this.options.guardRunner.onGuardPatch(patch);
2341
+ }
2342
+ catch {
2343
+ // best-effort — disk/runtime state must not take down the guard loop
2344
+ }
2345
+ }
2346
+ createGuardLogger() {
2347
+ return createLogger({ module: "service-guard" });
2348
+ }
2349
+ }
2350
+
2351
+ const WORKER_SCRIPT = "./dist/processes/worker.js";
2352
+ const GUARD_SCRIPT = "./dist/processes/guard.js";
2353
+ const SCHEDULER_SCRIPT = "./dist/processes/scheduler.js";
2354
+ function defaultSpawnEnv(label) {
2355
+ return { LOG_PERSIST_PROCESS: label };
2356
+ }
2357
+ /**
2358
+ * Guard-owned scheduler + worker spawn/stop, plus main-owned Guard spawn.
2359
+ * `stopBusinessNodes` must run in the Guard process (that instance's children only).
2360
+ * `stopChildProcesses` is for main shutdown (`extraPids` for Guard-spawned PIDs).
2361
+ */
2362
+ function createL3ChildLauncher(options) {
2363
+ const { childProcesses, readRuntimeState, writeRuntimeState } = options;
2364
+ const spawnEnv = options.spawnEnv ?? defaultSpawnEnv;
2365
+ function nowIso() {
2366
+ return new Date().toISOString();
2367
+ }
2368
+ async function spawnTaskWorker(reason) {
2369
+ const { child, created } = childProcesses.spawn("worker", WORKER_SCRIPT, {
2370
+ env: spawnEnv("worker"),
2371
+ });
2372
+ if (!created) {
2373
+ return child;
2374
+ }
2375
+ const current = await readRuntimeState();
2376
+ await writeRuntimeState({
2377
+ worker: {
2378
+ pid: child.pid ?? null,
2379
+ spawnedAt: nowIso(),
2380
+ ready: false,
2381
+ registeredTasks: [],
2382
+ readyAt: null,
2383
+ },
2384
+ guard: {
2385
+ lastSpawnAt: nowIso(),
2386
+ lastDecision: reason,
2387
+ spawnCount: current.guard.spawnCount + 1,
2388
+ },
2389
+ });
2390
+ return child;
2391
+ }
2392
+ function spawnGuard() {
2393
+ return childProcesses.spawn("guard", GUARD_SCRIPT, { env: spawnEnv("guard") }).child;
2394
+ }
2395
+ async function spawnScheduler(reason) {
2396
+ const { child, created } = childProcesses.spawn("scheduler", SCHEDULER_SCRIPT, {
2397
+ env: spawnEnv("scheduler"),
2398
+ });
2399
+ if (!created) {
2400
+ return child;
2401
+ }
2402
+ await writeRuntimeState({
2403
+ scheduler: {
2404
+ pid: child.pid ?? null,
2405
+ spawnedAt: nowIso(),
2406
+ ready: false,
2407
+ readyAt: null,
2408
+ },
2409
+ guard: {
2410
+ lastDecision: reason,
2411
+ },
2412
+ });
2413
+ return child;
2414
+ }
2415
+ async function spawnBusinessNodes(reason) {
2416
+ await spawnScheduler(reason);
2417
+ await spawnTaskWorker(reason);
2418
+ }
2419
+ async function stopBusinessNodes() {
2420
+ await childProcesses.stopAll();
2421
+ await writeRuntimeState({
2422
+ worker: {
2423
+ pid: null,
2424
+ ready: false,
2425
+ readyAt: null,
2426
+ registeredTasks: [],
2427
+ },
2428
+ scheduler: {
2429
+ pid: null,
2430
+ ready: false,
2431
+ readyAt: null,
2432
+ },
2433
+ });
2434
+ }
2435
+ async function isBusinessReady() {
2436
+ const state = await readRuntimeState();
2437
+ return state.worker.ready && state.scheduler.ready;
2438
+ }
2439
+ async function stopChildProcesses() {
2440
+ const state = await readRuntimeState();
2441
+ const extraPids = [state.worker.pid, state.scheduler.pid].filter((pid) => pid != null);
2442
+ await childProcesses.stopAll({ extraPids });
2443
+ }
2444
+ return {
2445
+ spawnGuard,
2446
+ spawnTaskWorker,
2447
+ spawnScheduler,
2448
+ spawnBusinessNodes,
2449
+ stopBusinessNodes,
2450
+ isBusinessReady,
2451
+ stopChildProcesses,
2452
+ };
2453
+ }
2454
+
2455
+ /**
2456
+ * Claim a registry slot, or `process.exit(1)` when no EMPTY slot remains.
2457
+ * Other errors propagate.
2458
+ */
2459
+ async function claimRegistrySlotOrExit(input) {
2460
+ try {
2461
+ return await claimServiceRegistrySlot({
2462
+ serviceValue: input.serviceValue,
2463
+ createInstance: input.createInstance,
2464
+ });
2465
+ }
2466
+ catch (error) {
2467
+ if (error instanceof ServiceRegistryError && error.code === "SERVICE_SLOTS_FULL") {
2468
+ input.logger.critical("No EMPTY registry slot — refusing to start", {
2469
+ topic: input.logTopic,
2470
+ data: { serviceValue: input.serviceValue, code: error.code },
2471
+ });
2472
+ process.exit(1);
2473
+ }
2474
+ throw error;
2091
2475
  }
2092
2476
  }
2093
2477
 
@@ -2146,22 +2530,12 @@ function createServiceHost(options) {
2146
2530
  if (options.prepare != null) {
2147
2531
  await options.prepare();
2148
2532
  }
2149
- try {
2150
- session = await claimServiceRegistrySlot({
2151
- serviceValue: options.serviceValue,
2152
- createInstance: async () => unwrapService(await options.createInstance()),
2153
- });
2154
- }
2155
- catch (error) {
2156
- if (error instanceof ServiceRegistryError && error.code === "SERVICE_SLOTS_FULL") {
2157
- logger.critical("No EMPTY registry slot — refusing to start", {
2158
- topic: logTopic,
2159
- data: { serviceValue: options.serviceValue, code: error.code },
2160
- });
2161
- process.exit(1);
2162
- }
2163
- throw error;
2164
- }
2533
+ session = await claimRegistrySlotOrExit({
2534
+ serviceValue: options.serviceValue,
2535
+ createInstance: async () => unwrapService(await options.createInstance()),
2536
+ logger,
2537
+ logTopic,
2538
+ });
2165
2539
  const hostCtx = { service: session.service, session, serviceId: session.service.id };
2166
2540
  if (options.onRegistryClaimed != null) {
2167
2541
  await options.onRegistryClaimed(hostCtx);
@@ -2220,23 +2594,12 @@ async function runSingleProcessService(options) {
2220
2594
  if (options.prepare != null) {
2221
2595
  await options.prepare();
2222
2596
  }
2223
- let session = null;
2224
- try {
2225
- session = await claimServiceRegistrySlot({
2226
- serviceValue: options.serviceValue,
2227
- createInstance: options.createInstance,
2228
- });
2229
- }
2230
- catch (error) {
2231
- if (error instanceof ServiceRegistryError && error.code === "SERVICE_SLOTS_FULL") {
2232
- logger.critical("No EMPTY registry slot — refusing to start", {
2233
- topic: logTopic,
2234
- data: { serviceValue: options.serviceValue, code: error.code },
2235
- });
2236
- process.exit(1);
2237
- }
2238
- throw error;
2239
- }
2597
+ const session = await claimRegistrySlotOrExit({
2598
+ serviceValue: options.serviceValue,
2599
+ createInstance: options.createInstance,
2600
+ logger,
2601
+ logTopic,
2602
+ });
2240
2603
  try {
2241
2604
  await options.run({ service: session.service, session });
2242
2605
  }
@@ -3673,5 +4036,5 @@ const patchTriggerFired = validateWithSchema(patchTriggerFiredSchema, "patchTrig
3673
4036
  }
3674
4037
  });
3675
4038
 
3676
- export { BaseNodeRuntime, COLLECT_LOG_RUNNER_KEY, CategoryCommand, CategoryNode, CategoryRepo, CategoryTrigger, CommandType, EMPTY_REGISTRY_SLOT_RUNTIME_STATE, LOG_SPOOL_SERVICE_ID_ENV, LOG_TOPIC_TRIGGER, LogLevel, MAX_POLL_TARGET_LIST_SIZE, ManagedChildProcesses, NodeStatus, RepoManager, ResultCode, SERVICE_GUARD_RUNNER_KEY, ServiceGuardNode, ServiceRegistryError, TASK_LOCAL_PACKAGE_CONFIG_FILENAME, TASK_REPO_SCRIPT_CATEGORY, TASK_REPO_USAGE, TASK_RUNNER_CONFIG_DIR, TASK_RUNNER_ROOT_CONFIG_FILENAME, TASK_RUNNER_ROOT_CONFIG_LEGACY_RELATIVE_PATH, TASK_RUNNER_ROOT_CONFIG_RELATIVE_PATH, TRIGGER_LOOP_INTERVAL_MS, TRIGGER_RUNNER_DEFAULT_RETRY_COUNT, TRIGGER_RUNNER_DEFAULT_RETRY_DELAY_MS, TaskManager, TaskNode, TaskRepoValidation, TaskStatus, TaskStatusAction, TriggerManager, TriggerNode, TriggerStatus, applyRegistrySlotGuardStep, buildNodeImportArgs, claimServiceRegistrySlot, createJsonFileStateStore, createLogSpoolCoordinator, createLogger, createManagedChildProcesses, createPathsExistCheck, createRequiredEnvCheck, createServiceHost, createServiceRuntimeStateStore, createSupabaseReachableCheck, createTarget, enableLogSpoolFromEnvInChild, ensureLogSpoolFromEnv, evaluateBusyNodeLiveness, generateResponse, getConfigFileDir, getLogSpoolStats, getMainLogSpoolWriterStats, getPollCommandList, getPollCommandListSchema, getPossibleTarget, getScanRemoteRepoValues, getWorkerSpawnCooldownLastAt, importJsConfigModule, initSupabaseFromStandardEnv, isChildProcessRunning, isLogEntry, isLogSpoolEnabled, isOptimisticLockResponse, loadCachedJsConfigModule, loadEnvFiles, logManager, logSpoolEnabledFromEnv, markWorkerSpawned, nodeBufferToArrayBuffer, patchChangeNodeStatus, patchChangeTaskStatus, patchClaimTask, patchNodeHeartBeat, patchServiceRuntime, patchStopNode, patchTriggerFired, patchTriggerFiredSchema, pathExists, pollTargetList, pollUntil, postCommand, postCommandSchema, postRegisterNode, postTaskSchema, postTaskWithValidation, postTrigger, postTriggerSchema, publicBaseUrlFromEnv, readAndVerifySourceFile, readEnv, registerCollectLogRunner, registerServiceGuardRunner, registrySlotRuntimePatchFromGuardResult, requireEnv, resolveFirstExistingPath, resolveLogSpoolRoot, resolvePathFromBaseDir, resolvePathFromConfigFile, resolvePathFromCwd, runCollectLogTick, runReadinessChecks, runReadinessGate, runRegistrySlotGuardCheck, runServiceGuardTick, runSingleProcessService, scanEnabledTriggers, scanEnabledTriggersSchema, scanTargetList, sha256Hex, shutdownLogSpool, shutdownLogSpoolFromEnv, spawnTsxChild, supabase, toFileImportHref, updateTargetDetails, validateTaskRepoAndParams, validateWithSchema, waitForServiceReady };
4039
+ export { BaseNodeRuntime, COLLECT_LOG_RUNNER_KEY, CategoryCommand, CategoryNode, CategoryRepo, CategoryTrigger, CommandType, EMPTY_REGISTRY_SLOT_RUNTIME_STATE, LOG_SPOOL_SERVICE_ID_ENV, LOG_TOPIC_TRIGGER, LogLevel, MAX_POLL_TARGET_LIST_SIZE, ManagedChildProcesses, NodeStatus, RepoManager, ResultCode, SERVICE_GUARD_RUNNER_KEY, ServiceGuardNode, ServiceRegistryError, TASK_LOCAL_PACKAGE_CONFIG_FILENAME, TASK_REPO_SCRIPT_CATEGORY, TASK_REPO_USAGE, TASK_RUNNER_CONFIG_DIR, TASK_RUNNER_ROOT_CONFIG_FILENAME, TASK_RUNNER_ROOT_CONFIG_LEGACY_RELATIVE_PATH, TASK_RUNNER_ROOT_CONFIG_RELATIVE_PATH, TRIGGER_LOOP_INTERVAL_MS, TRIGGER_RUNNER_DEFAULT_RETRY_COUNT, TRIGGER_RUNNER_DEFAULT_RETRY_DELAY_MS, TaskManager, TaskNode, TaskRepoValidation, TaskStatus, TaskStatusAction, TriggerManager, TriggerNode, TriggerStatus, applyRegistrySlotGuardStep, buildNodeImportArgs, claimServiceRegistrySlot, createJsonFileStateStore, createL3ChildLauncher, createLogSpoolCoordinator, createLogger, createManagedChildProcesses, createPathsExistCheck, createRequiredEnvCheck, createServiceHost, createServiceRuntimeStateStore, createSupabaseReachableCheck, createTarget, enableLogSpoolFromEnvInChild, ensureLogSpoolFromEnv, evaluateBusyNodeLiveness, generateResponse, getConfigFileDir, getLogSpoolStats, getMainLogSpoolWriterStats, getPollCommandList, getPollCommandListSchema, getPossibleTarget, getScanRemoteRepoValues, guardRetryAfterSec, importJsConfigModule, initSupabaseFromStandardEnv, isChildProcessRunning, isGuardAvailable, isLogEntry, isLogSpoolEnabled, isOptimisticLockResponse, loadCachedJsConfigModule, loadEnvFiles, logManager, logSpoolEnabledFromEnv, markWorkerSpawned, nodeBufferToArrayBuffer, patchChangeNodeStatus, patchChangeTaskStatus, patchClaimTask, patchNodeHeartBeat, patchServiceRuntime, patchStopNode, patchTriggerFired, patchTriggerFiredSchema, pathExists, pollTargetList, pollUntil, postCommand, postCommandSchema, postRegisterNode, postTaskSchema, postTaskWithValidation, postTrigger, postTriggerSchema, publicBaseUrlFromEnv, readAndVerifySourceFile, readEnv, registerCollectLogRunner, registerServiceGuardRunner, registrySlotRuntimePatchFromGuardResult, requireEnv, resolveFirstExistingPath, resolveLogSpoolRoot, resolvePathFromBaseDir, resolvePathFromConfigFile, resolvePathFromCwd, runCollectLogTick, runReadinessChecks, runReadinessGate, runRegistrySlotGuardCheck, runServiceGuardTick, runSingleProcessService, scanEnabledTriggers, scanEnabledTriggersSchema, scanTargetList, sha256Hex, shutdownLogSpool, shutdownLogSpoolFromEnv, spawnTsxChild, supabase, toFileImportHref, updateTargetDetails, validateTaskRepoAndParams, validateWithSchema, waitForServiceReady };
3677
4040
  //# sourceMappingURL=node.js.map