target-supabase-sdk 0.2.13 → 0.2.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +1 -1
- package/dist/chunks/{enable-DqnfLPsQ.js → enable-C7lFooTG.js} +2 -2
- package/dist/chunks/{enable-DqnfLPsQ.js.map → enable-C7lFooTG.js.map} +1 -1
- package/dist/chunks/{task-queue-6X3aglJh.js → task-queue-qSnsfiIg.js} +5 -3
- package/dist/chunks/task-queue-qSnsfiIg.js.map +1 -0
- package/dist/node/env/index.d.ts +1 -1
- package/dist/node/env/index.d.ts.map +1 -1
- package/dist/node/node-runtime.base.d.ts +14 -1
- package/dist/node/node-runtime.base.d.ts.map +1 -1
- package/dist/node/node.api.d.ts.map +1 -1
- package/dist/node/node.constant.d.ts +3 -0
- package/dist/node/node.constant.d.ts.map +1 -0
- package/dist/node/node.utils.d.ts +0 -2
- package/dist/node/node.utils.d.ts.map +1 -1
- package/dist/node/readiness/index.d.ts +2 -0
- package/dist/node/readiness/index.d.ts.map +1 -1
- package/dist/node/{service-guard → readiness}/run-readiness-gate.d.ts +1 -1
- package/dist/node/readiness/run-readiness-gate.d.ts.map +1 -0
- package/dist/node/readiness/service-ready-gate.d.ts.map +1 -1
- package/dist/node/runtime-state/create-service-runtime-state-store.d.ts.map +1 -1
- package/dist/node/runtime-state/service-runtime-state.types.d.ts +16 -0
- package/dist/node/runtime-state/service-runtime-state.types.d.ts.map +1 -1
- package/dist/node/service-guard/index.d.ts +4 -4
- package/dist/node/service-guard/index.d.ts.map +1 -1
- package/dist/node/service-guard/register-service-guard-runner.d.ts +1 -2
- package/dist/node/service-guard/register-service-guard-runner.d.ts.map +1 -1
- package/dist/node/service-guard/registry-slot-guard-step.d.ts.map +1 -0
- package/dist/node/service-guard/run-service-guard-tick.d.ts.map +1 -1
- package/dist/node/service-guard/service-guard-node.d.ts +25 -2
- package/dist/node/service-guard/service-guard-node.d.ts.map +1 -1
- package/dist/node/service-guard/service-guard.interface.d.ts +25 -10
- package/dist/node/service-guard/service-guard.interface.d.ts.map +1 -1
- package/dist/node/service-host/claim-registry-slot.d.ts +15 -0
- package/dist/node/service-host/claim-registry-slot.d.ts.map +1 -0
- package/dist/node/service-host/create-l3-child-launcher.d.ts +38 -0
- package/dist/node/service-host/create-l3-child-launcher.d.ts.map +1 -0
- package/dist/node/service-host/index.d.ts +2 -5
- package/dist/node/service-host/index.d.ts.map +1 -1
- package/dist/node/service-host/service-host.d.ts +3 -3
- package/dist/node/service-host/service-host.d.ts.map +1 -1
- package/dist/node/service-host/single-process-service.d.ts +3 -3
- package/dist/node/service-host/single-process-service.d.ts.map +1 -1
- package/dist/node.d.ts +8 -6
- package/dist/node.d.ts.map +1 -1
- package/dist/node.js +627 -264
- package/dist/node.js.map +1 -1
- package/dist/preload.js +1 -1
- package/package.json +1 -1
- package/src/node/env/index.ts +0 -4
- package/src/node/node-runtime.base.ts +33 -5
- package/src/node/node.api.ts +1 -1
- package/src/node/node.constant.ts +2 -0
- package/src/node/node.utils.ts +2 -2
- package/src/node/readiness/index.ts +2 -0
- package/src/node/{service-guard → readiness}/run-readiness-gate.ts +2 -2
- package/src/node/readiness/service-ready-gate.ts +27 -23
- package/src/node/runtime-state/create-service-runtime-state-store.ts +11 -0
- package/src/node/runtime-state/service-runtime-state.types.ts +12 -0
- package/src/node/service-guard/index.ts +7 -8
- package/src/node/service-guard/register-service-guard-runner.ts +0 -2
- package/src/node/service-guard/run-service-guard-tick.ts +5 -2
- package/src/node/service-guard/service-guard-node.ts +229 -6
- package/src/node/service-guard/service-guard.interface.ts +35 -10
- package/src/node/service-host/claim-registry-slot.ts +33 -0
- package/src/node/service-host/create-l3-child-launcher.ts +157 -0
- package/src/node/service-host/index.ts +5 -24
- package/src/node/service-host/service-host.ts +11 -18
- package/src/node/service-host/single-process-service.ts +10 -20
- package/src/node.ts +21 -18
- package/dist/chunks/task-queue-6X3aglJh.js.map +0 -1
- package/dist/node/service-guard/run-readiness-gate.d.ts.map +0 -1
- package/dist/node/service-host/registry-slot-guard-step.d.ts.map +0 -1
- /package/dist/node/{service-host → service-guard}/registry-slot-guard-step.d.ts +0 -0
- /package/src/node/{service-host → service-guard}/registry-slot-guard-step.ts +0 -0
package/dist/node.js
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { M as MAX_POLL_TARGET_LIST_SIZE, V as validateWithSchema, g as createLogger, z as pollTargetList, k as generateResponse, h as createTarget, K as supabase, w as logManager, X as createScope, d as LogLevel, Y as patchScope, J as scanTargetList, Z as withModule, l as getPossibleTarget, t as isOptimisticLockResponse, R as updateTargetDetails } from './chunks/list.api-BzbTJBsM.js';
|
|
3
3
|
export { B as BaseValidator, C as CREATE_TARGET_ALREADY_EXISTS_MESSAGE, a as CategoryList, L as LOG_BATCH_LOADER_KEY, b as LOG_MIN_LEVEL_ENV_KEY, c as LOG_PERSIST_LOADER_KEY, e as MAX_TARGET_LIST_PAGE_SIZE, O as OPTIMISTIC_LOCK_ERROR_CODE, f as OPTIMISTIC_LOCK_FAILED_MESSAGE, P as PostListValidator, S as StatusCode, U as UPDATE_TARGET_NOT_FOUND_MESSAGE, i as deleteTarget, j as formatZodError, m as getTarget, n as getTargetList, o as getTargetTotalCount, p as handleSupabaseError, q as isCreateTargetAlreadyExistsError, r as isDevEnvironment, s as isOptimisticLockError, u as logDev, v as logDevGroup, x as logMinLevelFromEnv, y as patchTarget, A as postListCreate, D as postLogListCreate, E as postTarget, F as postTargetPayloadSchema, G as queryFilterSchema, H as resolveLogMinLevel, I as safeParseWithSchema, N as targetDraftSchema, Q as updateTarget, T as validateWith } from './chunks/list.api-BzbTJBsM.js';
|
|
4
|
-
import { l as loadEnvFiles, b as readEnv, c as requireEnv, d as resolveLogSpoolSpawnEnvForLabel, e as resolveLogSpoolConfig, f as resolveLogSpoolRoot, g as resolveAllLogSpoolProcessRoles, h as logSpoolProcessDir, s as spoolLogger, i as parseLogSpoolBatchFilename, j as computeLogBatchIdempotencyKey, k as buildLogListDraft, n as postLogBatch, a as logSpoolEnabledFromEnv, o as logSpoolConfigFromEnv, L as LOG_SPOOL_SERVICE_ID_ENV, q as ensureLogSpoolFromEnv, t as shutdownLogSpoolFromEnv, u as getLogSpoolStats } from './chunks/enable-
|
|
5
|
-
export { C as CLI_PROD_FLAG, S as SERVICE_ENV_PROFILE_ENV_KEY, w as SERVICE_ENV_PROFILE_PROD, x as buildLogSpoolSpawnEnv, y as envBool, z as envInt, A as envMs, B as envNumber, D as envPort, E as envProfileFromProcess, F as
|
|
4
|
+
import { l as loadEnvFiles, b as readEnv, c as requireEnv, d as resolveLogSpoolSpawnEnvForLabel, e as resolveLogSpoolConfig, f as resolveLogSpoolRoot, g as resolveAllLogSpoolProcessRoles, h as logSpoolProcessDir, s as spoolLogger, i as parseLogSpoolBatchFilename, j as computeLogBatchIdempotencyKey, k as buildLogListDraft, n as postLogBatch, a as logSpoolEnabledFromEnv, o as logSpoolConfigFromEnv, L as LOG_SPOOL_SERVICE_ID_ENV, q as ensureLogSpoolFromEnv, t as shutdownLogSpoolFromEnv, u as getLogSpoolStats } from './chunks/enable-C7lFooTG.js';
|
|
5
|
+
export { C as CLI_PROD_FLAG, S as SERVICE_ENV_PROFILE_ENV_KEY, w as SERVICE_ENV_PROFILE_PROD, x as buildLogSpoolSpawnEnv, y as envBool, z as envInt, A as envMs, B as envNumber, D as envPort, E as envProfileFromProcess, F as resolveDefaultEnvFiles, r as resolveProjectRootByPackageName, G as resolveProjectRootFromModule, v as validateLogSpoolPreloadEnv } from './chunks/enable-C7lFooTG.js';
|
|
6
6
|
import { randomBytes, createHash } from 'node:crypto';
|
|
7
7
|
import { readFile, unlink, mkdir, writeFile, rename, access, stat, readdir, constants } from 'node:fs/promises';
|
|
8
8
|
import { dirname, resolve, join, isAbsolute } from 'node:path';
|
|
9
|
-
import { bh as sha256Hex, bm as getErrorMessage, aL as postRegisterNode, ap as patchChangeNodeStatus, N as NodeStatus, aA as patchStopNode, ax as patchNodeHeartBeat, E as EMPTY_REGISTRY_SLOT_RUNTIME_STATE,
|
|
10
|
-
export { A as ApiMethod, C as CategoryConfig, a as CategoryExtraction, b as CategoryLink, d as CategoryParcel, f as CategoryService, g as CategoryTask, D as DEFAULT_SYSTEM_REGISTRY_SEED_SLOTS, L as LINK_MANIFEST_VERSION, h as LIST_MANIFEST_VERSION, P as POST_TASK_DEFAULT_MAX_ATTEMPTS, i as ParcelManager, j as PostLinkValidator, S as ServiceLifecycleStatus, l as ServiceSlotStatus, T as TARGET_SYSTEM_REGISTRY_KEY, p as apiDetailsSchema, q as appendSystemRegistrySlots, r as assertRegistrySlotAvailable, s as assertRegistrySlotOwner, t as buildEmptyServiceSlots, u as buildLinkTargetDraft, v as buildListTargetDraft, w as buildSystemRegistryConfigDetails, y as classifyNetworkError, z as countTasksByType, B as createActiveServiceLifecycle, F as createClaimedRegistrySlotRuntimeState, G as createStorageProviderRegistry, H as createUploadTracker, I as decodeLogBatch, J as decodeLogBatchList, K as defaultL3ServiceDetails, M as deleteParcel, O as deleteParcelSchema, U as fetchBinaryWithRetry, V as fetchWithRetry, W as fieldDefinitionSchema, X as flattenLogBatch, Y as flattenLogBatches, Z as formatNetworkError, _ as getApi, $ as getApiSchema, a0 as getConfig, a1 as getConfigSchema, a2 as getExtraction, a3 as getExtractionSchema, a4 as getParcel, a5 as getParcelSchema, a7 as getScanRemoteRepoValuesSchema, a8 as getService, a9 as getServiceSchema, aa as getTargetSystemRegistry, ab as getValueAtPath, ac as installChunkFetchRegistry, ad as isHttpUrl, ae as isLocalFilesystemPath, af as isLogBatchList, ah as isOpaqueChunkUrl, ai as isRetryableHttpStatus, aj as linkTargetDraftBuildInputSchema, ak as parcelDetailsSchema, al as parseLogBatchMeta, am as parseProviderPrefixedUrl, an as parseServiceSlot, ao as parseServiceSlots, aq as patchChangeNodeStatusSchema, as as patchChangeTaskStatusSchema, au as patchClaimTaskSchema, av as patchExtractionObjects, aw as patchExtractionObjectsSchema, ay as patchNodeHeartBeatSchema, aB as patchStopNodeSchema, aC as patchTaskProgress, aD as patchTaskProgressSchema, aE as postApi, aF as postApiSchema, aG as postExtraction, aH as postExtractionSchema, aI as postLinkCreate, aJ as postParcel, aK as postParcelSchema, aM as postRegisterNodeSchema, aN as postService, aO as postServiceInstance, aP as postServiceSchema, aQ as postSystemRegistryConfig, aR as postSystemRegistryConfigSchema, aS as postTask, aU as postTaskWithRetry, aV as publishParcel, aW as registerProviderChunkResolver, aX as registerService, aY as registerServiceAtStartup, a_ as releaseSystemRegistrySlots, a$ as releaseSystemRegistrySlotsByServiceId, b0 as requestHasBody, b1 as resetSystemRegistryConfig, b2 as resetSystemRegistryConfigSchema, b3 as resolveActiveRegistryServiceId, b4 as resolveFetchUrl, b5 as restoreParcel, b6 as restoreParcelById, b8 as sanitizeFileName, b9 as sanitizeParcelFileName, ba as scanExtractionList, bb as scanExtractionListSchema, bc as schemaDefinitionSchema, bd as serviceDetailsSchema, be as serviceLifecycleSchema, bf as serviceNodeSnapshotSchema, bg as serviceRuntimeSchema, bi as summarizeTaskQueue, bj as systemRegistrySeedSlotSchema, bk as unregisterService, bl as unregisterServiceAtShutdown } from './chunks/task-queue-
|
|
9
|
+
import { bh as sha256Hex, bm as getErrorMessage, bn as LOG_TOPIC_NODE, aL as postRegisterNode, ap as patchChangeNodeStatus, N as NodeStatus, aA as patchStopNode, ax as patchNodeHeartBeat, E as EMPTY_REGISTRY_SLOT_RUNTIME_STATE, b7 as runRegistrySlotGuardCheck, aZ as registrySlotRuntimePatchFromGuardResult, c as CategoryNode, Q as evaluateBusyNodeLiveness, az as patchServiceRuntime, ag as isLogEntry, x as claimServiceRegistrySlot, k as ServiceRegistryError, e as CategoryRepo, R as ResultCode, bo as LOG_TOPIC_TASK, m as TASK_REPO_USAGE, a6 as getScanRemoteRepoValues, at as patchClaimTask, bp as CRITICAL_UNKNOWN_TASK_TRACE_ID, ar as patchChangeTaskStatus, o as TaskStatusAction, bq as toError, aT as postTaskSchema, br as createTaskRow, n as TaskStatus } from './chunks/task-queue-qSnsfiIg.js';
|
|
10
|
+
export { A as ApiMethod, C as CategoryConfig, a as CategoryExtraction, b as CategoryLink, d as CategoryParcel, f as CategoryService, g as CategoryTask, D as DEFAULT_SYSTEM_REGISTRY_SEED_SLOTS, L as LINK_MANIFEST_VERSION, h as LIST_MANIFEST_VERSION, P as POST_TASK_DEFAULT_MAX_ATTEMPTS, i as ParcelManager, j as PostLinkValidator, S as ServiceLifecycleStatus, l as ServiceSlotStatus, T as TARGET_SYSTEM_REGISTRY_KEY, p as apiDetailsSchema, q as appendSystemRegistrySlots, r as assertRegistrySlotAvailable, s as assertRegistrySlotOwner, t as buildEmptyServiceSlots, u as buildLinkTargetDraft, v as buildListTargetDraft, w as buildSystemRegistryConfigDetails, y as classifyNetworkError, z as countTasksByType, B as createActiveServiceLifecycle, F as createClaimedRegistrySlotRuntimeState, G as createStorageProviderRegistry, H as createUploadTracker, I as decodeLogBatch, J as decodeLogBatchList, K as defaultL3ServiceDetails, M as deleteParcel, O as deleteParcelSchema, U as fetchBinaryWithRetry, V as fetchWithRetry, W as fieldDefinitionSchema, X as flattenLogBatch, Y as flattenLogBatches, Z as formatNetworkError, _ as getApi, $ as getApiSchema, a0 as getConfig, a1 as getConfigSchema, a2 as getExtraction, a3 as getExtractionSchema, a4 as getParcel, a5 as getParcelSchema, a7 as getScanRemoteRepoValuesSchema, a8 as getService, a9 as getServiceSchema, aa as getTargetSystemRegistry, ab as getValueAtPath, ac as installChunkFetchRegistry, ad as isHttpUrl, ae as isLocalFilesystemPath, af as isLogBatchList, ah as isOpaqueChunkUrl, ai as isRetryableHttpStatus, aj as linkTargetDraftBuildInputSchema, ak as parcelDetailsSchema, al as parseLogBatchMeta, am as parseProviderPrefixedUrl, an as parseServiceSlot, ao as parseServiceSlots, aq as patchChangeNodeStatusSchema, as as patchChangeTaskStatusSchema, au as patchClaimTaskSchema, av as patchExtractionObjects, aw as patchExtractionObjectsSchema, ay as patchNodeHeartBeatSchema, aB as patchStopNodeSchema, aC as patchTaskProgress, aD as patchTaskProgressSchema, aE as postApi, aF as postApiSchema, aG as postExtraction, aH as postExtractionSchema, aI as postLinkCreate, aJ as postParcel, aK as postParcelSchema, aM as postRegisterNodeSchema, aN as postService, aO as postServiceInstance, aP as postServiceSchema, aQ as postSystemRegistryConfig, aR as postSystemRegistryConfigSchema, aS as postTask, aU as postTaskWithRetry, aV as publishParcel, aW as registerProviderChunkResolver, aX as registerService, aY as registerServiceAtStartup, a_ as releaseSystemRegistrySlots, a$ as releaseSystemRegistrySlotsByServiceId, b0 as requestHasBody, b1 as resetSystemRegistryConfig, b2 as resetSystemRegistryConfigSchema, b3 as resolveActiveRegistryServiceId, b4 as resolveFetchUrl, b5 as restoreParcel, b6 as restoreParcelById, b8 as sanitizeFileName, b9 as sanitizeParcelFileName, ba as scanExtractionList, bb as scanExtractionListSchema, bc as schemaDefinitionSchema, bd as serviceDetailsSchema, be as serviceLifecycleSchema, bf as serviceNodeSnapshotSchema, bg as serviceRuntimeSchema, bi as summarizeTaskQueue, bj as systemRegistrySeedSlotSchema, bk as unregisterService, bl as unregisterServiceAtShutdown } from './chunks/task-queue-qSnsfiIg.js';
|
|
11
11
|
import { spawn } from 'node:child_process';
|
|
12
12
|
import { tmpdir } from 'node:os';
|
|
13
13
|
import { pathToFileURL } from 'node:url';
|
|
@@ -288,7 +288,6 @@ const formatHeartbeat = (timestamp) => {
|
|
|
288
288
|
return `${absolute} (${relative})`;
|
|
289
289
|
};
|
|
290
290
|
|
|
291
|
-
const LOG_TOPIC_NODE = "node";
|
|
292
291
|
const LOG_TOPIC_COMMAND = "command";
|
|
293
292
|
const LOG_TOPIC_PROCESS = "process";
|
|
294
293
|
/**
|
|
@@ -357,6 +356,22 @@ class BaseNodeRuntime {
|
|
|
357
356
|
getLoopIntervalMs() {
|
|
358
357
|
return getRandomInterval();
|
|
359
358
|
}
|
|
359
|
+
/**
|
|
360
|
+
* When true, the main loop skips commands and {@link runLoopSteps} (heartbeat only).
|
|
361
|
+
* Used by the L3 guard process while waiting for connectivity to recover.
|
|
362
|
+
*/
|
|
363
|
+
isSilenced() {
|
|
364
|
+
return false;
|
|
365
|
+
}
|
|
366
|
+
/**
|
|
367
|
+
* Consecutive heartbeat failures reached {@link HEARTBEAT_FAILURE_THRESHOLD}.
|
|
368
|
+
* Default: shut down the node. Override to wait for recovery instead of exiting.
|
|
369
|
+
*/
|
|
370
|
+
async onHealthCheckFailed(_ctx) {
|
|
371
|
+
this.requestShutdown("heartbeat:consecutive-failures");
|
|
372
|
+
}
|
|
373
|
+
/** Heartbeat succeeded while {@link isSilenced} is true. Default: no-op. */
|
|
374
|
+
async onHeartbeatRecovered(_ctx) { }
|
|
360
375
|
getLogNodeId() {
|
|
361
376
|
return this.localNodeId ?? BaseNodeRuntime.NODE_ID_PENDING_ASSIGNMENT;
|
|
362
377
|
}
|
|
@@ -513,7 +528,9 @@ class BaseNodeRuntime {
|
|
|
513
528
|
data: { nodeId, loopCount: this.loopCount },
|
|
514
529
|
});
|
|
515
530
|
try {
|
|
516
|
-
|
|
531
|
+
if (!this.isSilenced()) {
|
|
532
|
+
await this.batchCommand(loopCtx);
|
|
533
|
+
}
|
|
517
534
|
if (this.isShuttingDown()) {
|
|
518
535
|
iterLogger.debug("節點關閉中,跳過本輪剩餘步驟", { topic: LOG_TOPIC_NODE });
|
|
519
536
|
break;
|
|
@@ -523,7 +540,9 @@ class BaseNodeRuntime {
|
|
|
523
540
|
iterLogger.debug("節點關閉中,跳過本輪剩餘步驟", { topic: LOG_TOPIC_NODE });
|
|
524
541
|
break;
|
|
525
542
|
}
|
|
526
|
-
|
|
543
|
+
if (!this.isSilenced()) {
|
|
544
|
+
await this.runLoopSteps(loopCtx, heartbeatOk);
|
|
545
|
+
}
|
|
527
546
|
}
|
|
528
547
|
catch (error) {
|
|
529
548
|
iterLogger.critical(`主循環第 ${this.loopCount} 輪遇到未知錯誤`, {
|
|
@@ -635,11 +654,11 @@ class BaseNodeRuntime {
|
|
|
635
654
|
threshold: BaseNodeRuntime.HEARTBEAT_FAILURE_THRESHOLD,
|
|
636
655
|
};
|
|
637
656
|
if (failureCount >= BaseNodeRuntime.HEARTBEAT_FAILURE_THRESHOLD) {
|
|
638
|
-
logger.critical("
|
|
657
|
+
logger.critical("連續心跳失敗達上限", {
|
|
639
658
|
topic: LOG_TOPIC_NODE,
|
|
640
659
|
data: logData,
|
|
641
660
|
});
|
|
642
|
-
this.
|
|
661
|
+
await this.onHealthCheckFailed(ctx);
|
|
643
662
|
}
|
|
644
663
|
else {
|
|
645
664
|
logger.warn("心跳更新失敗", { topic: LOG_TOPIC_NODE, data: logData });
|
|
@@ -647,6 +666,9 @@ class BaseNodeRuntime {
|
|
|
647
666
|
return false;
|
|
648
667
|
}
|
|
649
668
|
this.consecutiveHeartbeatFailures = 0;
|
|
669
|
+
if (this.isSilenced()) {
|
|
670
|
+
await this.onHeartbeatRecovered(ctx);
|
|
671
|
+
}
|
|
650
672
|
// Success is the steady state every loop — do not INFO-spam; failures already warn/critical.
|
|
651
673
|
logger.debug("心跳更新成功", {
|
|
652
674
|
topic: LOG_TOPIC_NODE,
|
|
@@ -901,7 +923,7 @@ function createManagedChildProcesses(options) {
|
|
|
901
923
|
});
|
|
902
924
|
}
|
|
903
925
|
|
|
904
|
-
function sleep
|
|
926
|
+
function sleep(ms) {
|
|
905
927
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
906
928
|
}
|
|
907
929
|
/**
|
|
@@ -915,7 +937,7 @@ async function pollUntil(options) {
|
|
|
915
937
|
if (await options.until()) {
|
|
916
938
|
return;
|
|
917
939
|
}
|
|
918
|
-
await sleep
|
|
940
|
+
await sleep(intervalMs);
|
|
919
941
|
}
|
|
920
942
|
throw new Error(`pollUntil timeout after ${options.timeoutMs}ms`);
|
|
921
943
|
}
|
|
@@ -989,9 +1011,26 @@ async function runReadinessChecks(checks) {
|
|
|
989
1011
|
};
|
|
990
1012
|
}
|
|
991
1013
|
|
|
992
|
-
|
|
993
|
-
|
|
1014
|
+
/** Run readiness checks, persist report via callback, fail-fast when not ok. */
|
|
1015
|
+
async function runReadinessGate(input) {
|
|
1016
|
+
const { checks, onReport, logger, logTopic = "guard" } = input;
|
|
1017
|
+
logger.info("Running readiness checks", { topic: logTopic });
|
|
1018
|
+
const report = await runReadinessChecks(checks);
|
|
1019
|
+
await onReport(report);
|
|
1020
|
+
if (!report.ok) {
|
|
1021
|
+
logger.error("Readiness checks failed", {
|
|
1022
|
+
topic: logTopic,
|
|
1023
|
+
data: { message: report.message, checks: report.checks },
|
|
1024
|
+
});
|
|
1025
|
+
throw new Error(report.message ?? "Readiness checks failed");
|
|
1026
|
+
}
|
|
1027
|
+
logger.debug("Readiness checks passed", {
|
|
1028
|
+
topic: logTopic,
|
|
1029
|
+
data: { checks: report.checks.map((check) => check.name) },
|
|
1030
|
+
});
|
|
1031
|
+
return report;
|
|
994
1032
|
}
|
|
1033
|
+
|
|
995
1034
|
/**
|
|
996
1035
|
* Poll {@link ServiceReadyGate} until ready, fail fast on `failed`, or timeout.
|
|
997
1036
|
* Returns the final ready snapshot.
|
|
@@ -999,25 +1038,37 @@ function sleep(ms) {
|
|
|
999
1038
|
async function waitForServiceReady(gate, options) {
|
|
1000
1039
|
const pollMs = options.pollMs ?? 500;
|
|
1001
1040
|
const defaultFailureMessage = options.failureMessage ?? "Readiness checks failed";
|
|
1002
|
-
const deadline = Date.now() + options.timeoutMs;
|
|
1003
1041
|
let last = { failed: false, ready: false };
|
|
1004
|
-
|
|
1005
|
-
|
|
1042
|
+
try {
|
|
1043
|
+
await pollUntil({
|
|
1044
|
+
timeoutMs: options.timeoutMs,
|
|
1045
|
+
intervalMs: pollMs,
|
|
1046
|
+
until: async () => {
|
|
1047
|
+
last = await gate.read();
|
|
1048
|
+
if (last.failed) {
|
|
1049
|
+
throw new Error(last.failureMessage ?? defaultFailureMessage);
|
|
1050
|
+
}
|
|
1051
|
+
return last.ready;
|
|
1052
|
+
},
|
|
1053
|
+
});
|
|
1054
|
+
}
|
|
1055
|
+
catch (error) {
|
|
1006
1056
|
if (last.failed) {
|
|
1007
|
-
throw new Error(
|
|
1057
|
+
throw error instanceof Error ? error : new Error(defaultFailureMessage);
|
|
1008
1058
|
}
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
data: last.readyDetail ?? {},
|
|
1013
|
-
});
|
|
1014
|
-
return last;
|
|
1059
|
+
const isPollTimeout = error instanceof Error && error.message.startsWith("pollUntil timeout after");
|
|
1060
|
+
if (!isPollTimeout) {
|
|
1061
|
+
throw error;
|
|
1015
1062
|
}
|
|
1016
|
-
|
|
1063
|
+
const message = options.formatTimeoutError?.(last, options.timeoutMs) ??
|
|
1064
|
+
`Service ready timeout after ${options.timeoutMs}ms (failed=${last.failed}, ready=${last.ready})`;
|
|
1065
|
+
throw new Error(message);
|
|
1017
1066
|
}
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1067
|
+
options.logger?.info("service ready", {
|
|
1068
|
+
topic: "readiness",
|
|
1069
|
+
data: last.readyDetail ?? {},
|
|
1070
|
+
});
|
|
1071
|
+
return last;
|
|
1021
1072
|
}
|
|
1022
1073
|
|
|
1023
1074
|
const RUNTIME_STATE_SHARD_DIR = "runtime-state";
|
|
@@ -1181,11 +1232,22 @@ function createDefaultServiceRuntimeState(options) {
|
|
|
1181
1232
|
lastDecision: null,
|
|
1182
1233
|
lastSpawnAt: null,
|
|
1183
1234
|
spawnCount: 0,
|
|
1235
|
+
mode: "healthy",
|
|
1236
|
+
silentEnteredAt: null,
|
|
1237
|
+
silentLastHeartbeatAt: null,
|
|
1238
|
+
silentBackoffMs: 0,
|
|
1239
|
+
silentConsecutiveFailures: 0,
|
|
1240
|
+
silentRecoveryAttempt: 0,
|
|
1241
|
+
silentEvents: [],
|
|
1184
1242
|
};
|
|
1185
1243
|
const scheduler = {
|
|
1186
1244
|
lastTickAt: null,
|
|
1187
1245
|
lastPostedCount: 0,
|
|
1188
1246
|
lastFired: {},
|
|
1247
|
+
pid: null,
|
|
1248
|
+
spawnedAt: null,
|
|
1249
|
+
ready: false,
|
|
1250
|
+
readyAt: null,
|
|
1189
1251
|
};
|
|
1190
1252
|
const worker = {
|
|
1191
1253
|
pid: null,
|
|
@@ -1477,6 +1539,193 @@ async function runRunner(ctx, runner) {
|
|
|
1477
1539
|
return succeeded;
|
|
1478
1540
|
}
|
|
1479
1541
|
|
|
1542
|
+
/**
|
|
1543
|
+
* Run registry slot ownership check and optionally SIGTERM the parent main process.
|
|
1544
|
+
* Used by guard / supervisor TriggerNode runners across L3 services.
|
|
1545
|
+
*/
|
|
1546
|
+
async function applyRegistrySlotGuardStep(input) {
|
|
1547
|
+
const result = await runRegistrySlotGuardCheck({
|
|
1548
|
+
serviceValue: input.serviceValue,
|
|
1549
|
+
serviceId: input.serviceId,
|
|
1550
|
+
});
|
|
1551
|
+
const registryPatch = registrySlotRuntimePatchFromGuardResult(result, input.checkedAt);
|
|
1552
|
+
if (result.shouldShutdown) {
|
|
1553
|
+
input.logger.critical("Registry slot lost — shutting down service", {
|
|
1554
|
+
topic: input.logTopic ?? "guard",
|
|
1555
|
+
data: {
|
|
1556
|
+
traceId: input.traceId,
|
|
1557
|
+
serviceId: input.serviceId,
|
|
1558
|
+
serviceValue: input.serviceValue,
|
|
1559
|
+
},
|
|
1560
|
+
});
|
|
1561
|
+
process.kill(process.ppid, "SIGTERM");
|
|
1562
|
+
return { continueTick: false, registryPatch };
|
|
1563
|
+
}
|
|
1564
|
+
if (result.error != null) {
|
|
1565
|
+
input.logger.warn("Registry slot check failed (transient) — skipping enforcement this tick", {
|
|
1566
|
+
topic: input.logTopic ?? "guard",
|
|
1567
|
+
data: {
|
|
1568
|
+
traceId: input.traceId,
|
|
1569
|
+
serviceId: input.serviceId,
|
|
1570
|
+
message: result.error,
|
|
1571
|
+
},
|
|
1572
|
+
});
|
|
1573
|
+
}
|
|
1574
|
+
return { continueTick: true, registryPatch };
|
|
1575
|
+
}
|
|
1576
|
+
|
|
1577
|
+
let lastWorkerSpawnAt = 0;
|
|
1578
|
+
function getWorkerSpawnCooldownLastAt() {
|
|
1579
|
+
return lastWorkerSpawnAt;
|
|
1580
|
+
}
|
|
1581
|
+
function markWorkerSpawned(at = Date.now()) {
|
|
1582
|
+
lastWorkerSpawnAt = at;
|
|
1583
|
+
}
|
|
1584
|
+
|
|
1585
|
+
/**
|
|
1586
|
+
* One guard-runner tick: registry slot check, TaskNode liveness, optional worker
|
|
1587
|
+
* respawn, Service runtime heartbeat rollup.
|
|
1588
|
+
*/
|
|
1589
|
+
async function runServiceGuardTick(input) {
|
|
1590
|
+
const { serviceValue, serviceId, nodeId, loopTraceId, logger, logTopic = "guard", checkedAt, nowMs, taskNodeStaleMs, workerSpawnCooldownMs, maxNodes = 200, spawnWorker, getLastWorkerSpawnAt = getWorkerSpawnCooldownLastAt, markWorkerSpawned: markWorkerSpawnedAt = markWorkerSpawned, } = input;
|
|
1591
|
+
const continueTick = true;
|
|
1592
|
+
let registryPatch;
|
|
1593
|
+
let decision = "idle";
|
|
1594
|
+
if (serviceId != null) {
|
|
1595
|
+
const guard = await applyRegistrySlotGuardStep({
|
|
1596
|
+
serviceValue,
|
|
1597
|
+
serviceId,
|
|
1598
|
+
checkedAt,
|
|
1599
|
+
logger,
|
|
1600
|
+
logTopic,
|
|
1601
|
+
traceId: loopTraceId,
|
|
1602
|
+
});
|
|
1603
|
+
registryPatch = guard.registryPatch;
|
|
1604
|
+
if (!guard.continueTick) {
|
|
1605
|
+
return { continueTick: false, decision: "slot_lost", nodes: [], registryPatch };
|
|
1606
|
+
}
|
|
1607
|
+
decision = "slot_ok";
|
|
1608
|
+
}
|
|
1609
|
+
const { data: nodes = [] } = await scanTargetList({
|
|
1610
|
+
category: CategoryNode.NODE,
|
|
1611
|
+
maxRows: maxNodes,
|
|
1612
|
+
});
|
|
1613
|
+
const liveness = evaluateBusyNodeLiveness(nodes, {
|
|
1614
|
+
staleMs: taskNodeStaleMs,
|
|
1615
|
+
now: nowMs,
|
|
1616
|
+
excludeNodeId: nodeId,
|
|
1617
|
+
onlyFreshCandidates: true,
|
|
1618
|
+
});
|
|
1619
|
+
const ageMs = liveness.freshest?.ageMs ?? Number.POSITIVE_INFINITY;
|
|
1620
|
+
if (liveness.healthy) {
|
|
1621
|
+
decision = "healthy";
|
|
1622
|
+
}
|
|
1623
|
+
else if (nowMs - getLastWorkerSpawnAt() < workerSpawnCooldownMs) {
|
|
1624
|
+
decision = "spawn_cooldown";
|
|
1625
|
+
logger.warn("TaskNode unavailable but spawn cooldown active", {
|
|
1626
|
+
topic: logTopic,
|
|
1627
|
+
data: {
|
|
1628
|
+
traceId: loopTraceId,
|
|
1629
|
+
ageMs,
|
|
1630
|
+
staleMs: taskNodeStaleMs,
|
|
1631
|
+
cooldownMs: workerSpawnCooldownMs,
|
|
1632
|
+
},
|
|
1633
|
+
});
|
|
1634
|
+
}
|
|
1635
|
+
else {
|
|
1636
|
+
decision = "spawn_worker";
|
|
1637
|
+
logger.warn("TaskNode stale — restarting worker", {
|
|
1638
|
+
topic: logTopic,
|
|
1639
|
+
data: {
|
|
1640
|
+
traceId: loopTraceId,
|
|
1641
|
+
peerWorkerCount: liveness.busyWorkerCount,
|
|
1642
|
+
freshPeerCount: liveness.freshWorkerCount,
|
|
1643
|
+
staleMs: taskNodeStaleMs,
|
|
1644
|
+
},
|
|
1645
|
+
});
|
|
1646
|
+
await spawnWorker("guard:restart-unhealthy-tasknode");
|
|
1647
|
+
markWorkerSpawnedAt(nowMs);
|
|
1648
|
+
}
|
|
1649
|
+
try {
|
|
1650
|
+
await patchServiceRuntime({
|
|
1651
|
+
serviceValue,
|
|
1652
|
+
serviceId: serviceId ?? undefined,
|
|
1653
|
+
nodes,
|
|
1654
|
+
lastHeartBeat: nowMs,
|
|
1655
|
+
});
|
|
1656
|
+
}
|
|
1657
|
+
catch (error) {
|
|
1658
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1659
|
+
logger.warn("Failed to patch Service runtime rollup", {
|
|
1660
|
+
topic: logTopic,
|
|
1661
|
+
data: { traceId: loopTraceId, message },
|
|
1662
|
+
});
|
|
1663
|
+
}
|
|
1664
|
+
return { continueTick, decision, nodes, registryPatch };
|
|
1665
|
+
}
|
|
1666
|
+
|
|
1667
|
+
/** Default TriggerManager runner key for {@link registerServiceGuardRunner}. */
|
|
1668
|
+
const SERVICE_GUARD_RUNNER_KEY = "service-guard";
|
|
1669
|
+
/** True when the guard slice is serving traffic (`mode` omitted = legacy healthy). */
|
|
1670
|
+
function isGuardAvailable(guard) {
|
|
1671
|
+
return guard.mode == null || guard.mode === "healthy";
|
|
1672
|
+
}
|
|
1673
|
+
/**
|
|
1674
|
+
* Seconds for `Retry-After` when the guard is not serving traffic.
|
|
1675
|
+
* `null` when {@link isGuardAvailable} is true.
|
|
1676
|
+
*/
|
|
1677
|
+
function guardRetryAfterSec(guard) {
|
|
1678
|
+
if (isGuardAvailable(guard)) {
|
|
1679
|
+
return null;
|
|
1680
|
+
}
|
|
1681
|
+
const backoffMs = guard.silentBackoffMs > 0 ? guard.silentBackoffMs : 15_000;
|
|
1682
|
+
return Math.max(1, Math.ceil(backoffMs / 1000));
|
|
1683
|
+
}
|
|
1684
|
+
|
|
1685
|
+
async function runServiceGuardRunner(ctx, options) {
|
|
1686
|
+
const { loopTraceId, nodeId, logger } = ctx;
|
|
1687
|
+
const nowMs = options.now?.() ?? Date.now();
|
|
1688
|
+
const checkedAt = new Date(nowMs).toISOString();
|
|
1689
|
+
const serviceId = await options.getServiceId();
|
|
1690
|
+
const result = await runServiceGuardTick({
|
|
1691
|
+
serviceValue: options.serviceValue,
|
|
1692
|
+
serviceId,
|
|
1693
|
+
nodeId,
|
|
1694
|
+
loopTraceId,
|
|
1695
|
+
logger,
|
|
1696
|
+
logTopic: options.logTopic,
|
|
1697
|
+
checkedAt,
|
|
1698
|
+
nowMs,
|
|
1699
|
+
taskNodeStaleMs: options.taskNodeStaleMs,
|
|
1700
|
+
workerSpawnCooldownMs: options.workerSpawnCooldownMs,
|
|
1701
|
+
maxNodes: options.maxNodes,
|
|
1702
|
+
spawnWorker: options.spawnWorker,
|
|
1703
|
+
});
|
|
1704
|
+
if (result.registryPatch != null) {
|
|
1705
|
+
await options.onRegistryPatch(result.registryPatch);
|
|
1706
|
+
}
|
|
1707
|
+
if (!result.continueTick) {
|
|
1708
|
+
return;
|
|
1709
|
+
}
|
|
1710
|
+
await options.onGuardPatch({
|
|
1711
|
+
nodeId,
|
|
1712
|
+
lastCheckAt: checkedAt,
|
|
1713
|
+
lastDecision: result.decision,
|
|
1714
|
+
});
|
|
1715
|
+
if (options.onDecision != null) {
|
|
1716
|
+
await options.onDecision(result.decision);
|
|
1717
|
+
}
|
|
1718
|
+
}
|
|
1719
|
+
/** Register the standard L3 guard runner (registry slot + TaskNode + Service runtime). */
|
|
1720
|
+
function registerServiceGuardRunner(options) {
|
|
1721
|
+
TriggerManager.registerRunner({
|
|
1722
|
+
key: options.runnerKey ?? SERVICE_GUARD_RUNNER_KEY,
|
|
1723
|
+
intervalMs: options.intervalMs,
|
|
1724
|
+
initialDelayMs: options.initialDelayMs,
|
|
1725
|
+
fn: (ctx) => runServiceGuardRunner(ctx, options),
|
|
1726
|
+
});
|
|
1727
|
+
}
|
|
1728
|
+
|
|
1480
1729
|
const UPLOAD_LANE = "slow";
|
|
1481
1730
|
/** Per serviceId — guard collect-log tick runs GC at most once per `gc.intervalMs`. */
|
|
1482
1731
|
const lastGcAtMsByServiceId = new Map();
|
|
@@ -1786,205 +2035,13 @@ function registerCollectLogRunner(options) {
|
|
|
1786
2035
|
}
|
|
1787
2036
|
|
|
1788
2037
|
/**
|
|
1789
|
-
*
|
|
1790
|
-
*
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
});
|
|
1797
|
-
const registryPatch = registrySlotRuntimePatchFromGuardResult(result, input.checkedAt);
|
|
1798
|
-
if (result.shouldShutdown) {
|
|
1799
|
-
input.logger.critical("Registry slot lost — shutting down service", {
|
|
1800
|
-
topic: input.logTopic ?? "guard",
|
|
1801
|
-
data: {
|
|
1802
|
-
traceId: input.traceId,
|
|
1803
|
-
serviceId: input.serviceId,
|
|
1804
|
-
serviceValue: input.serviceValue,
|
|
1805
|
-
},
|
|
1806
|
-
});
|
|
1807
|
-
process.kill(process.ppid, "SIGTERM");
|
|
1808
|
-
return { continueTick: false, registryPatch };
|
|
1809
|
-
}
|
|
1810
|
-
if (result.error != null) {
|
|
1811
|
-
input.logger.warn("Registry slot check failed (transient) — skipping enforcement this tick", {
|
|
1812
|
-
topic: input.logTopic ?? "guard",
|
|
1813
|
-
data: {
|
|
1814
|
-
traceId: input.traceId,
|
|
1815
|
-
serviceId: input.serviceId,
|
|
1816
|
-
message: result.error,
|
|
1817
|
-
},
|
|
1818
|
-
});
|
|
1819
|
-
}
|
|
1820
|
-
return { continueTick: true, registryPatch };
|
|
1821
|
-
}
|
|
1822
|
-
|
|
1823
|
-
let lastWorkerSpawnAt = 0;
|
|
1824
|
-
function getWorkerSpawnCooldownLastAt() {
|
|
1825
|
-
return lastWorkerSpawnAt;
|
|
1826
|
-
}
|
|
1827
|
-
function markWorkerSpawned(at = Date.now()) {
|
|
1828
|
-
lastWorkerSpawnAt = at;
|
|
1829
|
-
}
|
|
1830
|
-
|
|
1831
|
-
/**
|
|
1832
|
-
* One guard-runner tick: registry slot check, TaskNode liveness, optional worker
|
|
1833
|
-
* respawn, Service runtime heartbeat rollup.
|
|
1834
|
-
*/
|
|
1835
|
-
async function runServiceGuardTick(input) {
|
|
1836
|
-
const { serviceValue, serviceId, nodeId, loopTraceId, logger, logTopic = "guard", checkedAt, nowMs, taskNodeStaleMs, workerSpawnCooldownMs, maxNodes = 200, spawnWorker, getLastWorkerSpawnAt = getWorkerSpawnCooldownLastAt, markWorkerSpawned: markWorkerSpawnedAt = markWorkerSpawned, } = input;
|
|
1837
|
-
const continueTick = true;
|
|
1838
|
-
let registryPatch;
|
|
1839
|
-
let decision = "idle";
|
|
1840
|
-
if (serviceId != null) {
|
|
1841
|
-
const guard = await applyRegistrySlotGuardStep({
|
|
1842
|
-
serviceValue,
|
|
1843
|
-
serviceId,
|
|
1844
|
-
checkedAt,
|
|
1845
|
-
logger,
|
|
1846
|
-
logTopic,
|
|
1847
|
-
traceId: loopTraceId,
|
|
1848
|
-
});
|
|
1849
|
-
registryPatch = guard.registryPatch;
|
|
1850
|
-
if (!guard.continueTick) {
|
|
1851
|
-
return { continueTick: false, decision: "slot_lost", nodes: [], registryPatch };
|
|
1852
|
-
}
|
|
1853
|
-
decision = "slot_ok";
|
|
1854
|
-
}
|
|
1855
|
-
const { data: nodes = [] } = await scanTargetList({
|
|
1856
|
-
category: CategoryNode.NODE,
|
|
1857
|
-
maxRows: maxNodes,
|
|
1858
|
-
});
|
|
1859
|
-
const liveness = evaluateBusyNodeLiveness(nodes, {
|
|
1860
|
-
staleMs: taskNodeStaleMs,
|
|
1861
|
-
now: nowMs,
|
|
1862
|
-
excludeNodeId: nodeId,
|
|
1863
|
-
onlyFreshCandidates: true,
|
|
1864
|
-
});
|
|
1865
|
-
const ageMs = liveness.freshest?.ageMs ?? Number.POSITIVE_INFINITY;
|
|
1866
|
-
if (liveness.healthy) {
|
|
1867
|
-
decision = "healthy";
|
|
1868
|
-
}
|
|
1869
|
-
else if (nowMs - getLastWorkerSpawnAt() < workerSpawnCooldownMs) {
|
|
1870
|
-
decision = "spawn_cooldown";
|
|
1871
|
-
logger.warn("TaskNode unavailable but spawn cooldown active", {
|
|
1872
|
-
topic: logTopic,
|
|
1873
|
-
data: {
|
|
1874
|
-
traceId: loopTraceId,
|
|
1875
|
-
ageMs,
|
|
1876
|
-
staleMs: taskNodeStaleMs,
|
|
1877
|
-
cooldownMs: workerSpawnCooldownMs,
|
|
1878
|
-
},
|
|
1879
|
-
});
|
|
1880
|
-
}
|
|
1881
|
-
else {
|
|
1882
|
-
decision = "spawn_worker";
|
|
1883
|
-
logger.warn("TaskNode stale — restarting worker", {
|
|
1884
|
-
topic: logTopic,
|
|
1885
|
-
data: {
|
|
1886
|
-
traceId: loopTraceId,
|
|
1887
|
-
peerWorkerCount: liveness.busyWorkerCount,
|
|
1888
|
-
freshPeerCount: liveness.freshWorkerCount,
|
|
1889
|
-
staleMs: taskNodeStaleMs,
|
|
1890
|
-
},
|
|
1891
|
-
});
|
|
1892
|
-
await spawnWorker("guard:restart-unhealthy-tasknode");
|
|
1893
|
-
markWorkerSpawnedAt(nowMs);
|
|
1894
|
-
}
|
|
1895
|
-
try {
|
|
1896
|
-
await patchServiceRuntime({
|
|
1897
|
-
serviceValue,
|
|
1898
|
-
serviceId: serviceId ?? undefined,
|
|
1899
|
-
nodes,
|
|
1900
|
-
lastHeartBeat: nowMs,
|
|
1901
|
-
});
|
|
1902
|
-
}
|
|
1903
|
-
catch (error) {
|
|
1904
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
1905
|
-
logger.warn("Failed to patch Service runtime rollup", {
|
|
1906
|
-
topic: logTopic,
|
|
1907
|
-
data: { traceId: loopTraceId, message },
|
|
1908
|
-
});
|
|
1909
|
-
}
|
|
1910
|
-
return { continueTick, decision, nodes, registryPatch };
|
|
1911
|
-
}
|
|
1912
|
-
|
|
1913
|
-
/** Default TriggerManager runner key for {@link registerServiceGuardRunner}. */
|
|
1914
|
-
const SERVICE_GUARD_RUNNER_KEY = "service-guard";
|
|
1915
|
-
|
|
1916
|
-
async function runServiceGuardRunner(ctx, options) {
|
|
1917
|
-
const { loopTraceId, nodeId, logger } = ctx;
|
|
1918
|
-
const nowMs = options.now?.() ?? Date.now();
|
|
1919
|
-
const checkedAt = new Date(nowMs).toISOString();
|
|
1920
|
-
const serviceId = await options.getServiceId();
|
|
1921
|
-
const result = await runServiceGuardTick({
|
|
1922
|
-
serviceValue: options.serviceValue,
|
|
1923
|
-
serviceId,
|
|
1924
|
-
nodeId,
|
|
1925
|
-
loopTraceId,
|
|
1926
|
-
logger,
|
|
1927
|
-
logTopic: options.logTopic,
|
|
1928
|
-
checkedAt,
|
|
1929
|
-
nowMs,
|
|
1930
|
-
taskNodeStaleMs: options.taskNodeStaleMs,
|
|
1931
|
-
workerSpawnCooldownMs: options.workerSpawnCooldownMs,
|
|
1932
|
-
maxNodes: options.maxNodes,
|
|
1933
|
-
spawnWorker: options.spawnWorker,
|
|
1934
|
-
});
|
|
1935
|
-
if (result.registryPatch != null) {
|
|
1936
|
-
await options.onRegistryPatch(result.registryPatch);
|
|
1937
|
-
}
|
|
1938
|
-
if (!result.continueTick) {
|
|
1939
|
-
return;
|
|
1940
|
-
}
|
|
1941
|
-
await options.onGuardPatch({
|
|
1942
|
-
nodeId,
|
|
1943
|
-
lastCheckAt: checkedAt,
|
|
1944
|
-
lastDecision: result.decision,
|
|
1945
|
-
});
|
|
1946
|
-
if (options.onDecision != null) {
|
|
1947
|
-
await options.onDecision(result.decision);
|
|
1948
|
-
}
|
|
1949
|
-
}
|
|
1950
|
-
/** Register the standard L3 guard runner (registry slot + TaskNode + Service runtime). */
|
|
1951
|
-
function registerServiceGuardRunner(options) {
|
|
1952
|
-
TriggerManager.registerRunner({
|
|
1953
|
-
key: options.runnerKey ?? SERVICE_GUARD_RUNNER_KEY,
|
|
1954
|
-
intervalMs: options.intervalMs,
|
|
1955
|
-
initialDelayMs: options.initialDelayMs,
|
|
1956
|
-
fn: (ctx) => runServiceGuardRunner(ctx, options),
|
|
1957
|
-
});
|
|
1958
|
-
}
|
|
1959
|
-
|
|
1960
|
-
/** Run readiness checks, persist report via callback, fail-fast when not ok. */
|
|
1961
|
-
async function runReadinessGate(input) {
|
|
1962
|
-
const { checks, onReport, logger, logTopic = "guard" } = input;
|
|
1963
|
-
logger.info("Running readiness checks", { topic: logTopic });
|
|
1964
|
-
const report = await runReadinessChecks(checks);
|
|
1965
|
-
await onReport(report);
|
|
1966
|
-
if (!report.ok) {
|
|
1967
|
-
logger.error("Readiness checks failed", {
|
|
1968
|
-
topic: logTopic,
|
|
1969
|
-
data: { message: report.message, checks: report.checks },
|
|
1970
|
-
});
|
|
1971
|
-
throw new Error(report.message ?? "Readiness checks failed");
|
|
1972
|
-
}
|
|
1973
|
-
logger.debug("Readiness checks passed", {
|
|
1974
|
-
topic: logTopic,
|
|
1975
|
-
data: { checks: report.checks.map((check) => check.name) },
|
|
1976
|
-
});
|
|
1977
|
-
return report;
|
|
1978
|
-
}
|
|
1979
|
-
|
|
1980
|
-
/**
|
|
1981
|
-
* Local interval scheduler node.
|
|
1982
|
-
* Main loop: commands → heartbeat → {@link TriggerManager.tick}.
|
|
1983
|
-
*
|
|
1984
|
-
* Register runners via {@link TriggerManager.registerRunner} before {@link start}.
|
|
1985
|
-
* Does not scan Supabase trigger rows — see `trigger.api.ts` for remote admin APIs.
|
|
1986
|
-
*
|
|
1987
|
-
* Deploy **one** TriggerNode per logical scheduler — multiple processes duplicate runner work.
|
|
2038
|
+
* Local interval scheduler node.
|
|
2039
|
+
* Main loop: commands → heartbeat → {@link TriggerManager.tick}.
|
|
2040
|
+
*
|
|
2041
|
+
* Register runners via {@link TriggerManager.registerRunner} before {@link start}.
|
|
2042
|
+
* Does not scan Supabase trigger rows — see `trigger.api.ts` for remote admin APIs.
|
|
2043
|
+
*
|
|
2044
|
+
* Deploy **one** TriggerNode per logical scheduler — multiple processes duplicate runner work.
|
|
1988
2045
|
*/
|
|
1989
2046
|
class TriggerNode extends BaseNodeRuntime {
|
|
1990
2047
|
requireRunners;
|
|
@@ -2047,27 +2104,39 @@ class TriggerNode extends BaseNodeRuntime {
|
|
|
2047
2104
|
}
|
|
2048
2105
|
}
|
|
2049
2106
|
|
|
2107
|
+
const SILENT_BACKOFF_MS = [15_000, 30_000, 60_000, 120_000, 240_000, 300_000];
|
|
2108
|
+
const DEFAULT_BUSINESS_READY_TIMEOUT_MS = 180_000;
|
|
2050
2109
|
/**
|
|
2051
|
-
* L3 guard process TriggerNode: readiness gate,
|
|
2110
|
+
* L3 guard process TriggerNode: readiness gate, business-node spawn owner,
|
|
2052
2111
|
* and {@link registerServiceGuardRunner} tick (registry slot + liveness + runtime).
|
|
2053
2112
|
*
|
|
2113
|
+
* Consecutive heartbeat failure enters silent mode (no process exit). Recovery
|
|
2114
|
+
* respawns business nodes; leaving silent requires heartbeat + all nodes ready.
|
|
2115
|
+
*
|
|
2054
2116
|
* Use {@link ServiceGuardNode.create} so the guard runner is registered before `start()`.
|
|
2055
2117
|
*/
|
|
2056
2118
|
class ServiceGuardNode extends TriggerNode {
|
|
2057
2119
|
options;
|
|
2120
|
+
businessReadyTimeoutMs;
|
|
2121
|
+
mode = "healthy";
|
|
2122
|
+
silentBackoffIndex = 0;
|
|
2123
|
+
silentEvents = [];
|
|
2124
|
+
recoverPromise = null;
|
|
2058
2125
|
constructor(options) {
|
|
2059
2126
|
super({
|
|
2060
2127
|
requireRunners: true,
|
|
2061
2128
|
beforeProcessExit: options.beforeProcessExit,
|
|
2062
2129
|
});
|
|
2063
2130
|
this.options = options;
|
|
2131
|
+
this.businessReadyTimeoutMs = options.businessReadyTimeoutMs ?? DEFAULT_BUSINESS_READY_TIMEOUT_MS;
|
|
2064
2132
|
}
|
|
2065
2133
|
/** Register the guard runner and return a ready-to-start node. */
|
|
2066
2134
|
static create(options) {
|
|
2067
2135
|
registerServiceGuardRunner({
|
|
2136
|
+
...options.guardRunner,
|
|
2068
2137
|
serviceValue: options.serviceValue,
|
|
2069
2138
|
logTopic: options.logTopic,
|
|
2070
|
-
|
|
2139
|
+
spawnWorker: options.spawnBusinessNodes,
|
|
2071
2140
|
});
|
|
2072
2141
|
registerCollectLogRunner({
|
|
2073
2142
|
serviceValue: options.serviceValue,
|
|
@@ -2075,6 +2144,34 @@ class ServiceGuardNode extends TriggerNode {
|
|
|
2075
2144
|
});
|
|
2076
2145
|
return new ServiceGuardNode(options);
|
|
2077
2146
|
}
|
|
2147
|
+
isSilenced() {
|
|
2148
|
+
return this.mode !== "healthy";
|
|
2149
|
+
}
|
|
2150
|
+
getLoopIntervalMs() {
|
|
2151
|
+
if (this.mode === "silent") {
|
|
2152
|
+
return SILENT_BACKOFF_MS[this.silentBackoffIndex] ?? 300_000;
|
|
2153
|
+
}
|
|
2154
|
+
return super.getLoopIntervalMs();
|
|
2155
|
+
}
|
|
2156
|
+
async onHealthCheckFailed(_ctx) {
|
|
2157
|
+
if (this.mode !== "healthy") {
|
|
2158
|
+
this.bumpSilentBackoff();
|
|
2159
|
+
await this.persistGuard({
|
|
2160
|
+
lastDecision: "silent_heartbeat_fail",
|
|
2161
|
+
silentConsecutiveFailures: this.silentConsecutiveFailures(),
|
|
2162
|
+
silentBackoffMs: this.getLoopIntervalMs(),
|
|
2163
|
+
silentLastHeartbeatAt: new Date().toISOString(),
|
|
2164
|
+
});
|
|
2165
|
+
return;
|
|
2166
|
+
}
|
|
2167
|
+
await this.enterSilent("heartbeat:consecutive-failures");
|
|
2168
|
+
}
|
|
2169
|
+
async onHeartbeatRecovered(ctx) {
|
|
2170
|
+
if (this.mode !== "silent") {
|
|
2171
|
+
return;
|
|
2172
|
+
}
|
|
2173
|
+
await this.recoverFromSilent(ctx);
|
|
2174
|
+
}
|
|
2078
2175
|
async onBeforeRegisterNode(nodeLogger) {
|
|
2079
2176
|
await runReadinessGate({
|
|
2080
2177
|
checks: this.options.readinessChecks,
|
|
@@ -2083,11 +2180,298 @@ class ServiceGuardNode extends TriggerNode {
|
|
|
2083
2180
|
logTopic: this.options.logTopic,
|
|
2084
2181
|
});
|
|
2085
2182
|
await super.onBeforeRegisterNode(nodeLogger);
|
|
2086
|
-
nodeLogger.info("Spawning
|
|
2183
|
+
nodeLogger.info("Spawning business nodes", {
|
|
2087
2184
|
topic: this.options.logTopic ?? "guard",
|
|
2088
2185
|
});
|
|
2089
|
-
await this.options.
|
|
2186
|
+
await this.options.spawnBusinessNodes("guard:readiness-passed");
|
|
2187
|
+
markWorkerSpawned();
|
|
2090
2188
|
this.options.onWorkerSpawned?.();
|
|
2189
|
+
await this.waitUntilBusinessReady();
|
|
2190
|
+
await this.persistGuard({
|
|
2191
|
+
mode: "healthy",
|
|
2192
|
+
lastDecision: "bootstrap_ready",
|
|
2193
|
+
});
|
|
2194
|
+
}
|
|
2195
|
+
async runLoopSteps(ctx, heartbeatOk) {
|
|
2196
|
+
await super.runLoopSteps(ctx, heartbeatOk);
|
|
2197
|
+
if (this.mode !== "healthy" || !heartbeatOk) {
|
|
2198
|
+
return;
|
|
2199
|
+
}
|
|
2200
|
+
// Idempotent — respawn exited children without entering silent (worker stale is the runner's job).
|
|
2201
|
+
try {
|
|
2202
|
+
await this.options.spawnBusinessNodes("guard:ensure-business");
|
|
2203
|
+
}
|
|
2204
|
+
catch (error) {
|
|
2205
|
+
this.createGuardLogger().warn("ensure-business spawn failed", {
|
|
2206
|
+
topic: this.options.logTopic ?? "guard",
|
|
2207
|
+
data: { error: getErrorMessage(error) },
|
|
2208
|
+
});
|
|
2209
|
+
}
|
|
2210
|
+
}
|
|
2211
|
+
silentConsecutiveFailures() {
|
|
2212
|
+
return this.silentBackoffIndex + 1;
|
|
2213
|
+
}
|
|
2214
|
+
bumpSilentBackoff() {
|
|
2215
|
+
if (this.silentBackoffIndex < SILENT_BACKOFF_MS.length - 1) {
|
|
2216
|
+
this.silentBackoffIndex += 1;
|
|
2217
|
+
}
|
|
2218
|
+
}
|
|
2219
|
+
async enterSilent(reason) {
|
|
2220
|
+
this.mode = "silent";
|
|
2221
|
+
this.silentBackoffIndex = 0;
|
|
2222
|
+
const enteredAt = new Date().toISOString();
|
|
2223
|
+
this.pushEvent("enter", reason);
|
|
2224
|
+
await this.persistGuard({
|
|
2225
|
+
mode: "silent",
|
|
2226
|
+
lastDecision: `silent:${reason}`,
|
|
2227
|
+
silentEnteredAt: enteredAt,
|
|
2228
|
+
silentLastHeartbeatAt: enteredAt,
|
|
2229
|
+
silentBackoffMs: SILENT_BACKOFF_MS[0],
|
|
2230
|
+
silentConsecutiveFailures: 1,
|
|
2231
|
+
silentRecoveryAttempt: 0,
|
|
2232
|
+
silentEvents: this.silentEvents,
|
|
2233
|
+
});
|
|
2234
|
+
const logger = this.createGuardLogger();
|
|
2235
|
+
logger.warn("Entering silent mode — service unavailable until heartbeat + business nodes recover", {
|
|
2236
|
+
topic: this.options.logTopic ?? "guard",
|
|
2237
|
+
data: { reason },
|
|
2238
|
+
});
|
|
2239
|
+
try {
|
|
2240
|
+
await this.options.stopBusinessNodes();
|
|
2241
|
+
}
|
|
2242
|
+
catch (error) {
|
|
2243
|
+
logger.warn("stopBusinessNodes failed (best-effort)", {
|
|
2244
|
+
topic: this.options.logTopic ?? "guard",
|
|
2245
|
+
data: { error: getErrorMessage(error) },
|
|
2246
|
+
});
|
|
2247
|
+
}
|
|
2248
|
+
}
|
|
2249
|
+
async recoverFromSilent(ctx) {
|
|
2250
|
+
if (this.recoverPromise != null) {
|
|
2251
|
+
return this.recoverPromise;
|
|
2252
|
+
}
|
|
2253
|
+
this.recoverPromise = this.runRecover(ctx).finally(() => {
|
|
2254
|
+
this.recoverPromise = null;
|
|
2255
|
+
});
|
|
2256
|
+
return this.recoverPromise;
|
|
2257
|
+
}
|
|
2258
|
+
async runRecover(ctx) {
|
|
2259
|
+
this.mode = "recovering";
|
|
2260
|
+
this.pushEvent("recover_start", ctx.loopTraceId);
|
|
2261
|
+
await this.persistGuard({
|
|
2262
|
+
mode: "recovering",
|
|
2263
|
+
lastDecision: "recovering",
|
|
2264
|
+
silentLastHeartbeatAt: new Date().toISOString(),
|
|
2265
|
+
silentRecoveryAttempt: this.silentEvents.filter((event) => event.type === "recover_start").length,
|
|
2266
|
+
silentEvents: this.silentEvents,
|
|
2267
|
+
});
|
|
2268
|
+
const logger = this.createGuardLogger();
|
|
2269
|
+
logger.info("Heartbeat restored — recovering business nodes", {
|
|
2270
|
+
topic: this.options.logTopic ?? "guard",
|
|
2271
|
+
});
|
|
2272
|
+
try {
|
|
2273
|
+
await this.options.stopBusinessNodes();
|
|
2274
|
+
}
|
|
2275
|
+
catch (error) {
|
|
2276
|
+
logger.warn("stopBusinessNodes failed before respawn", {
|
|
2277
|
+
topic: this.options.logTopic ?? "guard",
|
|
2278
|
+
data: { error: getErrorMessage(error) },
|
|
2279
|
+
});
|
|
2280
|
+
}
|
|
2281
|
+
try {
|
|
2282
|
+
await this.options.spawnBusinessNodes("guard:silent-recover");
|
|
2283
|
+
markWorkerSpawned();
|
|
2284
|
+
await this.waitUntilBusinessReady();
|
|
2285
|
+
}
|
|
2286
|
+
catch (error) {
|
|
2287
|
+
this.pushEvent("recover_fail", getErrorMessage(error));
|
|
2288
|
+
this.mode = "silent";
|
|
2289
|
+
this.bumpSilentBackoff();
|
|
2290
|
+
await this.persistGuard({
|
|
2291
|
+
mode: "silent",
|
|
2292
|
+
lastDecision: "recover_fail",
|
|
2293
|
+
silentBackoffMs: this.getLoopIntervalMs(),
|
|
2294
|
+
silentEvents: this.silentEvents,
|
|
2295
|
+
});
|
|
2296
|
+
logger.warn("Recovery failed — returning to silent", {
|
|
2297
|
+
topic: this.options.logTopic ?? "guard",
|
|
2298
|
+
data: { error: getErrorMessage(error) },
|
|
2299
|
+
});
|
|
2300
|
+
try {
|
|
2301
|
+
await this.options.stopBusinessNodes();
|
|
2302
|
+
}
|
|
2303
|
+
catch {
|
|
2304
|
+
// best-effort — leftover children are stopped again on the next recover
|
|
2305
|
+
}
|
|
2306
|
+
return;
|
|
2307
|
+
}
|
|
2308
|
+
this.mode = "healthy";
|
|
2309
|
+
this.silentBackoffIndex = 0;
|
|
2310
|
+
this.pushEvent("recover_ok");
|
|
2311
|
+
await this.persistGuard({
|
|
2312
|
+
mode: "healthy",
|
|
2313
|
+
lastDecision: "recover_ok",
|
|
2314
|
+
silentEnteredAt: null,
|
|
2315
|
+
silentBackoffMs: 0,
|
|
2316
|
+
silentConsecutiveFailures: 0,
|
|
2317
|
+
silentEvents: this.silentEvents,
|
|
2318
|
+
});
|
|
2319
|
+
logger.success("Left silent mode — business nodes ready", {
|
|
2320
|
+
topic: this.options.logTopic ?? "guard",
|
|
2321
|
+
});
|
|
2322
|
+
}
|
|
2323
|
+
async waitUntilBusinessReady() {
|
|
2324
|
+
try {
|
|
2325
|
+
await pollUntil({
|
|
2326
|
+
timeoutMs: this.businessReadyTimeoutMs,
|
|
2327
|
+
intervalMs: 500,
|
|
2328
|
+
until: () => this.options.isBusinessReady(),
|
|
2329
|
+
});
|
|
2330
|
+
}
|
|
2331
|
+
catch {
|
|
2332
|
+
throw new Error(`Business nodes not ready within ${this.businessReadyTimeoutMs}ms`);
|
|
2333
|
+
}
|
|
2334
|
+
}
|
|
2335
|
+
pushEvent(type, detail) {
|
|
2336
|
+
this.silentEvents = [...this.silentEvents, { at: new Date().toISOString(), type, detail }].slice(-32);
|
|
2337
|
+
}
|
|
2338
|
+
async persistGuard(patch) {
|
|
2339
|
+
try {
|
|
2340
|
+
await this.options.guardRunner.onGuardPatch(patch);
|
|
2341
|
+
}
|
|
2342
|
+
catch {
|
|
2343
|
+
// best-effort — disk/runtime state must not take down the guard loop
|
|
2344
|
+
}
|
|
2345
|
+
}
|
|
2346
|
+
createGuardLogger() {
|
|
2347
|
+
return createLogger({ module: "service-guard" });
|
|
2348
|
+
}
|
|
2349
|
+
}
|
|
2350
|
+
|
|
2351
|
+
const WORKER_SCRIPT = "./dist/processes/worker.js";
|
|
2352
|
+
const GUARD_SCRIPT = "./dist/processes/guard.js";
|
|
2353
|
+
const SCHEDULER_SCRIPT = "./dist/processes/scheduler.js";
|
|
2354
|
+
function defaultSpawnEnv(label) {
|
|
2355
|
+
return { LOG_PERSIST_PROCESS: label };
|
|
2356
|
+
}
|
|
2357
|
+
/**
|
|
2358
|
+
* Guard-owned scheduler + worker spawn/stop, plus main-owned Guard spawn.
|
|
2359
|
+
* `stopBusinessNodes` must run in the Guard process (that instance's children only).
|
|
2360
|
+
* `stopChildProcesses` is for main shutdown (`extraPids` for Guard-spawned PIDs).
|
|
2361
|
+
*/
|
|
2362
|
+
function createL3ChildLauncher(options) {
|
|
2363
|
+
const { childProcesses, readRuntimeState, writeRuntimeState } = options;
|
|
2364
|
+
const spawnEnv = options.spawnEnv ?? defaultSpawnEnv;
|
|
2365
|
+
function nowIso() {
|
|
2366
|
+
return new Date().toISOString();
|
|
2367
|
+
}
|
|
2368
|
+
async function spawnTaskWorker(reason) {
|
|
2369
|
+
const { child, created } = childProcesses.spawn("worker", WORKER_SCRIPT, {
|
|
2370
|
+
env: spawnEnv("worker"),
|
|
2371
|
+
});
|
|
2372
|
+
if (!created) {
|
|
2373
|
+
return child;
|
|
2374
|
+
}
|
|
2375
|
+
const current = await readRuntimeState();
|
|
2376
|
+
await writeRuntimeState({
|
|
2377
|
+
worker: {
|
|
2378
|
+
pid: child.pid ?? null,
|
|
2379
|
+
spawnedAt: nowIso(),
|
|
2380
|
+
ready: false,
|
|
2381
|
+
registeredTasks: [],
|
|
2382
|
+
readyAt: null,
|
|
2383
|
+
},
|
|
2384
|
+
guard: {
|
|
2385
|
+
lastSpawnAt: nowIso(),
|
|
2386
|
+
lastDecision: reason,
|
|
2387
|
+
spawnCount: current.guard.spawnCount + 1,
|
|
2388
|
+
},
|
|
2389
|
+
});
|
|
2390
|
+
return child;
|
|
2391
|
+
}
|
|
2392
|
+
function spawnGuard() {
|
|
2393
|
+
return childProcesses.spawn("guard", GUARD_SCRIPT, { env: spawnEnv("guard") }).child;
|
|
2394
|
+
}
|
|
2395
|
+
async function spawnScheduler(reason) {
|
|
2396
|
+
const { child, created } = childProcesses.spawn("scheduler", SCHEDULER_SCRIPT, {
|
|
2397
|
+
env: spawnEnv("scheduler"),
|
|
2398
|
+
});
|
|
2399
|
+
if (!created) {
|
|
2400
|
+
return child;
|
|
2401
|
+
}
|
|
2402
|
+
await writeRuntimeState({
|
|
2403
|
+
scheduler: {
|
|
2404
|
+
pid: child.pid ?? null,
|
|
2405
|
+
spawnedAt: nowIso(),
|
|
2406
|
+
ready: false,
|
|
2407
|
+
readyAt: null,
|
|
2408
|
+
},
|
|
2409
|
+
guard: {
|
|
2410
|
+
lastDecision: reason,
|
|
2411
|
+
},
|
|
2412
|
+
});
|
|
2413
|
+
return child;
|
|
2414
|
+
}
|
|
2415
|
+
async function spawnBusinessNodes(reason) {
|
|
2416
|
+
await spawnScheduler(reason);
|
|
2417
|
+
await spawnTaskWorker(reason);
|
|
2418
|
+
}
|
|
2419
|
+
async function stopBusinessNodes() {
|
|
2420
|
+
await childProcesses.stopAll();
|
|
2421
|
+
await writeRuntimeState({
|
|
2422
|
+
worker: {
|
|
2423
|
+
pid: null,
|
|
2424
|
+
ready: false,
|
|
2425
|
+
readyAt: null,
|
|
2426
|
+
registeredTasks: [],
|
|
2427
|
+
},
|
|
2428
|
+
scheduler: {
|
|
2429
|
+
pid: null,
|
|
2430
|
+
ready: false,
|
|
2431
|
+
readyAt: null,
|
|
2432
|
+
},
|
|
2433
|
+
});
|
|
2434
|
+
}
|
|
2435
|
+
async function isBusinessReady() {
|
|
2436
|
+
const state = await readRuntimeState();
|
|
2437
|
+
return state.worker.ready && state.scheduler.ready;
|
|
2438
|
+
}
|
|
2439
|
+
async function stopChildProcesses() {
|
|
2440
|
+
const state = await readRuntimeState();
|
|
2441
|
+
const extraPids = [state.worker.pid, state.scheduler.pid].filter((pid) => pid != null);
|
|
2442
|
+
await childProcesses.stopAll({ extraPids });
|
|
2443
|
+
}
|
|
2444
|
+
return {
|
|
2445
|
+
spawnGuard,
|
|
2446
|
+
spawnTaskWorker,
|
|
2447
|
+
spawnScheduler,
|
|
2448
|
+
spawnBusinessNodes,
|
|
2449
|
+
stopBusinessNodes,
|
|
2450
|
+
isBusinessReady,
|
|
2451
|
+
stopChildProcesses,
|
|
2452
|
+
};
|
|
2453
|
+
}
|
|
2454
|
+
|
|
2455
|
+
/**
|
|
2456
|
+
* Claim a registry slot, or `process.exit(1)` when no EMPTY slot remains.
|
|
2457
|
+
* Other errors propagate.
|
|
2458
|
+
*/
|
|
2459
|
+
async function claimRegistrySlotOrExit(input) {
|
|
2460
|
+
try {
|
|
2461
|
+
return await claimServiceRegistrySlot({
|
|
2462
|
+
serviceValue: input.serviceValue,
|
|
2463
|
+
createInstance: input.createInstance,
|
|
2464
|
+
});
|
|
2465
|
+
}
|
|
2466
|
+
catch (error) {
|
|
2467
|
+
if (error instanceof ServiceRegistryError && error.code === "SERVICE_SLOTS_FULL") {
|
|
2468
|
+
input.logger.critical("No EMPTY registry slot — refusing to start", {
|
|
2469
|
+
topic: input.logTopic,
|
|
2470
|
+
data: { serviceValue: input.serviceValue, code: error.code },
|
|
2471
|
+
});
|
|
2472
|
+
process.exit(1);
|
|
2473
|
+
}
|
|
2474
|
+
throw error;
|
|
2091
2475
|
}
|
|
2092
2476
|
}
|
|
2093
2477
|
|
|
@@ -2146,22 +2530,12 @@ function createServiceHost(options) {
|
|
|
2146
2530
|
if (options.prepare != null) {
|
|
2147
2531
|
await options.prepare();
|
|
2148
2532
|
}
|
|
2149
|
-
|
|
2150
|
-
|
|
2151
|
-
|
|
2152
|
-
|
|
2153
|
-
|
|
2154
|
-
}
|
|
2155
|
-
catch (error) {
|
|
2156
|
-
if (error instanceof ServiceRegistryError && error.code === "SERVICE_SLOTS_FULL") {
|
|
2157
|
-
logger.critical("No EMPTY registry slot — refusing to start", {
|
|
2158
|
-
topic: logTopic,
|
|
2159
|
-
data: { serviceValue: options.serviceValue, code: error.code },
|
|
2160
|
-
});
|
|
2161
|
-
process.exit(1);
|
|
2162
|
-
}
|
|
2163
|
-
throw error;
|
|
2164
|
-
}
|
|
2533
|
+
session = await claimRegistrySlotOrExit({
|
|
2534
|
+
serviceValue: options.serviceValue,
|
|
2535
|
+
createInstance: async () => unwrapService(await options.createInstance()),
|
|
2536
|
+
logger,
|
|
2537
|
+
logTopic,
|
|
2538
|
+
});
|
|
2165
2539
|
const hostCtx = { service: session.service, session, serviceId: session.service.id };
|
|
2166
2540
|
if (options.onRegistryClaimed != null) {
|
|
2167
2541
|
await options.onRegistryClaimed(hostCtx);
|
|
@@ -2220,23 +2594,12 @@ async function runSingleProcessService(options) {
|
|
|
2220
2594
|
if (options.prepare != null) {
|
|
2221
2595
|
await options.prepare();
|
|
2222
2596
|
}
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
}
|
|
2230
|
-
catch (error) {
|
|
2231
|
-
if (error instanceof ServiceRegistryError && error.code === "SERVICE_SLOTS_FULL") {
|
|
2232
|
-
logger.critical("No EMPTY registry slot — refusing to start", {
|
|
2233
|
-
topic: logTopic,
|
|
2234
|
-
data: { serviceValue: options.serviceValue, code: error.code },
|
|
2235
|
-
});
|
|
2236
|
-
process.exit(1);
|
|
2237
|
-
}
|
|
2238
|
-
throw error;
|
|
2239
|
-
}
|
|
2597
|
+
const session = await claimRegistrySlotOrExit({
|
|
2598
|
+
serviceValue: options.serviceValue,
|
|
2599
|
+
createInstance: options.createInstance,
|
|
2600
|
+
logger,
|
|
2601
|
+
logTopic,
|
|
2602
|
+
});
|
|
2240
2603
|
try {
|
|
2241
2604
|
await options.run({ service: session.service, session });
|
|
2242
2605
|
}
|
|
@@ -3673,5 +4036,5 @@ const patchTriggerFired = validateWithSchema(patchTriggerFiredSchema, "patchTrig
|
|
|
3673
4036
|
}
|
|
3674
4037
|
});
|
|
3675
4038
|
|
|
3676
|
-
export { BaseNodeRuntime, COLLECT_LOG_RUNNER_KEY, CategoryCommand, CategoryNode, CategoryRepo, CategoryTrigger, CommandType, EMPTY_REGISTRY_SLOT_RUNTIME_STATE, LOG_SPOOL_SERVICE_ID_ENV, LOG_TOPIC_TRIGGER, LogLevel, MAX_POLL_TARGET_LIST_SIZE, ManagedChildProcesses, NodeStatus, RepoManager, ResultCode, SERVICE_GUARD_RUNNER_KEY, ServiceGuardNode, ServiceRegistryError, TASK_LOCAL_PACKAGE_CONFIG_FILENAME, TASK_REPO_SCRIPT_CATEGORY, TASK_REPO_USAGE, TASK_RUNNER_CONFIG_DIR, TASK_RUNNER_ROOT_CONFIG_FILENAME, TASK_RUNNER_ROOT_CONFIG_LEGACY_RELATIVE_PATH, TASK_RUNNER_ROOT_CONFIG_RELATIVE_PATH, TRIGGER_LOOP_INTERVAL_MS, TRIGGER_RUNNER_DEFAULT_RETRY_COUNT, TRIGGER_RUNNER_DEFAULT_RETRY_DELAY_MS, TaskManager, TaskNode, TaskRepoValidation, TaskStatus, TaskStatusAction, TriggerManager, TriggerNode, TriggerStatus, applyRegistrySlotGuardStep, buildNodeImportArgs, claimServiceRegistrySlot, createJsonFileStateStore, createLogSpoolCoordinator, createLogger, createManagedChildProcesses, createPathsExistCheck, createRequiredEnvCheck, createServiceHost, createServiceRuntimeStateStore, createSupabaseReachableCheck, createTarget, enableLogSpoolFromEnvInChild, ensureLogSpoolFromEnv, evaluateBusyNodeLiveness, generateResponse, getConfigFileDir, getLogSpoolStats, getMainLogSpoolWriterStats, getPollCommandList, getPollCommandListSchema, getPossibleTarget, getScanRemoteRepoValues,
|
|
4039
|
+
export { BaseNodeRuntime, COLLECT_LOG_RUNNER_KEY, CategoryCommand, CategoryNode, CategoryRepo, CategoryTrigger, CommandType, EMPTY_REGISTRY_SLOT_RUNTIME_STATE, LOG_SPOOL_SERVICE_ID_ENV, LOG_TOPIC_TRIGGER, LogLevel, MAX_POLL_TARGET_LIST_SIZE, ManagedChildProcesses, NodeStatus, RepoManager, ResultCode, SERVICE_GUARD_RUNNER_KEY, ServiceGuardNode, ServiceRegistryError, TASK_LOCAL_PACKAGE_CONFIG_FILENAME, TASK_REPO_SCRIPT_CATEGORY, TASK_REPO_USAGE, TASK_RUNNER_CONFIG_DIR, TASK_RUNNER_ROOT_CONFIG_FILENAME, TASK_RUNNER_ROOT_CONFIG_LEGACY_RELATIVE_PATH, TASK_RUNNER_ROOT_CONFIG_RELATIVE_PATH, TRIGGER_LOOP_INTERVAL_MS, TRIGGER_RUNNER_DEFAULT_RETRY_COUNT, TRIGGER_RUNNER_DEFAULT_RETRY_DELAY_MS, TaskManager, TaskNode, TaskRepoValidation, TaskStatus, TaskStatusAction, TriggerManager, TriggerNode, TriggerStatus, applyRegistrySlotGuardStep, buildNodeImportArgs, claimServiceRegistrySlot, createJsonFileStateStore, createL3ChildLauncher, createLogSpoolCoordinator, createLogger, createManagedChildProcesses, createPathsExistCheck, createRequiredEnvCheck, createServiceHost, createServiceRuntimeStateStore, createSupabaseReachableCheck, createTarget, enableLogSpoolFromEnvInChild, ensureLogSpoolFromEnv, evaluateBusyNodeLiveness, generateResponse, getConfigFileDir, getLogSpoolStats, getMainLogSpoolWriterStats, getPollCommandList, getPollCommandListSchema, getPossibleTarget, getScanRemoteRepoValues, guardRetryAfterSec, importJsConfigModule, initSupabaseFromStandardEnv, isChildProcessRunning, isGuardAvailable, isLogEntry, isLogSpoolEnabled, isOptimisticLockResponse, loadCachedJsConfigModule, loadEnvFiles, logManager, logSpoolEnabledFromEnv, markWorkerSpawned, nodeBufferToArrayBuffer, patchChangeNodeStatus, patchChangeTaskStatus, patchClaimTask, patchNodeHeartBeat, patchServiceRuntime, patchStopNode, patchTriggerFired, patchTriggerFiredSchema, pathExists, pollTargetList, pollUntil, postCommand, postCommandSchema, postRegisterNode, postTaskSchema, postTaskWithValidation, postTrigger, postTriggerSchema, publicBaseUrlFromEnv, readAndVerifySourceFile, readEnv, registerCollectLogRunner, registerServiceGuardRunner, registrySlotRuntimePatchFromGuardResult, requireEnv, resolveFirstExistingPath, resolveLogSpoolRoot, resolvePathFromBaseDir, resolvePathFromConfigFile, resolvePathFromCwd, runCollectLogTick, runReadinessChecks, runReadinessGate, runRegistrySlotGuardCheck, runServiceGuardTick, runSingleProcessService, scanEnabledTriggers, scanEnabledTriggersSchema, scanTargetList, sha256Hex, shutdownLogSpool, shutdownLogSpoolFromEnv, spawnTsxChild, supabase, toFileImportHref, updateTargetDetails, validateTaskRepoAndParams, validateWithSchema, waitForServiceReady };
|
|
3677
4040
|
//# sourceMappingURL=node.js.map
|