target-supabase-sdk 0.2.13 → 0.2.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -0
- package/dist/browser.d.ts +1 -0
- package/dist/browser.d.ts.map +1 -1
- package/dist/browser.js +1 -1
- package/dist/chunks/{enable-DqnfLPsQ.js → enable-C7lFooTG.js} +2 -2
- package/dist/chunks/{enable-DqnfLPsQ.js.map → enable-C7lFooTG.js.map} +1 -1
- package/dist/chunks/{task-queue-6X3aglJh.js → task-queue-DCY_0Qj0.js} +32 -3
- package/dist/chunks/task-queue-DCY_0Qj0.js.map +1 -0
- package/dist/node/env/index.d.ts +1 -1
- package/dist/node/env/index.d.ts.map +1 -1
- package/dist/node/node-runtime.base.d.ts +14 -1
- package/dist/node/node-runtime.base.d.ts.map +1 -1
- package/dist/node/node.api.d.ts.map +1 -1
- package/dist/node/node.constant.d.ts +3 -0
- package/dist/node/node.constant.d.ts.map +1 -0
- package/dist/node/node.utils.d.ts +0 -2
- package/dist/node/node.utils.d.ts.map +1 -1
- package/dist/node/readiness/index.d.ts +2 -0
- package/dist/node/readiness/index.d.ts.map +1 -1
- package/dist/node/{service-guard → readiness}/run-readiness-gate.d.ts +1 -1
- package/dist/node/readiness/run-readiness-gate.d.ts.map +1 -0
- package/dist/node/readiness/service-ready-gate.d.ts.map +1 -1
- package/dist/node/runtime-state/create-service-runtime-state-store.d.ts.map +1 -1
- package/dist/node/runtime-state/service-runtime-state.types.d.ts +16 -0
- package/dist/node/runtime-state/service-runtime-state.types.d.ts.map +1 -1
- package/dist/node/service-guard/index.d.ts +4 -4
- package/dist/node/service-guard/index.d.ts.map +1 -1
- package/dist/node/service-guard/register-service-guard-runner.d.ts +1 -2
- package/dist/node/service-guard/register-service-guard-runner.d.ts.map +1 -1
- package/dist/node/service-guard/registry-slot-guard-step.d.ts.map +1 -0
- package/dist/node/service-guard/run-service-guard-tick.d.ts.map +1 -1
- package/dist/node/service-guard/service-guard-node.d.ts +25 -2
- package/dist/node/service-guard/service-guard-node.d.ts.map +1 -1
- package/dist/node/service-guard/service-guard.interface.d.ts +25 -10
- package/dist/node/service-guard/service-guard.interface.d.ts.map +1 -1
- package/dist/node/service-host/claim-registry-slot.d.ts +15 -0
- package/dist/node/service-host/claim-registry-slot.d.ts.map +1 -0
- package/dist/node/service-host/create-l3-child-launcher.d.ts +38 -0
- package/dist/node/service-host/create-l3-child-launcher.d.ts.map +1 -0
- package/dist/node/service-host/index.d.ts +2 -5
- package/dist/node/service-host/index.d.ts.map +1 -1
- package/dist/node/service-host/service-host.d.ts +3 -3
- package/dist/node/service-host/service-host.d.ts.map +1 -1
- package/dist/node/service-host/single-process-service.d.ts +3 -3
- package/dist/node/service-host/single-process-service.d.ts.map +1 -1
- package/dist/node.d.ts +8 -6
- package/dist/node.d.ts.map +1 -1
- package/dist/node.js +627 -264
- package/dist/node.js.map +1 -1
- package/dist/preload.js +1 -1
- package/dist/shared/utils/postgrest.utils.d.ts +23 -0
- package/dist/shared/utils/postgrest.utils.d.ts.map +1 -0
- package/package.json +1 -1
- package/src/browser.ts +1 -0
- package/src/node/env/index.ts +0 -4
- package/src/node/node-runtime.base.ts +33 -5
- package/src/node/node.api.ts +1 -1
- package/src/node/node.constant.ts +2 -0
- package/src/node/node.utils.ts +2 -2
- package/src/node/readiness/index.ts +2 -0
- package/src/node/{service-guard → readiness}/run-readiness-gate.ts +2 -2
- package/src/node/readiness/service-ready-gate.ts +27 -23
- package/src/node/runtime-state/create-service-runtime-state-store.ts +11 -0
- package/src/node/runtime-state/service-runtime-state.types.ts +12 -0
- package/src/node/service-guard/index.ts +7 -8
- package/src/node/service-guard/register-service-guard-runner.ts +0 -2
- package/src/node/service-guard/run-service-guard-tick.ts +5 -2
- package/src/node/service-guard/service-guard-node.ts +229 -6
- package/src/node/service-guard/service-guard.interface.ts +35 -10
- package/src/node/service-host/claim-registry-slot.ts +33 -0
- package/src/node/service-host/create-l3-child-launcher.ts +157 -0
- package/src/node/service-host/index.ts +5 -24
- package/src/node/service-host/service-host.ts +11 -18
- package/src/node/service-host/single-process-service.ts +10 -20
- package/src/node.ts +21 -18
- package/src/shared/utils/postgrest.utils.ts +28 -0
- package/dist/chunks/task-queue-6X3aglJh.js.map +0 -1
- package/dist/node/service-guard/run-readiness-gate.d.ts.map +0 -1
- package/dist/node/service-host/registry-slot-guard-step.d.ts.map +0 -1
- /package/dist/node/{service-host → service-guard}/registry-slot-guard-step.d.ts +0 -0
- /package/src/node/{service-host → service-guard}/registry-slot-guard-step.ts +0 -0
package/dist/preload.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { resolve } from 'node:path';
|
|
2
2
|
import { w as logManager, H as resolveLogMinLevel } from './chunks/list.api-BzbTJBsM.js';
|
|
3
|
-
import { r as resolveProjectRootByPackageName, p as pinEnvProfileFromArgv, l as loadEnvFiles, v as validateLogSpoolPreloadEnv, a as logSpoolEnabledFromEnv, m as mergeLogSpoolExtraProcessRolesEnv } from './chunks/enable-
|
|
3
|
+
import { r as resolveProjectRootByPackageName, p as pinEnvProfileFromArgv, l as loadEnvFiles, v as validateLogSpoolPreloadEnv, a as logSpoolEnabledFromEnv, m as mergeLogSpoolExtraProcessRolesEnv } from './chunks/enable-C7lFooTG.js';
|
|
4
4
|
import 'zod';
|
|
5
5
|
import 'lodash-es';
|
|
6
6
|
import '@supabase/supabase-js';
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PostgREST / Supabase JS client helpers for `target` table filters.
|
|
3
|
+
*
|
|
4
|
+
* @see `.cursor/skills/postgrest-query-filters/SKILL.md`
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* Escape `%`, `_`, and `\` for PostgREST `ilike` patterns.
|
|
8
|
+
* Also strips commas — they break `.or()` filter strings when multiple columns are searched.
|
|
9
|
+
*/
|
|
10
|
+
export declare function escapeIlikePattern(raw: string): string;
|
|
11
|
+
/**
|
|
12
|
+
* Format a JS string array as a PostgreSQL `text[]` literal for PostgREST filter values.
|
|
13
|
+
*
|
|
14
|
+
* Required when tag values contain `:` or other special characters, and when using
|
|
15
|
+
* `.filter(column, "not.ov", literal)` — `.not("column", "ov", jsArray)` stringifies
|
|
16
|
+
* with `Array#toString()` and produces an invalid array literal.
|
|
17
|
+
*
|
|
18
|
+
* @example
|
|
19
|
+
* toPostgrestTextArrayLiteral(["stop-word:base", "small-word"])
|
|
20
|
+
* // → '{"stop-word:base","small-word"}'
|
|
21
|
+
*/
|
|
22
|
+
export declare function toPostgrestTextArrayLiteral(values: readonly string[]): string;
|
|
23
|
+
//# sourceMappingURL=postgrest.utils.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"postgrest.utils.d.ts","sourceRoot":"","sources":["../../../src/shared/utils/postgrest.utils.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAEtD;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,2BAA2B,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAE7E"}
|
package/package.json
CHANGED
package/src/browser.ts
CHANGED
|
@@ -233,6 +233,7 @@ export {
|
|
|
233
233
|
classifyNetworkError,
|
|
234
234
|
formatNetworkError,
|
|
235
235
|
} from "./shared/utils/network-error";
|
|
236
|
+
export { escapeIlikePattern, toPostgrestTextArrayLiteral } from "./shared/utils/postgrest.utils";
|
|
236
237
|
export type { SanitizeFileNameOptions } from "./shared/utils/safe-file-name";
|
|
237
238
|
export { sanitizeFileName, sanitizeParcelFileName } from "./shared/utils/safe-file-name";
|
|
238
239
|
export { sha256Hex } from "./shared/utils/sha256";
|
package/src/node/env/index.ts
CHANGED
|
@@ -12,9 +12,12 @@ import {
|
|
|
12
12
|
} from "../shared/log";
|
|
13
13
|
import { getErrorMessage } from "../shared/utils/error.utils";
|
|
14
14
|
import { patchChangeNodeStatus, patchNodeHeartBeat, patchStopNode, postRegisterNode } from "./node.api";
|
|
15
|
+
import { LOG_TOPIC_NODE } from "./node.constant";
|
|
15
16
|
import { NodeStatus } from "./node.interface";
|
|
16
17
|
import { formatHeartbeat, getRandomInterval } from "./node.utils";
|
|
17
18
|
|
|
19
|
+
export { LOG_TOPIC_NODE };
|
|
20
|
+
|
|
18
21
|
/** 主循环单轮运行时上下文 — 由 start() 创建并向下传递 */
|
|
19
22
|
export interface NodeLoopContext {
|
|
20
23
|
loopTraceId: string;
|
|
@@ -31,7 +34,6 @@ export interface BaseNodeRuntimeOptions {
|
|
|
31
34
|
beforeProcessExit?: () => void | Promise<void>;
|
|
32
35
|
}
|
|
33
36
|
|
|
34
|
-
export const LOG_TOPIC_NODE = "node";
|
|
35
37
|
const LOG_TOPIC_COMMAND = "command";
|
|
36
38
|
const LOG_TOPIC_PROCESS = "process";
|
|
37
39
|
|
|
@@ -118,6 +120,25 @@ abstract class BaseNodeRuntime {
|
|
|
118
120
|
return getRandomInterval();
|
|
119
121
|
}
|
|
120
122
|
|
|
123
|
+
/**
|
|
124
|
+
* When true, the main loop skips commands and {@link runLoopSteps} (heartbeat only).
|
|
125
|
+
* Used by the L3 guard process while waiting for connectivity to recover.
|
|
126
|
+
*/
|
|
127
|
+
protected isSilenced(): boolean {
|
|
128
|
+
return false;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Consecutive heartbeat failures reached {@link HEARTBEAT_FAILURE_THRESHOLD}.
|
|
133
|
+
* Default: shut down the node. Override to wait for recovery instead of exiting.
|
|
134
|
+
*/
|
|
135
|
+
protected async onHealthCheckFailed(_ctx: NodeLoopContext): Promise<void> {
|
|
136
|
+
this.requestShutdown("heartbeat:consecutive-failures");
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** Heartbeat succeeded while {@link isSilenced} is true. Default: no-op. */
|
|
140
|
+
protected async onHeartbeatRecovered(_ctx: NodeLoopContext): Promise<void> {}
|
|
141
|
+
|
|
121
142
|
private getLogNodeId(): string {
|
|
122
143
|
return this.localNodeId ?? BaseNodeRuntime.NODE_ID_PENDING_ASSIGNMENT;
|
|
123
144
|
}
|
|
@@ -284,7 +305,9 @@ abstract class BaseNodeRuntime {
|
|
|
284
305
|
data: { nodeId, loopCount: this.loopCount },
|
|
285
306
|
});
|
|
286
307
|
try {
|
|
287
|
-
|
|
308
|
+
if (!this.isSilenced()) {
|
|
309
|
+
await this.batchCommand(loopCtx);
|
|
310
|
+
}
|
|
288
311
|
if (this.isShuttingDown()) {
|
|
289
312
|
iterLogger.debug("節點關閉中,跳過本輪剩餘步驟", { topic: LOG_TOPIC_NODE });
|
|
290
313
|
break;
|
|
@@ -294,7 +317,9 @@ abstract class BaseNodeRuntime {
|
|
|
294
317
|
iterLogger.debug("節點關閉中,跳過本輪剩餘步驟", { topic: LOG_TOPIC_NODE });
|
|
295
318
|
break;
|
|
296
319
|
}
|
|
297
|
-
|
|
320
|
+
if (!this.isSilenced()) {
|
|
321
|
+
await this.runLoopSteps(loopCtx, heartbeatOk);
|
|
322
|
+
}
|
|
298
323
|
} catch (error) {
|
|
299
324
|
iterLogger.critical(`主循環第 ${this.loopCount} 輪遇到未知錯誤`, {
|
|
300
325
|
topic: LOG_TOPIC_NODE,
|
|
@@ -410,11 +435,11 @@ abstract class BaseNodeRuntime {
|
|
|
410
435
|
threshold: BaseNodeRuntime.HEARTBEAT_FAILURE_THRESHOLD,
|
|
411
436
|
};
|
|
412
437
|
if (failureCount >= BaseNodeRuntime.HEARTBEAT_FAILURE_THRESHOLD) {
|
|
413
|
-
logger.critical("
|
|
438
|
+
logger.critical("連續心跳失敗達上限", {
|
|
414
439
|
topic: LOG_TOPIC_NODE,
|
|
415
440
|
data: logData,
|
|
416
441
|
});
|
|
417
|
-
this.
|
|
442
|
+
await this.onHealthCheckFailed(ctx);
|
|
418
443
|
} else {
|
|
419
444
|
logger.warn("心跳更新失敗", { topic: LOG_TOPIC_NODE, data: logData });
|
|
420
445
|
}
|
|
@@ -422,6 +447,9 @@ abstract class BaseNodeRuntime {
|
|
|
422
447
|
}
|
|
423
448
|
|
|
424
449
|
this.consecutiveHeartbeatFailures = 0;
|
|
450
|
+
if (this.isSilenced()) {
|
|
451
|
+
await this.onHeartbeatRecovered(ctx);
|
|
452
|
+
}
|
|
425
453
|
// Success is the steady state every loop — do not INFO-spam; failures already warn/critical.
|
|
426
454
|
logger.debug("心跳更新成功", {
|
|
427
455
|
topic: LOG_TOPIC_NODE,
|
package/src/node/node.api.ts
CHANGED
|
@@ -2,10 +2,10 @@ import { z } from "zod";
|
|
|
2
2
|
import { createTarget, type QueryFilter, updateTargetDetails, validateWithSchema } from "../core.api";
|
|
3
3
|
import { generateResponse } from "../core.interface";
|
|
4
4
|
import { createLogger } from "../shared/log";
|
|
5
|
+
import { LOG_TOPIC_NODE } from "./node.constant";
|
|
5
6
|
import { CategoryNode, type Node, type NodeDetails, NodeStatus } from "./node.interface";
|
|
6
7
|
|
|
7
8
|
const NODE_STATUS_FIELD = "details->>status" as const;
|
|
8
|
-
const LOG_TOPIC_NODE = "node";
|
|
9
9
|
|
|
10
10
|
const nodeIdSchema = z.string().trim().min(1);
|
|
11
11
|
const traceIdSchema = z.string().trim().min(1).optional();
|
package/src/node/node.utils.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
|
|
1
|
+
const formatTimestamp = (timestamp: number): string => {
|
|
2
2
|
return new Intl.DateTimeFormat("zh-CN", {
|
|
3
3
|
year: "numeric",
|
|
4
4
|
month: "2-digit",
|
|
@@ -11,7 +11,7 @@ export const formatTimestamp = (timestamp: number): string => {
|
|
|
11
11
|
}).format(timestamp);
|
|
12
12
|
};
|
|
13
13
|
|
|
14
|
-
|
|
14
|
+
const formatRelativeTime = (timestamp: number): string => {
|
|
15
15
|
const now = Date.now();
|
|
16
16
|
const diff = now - timestamp;
|
|
17
17
|
|
|
@@ -12,6 +12,8 @@ export {
|
|
|
12
12
|
createSupabaseReachableCheck,
|
|
13
13
|
} from "./readiness-checks";
|
|
14
14
|
export { runReadinessChecks } from "./run-readiness-checks";
|
|
15
|
+
export type { RunReadinessGateInput } from "./run-readiness-gate";
|
|
16
|
+
export { runReadinessGate } from "./run-readiness-gate";
|
|
15
17
|
export type {
|
|
16
18
|
ServiceReadyGate,
|
|
17
19
|
ServiceReadySnapshot,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { LoggerWithScope } from "../../shared/log";
|
|
2
|
-
import type { ReadinessCheck, ReadinessReport } from "
|
|
3
|
-
import { runReadinessChecks } from "
|
|
2
|
+
import type { ReadinessCheck, ReadinessReport } from "./readiness.types";
|
|
3
|
+
import { runReadinessChecks } from "./run-readiness-checks";
|
|
4
4
|
|
|
5
5
|
export interface RunReadinessGateInput {
|
|
6
6
|
checks: ReadinessCheck[];
|
|
@@ -1,8 +1,5 @@
|
|
|
1
1
|
import type { LoggerWithScope } from "../../shared/log";
|
|
2
|
-
|
|
3
|
-
function sleep(ms: number): Promise<void> {
|
|
4
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
5
|
-
}
|
|
2
|
+
import { pollUntil } from "./poll-until";
|
|
6
3
|
|
|
7
4
|
export interface ServiceReadySnapshot {
|
|
8
5
|
failed: boolean;
|
|
@@ -36,30 +33,37 @@ export async function waitForServiceReady(
|
|
|
36
33
|
): Promise<ServiceReadySnapshot> {
|
|
37
34
|
const pollMs = options.pollMs ?? 500;
|
|
38
35
|
const defaultFailureMessage = options.failureMessage ?? "Readiness checks failed";
|
|
39
|
-
const deadline = Date.now() + options.timeoutMs;
|
|
40
36
|
let last: ServiceReadySnapshot = { failed: false, ready: false };
|
|
41
37
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
38
|
+
try {
|
|
39
|
+
await pollUntil({
|
|
40
|
+
timeoutMs: options.timeoutMs,
|
|
41
|
+
intervalMs: pollMs,
|
|
42
|
+
until: async () => {
|
|
43
|
+
last = await gate.read();
|
|
44
|
+
if (last.failed) {
|
|
45
|
+
throw new Error(last.failureMessage ?? defaultFailureMessage);
|
|
46
|
+
}
|
|
47
|
+
return last.ready;
|
|
48
|
+
},
|
|
49
|
+
});
|
|
50
|
+
} catch (error) {
|
|
45
51
|
if (last.failed) {
|
|
46
|
-
throw new Error(
|
|
52
|
+
throw error instanceof Error ? error : new Error(defaultFailureMessage);
|
|
47
53
|
}
|
|
48
|
-
|
|
49
|
-
if (
|
|
50
|
-
|
|
51
|
-
topic: "readiness",
|
|
52
|
-
data: last.readyDetail ?? {},
|
|
53
|
-
});
|
|
54
|
-
return last;
|
|
54
|
+
const isPollTimeout = error instanceof Error && error.message.startsWith("pollUntil timeout after");
|
|
55
|
+
if (!isPollTimeout) {
|
|
56
|
+
throw error;
|
|
55
57
|
}
|
|
56
|
-
|
|
57
|
-
|
|
58
|
+
const message =
|
|
59
|
+
options.formatTimeoutError?.(last, options.timeoutMs) ??
|
|
60
|
+
`Service ready timeout after ${options.timeoutMs}ms (failed=${last.failed}, ready=${last.ready})`;
|
|
61
|
+
throw new Error(message);
|
|
58
62
|
}
|
|
59
63
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
64
|
+
options.logger?.info("service ready", {
|
|
65
|
+
topic: "readiness",
|
|
66
|
+
data: last.readyDetail ?? {},
|
|
67
|
+
});
|
|
68
|
+
return last;
|
|
65
69
|
}
|
|
@@ -58,11 +58,22 @@ function createDefaultServiceRuntimeState<TExtraSlices extends Record<string, ob
|
|
|
58
58
|
lastDecision: null,
|
|
59
59
|
lastSpawnAt: null,
|
|
60
60
|
spawnCount: 0,
|
|
61
|
+
mode: "healthy",
|
|
62
|
+
silentEnteredAt: null,
|
|
63
|
+
silentLastHeartbeatAt: null,
|
|
64
|
+
silentBackoffMs: 0,
|
|
65
|
+
silentConsecutiveFailures: 0,
|
|
66
|
+
silentRecoveryAttempt: 0,
|
|
67
|
+
silentEvents: [],
|
|
61
68
|
};
|
|
62
69
|
const scheduler: SchedulerRuntimeSlice = {
|
|
63
70
|
lastTickAt: null,
|
|
64
71
|
lastPostedCount: 0,
|
|
65
72
|
lastFired: {},
|
|
73
|
+
pid: null,
|
|
74
|
+
spawnedAt: null,
|
|
75
|
+
ready: false,
|
|
76
|
+
readyAt: null,
|
|
66
77
|
};
|
|
67
78
|
const worker: WorkerRuntimeSlice = {
|
|
68
79
|
pid: null,
|
|
@@ -15,12 +15,24 @@ export interface GuardRuntimeSlice {
|
|
|
15
15
|
lastDecision: string | null;
|
|
16
16
|
lastSpawnAt: string | null;
|
|
17
17
|
spawnCount: number;
|
|
18
|
+
/** `healthy` when omitted (legacy shards). Silent: heartbeat-only wait. Recovering: respawn business nodes. */
|
|
19
|
+
mode: "healthy" | "silent" | "recovering";
|
|
20
|
+
silentEnteredAt: string | null;
|
|
21
|
+
silentLastHeartbeatAt: string | null;
|
|
22
|
+
silentBackoffMs: number;
|
|
23
|
+
silentConsecutiveFailures: number;
|
|
24
|
+
silentRecoveryAttempt: number;
|
|
25
|
+
silentEvents: { at: string; type: string; detail?: string }[];
|
|
18
26
|
}
|
|
19
27
|
|
|
20
28
|
export interface SchedulerRuntimeSlice {
|
|
21
29
|
lastTickAt: string | null;
|
|
22
30
|
lastPostedCount: number;
|
|
23
31
|
lastFired: Record<string, string>;
|
|
32
|
+
pid: number | null;
|
|
33
|
+
spawnedAt: string | null;
|
|
34
|
+
ready: boolean;
|
|
35
|
+
readyAt: string | null;
|
|
24
36
|
}
|
|
25
37
|
|
|
26
38
|
export interface WorkerRuntimeSlice {
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
export { registerServiceGuardRunner } from "./register-service-guard-runner";
|
|
2
|
-
export type {
|
|
3
|
-
|
|
2
|
+
export type {
|
|
3
|
+
ApplyRegistrySlotGuardInput,
|
|
4
|
+
ApplyRegistrySlotGuardResult,
|
|
5
|
+
} from "./registry-slot-guard-step";
|
|
6
|
+
export { applyRegistrySlotGuardStep } from "./registry-slot-guard-step";
|
|
4
7
|
export { runServiceGuardTick } from "./run-service-guard-tick";
|
|
5
8
|
export type {
|
|
6
9
|
RegisterServiceGuardRunnerOptions,
|
|
@@ -8,10 +11,6 @@ export type {
|
|
|
8
11
|
ServiceGuardTickInput,
|
|
9
12
|
ServiceGuardTickResult,
|
|
10
13
|
} from "./service-guard.interface";
|
|
11
|
-
export { SERVICE_GUARD_RUNNER_KEY } from "./service-guard.interface";
|
|
14
|
+
export { guardRetryAfterSec, isGuardAvailable, SERVICE_GUARD_RUNNER_KEY } from "./service-guard.interface";
|
|
12
15
|
export { ServiceGuardNode } from "./service-guard-node";
|
|
13
|
-
export {
|
|
14
|
-
getWorkerSpawnCooldownLastAt,
|
|
15
|
-
markWorkerSpawned,
|
|
16
|
-
resetWorkerSpawnCooldownForTests,
|
|
17
|
-
} from "./worker-spawn-cooldown";
|
|
16
|
+
export { markWorkerSpawned } from "./worker-spawn-cooldown";
|
|
@@ -1,5 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { scanTargetList } from "../../core.api";
|
|
2
|
+
import { patchServiceRuntime } from "../../service/registry.service";
|
|
3
|
+
import { CategoryNode, type Node } from "../node.interface";
|
|
4
|
+
import { evaluateBusyNodeLiveness } from "../node-liveness";
|
|
5
|
+
import { applyRegistrySlotGuardStep } from "./registry-slot-guard-step";
|
|
3
6
|
import type { ServiceGuardTickInput, ServiceGuardTickResult } from "./service-guard.interface";
|
|
4
7
|
import { getWorkerSpawnCooldownLastAt, markWorkerSpawned } from "./worker-spawn-cooldown";
|
|
5
8
|
|
|
@@ -1,18 +1,35 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { createLogger, type LoggerWithScope } from "../../shared/log";
|
|
2
2
|
import { registerCollectLogRunner } from "../../shared/log/spool/register-collect-log-runner";
|
|
3
|
+
import { getErrorMessage } from "../../shared/utils/error.utils";
|
|
3
4
|
import { TriggerNode } from "../../trigger/trigger-node";
|
|
5
|
+
import type { NodeLoopContext } from "../node-runtime.base";
|
|
6
|
+
import { pollUntil } from "../readiness/poll-until";
|
|
7
|
+
import { runReadinessGate } from "../readiness/run-readiness-gate";
|
|
8
|
+
import type { GuardRuntimeSlice } from "../runtime-state/service-runtime-state.types";
|
|
4
9
|
import { registerServiceGuardRunner } from "./register-service-guard-runner";
|
|
5
|
-
import { runReadinessGate } from "./run-readiness-gate";
|
|
6
10
|
import type { ServiceGuardNodeOptions } from "./service-guard.interface";
|
|
11
|
+
import { markWorkerSpawned } from "./worker-spawn-cooldown";
|
|
12
|
+
|
|
13
|
+
const SILENT_BACKOFF_MS = [15_000, 30_000, 60_000, 120_000, 240_000, 300_000] as const;
|
|
14
|
+
const SILENT_EVENT_LIMIT = 32;
|
|
15
|
+
const DEFAULT_BUSINESS_READY_TIMEOUT_MS = 180_000;
|
|
7
16
|
|
|
8
17
|
/**
|
|
9
|
-
* L3 guard process TriggerNode: readiness gate,
|
|
18
|
+
* L3 guard process TriggerNode: readiness gate, business-node spawn owner,
|
|
10
19
|
* and {@link registerServiceGuardRunner} tick (registry slot + liveness + runtime).
|
|
11
20
|
*
|
|
21
|
+
* Consecutive heartbeat failure enters silent mode (no process exit). Recovery
|
|
22
|
+
* respawns business nodes; leaving silent requires heartbeat + all nodes ready.
|
|
23
|
+
*
|
|
12
24
|
* Use {@link ServiceGuardNode.create} so the guard runner is registered before `start()`.
|
|
13
25
|
*/
|
|
14
26
|
class ServiceGuardNode extends TriggerNode {
|
|
15
27
|
private readonly options: ServiceGuardNodeOptions;
|
|
28
|
+
private readonly businessReadyTimeoutMs: number;
|
|
29
|
+
private mode: GuardRuntimeSlice["mode"] = "healthy";
|
|
30
|
+
private silentBackoffIndex = 0;
|
|
31
|
+
private silentEvents: GuardRuntimeSlice["silentEvents"] = [];
|
|
32
|
+
private recoverPromise: Promise<void> | null = null;
|
|
16
33
|
|
|
17
34
|
private constructor(options: ServiceGuardNodeOptions) {
|
|
18
35
|
super({
|
|
@@ -20,14 +37,16 @@ class ServiceGuardNode extends TriggerNode {
|
|
|
20
37
|
beforeProcessExit: options.beforeProcessExit,
|
|
21
38
|
});
|
|
22
39
|
this.options = options;
|
|
40
|
+
this.businessReadyTimeoutMs = options.businessReadyTimeoutMs ?? DEFAULT_BUSINESS_READY_TIMEOUT_MS;
|
|
23
41
|
}
|
|
24
42
|
|
|
25
43
|
/** Register the guard runner and return a ready-to-start node. */
|
|
26
44
|
static create(options: ServiceGuardNodeOptions): ServiceGuardNode {
|
|
27
45
|
registerServiceGuardRunner({
|
|
46
|
+
...options.guardRunner,
|
|
28
47
|
serviceValue: options.serviceValue,
|
|
29
48
|
logTopic: options.logTopic,
|
|
30
|
-
|
|
49
|
+
spawnWorker: options.spawnBusinessNodes,
|
|
31
50
|
});
|
|
32
51
|
registerCollectLogRunner({
|
|
33
52
|
serviceValue: options.serviceValue,
|
|
@@ -36,6 +55,38 @@ class ServiceGuardNode extends TriggerNode {
|
|
|
36
55
|
return new ServiceGuardNode(options);
|
|
37
56
|
}
|
|
38
57
|
|
|
58
|
+
protected isSilenced(): boolean {
|
|
59
|
+
return this.mode !== "healthy";
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
protected getLoopIntervalMs(): number {
|
|
63
|
+
if (this.mode === "silent") {
|
|
64
|
+
return SILENT_BACKOFF_MS[this.silentBackoffIndex] ?? 300_000;
|
|
65
|
+
}
|
|
66
|
+
return super.getLoopIntervalMs();
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
protected async onHealthCheckFailed(_ctx: NodeLoopContext): Promise<void> {
|
|
70
|
+
if (this.mode !== "healthy") {
|
|
71
|
+
this.bumpSilentBackoff();
|
|
72
|
+
await this.persistGuard({
|
|
73
|
+
lastDecision: "silent_heartbeat_fail",
|
|
74
|
+
silentConsecutiveFailures: this.silentConsecutiveFailures(),
|
|
75
|
+
silentBackoffMs: this.getLoopIntervalMs(),
|
|
76
|
+
silentLastHeartbeatAt: new Date().toISOString(),
|
|
77
|
+
});
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
await this.enterSilent("heartbeat:consecutive-failures");
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
protected async onHeartbeatRecovered(ctx: NodeLoopContext): Promise<void> {
|
|
84
|
+
if (this.mode !== "silent") {
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
await this.recoverFromSilent(ctx);
|
|
88
|
+
}
|
|
89
|
+
|
|
39
90
|
protected async onBeforeRegisterNode(nodeLogger: LoggerWithScope): Promise<void> {
|
|
40
91
|
await runReadinessGate({
|
|
41
92
|
checks: this.options.readinessChecks,
|
|
@@ -46,11 +97,183 @@ class ServiceGuardNode extends TriggerNode {
|
|
|
46
97
|
|
|
47
98
|
await super.onBeforeRegisterNode(nodeLogger);
|
|
48
99
|
|
|
49
|
-
nodeLogger.info("Spawning
|
|
100
|
+
nodeLogger.info("Spawning business nodes", {
|
|
50
101
|
topic: this.options.logTopic ?? "guard",
|
|
51
102
|
});
|
|
52
|
-
await this.options.
|
|
103
|
+
await this.options.spawnBusinessNodes("guard:readiness-passed");
|
|
104
|
+
markWorkerSpawned();
|
|
53
105
|
this.options.onWorkerSpawned?.();
|
|
106
|
+
|
|
107
|
+
await this.waitUntilBusinessReady();
|
|
108
|
+
await this.persistGuard({
|
|
109
|
+
mode: "healthy",
|
|
110
|
+
lastDecision: "bootstrap_ready",
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
protected async runLoopSteps(ctx: NodeLoopContext, heartbeatOk: boolean): Promise<void> {
|
|
115
|
+
await super.runLoopSteps(ctx, heartbeatOk);
|
|
116
|
+
if (this.mode !== "healthy" || !heartbeatOk) {
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
// Idempotent — respawn exited children without entering silent (worker stale is the runner's job).
|
|
120
|
+
try {
|
|
121
|
+
await this.options.spawnBusinessNodes("guard:ensure-business");
|
|
122
|
+
} catch (error) {
|
|
123
|
+
this.createGuardLogger().warn("ensure-business spawn failed", {
|
|
124
|
+
topic: this.options.logTopic ?? "guard",
|
|
125
|
+
data: { error: getErrorMessage(error) },
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
private silentConsecutiveFailures(): number {
|
|
131
|
+
return this.silentBackoffIndex + 1;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
private bumpSilentBackoff(): void {
|
|
135
|
+
if (this.silentBackoffIndex < SILENT_BACKOFF_MS.length - 1) {
|
|
136
|
+
this.silentBackoffIndex += 1;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
private async enterSilent(reason: string): Promise<void> {
|
|
141
|
+
this.mode = "silent";
|
|
142
|
+
this.silentBackoffIndex = 0;
|
|
143
|
+
const enteredAt = new Date().toISOString();
|
|
144
|
+
this.pushEvent("enter", reason);
|
|
145
|
+
await this.persistGuard({
|
|
146
|
+
mode: "silent",
|
|
147
|
+
lastDecision: `silent:${reason}`,
|
|
148
|
+
silentEnteredAt: enteredAt,
|
|
149
|
+
silentLastHeartbeatAt: enteredAt,
|
|
150
|
+
silentBackoffMs: SILENT_BACKOFF_MS[0],
|
|
151
|
+
silentConsecutiveFailures: 1,
|
|
152
|
+
silentRecoveryAttempt: 0,
|
|
153
|
+
silentEvents: this.silentEvents,
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
const logger = this.createGuardLogger();
|
|
157
|
+
logger.warn("Entering silent mode — service unavailable until heartbeat + business nodes recover", {
|
|
158
|
+
topic: this.options.logTopic ?? "guard",
|
|
159
|
+
data: { reason },
|
|
160
|
+
});
|
|
161
|
+
|
|
162
|
+
try {
|
|
163
|
+
await this.options.stopBusinessNodes();
|
|
164
|
+
} catch (error) {
|
|
165
|
+
logger.warn("stopBusinessNodes failed (best-effort)", {
|
|
166
|
+
topic: this.options.logTopic ?? "guard",
|
|
167
|
+
data: { error: getErrorMessage(error) },
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
private async recoverFromSilent(ctx: NodeLoopContext): Promise<void> {
|
|
173
|
+
if (this.recoverPromise != null) {
|
|
174
|
+
return this.recoverPromise;
|
|
175
|
+
}
|
|
176
|
+
this.recoverPromise = this.runRecover(ctx).finally(() => {
|
|
177
|
+
this.recoverPromise = null;
|
|
178
|
+
});
|
|
179
|
+
return this.recoverPromise;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
private async runRecover(ctx: NodeLoopContext): Promise<void> {
|
|
183
|
+
this.mode = "recovering";
|
|
184
|
+
this.pushEvent("recover_start", ctx.loopTraceId);
|
|
185
|
+
await this.persistGuard({
|
|
186
|
+
mode: "recovering",
|
|
187
|
+
lastDecision: "recovering",
|
|
188
|
+
silentLastHeartbeatAt: new Date().toISOString(),
|
|
189
|
+
silentRecoveryAttempt: this.silentEvents.filter((event) => event.type === "recover_start").length,
|
|
190
|
+
silentEvents: this.silentEvents,
|
|
191
|
+
});
|
|
192
|
+
|
|
193
|
+
const logger = this.createGuardLogger();
|
|
194
|
+
logger.info("Heartbeat restored — recovering business nodes", {
|
|
195
|
+
topic: this.options.logTopic ?? "guard",
|
|
196
|
+
});
|
|
197
|
+
|
|
198
|
+
try {
|
|
199
|
+
await this.options.stopBusinessNodes();
|
|
200
|
+
} catch (error) {
|
|
201
|
+
logger.warn("stopBusinessNodes failed before respawn", {
|
|
202
|
+
topic: this.options.logTopic ?? "guard",
|
|
203
|
+
data: { error: getErrorMessage(error) },
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
try {
|
|
208
|
+
await this.options.spawnBusinessNodes("guard:silent-recover");
|
|
209
|
+
markWorkerSpawned();
|
|
210
|
+
await this.waitUntilBusinessReady();
|
|
211
|
+
} catch (error) {
|
|
212
|
+
this.pushEvent("recover_fail", getErrorMessage(error));
|
|
213
|
+
this.mode = "silent";
|
|
214
|
+
this.bumpSilentBackoff();
|
|
215
|
+
await this.persistGuard({
|
|
216
|
+
mode: "silent",
|
|
217
|
+
lastDecision: "recover_fail",
|
|
218
|
+
silentBackoffMs: this.getLoopIntervalMs(),
|
|
219
|
+
silentEvents: this.silentEvents,
|
|
220
|
+
});
|
|
221
|
+
logger.warn("Recovery failed — returning to silent", {
|
|
222
|
+
topic: this.options.logTopic ?? "guard",
|
|
223
|
+
data: { error: getErrorMessage(error) },
|
|
224
|
+
});
|
|
225
|
+
try {
|
|
226
|
+
await this.options.stopBusinessNodes();
|
|
227
|
+
} catch {
|
|
228
|
+
// best-effort — leftover children are stopped again on the next recover
|
|
229
|
+
}
|
|
230
|
+
return;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
this.mode = "healthy";
|
|
234
|
+
this.silentBackoffIndex = 0;
|
|
235
|
+
this.pushEvent("recover_ok");
|
|
236
|
+
await this.persistGuard({
|
|
237
|
+
mode: "healthy",
|
|
238
|
+
lastDecision: "recover_ok",
|
|
239
|
+
silentEnteredAt: null,
|
|
240
|
+
silentBackoffMs: 0,
|
|
241
|
+
silentConsecutiveFailures: 0,
|
|
242
|
+
silentEvents: this.silentEvents,
|
|
243
|
+
});
|
|
244
|
+
logger.success("Left silent mode — business nodes ready", {
|
|
245
|
+
topic: this.options.logTopic ?? "guard",
|
|
246
|
+
});
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
private async waitUntilBusinessReady(): Promise<void> {
|
|
250
|
+
try {
|
|
251
|
+
await pollUntil({
|
|
252
|
+
timeoutMs: this.businessReadyTimeoutMs,
|
|
253
|
+
intervalMs: 500,
|
|
254
|
+
until: () => this.options.isBusinessReady(),
|
|
255
|
+
});
|
|
256
|
+
} catch {
|
|
257
|
+
throw new Error(`Business nodes not ready within ${this.businessReadyTimeoutMs}ms`);
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
private pushEvent(type: string, detail?: string): void {
|
|
262
|
+
this.silentEvents = [...this.silentEvents, { at: new Date().toISOString(), type, detail }].slice(
|
|
263
|
+
-SILENT_EVENT_LIMIT,
|
|
264
|
+
);
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
private async persistGuard(patch: Partial<GuardRuntimeSlice>): Promise<void> {
|
|
268
|
+
try {
|
|
269
|
+
await this.options.guardRunner.onGuardPatch(patch);
|
|
270
|
+
} catch {
|
|
271
|
+
// best-effort — disk/runtime state must not take down the guard loop
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
private createGuardLogger(): LoggerWithScope {
|
|
276
|
+
return createLogger({ module: "service-guard" });
|
|
54
277
|
}
|
|
55
278
|
}
|
|
56
279
|
|