target-supabase-sdk 0.2.12 → 0.2.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.d.ts +1 -1
- package/dist/browser.d.ts.map +1 -1
- package/dist/browser.js +1 -1
- package/dist/chunks/enable-C7lFooTG.js +874 -0
- package/dist/chunks/enable-C7lFooTG.js.map +1 -0
- package/dist/chunks/list.api-BzbTJBsM.js.map +1 -1
- package/dist/chunks/{task-queue-H4mpSW8R.js → task-queue-qSnsfiIg.js} +111 -4
- package/dist/chunks/task-queue-qSnsfiIg.js.map +1 -0
- package/dist/node/env/index.d.ts +1 -1
- package/dist/node/env/index.d.ts.map +1 -1
- package/dist/node/node-runtime.base.d.ts +14 -1
- package/dist/node/node-runtime.base.d.ts.map +1 -1
- package/dist/node/node.api.d.ts.map +1 -1
- package/dist/node/node.constant.d.ts +3 -0
- package/dist/node/node.constant.d.ts.map +1 -0
- package/dist/node/node.utils.d.ts +0 -2
- package/dist/node/node.utils.d.ts.map +1 -1
- package/dist/node/preload/run-service-preload.d.ts.map +1 -1
- package/dist/node/preload/service-preload.interface.d.ts +3 -1
- package/dist/node/preload/service-preload.interface.d.ts.map +1 -1
- package/dist/node/process/managed-child-processes.d.ts.map +1 -1
- package/dist/node/readiness/index.d.ts +2 -0
- package/dist/node/readiness/index.d.ts.map +1 -1
- package/dist/node/{service-guard → readiness}/run-readiness-gate.d.ts +1 -1
- package/dist/node/readiness/run-readiness-gate.d.ts.map +1 -0
- package/dist/node/readiness/service-ready-gate.d.ts.map +1 -1
- package/dist/node/runtime-state/create-service-runtime-state-store.d.ts.map +1 -1
- package/dist/node/runtime-state/service-runtime-state.types.d.ts +16 -0
- package/dist/node/runtime-state/service-runtime-state.types.d.ts.map +1 -1
- package/dist/node/service-guard/index.d.ts +4 -4
- package/dist/node/service-guard/index.d.ts.map +1 -1
- package/dist/node/service-guard/register-service-guard-runner.d.ts +1 -2
- package/dist/node/service-guard/register-service-guard-runner.d.ts.map +1 -1
- package/dist/node/service-guard/registry-slot-guard-step.d.ts.map +1 -0
- package/dist/node/service-guard/run-service-guard-tick.d.ts.map +1 -1
- package/dist/node/service-guard/service-guard-node.d.ts +25 -2
- package/dist/node/service-guard/service-guard-node.d.ts.map +1 -1
- package/dist/node/service-guard/service-guard.interface.d.ts +25 -10
- package/dist/node/service-guard/service-guard.interface.d.ts.map +1 -1
- package/dist/node/service-host/claim-registry-slot.d.ts +15 -0
- package/dist/node/service-host/claim-registry-slot.d.ts.map +1 -0
- package/dist/node/service-host/create-l3-child-launcher.d.ts +38 -0
- package/dist/node/service-host/create-l3-child-launcher.d.ts.map +1 -0
- package/dist/node/service-host/index.d.ts +2 -4
- package/dist/node/service-host/index.d.ts.map +1 -1
- package/dist/node/service-host/service-host.d.ts +8 -4
- package/dist/node/service-host/service-host.d.ts.map +1 -1
- package/dist/node/service-host/single-process-service.d.ts +3 -3
- package/dist/node/service-host/single-process-service.d.ts.map +1 -1
- package/dist/node.d.ts +17 -10
- package/dist/node.d.ts.map +1 -1
- package/dist/node.js +927 -119
- package/dist/node.js.map +1 -1
- package/dist/preload.js +9 -6
- package/dist/preload.js.map +1 -1
- package/dist/service/index.d.ts +2 -2
- package/dist/service/index.d.ts.map +1 -1
- package/dist/service/registry.service.d.ts +19 -0
- package/dist/service/registry.service.d.ts.map +1 -1
- package/dist/shared/log/core/create-logger.d.ts.map +1 -0
- package/dist/shared/log/{log-batch.d.ts → core/log-batch.d.ts} +4 -4
- package/dist/shared/log/core/log-batch.d.ts.map +1 -0
- package/dist/shared/log/core/log-manager.d.ts.map +1 -0
- package/dist/shared/log/core/log-min-level.d.ts.map +1 -0
- package/dist/shared/log/core/log-scope.d.ts.map +1 -0
- package/dist/shared/log/index.d.ts +13 -13
- package/dist/shared/log/index.d.ts.map +1 -1
- package/dist/shared/log/spool/collector.d.ts +21 -0
- package/dist/shared/log/spool/collector.d.ts.map +1 -0
- package/dist/shared/log/spool/config.d.ts +7 -0
- package/dist/shared/log/spool/config.d.ts.map +1 -0
- package/dist/shared/log/spool/coordinator.d.ts +18 -0
- package/dist/shared/log/spool/coordinator.d.ts.map +1 -0
- package/dist/shared/log/spool/enable.d.ts +23 -0
- package/dist/shared/log/spool/enable.d.ts.map +1 -0
- package/dist/shared/log/spool/file.d.ts +21 -0
- package/dist/shared/log/spool/file.d.ts.map +1 -0
- package/dist/shared/log/spool/interface.d.ts +55 -0
- package/dist/shared/log/spool/interface.d.ts.map +1 -0
- package/dist/shared/log/spool/paths.d.ts +6 -0
- package/dist/shared/log/spool/paths.d.ts.map +1 -0
- package/dist/shared/log/spool/process-roles.d.ts +17 -0
- package/dist/shared/log/spool/process-roles.d.ts.map +1 -0
- package/dist/shared/log/spool/register-collect-log-runner.d.ts +15 -0
- package/dist/shared/log/spool/register-collect-log-runner.d.ts.map +1 -0
- package/dist/shared/log/spool/service-lifecycle.d.ts +13 -0
- package/dist/shared/log/spool/service-lifecycle.d.ts.map +1 -0
- package/dist/shared/log/spool/writer.d.ts +32 -0
- package/dist/shared/log/spool/writer.d.ts.map +1 -0
- package/dist/shared/log/{log-persist-batch-id.d.ts → upload/batch-id.d.ts} +3 -3
- package/dist/shared/log/upload/batch-id.d.ts.map +1 -0
- package/dist/shared/log/upload/env.d.ts +7 -0
- package/dist/shared/log/upload/env.d.ts.map +1 -0
- package/dist/shared/log/{log-persist-flush.d.ts → upload/flush.d.ts} +3 -3
- package/dist/shared/log/upload/flush.d.ts.map +1 -0
- package/dist/shared/log/{log-persist-hook.d.ts → upload/hook.d.ts} +2 -2
- package/dist/shared/log/upload/hook.d.ts.map +1 -0
- package/dist/shared/log/upload/interface.d.ts +33 -0
- package/dist/shared/log/upload/interface.d.ts.map +1 -0
- package/dist/shared/log/upload/logger.d.ts +22 -0
- package/dist/shared/log/upload/logger.d.ts.map +1 -0
- package/package.json +1 -1
- package/src/browser.ts +1 -0
- package/src/node/env/index.ts +0 -4
- package/src/node/node-runtime.base.ts +33 -5
- package/src/node/node.api.ts +1 -1
- package/src/node/node.constant.ts +2 -0
- package/src/node/node.utils.ts +2 -2
- package/src/node/preload/run-service-preload.ts +11 -6
- package/src/node/preload/service-preload.interface.ts +3 -1
- package/src/node/process/create-managed-child-processes.ts +1 -1
- package/src/node/process/managed-child-processes.ts +8 -1
- package/src/node/readiness/index.ts +2 -0
- package/src/node/{service-guard → readiness}/run-readiness-gate.ts +2 -2
- package/src/node/readiness/service-ready-gate.ts +27 -23
- package/src/node/runtime-state/create-service-runtime-state-store.ts +11 -0
- package/src/node/runtime-state/service-runtime-state.types.ts +12 -0
- package/src/node/service-guard/index.ts +7 -8
- package/src/node/service-guard/register-service-guard-runner.ts +0 -2
- package/src/node/service-guard/run-service-guard-tick.ts +5 -2
- package/src/node/service-guard/service-guard-node.ts +234 -6
- package/src/node/service-guard/service-guard.interface.ts +35 -10
- package/src/node/service-host/claim-registry-slot.ts +33 -0
- package/src/node/service-host/create-l3-child-launcher.ts +157 -0
- package/src/node/service-host/index.ts +5 -20
- package/src/node/service-host/service-host.ts +33 -21
- package/src/node/service-host/single-process-service.ts +10 -20
- package/src/node.ts +45 -35
- package/src/service/index.ts +3 -0
- package/src/service/registry.service.ts +142 -0
- package/src/shared/log/README.md +206 -0
- package/src/shared/log/{log-batch.ts → core/log-batch.ts} +4 -4
- package/src/shared/log/{log-manager.ts → core/log-manager.ts} +3 -3
- package/src/shared/log/{log-min-level.ts → core/log-min-level.ts} +1 -1
- package/src/shared/log/{log-scope.ts → core/log-scope.ts} +1 -1
- package/src/shared/log/index.ts +13 -18
- package/src/shared/log/spool/collector.ts +330 -0
- package/src/shared/log/spool/config.ts +56 -0
- package/src/shared/log/spool/coordinator.ts +55 -0
- package/src/shared/log/spool/enable.ts +136 -0
- package/src/shared/log/spool/file.ts +75 -0
- package/src/shared/log/spool/interface.ts +62 -0
- package/src/shared/log/spool/paths.ts +30 -0
- package/src/shared/log/spool/process-roles.ts +75 -0
- package/src/shared/log/spool/register-collect-log-runner.ts +63 -0
- package/src/shared/log/spool/service-lifecycle.ts +35 -0
- package/src/shared/log/spool/writer.ts +273 -0
- package/src/shared/log/{log-persist-batch-id.ts → upload/batch-id.ts} +2 -2
- package/src/shared/log/upload/env.ts +26 -0
- package/src/shared/log/{log-persist-flush.ts → upload/flush.ts} +8 -7
- package/src/shared/log/{log-persist-hook.ts → upload/hook.ts} +1 -1
- package/src/shared/log/upload/interface.ts +38 -0
- package/src/shared/log/upload/logger.ts +37 -0
- package/dist/chunks/enable-log-persist-Dl55_mv2.js +0 -1674
- package/dist/chunks/enable-log-persist-Dl55_mv2.js.map +0 -1
- package/dist/chunks/task-queue-H4mpSW8R.js.map +0 -1
- package/dist/node/service-guard/run-readiness-gate.d.ts.map +0 -1
- package/dist/node/service-host/registry-slot-guard-step.d.ts.map +0 -1
- package/dist/shared/log/create-logger.d.ts.map +0 -1
- package/dist/shared/log/enable-log-persist.d.ts +0 -28
- package/dist/shared/log/enable-log-persist.d.ts.map +0 -1
- package/dist/shared/log/log-batch.d.ts.map +0 -1
- package/dist/shared/log/log-manager.d.ts.map +0 -1
- package/dist/shared/log/log-min-level.d.ts.map +0 -1
- package/dist/shared/log/log-persist-batch-id.d.ts.map +0 -1
- package/dist/shared/log/log-persist-coordinator.d.ts +0 -37
- package/dist/shared/log/log-persist-coordinator.d.ts.map +0 -1
- package/dist/shared/log/log-persist-errors.d.ts +0 -4
- package/dist/shared/log/log-persist-errors.d.ts.map +0 -1
- package/dist/shared/log/log-persist-flush.d.ts.map +0 -1
- package/dist/shared/log/log-persist-hook.d.ts.map +0 -1
- package/dist/shared/log/log-persist-logger.d.ts +0 -16
- package/dist/shared/log/log-persist-logger.d.ts.map +0 -1
- package/dist/shared/log/log-persist-registry.d.ts +0 -18
- package/dist/shared/log/log-persist-registry.d.ts.map +0 -1
- package/dist/shared/log/log-persist.config.d.ts +0 -9
- package/dist/shared/log/log-persist.config.d.ts.map +0 -1
- package/dist/shared/log/log-persist.d.ts +0 -62
- package/dist/shared/log/log-persist.d.ts.map +0 -1
- package/dist/shared/log/log-persist.interface.d.ts +0 -132
- package/dist/shared/log/log-persist.interface.d.ts.map +0 -1
- package/dist/shared/log/log-scope.d.ts.map +0 -1
- package/src/shared/log/enable-log-persist.ts +0 -225
- package/src/shared/log/log-persist-coordinator.ts +0 -107
- package/src/shared/log/log-persist-errors.ts +0 -16
- package/src/shared/log/log-persist-logger.ts +0 -61
- package/src/shared/log/log-persist-registry.ts +0 -252
- package/src/shared/log/log-persist.config.ts +0 -115
- package/src/shared/log/log-persist.interface.ts +0 -150
- package/src/shared/log/log-persist.ts +0 -822
- /package/dist/node/{service-host → service-guard}/registry-slot-guard-step.d.ts +0 -0
- /package/dist/shared/log/{create-logger.d.ts → core/create-logger.d.ts} +0 -0
- /package/dist/shared/log/{log-manager.d.ts → core/log-manager.d.ts} +0 -0
- /package/dist/shared/log/{log-min-level.d.ts → core/log-min-level.d.ts} +0 -0
- /package/dist/shared/log/{log-scope.d.ts → core/log-scope.d.ts} +0 -0
- /package/src/node/{service-host → service-guard}/registry-slot-guard-step.ts +0 -0
- /package/src/shared/log/{create-logger.ts → core/create-logger.ts} +0 -0
package/dist/node.js
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { M as MAX_POLL_TARGET_LIST_SIZE, V as validateWithSchema, g as createLogger, z as pollTargetList, k as generateResponse, h as createTarget, K as supabase, w as logManager, X as createScope, d as LogLevel, Y as patchScope, J as scanTargetList, Z as withModule, l as getPossibleTarget, t as isOptimisticLockResponse, R as updateTargetDetails } from './chunks/list.api-BzbTJBsM.js';
|
|
3
3
|
export { B as BaseValidator, C as CREATE_TARGET_ALREADY_EXISTS_MESSAGE, a as CategoryList, L as LOG_BATCH_LOADER_KEY, b as LOG_MIN_LEVEL_ENV_KEY, c as LOG_PERSIST_LOADER_KEY, e as MAX_TARGET_LIST_PAGE_SIZE, O as OPTIMISTIC_LOCK_ERROR_CODE, f as OPTIMISTIC_LOCK_FAILED_MESSAGE, P as PostListValidator, S as StatusCode, U as UPDATE_TARGET_NOT_FOUND_MESSAGE, i as deleteTarget, j as formatZodError, m as getTarget, n as getTargetList, o as getTargetTotalCount, p as handleSupabaseError, q as isCreateTargetAlreadyExistsError, r as isDevEnvironment, s as isOptimisticLockError, u as logDev, v as logDevGroup, x as logMinLevelFromEnv, y as patchTarget, A as postListCreate, D as postLogListCreate, E as postTarget, F as postTargetPayloadSchema, G as queryFilterSchema, H as resolveLogMinLevel, I as safeParseWithSchema, N as targetDraftSchema, Q as updateTarget, T as validateWith } from './chunks/list.api-BzbTJBsM.js';
|
|
4
|
-
import { l as loadEnvFiles, b as readEnv, c as requireEnv, d as
|
|
5
|
-
export { C as CLI_PROD_FLAG, S as SERVICE_ENV_PROFILE_ENV_KEY,
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
|
|
4
|
+
import { l as loadEnvFiles, b as readEnv, c as requireEnv, d as resolveLogSpoolSpawnEnvForLabel, e as resolveLogSpoolConfig, f as resolveLogSpoolRoot, g as resolveAllLogSpoolProcessRoles, h as logSpoolProcessDir, s as spoolLogger, i as parseLogSpoolBatchFilename, j as computeLogBatchIdempotencyKey, k as buildLogListDraft, n as postLogBatch, a as logSpoolEnabledFromEnv, o as logSpoolConfigFromEnv, L as LOG_SPOOL_SERVICE_ID_ENV, q as ensureLogSpoolFromEnv, t as shutdownLogSpoolFromEnv, u as getLogSpoolStats } from './chunks/enable-C7lFooTG.js';
|
|
5
|
+
export { C as CLI_PROD_FLAG, S as SERVICE_ENV_PROFILE_ENV_KEY, w as SERVICE_ENV_PROFILE_PROD, x as buildLogSpoolSpawnEnv, y as envBool, z as envInt, A as envMs, B as envNumber, D as envPort, E as envProfileFromProcess, F as resolveDefaultEnvFiles, r as resolveProjectRootByPackageName, G as resolveProjectRootFromModule, v as validateLogSpoolPreloadEnv } from './chunks/enable-C7lFooTG.js';
|
|
6
|
+
import { randomBytes, createHash } from 'node:crypto';
|
|
7
|
+
import { readFile, unlink, mkdir, writeFile, rename, access, stat, readdir, constants } from 'node:fs/promises';
|
|
8
|
+
import { dirname, resolve, join, isAbsolute } from 'node:path';
|
|
9
|
+
import { bh as sha256Hex, bm as getErrorMessage, bn as LOG_TOPIC_NODE, aL as postRegisterNode, ap as patchChangeNodeStatus, N as NodeStatus, aA as patchStopNode, ax as patchNodeHeartBeat, E as EMPTY_REGISTRY_SLOT_RUNTIME_STATE, b7 as runRegistrySlotGuardCheck, aZ as registrySlotRuntimePatchFromGuardResult, c as CategoryNode, Q as evaluateBusyNodeLiveness, az as patchServiceRuntime, ag as isLogEntry, x as claimServiceRegistrySlot, k as ServiceRegistryError, e as CategoryRepo, R as ResultCode, bo as LOG_TOPIC_TASK, m as TASK_REPO_USAGE, a6 as getScanRemoteRepoValues, at as patchClaimTask, bp as CRITICAL_UNKNOWN_TASK_TRACE_ID, ar as patchChangeTaskStatus, o as TaskStatusAction, bq as toError, aT as postTaskSchema, br as createTaskRow, n as TaskStatus } from './chunks/task-queue-qSnsfiIg.js';
|
|
10
|
+
export { A as ApiMethod, C as CategoryConfig, a as CategoryExtraction, b as CategoryLink, d as CategoryParcel, f as CategoryService, g as CategoryTask, D as DEFAULT_SYSTEM_REGISTRY_SEED_SLOTS, L as LINK_MANIFEST_VERSION, h as LIST_MANIFEST_VERSION, P as POST_TASK_DEFAULT_MAX_ATTEMPTS, i as ParcelManager, j as PostLinkValidator, S as ServiceLifecycleStatus, l as ServiceSlotStatus, T as TARGET_SYSTEM_REGISTRY_KEY, p as apiDetailsSchema, q as appendSystemRegistrySlots, r as assertRegistrySlotAvailable, s as assertRegistrySlotOwner, t as buildEmptyServiceSlots, u as buildLinkTargetDraft, v as buildListTargetDraft, w as buildSystemRegistryConfigDetails, y as classifyNetworkError, z as countTasksByType, B as createActiveServiceLifecycle, F as createClaimedRegistrySlotRuntimeState, G as createStorageProviderRegistry, H as createUploadTracker, I as decodeLogBatch, J as decodeLogBatchList, K as defaultL3ServiceDetails, M as deleteParcel, O as deleteParcelSchema, U as fetchBinaryWithRetry, V as fetchWithRetry, W as fieldDefinitionSchema, X as flattenLogBatch, Y as flattenLogBatches, Z as formatNetworkError, _ as getApi, $ as getApiSchema, a0 as getConfig, a1 as getConfigSchema, a2 as getExtraction, a3 as getExtractionSchema, a4 as getParcel, a5 as getParcelSchema, a7 as getScanRemoteRepoValuesSchema, a8 as getService, a9 as getServiceSchema, aa as getTargetSystemRegistry, ab as getValueAtPath, ac as installChunkFetchRegistry, ad as isHttpUrl, ae as isLocalFilesystemPath, af as isLogBatchList, ah as isOpaqueChunkUrl, ai as isRetryableHttpStatus, aj as linkTargetDraftBuildInputSchema, ak as parcelDetailsSchema, al as parseLogBatchMeta, am as parseProviderPrefixedUrl, an as parseServiceSlot, ao as parseServiceSlots, aq as patchChangeNodeStatusSchema, as as patchChangeTaskStatusSchema, au as patchClaimTaskSchema, av as patchExtractionObjects, aw as patchExtractionObjectsSchema, ay as patchNodeHeartBeatSchema, aB as patchStopNodeSchema, aC as patchTaskProgress, aD as patchTaskProgressSchema, aE as postApi, aF as postApiSchema, aG as postExtraction, aH as postExtractionSchema, aI as postLinkCreate, aJ as postParcel, aK as postParcelSchema, aM as postRegisterNodeSchema, aN as postService, aO as postServiceInstance, aP as postServiceSchema, aQ as postSystemRegistryConfig, aR as postSystemRegistryConfigSchema, aS as postTask, aU as postTaskWithRetry, aV as publishParcel, aW as registerProviderChunkResolver, aX as registerService, aY as registerServiceAtStartup, a_ as releaseSystemRegistrySlots, a$ as releaseSystemRegistrySlotsByServiceId, b0 as requestHasBody, b1 as resetSystemRegistryConfig, b2 as resetSystemRegistryConfigSchema, b3 as resolveActiveRegistryServiceId, b4 as resolveFetchUrl, b5 as restoreParcel, b6 as restoreParcelById, b8 as sanitizeFileName, b9 as sanitizeParcelFileName, ba as scanExtractionList, bb as scanExtractionListSchema, bc as schemaDefinitionSchema, bd as serviceDetailsSchema, be as serviceLifecycleSchema, bf as serviceNodeSnapshotSchema, bg as serviceRuntimeSchema, bi as summarizeTaskQueue, bj as systemRegistrySeedSlotSchema, bk as unregisterService, bl as unregisterServiceAtShutdown } from './chunks/task-queue-qSnsfiIg.js';
|
|
10
11
|
import { spawn } from 'node:child_process';
|
|
11
|
-
import { createHash } from 'node:crypto';
|
|
12
12
|
import { tmpdir } from 'node:os';
|
|
13
13
|
import { pathToFileURL } from 'node:url';
|
|
14
14
|
import 'lodash-es';
|
|
@@ -119,6 +119,98 @@ function publicBaseUrlFromEnv(port, options) {
|
|
|
119
119
|
return `http://localhost:${port}`;
|
|
120
120
|
}
|
|
121
121
|
|
|
122
|
+
/** Per-file write queue — serializes read-modify-write for a single JSON path within one process. */
|
|
123
|
+
const writeChains = new Map();
|
|
124
|
+
function withSerializedFileWrites(filePath, operation) {
|
|
125
|
+
const previous = writeChains.get(filePath) ?? Promise.resolve();
|
|
126
|
+
const next = previous.then(operation, operation);
|
|
127
|
+
writeChains.set(filePath, next.then(() => undefined, () => undefined));
|
|
128
|
+
return next;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function mergeWithNested(defaults, current, patch, nestedKeys) {
|
|
132
|
+
const merged = {
|
|
133
|
+
...defaults,
|
|
134
|
+
...current,
|
|
135
|
+
...patch,
|
|
136
|
+
};
|
|
137
|
+
for (const key of nestedKeys) {
|
|
138
|
+
const defaultNested = defaults[key];
|
|
139
|
+
if (defaultNested != null && typeof defaultNested === "object") {
|
|
140
|
+
merged[key] = {
|
|
141
|
+
...defaultNested,
|
|
142
|
+
...current?.[key],
|
|
143
|
+
...patch[key],
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
return merged;
|
|
148
|
+
}
|
|
149
|
+
async function atomicWriteJson(filePath, data) {
|
|
150
|
+
await mkdir(dirname(filePath), { recursive: true });
|
|
151
|
+
const content = `${JSON.stringify(data, null, 2)}\n`;
|
|
152
|
+
JSON.parse(content);
|
|
153
|
+
const tempPath = `${filePath}.${process.pid}.${randomBytes(4).toString("hex")}.tmp`;
|
|
154
|
+
try {
|
|
155
|
+
await writeFile(tempPath, content, "utf8");
|
|
156
|
+
await rename(tempPath, filePath);
|
|
157
|
+
}
|
|
158
|
+
catch (error) {
|
|
159
|
+
await unlink(tempPath).catch(() => undefined);
|
|
160
|
+
throw error;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
/** JSON file-backed state with optional nested key merge and atomic writes. */
|
|
164
|
+
function createJsonFileStateStore(options) {
|
|
165
|
+
const { filePath, defaultState, nestedKeys = [], updatedAtKey } = options;
|
|
166
|
+
function touchUpdatedAt(state) {
|
|
167
|
+
if (updatedAtKey == null) {
|
|
168
|
+
return state;
|
|
169
|
+
}
|
|
170
|
+
return {
|
|
171
|
+
...state,
|
|
172
|
+
[updatedAtKey]: new Date().toISOString(),
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
async function readFromDisk() {
|
|
176
|
+
try {
|
|
177
|
+
const raw = await readFile(filePath, "utf8");
|
|
178
|
+
const parsed = JSON.parse(raw);
|
|
179
|
+
return mergeWithNested(defaultState, parsed, {}, nestedKeys);
|
|
180
|
+
}
|
|
181
|
+
catch (error) {
|
|
182
|
+
if (error.code === "ENOENT") {
|
|
183
|
+
return mergeWithNested(defaultState, {}, {}, nestedKeys);
|
|
184
|
+
}
|
|
185
|
+
if (error instanceof SyntaxError) {
|
|
186
|
+
await unlink(filePath).catch(() => undefined);
|
|
187
|
+
return mergeWithNested(defaultState, {}, {}, nestedKeys);
|
|
188
|
+
}
|
|
189
|
+
throw error;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
return {
|
|
193
|
+
read() {
|
|
194
|
+
return readFromDisk();
|
|
195
|
+
},
|
|
196
|
+
write(patch) {
|
|
197
|
+
return withSerializedFileWrites(filePath, async () => {
|
|
198
|
+
const current = await readFromDisk();
|
|
199
|
+
const next = touchUpdatedAt(mergeWithNested(defaultState, current, patch, nestedKeys));
|
|
200
|
+
await atomicWriteJson(filePath, next);
|
|
201
|
+
return next;
|
|
202
|
+
});
|
|
203
|
+
},
|
|
204
|
+
reset() {
|
|
205
|
+
return withSerializedFileWrites(filePath, async () => {
|
|
206
|
+
const next = touchUpdatedAt({ ...defaultState });
|
|
207
|
+
await atomicWriteJson(filePath, next);
|
|
208
|
+
return next;
|
|
209
|
+
});
|
|
210
|
+
},
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
|
|
122
214
|
/** Copy a Node.js Buffer view to a standalone ArrayBuffer. */
|
|
123
215
|
function nodeBufferToArrayBuffer(nodeBuffer) {
|
|
124
216
|
return nodeBuffer.buffer.slice(nodeBuffer.byteOffset, nodeBuffer.byteOffset + nodeBuffer.byteLength);
|
|
@@ -196,7 +288,6 @@ const formatHeartbeat = (timestamp) => {
|
|
|
196
288
|
return `${absolute} (${relative})`;
|
|
197
289
|
};
|
|
198
290
|
|
|
199
|
-
const LOG_TOPIC_NODE = "node";
|
|
200
291
|
const LOG_TOPIC_COMMAND = "command";
|
|
201
292
|
const LOG_TOPIC_PROCESS = "process";
|
|
202
293
|
/**
|
|
@@ -265,6 +356,22 @@ class BaseNodeRuntime {
|
|
|
265
356
|
getLoopIntervalMs() {
|
|
266
357
|
return getRandomInterval();
|
|
267
358
|
}
|
|
359
|
+
/**
|
|
360
|
+
* When true, the main loop skips commands and {@link runLoopSteps} (heartbeat only).
|
|
361
|
+
* Used by the L3 guard process while waiting for connectivity to recover.
|
|
362
|
+
*/
|
|
363
|
+
isSilenced() {
|
|
364
|
+
return false;
|
|
365
|
+
}
|
|
366
|
+
/**
|
|
367
|
+
* Consecutive heartbeat failures reached {@link HEARTBEAT_FAILURE_THRESHOLD}.
|
|
368
|
+
* Default: shut down the node. Override to wait for recovery instead of exiting.
|
|
369
|
+
*/
|
|
370
|
+
async onHealthCheckFailed(_ctx) {
|
|
371
|
+
this.requestShutdown("heartbeat:consecutive-failures");
|
|
372
|
+
}
|
|
373
|
+
/** Heartbeat succeeded while {@link isSilenced} is true. Default: no-op. */
|
|
374
|
+
async onHeartbeatRecovered(_ctx) { }
|
|
268
375
|
getLogNodeId() {
|
|
269
376
|
return this.localNodeId ?? BaseNodeRuntime.NODE_ID_PENDING_ASSIGNMENT;
|
|
270
377
|
}
|
|
@@ -421,7 +528,9 @@ class BaseNodeRuntime {
|
|
|
421
528
|
data: { nodeId, loopCount: this.loopCount },
|
|
422
529
|
});
|
|
423
530
|
try {
|
|
424
|
-
|
|
531
|
+
if (!this.isSilenced()) {
|
|
532
|
+
await this.batchCommand(loopCtx);
|
|
533
|
+
}
|
|
425
534
|
if (this.isShuttingDown()) {
|
|
426
535
|
iterLogger.debug("節點關閉中,跳過本輪剩餘步驟", { topic: LOG_TOPIC_NODE });
|
|
427
536
|
break;
|
|
@@ -431,7 +540,9 @@ class BaseNodeRuntime {
|
|
|
431
540
|
iterLogger.debug("節點關閉中,跳過本輪剩餘步驟", { topic: LOG_TOPIC_NODE });
|
|
432
541
|
break;
|
|
433
542
|
}
|
|
434
|
-
|
|
543
|
+
if (!this.isSilenced()) {
|
|
544
|
+
await this.runLoopSteps(loopCtx, heartbeatOk);
|
|
545
|
+
}
|
|
435
546
|
}
|
|
436
547
|
catch (error) {
|
|
437
548
|
iterLogger.critical(`主循環第 ${this.loopCount} 輪遇到未知錯誤`, {
|
|
@@ -543,11 +654,11 @@ class BaseNodeRuntime {
|
|
|
543
654
|
threshold: BaseNodeRuntime.HEARTBEAT_FAILURE_THRESHOLD,
|
|
544
655
|
};
|
|
545
656
|
if (failureCount >= BaseNodeRuntime.HEARTBEAT_FAILURE_THRESHOLD) {
|
|
546
|
-
logger.critical("
|
|
657
|
+
logger.critical("連續心跳失敗達上限", {
|
|
547
658
|
topic: LOG_TOPIC_NODE,
|
|
548
659
|
data: logData,
|
|
549
660
|
});
|
|
550
|
-
this.
|
|
661
|
+
await this.onHealthCheckFailed(ctx);
|
|
551
662
|
}
|
|
552
663
|
else {
|
|
553
664
|
logger.warn("心跳更新失敗", { topic: LOG_TOPIC_NODE, data: logData });
|
|
@@ -555,6 +666,9 @@ class BaseNodeRuntime {
|
|
|
555
666
|
return false;
|
|
556
667
|
}
|
|
557
668
|
this.consecutiveHeartbeatFailures = 0;
|
|
669
|
+
if (this.isSilenced()) {
|
|
670
|
+
await this.onHeartbeatRecovered(ctx);
|
|
671
|
+
}
|
|
558
672
|
// Success is the steady state every loop — do not INFO-spam; failures already warn/critical.
|
|
559
673
|
logger.debug("心跳更新成功", {
|
|
560
674
|
topic: LOG_TOPIC_NODE,
|
|
@@ -725,12 +839,16 @@ class ManagedChildProcesses {
|
|
|
725
839
|
if (existing != null) {
|
|
726
840
|
return { child: existing, created: false };
|
|
727
841
|
}
|
|
842
|
+
const logSpoolEnv = resolveLogSpoolSpawnEnvForLabel(label);
|
|
843
|
+
const childEnv = spawnOptions?.env != null
|
|
844
|
+
? { ...process.env, ...logSpoolEnv, ...spawnOptions.env }
|
|
845
|
+
: { ...process.env, ...logSpoolEnv };
|
|
728
846
|
const child = spawnTsxChild({
|
|
729
847
|
projectRoot: this.projectRoot,
|
|
730
848
|
entryScript,
|
|
731
849
|
preloadModules: this.preloadModules,
|
|
732
850
|
useTsx: spawnOptions?.useTsx ?? this.defaultUseTsx,
|
|
733
|
-
env:
|
|
851
|
+
env: childEnv,
|
|
734
852
|
});
|
|
735
853
|
child.on("exit", (code, signal) => {
|
|
736
854
|
this.logger?.info("child exited", { topic: "process", data: { label, code, signal } });
|
|
@@ -805,6 +923,25 @@ function createManagedChildProcesses(options) {
|
|
|
805
923
|
});
|
|
806
924
|
}
|
|
807
925
|
|
|
926
|
+
function sleep(ms) {
|
|
927
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
928
|
+
}
|
|
929
|
+
/**
|
|
930
|
+
* Poll `until()` until it returns true or `timeoutMs` elapses.
|
|
931
|
+
* @throws Error on timeout
|
|
932
|
+
*/
|
|
933
|
+
async function pollUntil(options) {
|
|
934
|
+
const intervalMs = options.intervalMs ?? 500;
|
|
935
|
+
const deadline = Date.now() + options.timeoutMs;
|
|
936
|
+
while (Date.now() < deadline) {
|
|
937
|
+
if (await options.until()) {
|
|
938
|
+
return;
|
|
939
|
+
}
|
|
940
|
+
await sleep(intervalMs);
|
|
941
|
+
}
|
|
942
|
+
throw new Error(`pollUntil timeout after ${options.timeoutMs}ms`);
|
|
943
|
+
}
|
|
944
|
+
|
|
808
945
|
/** Factory: all listed env keys must be non-empty. */
|
|
809
946
|
function createRequiredEnvCheck(name, keys, options) {
|
|
810
947
|
return () => {
|
|
@@ -874,9 +1011,26 @@ async function runReadinessChecks(checks) {
|
|
|
874
1011
|
};
|
|
875
1012
|
}
|
|
876
1013
|
|
|
877
|
-
|
|
878
|
-
|
|
1014
|
+
/** Run readiness checks, persist report via callback, fail-fast when not ok. */
|
|
1015
|
+
async function runReadinessGate(input) {
|
|
1016
|
+
const { checks, onReport, logger, logTopic = "guard" } = input;
|
|
1017
|
+
logger.info("Running readiness checks", { topic: logTopic });
|
|
1018
|
+
const report = await runReadinessChecks(checks);
|
|
1019
|
+
await onReport(report);
|
|
1020
|
+
if (!report.ok) {
|
|
1021
|
+
logger.error("Readiness checks failed", {
|
|
1022
|
+
topic: logTopic,
|
|
1023
|
+
data: { message: report.message, checks: report.checks },
|
|
1024
|
+
});
|
|
1025
|
+
throw new Error(report.message ?? "Readiness checks failed");
|
|
1026
|
+
}
|
|
1027
|
+
logger.debug("Readiness checks passed", {
|
|
1028
|
+
topic: logTopic,
|
|
1029
|
+
data: { checks: report.checks.map((check) => check.name) },
|
|
1030
|
+
});
|
|
1031
|
+
return report;
|
|
879
1032
|
}
|
|
1033
|
+
|
|
880
1034
|
/**
|
|
881
1035
|
* Poll {@link ServiceReadyGate} until ready, fail fast on `failed`, or timeout.
|
|
882
1036
|
* Returns the final ready snapshot.
|
|
@@ -884,25 +1038,37 @@ function sleep(ms) {
|
|
|
884
1038
|
async function waitForServiceReady(gate, options) {
|
|
885
1039
|
const pollMs = options.pollMs ?? 500;
|
|
886
1040
|
const defaultFailureMessage = options.failureMessage ?? "Readiness checks failed";
|
|
887
|
-
const deadline = Date.now() + options.timeoutMs;
|
|
888
1041
|
let last = { failed: false, ready: false };
|
|
889
|
-
|
|
890
|
-
|
|
1042
|
+
try {
|
|
1043
|
+
await pollUntil({
|
|
1044
|
+
timeoutMs: options.timeoutMs,
|
|
1045
|
+
intervalMs: pollMs,
|
|
1046
|
+
until: async () => {
|
|
1047
|
+
last = await gate.read();
|
|
1048
|
+
if (last.failed) {
|
|
1049
|
+
throw new Error(last.failureMessage ?? defaultFailureMessage);
|
|
1050
|
+
}
|
|
1051
|
+
return last.ready;
|
|
1052
|
+
},
|
|
1053
|
+
});
|
|
1054
|
+
}
|
|
1055
|
+
catch (error) {
|
|
891
1056
|
if (last.failed) {
|
|
892
|
-
throw new Error(
|
|
1057
|
+
throw error instanceof Error ? error : new Error(defaultFailureMessage);
|
|
893
1058
|
}
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
data: last.readyDetail ?? {},
|
|
898
|
-
});
|
|
899
|
-
return last;
|
|
1059
|
+
const isPollTimeout = error instanceof Error && error.message.startsWith("pollUntil timeout after");
|
|
1060
|
+
if (!isPollTimeout) {
|
|
1061
|
+
throw error;
|
|
900
1062
|
}
|
|
901
|
-
|
|
1063
|
+
const message = options.formatTimeoutError?.(last, options.timeoutMs) ??
|
|
1064
|
+
`Service ready timeout after ${options.timeoutMs}ms (failed=${last.failed}, ready=${last.ready})`;
|
|
1065
|
+
throw new Error(message);
|
|
902
1066
|
}
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
1067
|
+
options.logger?.info("service ready", {
|
|
1068
|
+
topic: "readiness",
|
|
1069
|
+
data: last.readyDetail ?? {},
|
|
1070
|
+
});
|
|
1071
|
+
return last;
|
|
906
1072
|
}
|
|
907
1073
|
|
|
908
1074
|
const RUNTIME_STATE_SHARD_DIR = "runtime-state";
|
|
@@ -1066,11 +1232,22 @@ function createDefaultServiceRuntimeState(options) {
|
|
|
1066
1232
|
lastDecision: null,
|
|
1067
1233
|
lastSpawnAt: null,
|
|
1068
1234
|
spawnCount: 0,
|
|
1235
|
+
mode: "healthy",
|
|
1236
|
+
silentEnteredAt: null,
|
|
1237
|
+
silentLastHeartbeatAt: null,
|
|
1238
|
+
silentBackoffMs: 0,
|
|
1239
|
+
silentConsecutiveFailures: 0,
|
|
1240
|
+
silentRecoveryAttempt: 0,
|
|
1241
|
+
silentEvents: [],
|
|
1069
1242
|
};
|
|
1070
1243
|
const scheduler = {
|
|
1071
1244
|
lastTickAt: null,
|
|
1072
1245
|
lastPostedCount: 0,
|
|
1073
1246
|
lastFired: {},
|
|
1247
|
+
pid: null,
|
|
1248
|
+
spawnedAt: null,
|
|
1249
|
+
ready: false,
|
|
1250
|
+
readyAt: null,
|
|
1074
1251
|
};
|
|
1075
1252
|
const worker = {
|
|
1076
1253
|
pid: null,
|
|
@@ -1489,6 +1666,21 @@ async function runServiceGuardTick(input) {
|
|
|
1489
1666
|
|
|
1490
1667
|
/** Default TriggerManager runner key for {@link registerServiceGuardRunner}. */
|
|
1491
1668
|
const SERVICE_GUARD_RUNNER_KEY = "service-guard";
|
|
1669
|
+
/** True when the guard slice is serving traffic (`mode` omitted = legacy healthy). */
|
|
1670
|
+
function isGuardAvailable(guard) {
|
|
1671
|
+
return guard.mode == null || guard.mode === "healthy";
|
|
1672
|
+
}
|
|
1673
|
+
/**
|
|
1674
|
+
* Seconds for `Retry-After` when the guard is not serving traffic.
|
|
1675
|
+
* `null` when {@link isGuardAvailable} is true.
|
|
1676
|
+
*/
|
|
1677
|
+
function guardRetryAfterSec(guard) {
|
|
1678
|
+
if (isGuardAvailable(guard)) {
|
|
1679
|
+
return null;
|
|
1680
|
+
}
|
|
1681
|
+
const backoffMs = guard.silentBackoffMs > 0 ? guard.silentBackoffMs : 15_000;
|
|
1682
|
+
return Math.max(1, Math.ceil(backoffMs / 1000));
|
|
1683
|
+
}
|
|
1492
1684
|
|
|
1493
1685
|
async function runServiceGuardRunner(ctx, options) {
|
|
1494
1686
|
const { loopTraceId, nodeId, logger } = ctx;
|
|
@@ -1534,24 +1726,312 @@ function registerServiceGuardRunner(options) {
|
|
|
1534
1726
|
});
|
|
1535
1727
|
}
|
|
1536
1728
|
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1729
|
+
const UPLOAD_LANE = "slow";
|
|
1730
|
+
/** Per serviceId — guard collect-log tick runs GC at most once per `gc.intervalMs`. */
|
|
1731
|
+
const lastGcAtMsByServiceId = new Map();
|
|
1732
|
+
async function pathExists$1(filePath) {
|
|
1733
|
+
try {
|
|
1734
|
+
await access(filePath, constants.F_OK);
|
|
1735
|
+
return true;
|
|
1736
|
+
}
|
|
1737
|
+
catch {
|
|
1738
|
+
return false;
|
|
1739
|
+
}
|
|
1740
|
+
}
|
|
1741
|
+
async function readBatchTmpFile(tmpPath) {
|
|
1742
|
+
try {
|
|
1743
|
+
const raw = await readFile(tmpPath, "utf8");
|
|
1744
|
+
const parsed = JSON.parse(raw);
|
|
1745
|
+
if (!Array.isArray(parsed.entries) || parsed.entries.length === 0) {
|
|
1746
|
+
return null;
|
|
1747
|
+
}
|
|
1748
|
+
const entries = parsed.entries.filter(isLogEntry);
|
|
1749
|
+
if (entries.length === 0) {
|
|
1750
|
+
return null;
|
|
1751
|
+
}
|
|
1752
|
+
return { entries, meta: parsed.meta };
|
|
1753
|
+
}
|
|
1754
|
+
catch {
|
|
1755
|
+
return null;
|
|
1756
|
+
}
|
|
1757
|
+
}
|
|
1758
|
+
function processMetaKey(serviceId, processRole) {
|
|
1759
|
+
return `${serviceId}:${processRole}`;
|
|
1760
|
+
}
|
|
1761
|
+
async function uploadTmpBatch(input) {
|
|
1762
|
+
const parsed = parseLogSpoolBatchFilename(input.fileName);
|
|
1763
|
+
if (parsed == null) {
|
|
1764
|
+
spoolLogger.warn("collect-log skipped — invalid tmp filename", { fileName: input.fileName });
|
|
1765
|
+
return "failed";
|
|
1766
|
+
}
|
|
1767
|
+
const jsonFileName = input.fileName.replace(/\.tmp$/, ".json");
|
|
1768
|
+
const jsonPath = join(logSpoolProcessDir(input.spoolRoot, input.serviceId, input.processRole), jsonFileName);
|
|
1769
|
+
if (await pathExists$1(jsonPath)) {
|
|
1770
|
+
try {
|
|
1771
|
+
await unlink(input.tmpPath);
|
|
1772
|
+
spoolLogger.info("collect-log removed orphan tmp (json exists)", {
|
|
1773
|
+
tmpPath: input.tmpPath,
|
|
1774
|
+
jsonPath,
|
|
1775
|
+
});
|
|
1776
|
+
}
|
|
1777
|
+
catch {
|
|
1778
|
+
// best-effort
|
|
1779
|
+
}
|
|
1780
|
+
return "skipped";
|
|
1781
|
+
}
|
|
1782
|
+
const batch = await readBatchTmpFile(input.tmpPath);
|
|
1783
|
+
if (batch == null) {
|
|
1784
|
+
spoolLogger.warn("collect-log skipped — tmp failed validation", {
|
|
1785
|
+
tmpPath: input.tmpPath,
|
|
1786
|
+
fileName: input.fileName,
|
|
1547
1787
|
});
|
|
1548
|
-
|
|
1788
|
+
try {
|
|
1789
|
+
await unlink(input.tmpPath);
|
|
1790
|
+
spoolLogger.info("collect-log removed invalid tmp", { tmpPath: input.tmpPath });
|
|
1791
|
+
}
|
|
1792
|
+
catch {
|
|
1793
|
+
// best-effort
|
|
1794
|
+
}
|
|
1795
|
+
return "failed";
|
|
1549
1796
|
}
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1797
|
+
const processKey = processMetaKey(input.serviceId, input.processRole);
|
|
1798
|
+
// TODO: `lane` is legacy transport metadata — scheduled for removal; fixed to "slow" for downstream decode compatibility.
|
|
1799
|
+
const idempotencyKey = computeLogBatchIdempotencyKey({
|
|
1800
|
+
service: input.serviceValue,
|
|
1801
|
+
process: processKey,
|
|
1802
|
+
lane: UPLOAD_LANE,
|
|
1803
|
+
entries: batch.entries,
|
|
1804
|
+
});
|
|
1805
|
+
const draft = buildLogListDraft({
|
|
1806
|
+
service: input.serviceValue,
|
|
1807
|
+
process: processKey,
|
|
1808
|
+
lane: UPLOAD_LANE,
|
|
1809
|
+
idempotencyKey,
|
|
1810
|
+
entries: batch.entries,
|
|
1811
|
+
});
|
|
1812
|
+
try {
|
|
1813
|
+
await postLogBatch(draft, input.postTimeoutMs);
|
|
1814
|
+
}
|
|
1815
|
+
catch (error) {
|
|
1816
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1817
|
+
spoolLogger.error("collect-log upload failed", {
|
|
1818
|
+
fileName: input.fileName,
|
|
1819
|
+
processRole: input.processRole,
|
|
1820
|
+
error: message,
|
|
1821
|
+
});
|
|
1822
|
+
return "failed";
|
|
1823
|
+
}
|
|
1824
|
+
try {
|
|
1825
|
+
await rename(input.tmpPath, jsonPath);
|
|
1826
|
+
}
|
|
1827
|
+
catch (error) {
|
|
1828
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1829
|
+
spoolLogger.error("collect-log rename tmp→json failed after upload", {
|
|
1830
|
+
tmpPath: input.tmpPath,
|
|
1831
|
+
jsonPath,
|
|
1832
|
+
error: message,
|
|
1833
|
+
});
|
|
1834
|
+
return "failed";
|
|
1835
|
+
}
|
|
1836
|
+
spoolLogger.info("collect-log uploaded batch", {
|
|
1837
|
+
processRole: input.processRole,
|
|
1838
|
+
fileName: jsonFileName,
|
|
1839
|
+
entryCount: batch.entries.length,
|
|
1840
|
+
idempotencyKey,
|
|
1841
|
+
});
|
|
1842
|
+
return "uploaded";
|
|
1843
|
+
}
|
|
1844
|
+
async function gcTmpFiles(input) {
|
|
1845
|
+
const processDir = logSpoolProcessDir(input.spoolRoot, input.serviceId, input.processRole);
|
|
1846
|
+
let removed = 0;
|
|
1847
|
+
try {
|
|
1848
|
+
const names = await readdir(processDir);
|
|
1849
|
+
for (const name of names) {
|
|
1850
|
+
if (!name.endsWith(".tmp")) {
|
|
1851
|
+
continue;
|
|
1852
|
+
}
|
|
1853
|
+
const parsed = parseLogSpoolBatchFilename(name);
|
|
1854
|
+
if (parsed == null) {
|
|
1855
|
+
continue;
|
|
1856
|
+
}
|
|
1857
|
+
if (input.nowMs - parsed.timestampMs <= input.tmpRetentionMs) {
|
|
1858
|
+
continue;
|
|
1859
|
+
}
|
|
1860
|
+
try {
|
|
1861
|
+
await unlink(join(processDir, name));
|
|
1862
|
+
removed += 1;
|
|
1863
|
+
spoolLogger.info("collect-log gc removed stale tmp", {
|
|
1864
|
+
fileName: name,
|
|
1865
|
+
processRole: input.processRole,
|
|
1866
|
+
});
|
|
1867
|
+
}
|
|
1868
|
+
catch {
|
|
1869
|
+
// best-effort
|
|
1870
|
+
}
|
|
1871
|
+
}
|
|
1872
|
+
}
|
|
1873
|
+
catch {
|
|
1874
|
+
// directory may not exist yet
|
|
1875
|
+
}
|
|
1876
|
+
return removed;
|
|
1877
|
+
}
|
|
1878
|
+
async function gcSyncedJsonFiles(input) {
|
|
1879
|
+
let removed = 0;
|
|
1880
|
+
for (const processRole of resolveAllLogSpoolProcessRoles()) {
|
|
1881
|
+
const processDir = logSpoolProcessDir(input.spoolRoot, input.serviceId, processRole);
|
|
1882
|
+
let names = [];
|
|
1883
|
+
try {
|
|
1884
|
+
names = await readdir(processDir);
|
|
1885
|
+
}
|
|
1886
|
+
catch {
|
|
1887
|
+
continue;
|
|
1888
|
+
}
|
|
1889
|
+
for (const name of names) {
|
|
1890
|
+
if (!name.endsWith(".json")) {
|
|
1891
|
+
continue;
|
|
1892
|
+
}
|
|
1893
|
+
if (parseLogSpoolBatchFilename(name) == null) {
|
|
1894
|
+
continue;
|
|
1895
|
+
}
|
|
1896
|
+
const jsonPath = join(processDir, name);
|
|
1897
|
+
try {
|
|
1898
|
+
const fileStat = await stat(jsonPath);
|
|
1899
|
+
if (input.nowMs - fileStat.mtimeMs <= input.syncedRetentionMs) {
|
|
1900
|
+
continue;
|
|
1901
|
+
}
|
|
1902
|
+
await unlink(jsonPath);
|
|
1903
|
+
removed += 1;
|
|
1904
|
+
spoolLogger.info("collect-log gc removed synced json", {
|
|
1905
|
+
fileName: name,
|
|
1906
|
+
processRole,
|
|
1907
|
+
});
|
|
1908
|
+
}
|
|
1909
|
+
catch {
|
|
1910
|
+
// best-effort
|
|
1911
|
+
}
|
|
1912
|
+
}
|
|
1913
|
+
}
|
|
1914
|
+
return removed;
|
|
1915
|
+
}
|
|
1916
|
+
function shouldRunGc(serviceId, nowMs, intervalMs) {
|
|
1917
|
+
const lastAt = lastGcAtMsByServiceId.get(serviceId) ?? 0;
|
|
1918
|
+
if (nowMs - lastAt < intervalMs) {
|
|
1919
|
+
return false;
|
|
1920
|
+
}
|
|
1921
|
+
lastGcAtMsByServiceId.set(serviceId, nowMs);
|
|
1922
|
+
return true;
|
|
1923
|
+
}
|
|
1924
|
+
/**
|
|
1925
|
+
* Guard-only: upload pending `.tmp` batches, rename to `.json`, then periodic GC.
|
|
1926
|
+
* Upload completion is implied by `.json` presence; no sync-state file.
|
|
1927
|
+
*/
|
|
1928
|
+
async function runCollectLogTick(input) {
|
|
1929
|
+
const config = resolveLogSpoolConfig(input.config);
|
|
1930
|
+
const spoolRoot = input.spoolRoot ?? resolveLogSpoolRoot();
|
|
1931
|
+
const nowMs = input.nowMs ?? Date.now();
|
|
1932
|
+
const result = {
|
|
1933
|
+
uploaded: 0,
|
|
1934
|
+
skipped: 0,
|
|
1935
|
+
failed: 0,
|
|
1936
|
+
gcTmpRemoved: 0,
|
|
1937
|
+
gcSyncedRemoved: 0,
|
|
1938
|
+
};
|
|
1939
|
+
for (const processRole of resolveAllLogSpoolProcessRoles()) {
|
|
1940
|
+
const processDir = logSpoolProcessDir(spoolRoot, input.serviceId, processRole);
|
|
1941
|
+
let names = [];
|
|
1942
|
+
try {
|
|
1943
|
+
names = await readdir(processDir);
|
|
1944
|
+
}
|
|
1945
|
+
catch {
|
|
1946
|
+
continue;
|
|
1947
|
+
}
|
|
1948
|
+
for (const name of names) {
|
|
1949
|
+
if (!name.endsWith(".tmp")) {
|
|
1950
|
+
continue;
|
|
1951
|
+
}
|
|
1952
|
+
const status = await uploadTmpBatch({
|
|
1953
|
+
serviceId: input.serviceId,
|
|
1954
|
+
serviceValue: input.serviceValue,
|
|
1955
|
+
processRole,
|
|
1956
|
+
tmpPath: join(processDir, name),
|
|
1957
|
+
fileName: name,
|
|
1958
|
+
spoolRoot,
|
|
1959
|
+
postTimeoutMs: config.writer.postTimeoutMs,
|
|
1960
|
+
});
|
|
1961
|
+
if (status === "uploaded") {
|
|
1962
|
+
result.uploaded += 1;
|
|
1963
|
+
}
|
|
1964
|
+
else if (status === "skipped") {
|
|
1965
|
+
result.skipped += 1;
|
|
1966
|
+
}
|
|
1967
|
+
else {
|
|
1968
|
+
result.failed += 1;
|
|
1969
|
+
}
|
|
1970
|
+
}
|
|
1971
|
+
}
|
|
1972
|
+
if (shouldRunGc(input.serviceId, nowMs, config.gc.intervalMs)) {
|
|
1973
|
+
for (const processRole of resolveAllLogSpoolProcessRoles()) {
|
|
1974
|
+
result.gcTmpRemoved += await gcTmpFiles({
|
|
1975
|
+
spoolRoot,
|
|
1976
|
+
serviceId: input.serviceId,
|
|
1977
|
+
processRole,
|
|
1978
|
+
tmpRetentionMs: config.gc.tmpRetentionMs,
|
|
1979
|
+
nowMs,
|
|
1980
|
+
});
|
|
1981
|
+
}
|
|
1982
|
+
result.gcSyncedRemoved = await gcSyncedJsonFiles({
|
|
1983
|
+
spoolRoot,
|
|
1984
|
+
serviceId: input.serviceId,
|
|
1985
|
+
syncedRetentionMs: config.gc.syncedRetentionMs,
|
|
1986
|
+
nowMs,
|
|
1987
|
+
});
|
|
1988
|
+
}
|
|
1989
|
+
if (result.uploaded > 0 || result.failed > 0) {
|
|
1990
|
+
spoolLogger.info("collect-log tick complete", { ...result, serviceId: input.serviceId });
|
|
1991
|
+
}
|
|
1992
|
+
return result;
|
|
1993
|
+
}
|
|
1994
|
+
|
|
1995
|
+
const COLLECT_LOG_RUNNER_KEY = "collect-log";
|
|
1996
|
+
/**
|
|
1997
|
+
* Guard process only — uploads `.tmp` spool batches and runs GC.
|
|
1998
|
+
* Register before {@link TriggerNode.start} (e.g. in {@link ServiceGuardNode.create}).
|
|
1999
|
+
*/
|
|
2000
|
+
function registerCollectLogRunner(options) {
|
|
2001
|
+
if (!logSpoolEnabledFromEnv()) {
|
|
2002
|
+
return;
|
|
2003
|
+
}
|
|
2004
|
+
const config = logSpoolConfigFromEnv();
|
|
2005
|
+
const intervalMs = options.intervalMs ?? config.collectIntervalMs;
|
|
2006
|
+
TriggerManager.registerRunner({
|
|
2007
|
+
key: options.runnerKey ?? COLLECT_LOG_RUNNER_KEY,
|
|
2008
|
+
intervalMs,
|
|
2009
|
+
initialDelayMs: options.initialDelayMs ?? 0,
|
|
2010
|
+
retryCount: 1,
|
|
2011
|
+
fn: async (ctx) => {
|
|
2012
|
+
const serviceId = await options.getServiceId();
|
|
2013
|
+
if (serviceId == null) {
|
|
2014
|
+
ctx.logger.warn("collect-log skipped — serviceId unavailable", { topic: LOG_TOPIC_TRIGGER });
|
|
2015
|
+
return;
|
|
2016
|
+
}
|
|
2017
|
+
const result = await runCollectLogTick({
|
|
2018
|
+
serviceId,
|
|
2019
|
+
serviceValue: options.serviceValue,
|
|
2020
|
+
spoolRoot: options.spoolRoot,
|
|
2021
|
+
config,
|
|
2022
|
+
});
|
|
2023
|
+
if (result.uploaded > 0 || result.failed > 0) {
|
|
2024
|
+
ctx.logger.info("collect-log runner finished", {
|
|
2025
|
+
topic: LOG_TOPIC_TRIGGER,
|
|
2026
|
+
data: result,
|
|
2027
|
+
});
|
|
2028
|
+
}
|
|
2029
|
+
},
|
|
2030
|
+
});
|
|
2031
|
+
spoolLogger.info("collect-log runner registered", {
|
|
2032
|
+
intervalMs,
|
|
2033
|
+
serviceValue: options.serviceValue,
|
|
1553
2034
|
});
|
|
1554
|
-
return report;
|
|
1555
2035
|
}
|
|
1556
2036
|
|
|
1557
2037
|
/**
|
|
@@ -1624,30 +2104,74 @@ class TriggerNode extends BaseNodeRuntime {
|
|
|
1624
2104
|
}
|
|
1625
2105
|
}
|
|
1626
2106
|
|
|
2107
|
+
const SILENT_BACKOFF_MS = [15_000, 30_000, 60_000, 120_000, 240_000, 300_000];
|
|
2108
|
+
const DEFAULT_BUSINESS_READY_TIMEOUT_MS = 180_000;
|
|
1627
2109
|
/**
|
|
1628
|
-
* L3 guard process TriggerNode: readiness gate,
|
|
2110
|
+
* L3 guard process TriggerNode: readiness gate, business-node spawn owner,
|
|
1629
2111
|
* and {@link registerServiceGuardRunner} tick (registry slot + liveness + runtime).
|
|
1630
2112
|
*
|
|
2113
|
+
* Consecutive heartbeat failure enters silent mode (no process exit). Recovery
|
|
2114
|
+
* respawns business nodes; leaving silent requires heartbeat + all nodes ready.
|
|
2115
|
+
*
|
|
1631
2116
|
* Use {@link ServiceGuardNode.create} so the guard runner is registered before `start()`.
|
|
1632
2117
|
*/
|
|
1633
2118
|
class ServiceGuardNode extends TriggerNode {
|
|
1634
2119
|
options;
|
|
2120
|
+
businessReadyTimeoutMs;
|
|
2121
|
+
mode = "healthy";
|
|
2122
|
+
silentBackoffIndex = 0;
|
|
2123
|
+
silentEvents = [];
|
|
2124
|
+
recoverPromise = null;
|
|
1635
2125
|
constructor(options) {
|
|
1636
2126
|
super({
|
|
1637
2127
|
requireRunners: true,
|
|
1638
2128
|
beforeProcessExit: options.beforeProcessExit,
|
|
1639
2129
|
});
|
|
1640
2130
|
this.options = options;
|
|
2131
|
+
this.businessReadyTimeoutMs = options.businessReadyTimeoutMs ?? DEFAULT_BUSINESS_READY_TIMEOUT_MS;
|
|
1641
2132
|
}
|
|
1642
2133
|
/** Register the guard runner and return a ready-to-start node. */
|
|
1643
2134
|
static create(options) {
|
|
1644
2135
|
registerServiceGuardRunner({
|
|
2136
|
+
...options.guardRunner,
|
|
1645
2137
|
serviceValue: options.serviceValue,
|
|
1646
2138
|
logTopic: options.logTopic,
|
|
1647
|
-
|
|
2139
|
+
spawnWorker: options.spawnBusinessNodes,
|
|
2140
|
+
});
|
|
2141
|
+
registerCollectLogRunner({
|
|
2142
|
+
serviceValue: options.serviceValue,
|
|
2143
|
+
getServiceId: options.guardRunner.getServiceId,
|
|
1648
2144
|
});
|
|
1649
2145
|
return new ServiceGuardNode(options);
|
|
1650
2146
|
}
|
|
2147
|
+
isSilenced() {
|
|
2148
|
+
return this.mode !== "healthy";
|
|
2149
|
+
}
|
|
2150
|
+
getLoopIntervalMs() {
|
|
2151
|
+
if (this.mode === "silent") {
|
|
2152
|
+
return SILENT_BACKOFF_MS[this.silentBackoffIndex] ?? 300_000;
|
|
2153
|
+
}
|
|
2154
|
+
return super.getLoopIntervalMs();
|
|
2155
|
+
}
|
|
2156
|
+
async onHealthCheckFailed(_ctx) {
|
|
2157
|
+
if (this.mode !== "healthy") {
|
|
2158
|
+
this.bumpSilentBackoff();
|
|
2159
|
+
await this.persistGuard({
|
|
2160
|
+
lastDecision: "silent_heartbeat_fail",
|
|
2161
|
+
silentConsecutiveFailures: this.silentConsecutiveFailures(),
|
|
2162
|
+
silentBackoffMs: this.getLoopIntervalMs(),
|
|
2163
|
+
silentLastHeartbeatAt: new Date().toISOString(),
|
|
2164
|
+
});
|
|
2165
|
+
return;
|
|
2166
|
+
}
|
|
2167
|
+
await this.enterSilent("heartbeat:consecutive-failures");
|
|
2168
|
+
}
|
|
2169
|
+
async onHeartbeatRecovered(ctx) {
|
|
2170
|
+
if (this.mode !== "silent") {
|
|
2171
|
+
return;
|
|
2172
|
+
}
|
|
2173
|
+
await this.recoverFromSilent(ctx);
|
|
2174
|
+
}
|
|
1651
2175
|
async onBeforeRegisterNode(nodeLogger) {
|
|
1652
2176
|
await runReadinessGate({
|
|
1653
2177
|
checks: this.options.readinessChecks,
|
|
@@ -1656,11 +2180,298 @@ class ServiceGuardNode extends TriggerNode {
|
|
|
1656
2180
|
logTopic: this.options.logTopic,
|
|
1657
2181
|
});
|
|
1658
2182
|
await super.onBeforeRegisterNode(nodeLogger);
|
|
1659
|
-
nodeLogger.info("Spawning
|
|
2183
|
+
nodeLogger.info("Spawning business nodes", {
|
|
1660
2184
|
topic: this.options.logTopic ?? "guard",
|
|
1661
2185
|
});
|
|
1662
|
-
await this.options.
|
|
2186
|
+
await this.options.spawnBusinessNodes("guard:readiness-passed");
|
|
2187
|
+
markWorkerSpawned();
|
|
1663
2188
|
this.options.onWorkerSpawned?.();
|
|
2189
|
+
await this.waitUntilBusinessReady();
|
|
2190
|
+
await this.persistGuard({
|
|
2191
|
+
mode: "healthy",
|
|
2192
|
+
lastDecision: "bootstrap_ready",
|
|
2193
|
+
});
|
|
2194
|
+
}
|
|
2195
|
+
async runLoopSteps(ctx, heartbeatOk) {
|
|
2196
|
+
await super.runLoopSteps(ctx, heartbeatOk);
|
|
2197
|
+
if (this.mode !== "healthy" || !heartbeatOk) {
|
|
2198
|
+
return;
|
|
2199
|
+
}
|
|
2200
|
+
// Idempotent — respawn exited children without entering silent (worker stale is the runner's job).
|
|
2201
|
+
try {
|
|
2202
|
+
await this.options.spawnBusinessNodes("guard:ensure-business");
|
|
2203
|
+
}
|
|
2204
|
+
catch (error) {
|
|
2205
|
+
this.createGuardLogger().warn("ensure-business spawn failed", {
|
|
2206
|
+
topic: this.options.logTopic ?? "guard",
|
|
2207
|
+
data: { error: getErrorMessage(error) },
|
|
2208
|
+
});
|
|
2209
|
+
}
|
|
2210
|
+
}
|
|
2211
|
+
silentConsecutiveFailures() {
|
|
2212
|
+
return this.silentBackoffIndex + 1;
|
|
2213
|
+
}
|
|
2214
|
+
bumpSilentBackoff() {
|
|
2215
|
+
if (this.silentBackoffIndex < SILENT_BACKOFF_MS.length - 1) {
|
|
2216
|
+
this.silentBackoffIndex += 1;
|
|
2217
|
+
}
|
|
2218
|
+
}
|
|
2219
|
+
async enterSilent(reason) {
|
|
2220
|
+
this.mode = "silent";
|
|
2221
|
+
this.silentBackoffIndex = 0;
|
|
2222
|
+
const enteredAt = new Date().toISOString();
|
|
2223
|
+
this.pushEvent("enter", reason);
|
|
2224
|
+
await this.persistGuard({
|
|
2225
|
+
mode: "silent",
|
|
2226
|
+
lastDecision: `silent:${reason}`,
|
|
2227
|
+
silentEnteredAt: enteredAt,
|
|
2228
|
+
silentLastHeartbeatAt: enteredAt,
|
|
2229
|
+
silentBackoffMs: SILENT_BACKOFF_MS[0],
|
|
2230
|
+
silentConsecutiveFailures: 1,
|
|
2231
|
+
silentRecoveryAttempt: 0,
|
|
2232
|
+
silentEvents: this.silentEvents,
|
|
2233
|
+
});
|
|
2234
|
+
const logger = this.createGuardLogger();
|
|
2235
|
+
logger.warn("Entering silent mode — service unavailable until heartbeat + business nodes recover", {
|
|
2236
|
+
topic: this.options.logTopic ?? "guard",
|
|
2237
|
+
data: { reason },
|
|
2238
|
+
});
|
|
2239
|
+
try {
|
|
2240
|
+
await this.options.stopBusinessNodes();
|
|
2241
|
+
}
|
|
2242
|
+
catch (error) {
|
|
2243
|
+
logger.warn("stopBusinessNodes failed (best-effort)", {
|
|
2244
|
+
topic: this.options.logTopic ?? "guard",
|
|
2245
|
+
data: { error: getErrorMessage(error) },
|
|
2246
|
+
});
|
|
2247
|
+
}
|
|
2248
|
+
}
|
|
2249
|
+
async recoverFromSilent(ctx) {
|
|
2250
|
+
if (this.recoverPromise != null) {
|
|
2251
|
+
return this.recoverPromise;
|
|
2252
|
+
}
|
|
2253
|
+
this.recoverPromise = this.runRecover(ctx).finally(() => {
|
|
2254
|
+
this.recoverPromise = null;
|
|
2255
|
+
});
|
|
2256
|
+
return this.recoverPromise;
|
|
2257
|
+
}
|
|
2258
|
+
async runRecover(ctx) {
|
|
2259
|
+
this.mode = "recovering";
|
|
2260
|
+
this.pushEvent("recover_start", ctx.loopTraceId);
|
|
2261
|
+
await this.persistGuard({
|
|
2262
|
+
mode: "recovering",
|
|
2263
|
+
lastDecision: "recovering",
|
|
2264
|
+
silentLastHeartbeatAt: new Date().toISOString(),
|
|
2265
|
+
silentRecoveryAttempt: this.silentEvents.filter((event) => event.type === "recover_start").length,
|
|
2266
|
+
silentEvents: this.silentEvents,
|
|
2267
|
+
});
|
|
2268
|
+
const logger = this.createGuardLogger();
|
|
2269
|
+
logger.info("Heartbeat restored — recovering business nodes", {
|
|
2270
|
+
topic: this.options.logTopic ?? "guard",
|
|
2271
|
+
});
|
|
2272
|
+
try {
|
|
2273
|
+
await this.options.stopBusinessNodes();
|
|
2274
|
+
}
|
|
2275
|
+
catch (error) {
|
|
2276
|
+
logger.warn("stopBusinessNodes failed before respawn", {
|
|
2277
|
+
topic: this.options.logTopic ?? "guard",
|
|
2278
|
+
data: { error: getErrorMessage(error) },
|
|
2279
|
+
});
|
|
2280
|
+
}
|
|
2281
|
+
try {
|
|
2282
|
+
await this.options.spawnBusinessNodes("guard:silent-recover");
|
|
2283
|
+
markWorkerSpawned();
|
|
2284
|
+
await this.waitUntilBusinessReady();
|
|
2285
|
+
}
|
|
2286
|
+
catch (error) {
|
|
2287
|
+
this.pushEvent("recover_fail", getErrorMessage(error));
|
|
2288
|
+
this.mode = "silent";
|
|
2289
|
+
this.bumpSilentBackoff();
|
|
2290
|
+
await this.persistGuard({
|
|
2291
|
+
mode: "silent",
|
|
2292
|
+
lastDecision: "recover_fail",
|
|
2293
|
+
silentBackoffMs: this.getLoopIntervalMs(),
|
|
2294
|
+
silentEvents: this.silentEvents,
|
|
2295
|
+
});
|
|
2296
|
+
logger.warn("Recovery failed — returning to silent", {
|
|
2297
|
+
topic: this.options.logTopic ?? "guard",
|
|
2298
|
+
data: { error: getErrorMessage(error) },
|
|
2299
|
+
});
|
|
2300
|
+
try {
|
|
2301
|
+
await this.options.stopBusinessNodes();
|
|
2302
|
+
}
|
|
2303
|
+
catch {
|
|
2304
|
+
// best-effort — leftover children are stopped again on the next recover
|
|
2305
|
+
}
|
|
2306
|
+
return;
|
|
2307
|
+
}
|
|
2308
|
+
this.mode = "healthy";
|
|
2309
|
+
this.silentBackoffIndex = 0;
|
|
2310
|
+
this.pushEvent("recover_ok");
|
|
2311
|
+
await this.persistGuard({
|
|
2312
|
+
mode: "healthy",
|
|
2313
|
+
lastDecision: "recover_ok",
|
|
2314
|
+
silentEnteredAt: null,
|
|
2315
|
+
silentBackoffMs: 0,
|
|
2316
|
+
silentConsecutiveFailures: 0,
|
|
2317
|
+
silentEvents: this.silentEvents,
|
|
2318
|
+
});
|
|
2319
|
+
logger.success("Left silent mode — business nodes ready", {
|
|
2320
|
+
topic: this.options.logTopic ?? "guard",
|
|
2321
|
+
});
|
|
2322
|
+
}
|
|
2323
|
+
async waitUntilBusinessReady() {
|
|
2324
|
+
try {
|
|
2325
|
+
await pollUntil({
|
|
2326
|
+
timeoutMs: this.businessReadyTimeoutMs,
|
|
2327
|
+
intervalMs: 500,
|
|
2328
|
+
until: () => this.options.isBusinessReady(),
|
|
2329
|
+
});
|
|
2330
|
+
}
|
|
2331
|
+
catch {
|
|
2332
|
+
throw new Error(`Business nodes not ready within ${this.businessReadyTimeoutMs}ms`);
|
|
2333
|
+
}
|
|
2334
|
+
}
|
|
2335
|
+
pushEvent(type, detail) {
|
|
2336
|
+
this.silentEvents = [...this.silentEvents, { at: new Date().toISOString(), type, detail }].slice(-32);
|
|
2337
|
+
}
|
|
2338
|
+
async persistGuard(patch) {
|
|
2339
|
+
try {
|
|
2340
|
+
await this.options.guardRunner.onGuardPatch(patch);
|
|
2341
|
+
}
|
|
2342
|
+
catch {
|
|
2343
|
+
// best-effort — disk/runtime state must not take down the guard loop
|
|
2344
|
+
}
|
|
2345
|
+
}
|
|
2346
|
+
createGuardLogger() {
|
|
2347
|
+
return createLogger({ module: "service-guard" });
|
|
2348
|
+
}
|
|
2349
|
+
}
|
|
2350
|
+
|
|
2351
|
+
const WORKER_SCRIPT = "./dist/processes/worker.js";
|
|
2352
|
+
const GUARD_SCRIPT = "./dist/processes/guard.js";
|
|
2353
|
+
const SCHEDULER_SCRIPT = "./dist/processes/scheduler.js";
|
|
2354
|
+
function defaultSpawnEnv(label) {
|
|
2355
|
+
return { LOG_PERSIST_PROCESS: label };
|
|
2356
|
+
}
|
|
2357
|
+
/**
|
|
2358
|
+
* Guard-owned scheduler + worker spawn/stop, plus main-owned Guard spawn.
|
|
2359
|
+
* `stopBusinessNodes` must run in the Guard process (that instance's children only).
|
|
2360
|
+
* `stopChildProcesses` is for main shutdown (`extraPids` for Guard-spawned PIDs).
|
|
2361
|
+
*/
|
|
2362
|
+
function createL3ChildLauncher(options) {
|
|
2363
|
+
const { childProcesses, readRuntimeState, writeRuntimeState } = options;
|
|
2364
|
+
const spawnEnv = options.spawnEnv ?? defaultSpawnEnv;
|
|
2365
|
+
function nowIso() {
|
|
2366
|
+
return new Date().toISOString();
|
|
2367
|
+
}
|
|
2368
|
+
async function spawnTaskWorker(reason) {
|
|
2369
|
+
const { child, created } = childProcesses.spawn("worker", WORKER_SCRIPT, {
|
|
2370
|
+
env: spawnEnv("worker"),
|
|
2371
|
+
});
|
|
2372
|
+
if (!created) {
|
|
2373
|
+
return child;
|
|
2374
|
+
}
|
|
2375
|
+
const current = await readRuntimeState();
|
|
2376
|
+
await writeRuntimeState({
|
|
2377
|
+
worker: {
|
|
2378
|
+
pid: child.pid ?? null,
|
|
2379
|
+
spawnedAt: nowIso(),
|
|
2380
|
+
ready: false,
|
|
2381
|
+
registeredTasks: [],
|
|
2382
|
+
readyAt: null,
|
|
2383
|
+
},
|
|
2384
|
+
guard: {
|
|
2385
|
+
lastSpawnAt: nowIso(),
|
|
2386
|
+
lastDecision: reason,
|
|
2387
|
+
spawnCount: current.guard.spawnCount + 1,
|
|
2388
|
+
},
|
|
2389
|
+
});
|
|
2390
|
+
return child;
|
|
2391
|
+
}
|
|
2392
|
+
function spawnGuard() {
|
|
2393
|
+
return childProcesses.spawn("guard", GUARD_SCRIPT, { env: spawnEnv("guard") }).child;
|
|
2394
|
+
}
|
|
2395
|
+
async function spawnScheduler(reason) {
|
|
2396
|
+
const { child, created } = childProcesses.spawn("scheduler", SCHEDULER_SCRIPT, {
|
|
2397
|
+
env: spawnEnv("scheduler"),
|
|
2398
|
+
});
|
|
2399
|
+
if (!created) {
|
|
2400
|
+
return child;
|
|
2401
|
+
}
|
|
2402
|
+
await writeRuntimeState({
|
|
2403
|
+
scheduler: {
|
|
2404
|
+
pid: child.pid ?? null,
|
|
2405
|
+
spawnedAt: nowIso(),
|
|
2406
|
+
ready: false,
|
|
2407
|
+
readyAt: null,
|
|
2408
|
+
},
|
|
2409
|
+
guard: {
|
|
2410
|
+
lastDecision: reason,
|
|
2411
|
+
},
|
|
2412
|
+
});
|
|
2413
|
+
return child;
|
|
2414
|
+
}
|
|
2415
|
+
async function spawnBusinessNodes(reason) {
|
|
2416
|
+
await spawnScheduler(reason);
|
|
2417
|
+
await spawnTaskWorker(reason);
|
|
2418
|
+
}
|
|
2419
|
+
async function stopBusinessNodes() {
|
|
2420
|
+
await childProcesses.stopAll();
|
|
2421
|
+
await writeRuntimeState({
|
|
2422
|
+
worker: {
|
|
2423
|
+
pid: null,
|
|
2424
|
+
ready: false,
|
|
2425
|
+
readyAt: null,
|
|
2426
|
+
registeredTasks: [],
|
|
2427
|
+
},
|
|
2428
|
+
scheduler: {
|
|
2429
|
+
pid: null,
|
|
2430
|
+
ready: false,
|
|
2431
|
+
readyAt: null,
|
|
2432
|
+
},
|
|
2433
|
+
});
|
|
2434
|
+
}
|
|
2435
|
+
async function isBusinessReady() {
|
|
2436
|
+
const state = await readRuntimeState();
|
|
2437
|
+
return state.worker.ready && state.scheduler.ready;
|
|
2438
|
+
}
|
|
2439
|
+
async function stopChildProcesses() {
|
|
2440
|
+
const state = await readRuntimeState();
|
|
2441
|
+
const extraPids = [state.worker.pid, state.scheduler.pid].filter((pid) => pid != null);
|
|
2442
|
+
await childProcesses.stopAll({ extraPids });
|
|
2443
|
+
}
|
|
2444
|
+
return {
|
|
2445
|
+
spawnGuard,
|
|
2446
|
+
spawnTaskWorker,
|
|
2447
|
+
spawnScheduler,
|
|
2448
|
+
spawnBusinessNodes,
|
|
2449
|
+
stopBusinessNodes,
|
|
2450
|
+
isBusinessReady,
|
|
2451
|
+
stopChildProcesses,
|
|
2452
|
+
};
|
|
2453
|
+
}
|
|
2454
|
+
|
|
2455
|
+
/**
|
|
2456
|
+
* Claim a registry slot, or `process.exit(1)` when no EMPTY slot remains.
|
|
2457
|
+
* Other errors propagate.
|
|
2458
|
+
*/
|
|
2459
|
+
async function claimRegistrySlotOrExit(input) {
|
|
2460
|
+
try {
|
|
2461
|
+
return await claimServiceRegistrySlot({
|
|
2462
|
+
serviceValue: input.serviceValue,
|
|
2463
|
+
createInstance: input.createInstance,
|
|
2464
|
+
});
|
|
2465
|
+
}
|
|
2466
|
+
catch (error) {
|
|
2467
|
+
if (error instanceof ServiceRegistryError && error.code === "SERVICE_SLOTS_FULL") {
|
|
2468
|
+
input.logger.critical("No EMPTY registry slot — refusing to start", {
|
|
2469
|
+
topic: input.logTopic,
|
|
2470
|
+
data: { serviceValue: input.serviceValue, code: error.code },
|
|
2471
|
+
});
|
|
2472
|
+
process.exit(1);
|
|
2473
|
+
}
|
|
2474
|
+
throw error;
|
|
1664
2475
|
}
|
|
1665
2476
|
}
|
|
1666
2477
|
|
|
@@ -1719,27 +2530,27 @@ function createServiceHost(options) {
|
|
|
1719
2530
|
if (options.prepare != null) {
|
|
1720
2531
|
await options.prepare();
|
|
1721
2532
|
}
|
|
1722
|
-
|
|
1723
|
-
|
|
2533
|
+
session = await claimRegistrySlotOrExit({
|
|
2534
|
+
serviceValue: options.serviceValue,
|
|
2535
|
+
createInstance: async () => unwrapService(await options.createInstance()),
|
|
2536
|
+
logger,
|
|
2537
|
+
logTopic,
|
|
2538
|
+
});
|
|
2539
|
+
const hostCtx = { service: session.service, session, serviceId: session.service.id };
|
|
2540
|
+
if (options.onRegistryClaimed != null) {
|
|
2541
|
+
await options.onRegistryClaimed(hostCtx);
|
|
2542
|
+
}
|
|
2543
|
+
const enableSpool = options.enableLogSpoolAfterClaim ?? true;
|
|
2544
|
+
if (enableSpool && logSpoolEnabledFromEnv()) {
|
|
2545
|
+
process.env[LOG_SPOOL_SERVICE_ID_ENV] = session.service.id;
|
|
2546
|
+
await ensureLogSpoolFromEnv({
|
|
2547
|
+
serviceId: session.service.id,
|
|
1724
2548
|
serviceValue: options.serviceValue,
|
|
1725
|
-
|
|
2549
|
+
processRole: "main",
|
|
1726
2550
|
});
|
|
1727
2551
|
}
|
|
1728
|
-
catch (error) {
|
|
1729
|
-
if (error instanceof ServiceRegistryError && error.code === "SERVICE_SLOTS_FULL") {
|
|
1730
|
-
logger.critical("No EMPTY registry slot — refusing to start", {
|
|
1731
|
-
topic: logTopic,
|
|
1732
|
-
data: { serviceValue: options.serviceValue, code: error.code },
|
|
1733
|
-
});
|
|
1734
|
-
process.exit(1);
|
|
1735
|
-
}
|
|
1736
|
-
throw error;
|
|
1737
|
-
}
|
|
1738
|
-
if (options.onRegistryClaimed != null) {
|
|
1739
|
-
await options.onRegistryClaimed({ service: session.service, session });
|
|
1740
|
-
}
|
|
1741
2552
|
if (options.startSupervisors != null) {
|
|
1742
|
-
await options.startSupervisors();
|
|
2553
|
+
await options.startSupervisors(hostCtx);
|
|
1743
2554
|
}
|
|
1744
2555
|
if (options.waitUntilReady != null) {
|
|
1745
2556
|
await options.waitUntilReady();
|
|
@@ -1783,23 +2594,12 @@ async function runSingleProcessService(options) {
|
|
|
1783
2594
|
if (options.prepare != null) {
|
|
1784
2595
|
await options.prepare();
|
|
1785
2596
|
}
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
}
|
|
1793
|
-
catch (error) {
|
|
1794
|
-
if (error instanceof ServiceRegistryError && error.code === "SERVICE_SLOTS_FULL") {
|
|
1795
|
-
logger.critical("No EMPTY registry slot — refusing to start", {
|
|
1796
|
-
topic: logTopic,
|
|
1797
|
-
data: { serviceValue: options.serviceValue, code: error.code },
|
|
1798
|
-
});
|
|
1799
|
-
process.exit(1);
|
|
1800
|
-
}
|
|
1801
|
-
throw error;
|
|
1802
|
-
}
|
|
2597
|
+
const session = await claimRegistrySlotOrExit({
|
|
2598
|
+
serviceValue: options.serviceValue,
|
|
2599
|
+
createInstance: options.createInstance,
|
|
2600
|
+
logger,
|
|
2601
|
+
logTopic,
|
|
2602
|
+
});
|
|
1803
2603
|
try {
|
|
1804
2604
|
await options.run({ service: session.service, session });
|
|
1805
2605
|
}
|
|
@@ -2265,58 +3065,66 @@ const RepoManager = {
|
|
|
2265
3065
|
clearRepoScriptModuleCache,
|
|
2266
3066
|
};
|
|
2267
3067
|
|
|
2268
|
-
|
|
2269
|
-
|
|
2270
|
-
* Each L3 service exports one instance from `src/startup/log-persist.ts`.
|
|
2271
|
-
*/
|
|
2272
|
-
function createLogPersistCoordinator(options) {
|
|
2273
|
-
const { service, registryFilePath, expectedProcesses, readyTimeoutMs } = options;
|
|
3068
|
+
function createLogSpoolCoordinator(options) {
|
|
3069
|
+
const { serviceValue, serviceId } = options;
|
|
2274
3070
|
return {
|
|
2275
3071
|
isEnabled() {
|
|
2276
|
-
return
|
|
3072
|
+
return logSpoolEnabledFromEnv();
|
|
2277
3073
|
},
|
|
2278
|
-
|
|
2279
|
-
return
|
|
3074
|
+
getServiceId() {
|
|
3075
|
+
return serviceId;
|
|
2280
3076
|
},
|
|
2281
3077
|
registerProcess(process) {
|
|
2282
|
-
return
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
});
|
|
2287
|
-
},
|
|
2288
|
-
waitForAllProcessesReady() {
|
|
2289
|
-
return waitForLogPersistReady({
|
|
2290
|
-
registryFilePath,
|
|
2291
|
-
service,
|
|
2292
|
-
expectedProcesses: [...expectedProcesses],
|
|
2293
|
-
timeoutMs: readyTimeoutMs,
|
|
2294
|
-
});
|
|
2295
|
-
},
|
|
2296
|
-
async snapshotProcessesReady() {
|
|
2297
|
-
if (!logPersistEnabledFromEnv()) {
|
|
2298
|
-
return null;
|
|
2299
|
-
}
|
|
2300
|
-
return snapshotLogPersistReady({
|
|
2301
|
-
registryFilePath,
|
|
2302
|
-
expectedProcesses: [...expectedProcesses],
|
|
3078
|
+
return ensureLogSpoolFromEnv({
|
|
3079
|
+
serviceValue,
|
|
3080
|
+
serviceId,
|
|
3081
|
+
processRole: process,
|
|
2303
3082
|
});
|
|
2304
3083
|
},
|
|
2305
|
-
|
|
2306
|
-
if (!
|
|
3084
|
+
getWriterStats() {
|
|
3085
|
+
if (!logSpoolEnabledFromEnv()) {
|
|
2307
3086
|
return null;
|
|
2308
3087
|
}
|
|
2309
|
-
return
|
|
3088
|
+
return getLogSpoolStats();
|
|
2310
3089
|
},
|
|
2311
|
-
async
|
|
2312
|
-
if (!
|
|
3090
|
+
async shutdownLogSpool() {
|
|
3091
|
+
if (!logSpoolEnabledFromEnv()) {
|
|
2313
3092
|
return;
|
|
2314
3093
|
}
|
|
2315
|
-
await
|
|
3094
|
+
await shutdownLogSpoolFromEnv();
|
|
2316
3095
|
},
|
|
2317
3096
|
};
|
|
2318
3097
|
}
|
|
2319
3098
|
|
|
3099
|
+
/** Whether file log spool is enabled (`LOG_PERSIST_ENABLED=true`). */
|
|
3100
|
+
function isLogSpoolEnabled(env = process.env) {
|
|
3101
|
+
return logSpoolEnabledFromEnv(env);
|
|
3102
|
+
}
|
|
3103
|
+
/**
|
|
3104
|
+
* Guard / scheduler / worker (and extra roles) — enable writer from spawn env.
|
|
3105
|
+
* Main is enabled by `createServiceHost` after registry claim.
|
|
3106
|
+
*/
|
|
3107
|
+
async function enableLogSpoolFromEnvInChild() {
|
|
3108
|
+
if (!logSpoolEnabledFromEnv()) {
|
|
3109
|
+
return;
|
|
3110
|
+
}
|
|
3111
|
+
await ensureLogSpoolFromEnv();
|
|
3112
|
+
}
|
|
3113
|
+
/** Writer buffer stats when spool enabled; `null` when disabled. Typical use: main observability. */
|
|
3114
|
+
function getMainLogSpoolWriterStats() {
|
|
3115
|
+
if (!logSpoolEnabledFromEnv()) {
|
|
3116
|
+
return null;
|
|
3117
|
+
}
|
|
3118
|
+
return getLogSpoolStats();
|
|
3119
|
+
}
|
|
3120
|
+
/** Flush memory buffer and disable writer when spool enabled; no-op otherwise. */
|
|
3121
|
+
async function shutdownLogSpool() {
|
|
3122
|
+
if (!logSpoolEnabledFromEnv()) {
|
|
3123
|
+
return;
|
|
3124
|
+
}
|
|
3125
|
+
await shutdownLogSpoolFromEnv();
|
|
3126
|
+
}
|
|
3127
|
+
|
|
2320
3128
|
const EMPTY_RESULT = {
|
|
2321
3129
|
status: "not_configured",
|
|
2322
3130
|
registered: [],
|
|
@@ -3228,5 +4036,5 @@ const patchTriggerFired = validateWithSchema(patchTriggerFiredSchema, "patchTrig
|
|
|
3228
4036
|
}
|
|
3229
4037
|
});
|
|
3230
4038
|
|
|
3231
|
-
export { BaseNodeRuntime, CategoryCommand, CategoryNode, CategoryRepo, CategoryTrigger, CommandType, EMPTY_REGISTRY_SLOT_RUNTIME_STATE, LOG_TOPIC_TRIGGER, LogLevel, MAX_POLL_TARGET_LIST_SIZE, ManagedChildProcesses, NodeStatus, RepoManager, ResultCode, SERVICE_GUARD_RUNNER_KEY, ServiceGuardNode, ServiceRegistryError, TASK_LOCAL_PACKAGE_CONFIG_FILENAME, TASK_REPO_SCRIPT_CATEGORY, TASK_REPO_USAGE, TASK_RUNNER_CONFIG_DIR, TASK_RUNNER_ROOT_CONFIG_FILENAME, TASK_RUNNER_ROOT_CONFIG_LEGACY_RELATIVE_PATH, TASK_RUNNER_ROOT_CONFIG_RELATIVE_PATH, TRIGGER_LOOP_INTERVAL_MS, TRIGGER_RUNNER_DEFAULT_RETRY_COUNT, TRIGGER_RUNNER_DEFAULT_RETRY_DELAY_MS, TaskManager, TaskNode, TaskRepoValidation, TaskStatus, TaskStatusAction, TriggerManager, TriggerNode, TriggerStatus, applyRegistrySlotGuardStep, buildNodeImportArgs, claimServiceRegistrySlot, createJsonFileStateStore,
|
|
4039
|
+
export { BaseNodeRuntime, COLLECT_LOG_RUNNER_KEY, CategoryCommand, CategoryNode, CategoryRepo, CategoryTrigger, CommandType, EMPTY_REGISTRY_SLOT_RUNTIME_STATE, LOG_SPOOL_SERVICE_ID_ENV, LOG_TOPIC_TRIGGER, LogLevel, MAX_POLL_TARGET_LIST_SIZE, ManagedChildProcesses, NodeStatus, RepoManager, ResultCode, SERVICE_GUARD_RUNNER_KEY, ServiceGuardNode, ServiceRegistryError, TASK_LOCAL_PACKAGE_CONFIG_FILENAME, TASK_REPO_SCRIPT_CATEGORY, TASK_REPO_USAGE, TASK_RUNNER_CONFIG_DIR, TASK_RUNNER_ROOT_CONFIG_FILENAME, TASK_RUNNER_ROOT_CONFIG_LEGACY_RELATIVE_PATH, TASK_RUNNER_ROOT_CONFIG_RELATIVE_PATH, TRIGGER_LOOP_INTERVAL_MS, TRIGGER_RUNNER_DEFAULT_RETRY_COUNT, TRIGGER_RUNNER_DEFAULT_RETRY_DELAY_MS, TaskManager, TaskNode, TaskRepoValidation, TaskStatus, TaskStatusAction, TriggerManager, TriggerNode, TriggerStatus, applyRegistrySlotGuardStep, buildNodeImportArgs, claimServiceRegistrySlot, createJsonFileStateStore, createL3ChildLauncher, createLogSpoolCoordinator, createLogger, createManagedChildProcesses, createPathsExistCheck, createRequiredEnvCheck, createServiceHost, createServiceRuntimeStateStore, createSupabaseReachableCheck, createTarget, enableLogSpoolFromEnvInChild, ensureLogSpoolFromEnv, evaluateBusyNodeLiveness, generateResponse, getConfigFileDir, getLogSpoolStats, getMainLogSpoolWriterStats, getPollCommandList, getPollCommandListSchema, getPossibleTarget, getScanRemoteRepoValues, guardRetryAfterSec, importJsConfigModule, initSupabaseFromStandardEnv, isChildProcessRunning, isGuardAvailable, isLogEntry, isLogSpoolEnabled, isOptimisticLockResponse, loadCachedJsConfigModule, loadEnvFiles, logManager, logSpoolEnabledFromEnv, markWorkerSpawned, nodeBufferToArrayBuffer, patchChangeNodeStatus, patchChangeTaskStatus, patchClaimTask, patchNodeHeartBeat, patchServiceRuntime, patchStopNode, patchTriggerFired, patchTriggerFiredSchema, pathExists, pollTargetList, pollUntil, postCommand, postCommandSchema, postRegisterNode, postTaskSchema, postTaskWithValidation, postTrigger, postTriggerSchema, publicBaseUrlFromEnv, readAndVerifySourceFile, readEnv, registerCollectLogRunner, registerServiceGuardRunner, registrySlotRuntimePatchFromGuardResult, requireEnv, resolveFirstExistingPath, resolveLogSpoolRoot, resolvePathFromBaseDir, resolvePathFromConfigFile, resolvePathFromCwd, runCollectLogTick, runReadinessChecks, runReadinessGate, runRegistrySlotGuardCheck, runServiceGuardTick, runSingleProcessService, scanEnabledTriggers, scanEnabledTriggersSchema, scanTargetList, sha256Hex, shutdownLogSpool, shutdownLogSpoolFromEnv, spawnTsxChild, supabase, toFileImportHref, updateTargetDetails, validateTaskRepoAndParams, validateWithSchema, waitForServiceReady };
|
|
3232
4040
|
//# sourceMappingURL=node.js.map
|