rollbridge 0.1.56 → 0.1.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -0
- package/TODO.md +12 -0
- package/changelog.d/20261013-recover-dead-guardian-state.md +1 -0
- package/changelog.d/20261025-guardian-re-register-dead-generation.md +13 -0
- package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +1 -0
- package/changelog.d/20261104-compose-service-units.md +21 -0
- package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +20 -0
- package/docs/cli.md +70 -1
- package/docs/compose-services.md +220 -0
- package/docs/config.md +55 -0
- package/docs/troubleshooting.md +24 -0
- package/package.json +1 -1
- package/src/cli.js +82 -4
- package/src/compose-unit.js +725 -0
- package/src/config.js +201 -2
- package/src/daemon.js +23 -8
- package/src/doctor.js +67 -0
- package/src/guardian-client.js +13 -2
- package/src/process-guardian.js +137 -17
- package/src/recover.js +128 -8
- package/test/completion.test.js +5 -2
- package/test/compose-config.test.js +158 -0
- package/test/compose-docker.test.js +297 -0
- package/test/compose-unit.test.js +415 -0
- package/test/fixtures/gated-app.js +50 -0
- package/test/guardian-client.test.js +70 -0
- package/test/recover.test.js +119 -0
- package/test/rollbridge.test.js +106 -6
package/src/config.js
CHANGED
|
@@ -24,7 +24,12 @@ import {pathToFileURL} from "node:url"
|
|
|
24
24
|
* @typedef {{forceStopTimeoutMs: number, processes: LegacyTakeoverProcessConfig[], screens: string[]}} LegacyTakeoverConfig
|
|
25
25
|
* @typedef {{keep: number, maxAgeMs: number}} ReleaseRetentionConfig
|
|
26
26
|
* @typedef {{reconnectGraceMs: number}} OwnerRecoveryConfig
|
|
27
|
-
* @typedef {{
|
|
27
|
+
* @typedef {{command: string | undefined, timeoutMs: number}} ComposeDrainConfig
|
|
28
|
+
* @typedef {{command: string | undefined, intervalMs: number, timeoutMs: number}} ComposeInFlightConfig
|
|
29
|
+
* @typedef {{healthcheck: boolean, intervalMs: number, readyCommand: string | undefined, readyUrl: string | undefined, startDelayMs: number, timeoutMs: number}} ComposeActivationConfig
|
|
30
|
+
* @typedef {{enabled: boolean}} ComposeRollbackConfig
|
|
31
|
+
* @typedef {{activation: ComposeActivationConfig, composeFile: string, drain: ComposeDrainConfig, envFile: string | undefined, id: string, image: string, inFlight: ComposeInFlightConfig, projectName: string | undefined, projectDirectory: string, recreateTimeoutMs: number, rollback: ComposeRollbackConfig, service: string, stopGracePeriodMs: number}} ComposeUnitConfig
|
|
32
|
+
* @typedef {{application: string, compose?: ComposeUnitConfig[], composeStatePath?: string, control: ControlConfig, legacyTakeover?: LegacyTakeoverConfig, ownerRecovery?: OwnerRecoveryConfig, processes: ProcessConfig[], proxy: ProxyConfig, releaseRetention: ReleaseRetentionConfig, statePath?: string}} RollbridgeConfig
|
|
28
33
|
* @typedef {{fix: string, message: string}} ConfigIssue
|
|
29
34
|
*/
|
|
30
35
|
|
|
@@ -72,6 +77,7 @@ export async function loadConfig(configPath) {
|
|
|
72
77
|
|
|
73
78
|
config.control.path = path.resolve(configDirectory, config.control.path)
|
|
74
79
|
if (config.statePath) config.statePath = path.resolve(configDirectory, config.statePath)
|
|
80
|
+
if (config.composeStatePath) config.composeStatePath = path.resolve(configDirectory, config.composeStatePath)
|
|
75
81
|
return config
|
|
76
82
|
}
|
|
77
83
|
|
|
@@ -158,13 +164,32 @@ export function validateConfig(rawConfig, configPath = process.cwd()) {
|
|
|
158
164
|
const ownerRecoverySource = source.ownerRecovery === undefined || source.ownerRecovery === null ? undefined : objectAt(source.ownerRecovery, "ownerRecovery", issues)
|
|
159
165
|
const reconnectGraceMs = ownerRecoverySource ? normalizeNumber(ownerRecoverySource.reconnectGraceMs, "ownerRecovery.reconnectGraceMs", issues, {default: 30000}) : 30000
|
|
160
166
|
const ownerRecovery = ownerRecoverySource ? {reconnectGraceMs: nonNegativeOrDefault(reconnectGraceMs, "ownerRecovery.reconnectGraceMs", issues, 30000, true)} : undefined
|
|
167
|
+
const compose = normalizeComposeUnits(source.compose, issues)
|
|
168
|
+
const composeStatePath = compose.length === 0 && source.composeStatePath === undefined && source.composeStatePath !== null
|
|
169
|
+
? undefined
|
|
170
|
+
: normalizeString(source.composeStatePath, "composeStatePath", issues, {default: path.resolve(path.dirname(configPath), `rollbridge-${application}.compose.state.json`)})
|
|
161
171
|
|
|
162
172
|
if (ownerRecovery && !statePath) issues.push({fix: "Configure statePath when ownerRecovery is enabled.", message: "ownerRecovery requires statePath"})
|
|
163
173
|
|
|
164
174
|
validateProcessSet(processes, issues)
|
|
165
175
|
validateActivationLifecycle(processes, ownerRecovery, statePath, issues)
|
|
176
|
+
validateComposeUnits(compose, processes, issues)
|
|
166
177
|
|
|
167
|
-
return {
|
|
178
|
+
return {
|
|
179
|
+
config: {
|
|
180
|
+
application,
|
|
181
|
+
...(compose.length > 0 ? {compose} : {}),
|
|
182
|
+
...(composeStatePath ? {composeStatePath} : {}),
|
|
183
|
+
control,
|
|
184
|
+
legacyTakeover,
|
|
185
|
+
ownerRecovery,
|
|
186
|
+
processes,
|
|
187
|
+
proxy,
|
|
188
|
+
releaseRetention,
|
|
189
|
+
statePath
|
|
190
|
+
},
|
|
191
|
+
issues
|
|
192
|
+
}
|
|
168
193
|
}
|
|
169
194
|
|
|
170
195
|
/**
|
|
@@ -518,6 +543,180 @@ function normalizeLegacyTakeoverProcess(value, index, issues) {
|
|
|
518
543
|
}]
|
|
519
544
|
}
|
|
520
545
|
|
|
546
|
+
/**
|
|
547
|
+
* @param {JsonValue} value - Raw Compose unit list.
|
|
548
|
+
* @param {ConfigIssue[]} issues - Issue collector.
|
|
549
|
+
* @returns {ComposeUnitConfig[]} Normalized Compose units (empty when the key is absent).
|
|
550
|
+
*/
|
|
551
|
+
function normalizeComposeUnits(value, issues) {
|
|
552
|
+
if (value === undefined || value === null) return []
|
|
553
|
+
|
|
554
|
+
if (!Array.isArray(value)) {
|
|
555
|
+
issues.push({fix: "Set compose to a list of Compose service units, or remove the key.", message: "compose must be an array"})
|
|
556
|
+
|
|
557
|
+
return []
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
if (value.length === 0) {
|
|
561
|
+
issues.push({fix: "Define at least one compose unit, or remove the compose key.", message: "compose must define at least one unit"})
|
|
562
|
+
|
|
563
|
+
return []
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
return value.map((unitSource, index) => normalizeComposeUnit(unitSource, index, issues))
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
/**
|
|
570
|
+
* @param {JsonValue} value - Raw Compose unit.
|
|
571
|
+
* @param {number} index - Unit index.
|
|
572
|
+
* @param {ConfigIssue[]} issues - Issue collector.
|
|
573
|
+
* @returns {ComposeUnitConfig} Normalized Compose unit.
|
|
574
|
+
*/
|
|
575
|
+
function normalizeComposeUnit(value, index, issues) {
|
|
576
|
+
const key = `compose[${index}]`
|
|
577
|
+
|
|
578
|
+
if (!isPlainObject(value)) {
|
|
579
|
+
issues.push({fix: `Define ${key} as a mapping with id, service, image, and projectDirectory.`, message: `${key} must be an object`})
|
|
580
|
+
|
|
581
|
+
return {
|
|
582
|
+
activation: {healthcheck: true, intervalMs: 1000, readyCommand: undefined, readyUrl: undefined, startDelayMs: 0, timeoutMs: 60000},
|
|
583
|
+
composeFile: "compose.yml",
|
|
584
|
+
drain: {command: undefined, timeoutMs: 0},
|
|
585
|
+
envFile: undefined,
|
|
586
|
+
id: "",
|
|
587
|
+
image: "",
|
|
588
|
+
inFlight: {command: undefined, intervalMs: 1000, timeoutMs: 0},
|
|
589
|
+
projectName: undefined,
|
|
590
|
+
projectDirectory: "",
|
|
591
|
+
recreateTimeoutMs: 120000,
|
|
592
|
+
rollback: {enabled: true},
|
|
593
|
+
service: "",
|
|
594
|
+
stopGracePeriodMs: 10000
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
const source = value
|
|
599
|
+
const stopGracePeriodMs = normalizePositiveNumber(source.stopGracePeriodMs, `${key}.stopGracePeriodMs`, issues, 10000)
|
|
600
|
+
const drainSource = source.drain === undefined || source.drain === null ? undefined : objectAt(source.drain, `${key}.drain`, issues, {})
|
|
601
|
+
const inFlightSource = source.inFlight === undefined || source.inFlight === null ? undefined : objectAt(source.inFlight, `${key}.inFlight`, issues, {})
|
|
602
|
+
const activationSource = source.activation === undefined || source.activation === null ? {} : objectAt(source.activation, `${key}.activation`, issues, {})
|
|
603
|
+
const rollbackSource = source.rollback === undefined || source.rollback === null ? undefined : objectAt(source.rollback, `${key}.rollback`, issues, {})
|
|
604
|
+
|
|
605
|
+
/** @type {ComposeUnitConfig} */
|
|
606
|
+
const unit = {
|
|
607
|
+
activation: {
|
|
608
|
+
healthcheck: normalizeBoolean(activationSource.healthcheck, `${key}.activation.healthcheck`, issues, true),
|
|
609
|
+
intervalMs: normalizePositiveNumber(activationSource.intervalMs, `${key}.activation.intervalMs`, issues, 1000),
|
|
610
|
+
readyCommand: activationSource.readyCommand === undefined ? undefined : normalizeString(activationSource.readyCommand, `${key}.activation.readyCommand`, issues, {nonEmpty: true}),
|
|
611
|
+
readyUrl: activationSource.readyUrl === undefined ? undefined : normalizeString(activationSource.readyUrl, `${key}.activation.readyUrl`, issues, {nonEmpty: true}),
|
|
612
|
+
startDelayMs: normalizeStartDelayMs(activationSource.startDelayMs, `${key}.activation.startDelayMs`, issues),
|
|
613
|
+
timeoutMs: normalizePositiveNumber(activationSource.timeoutMs, `${key}.activation.timeoutMs`, issues, 60000)
|
|
614
|
+
},
|
|
615
|
+
composeFile: normalizeString(source.composeFile, `${key}.composeFile`, issues, {default: "compose.yml"}),
|
|
616
|
+
drain: {
|
|
617
|
+
command: drainSource?.command === undefined ? undefined : normalizeString(drainSource.command, `${key}.drain.command`, issues, {nonEmpty: true}),
|
|
618
|
+
timeoutMs: normalizeNumber(drainSource?.timeoutMs, `${key}.drain.timeoutMs`, issues, {default: 60000})
|
|
619
|
+
},
|
|
620
|
+
envFile: source.envFile === undefined || source.envFile === null ? undefined : normalizeString(source.envFile, `${key}.envFile`, issues, {nonEmpty: true}),
|
|
621
|
+
id: normalizeString(source.id, `${key}.id`, issues),
|
|
622
|
+
image: normalizeString(source.image, `${key}.image`, issues),
|
|
623
|
+
inFlight: {
|
|
624
|
+
command: inFlightSource?.command === undefined ? undefined : normalizeString(inFlightSource.command, `${key}.inFlight.command`, issues, {nonEmpty: true}),
|
|
625
|
+
intervalMs: normalizePositiveNumber(inFlightSource?.intervalMs, `${key}.inFlight.intervalMs`, issues, 1000),
|
|
626
|
+
timeoutMs: normalizeNumber(inFlightSource?.timeoutMs, `${key}.inFlight.timeoutMs`, issues, {default: 0})
|
|
627
|
+
},
|
|
628
|
+
projectName: source.projectName === undefined || source.projectName === null ? undefined : normalizeString(source.projectName, `${key}.projectName`, issues, {nonEmpty: true}),
|
|
629
|
+
projectDirectory: normalizeString(source.projectDirectory, `${key}.projectDirectory`, issues),
|
|
630
|
+
recreateTimeoutMs: normalizePositiveNumber(source.recreateTimeoutMs, `${key}.recreateTimeoutMs`, issues, 120000),
|
|
631
|
+
rollback: {enabled: normalizeBoolean(rollbackSource?.enabled, `${key}.rollback.enabled`, issues, true)},
|
|
632
|
+
service: normalizeString(source.service, `${key}.service`, issues),
|
|
633
|
+
stopGracePeriodMs
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
if (unit.drain.command !== undefined && unit.drain.timeoutMs <= 0) {
|
|
637
|
+
issues.push({fix: `Set ${key}.drain.timeoutMs to a positive number of milliseconds; with 0 the quiescence step is skipped and ${key}.drain.command never runs.`, message: `${key}.drain.command requires a positive ${key}.drain.timeoutMs`})
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
if (unit.inFlight.command !== undefined && unit.inFlight.timeoutMs <= 0) {
|
|
641
|
+
issues.push({fix: `Set ${key}.inFlight.timeoutMs to how long the roll may wait for in-flight work, e.g. 120000; with 0 the gate is skipped and ${key}.inFlight.command never runs.`, message: `${key}.inFlight.command requires a positive ${key}.inFlight.timeoutMs`})
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
if (unit.image.length > 0 && !isComposeImageReference(unit.image)) {
|
|
645
|
+
issues.push({fix: `Set ${key}.image to an image reference pinned to a tag or digest, e.g. "registry.local/socketduct-gateway:1.4.0".`, message: `${key}.image "${unit.image}" is not a valid image reference`})
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
return unit
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
/**
|
|
652
|
+
* @param {JsonValue} value - Raw positive-number value.
|
|
653
|
+
* @param {string} key - Config key.
|
|
654
|
+
* @param {ConfigIssue[]} issues - Issue collector.
|
|
655
|
+
* @param {number} fallback - Default and fallback value.
|
|
656
|
+
* @returns {number} Positive number, or the fallback when invalid.
|
|
657
|
+
*/
|
|
658
|
+
function normalizePositiveNumber(value, key, issues, fallback) {
|
|
659
|
+
const number = normalizeNumber(value, key, issues, {default: fallback})
|
|
660
|
+
|
|
661
|
+
if (!(number > 0)) {
|
|
662
|
+
issues.push({fix: `Set ${key} to a positive number of milliseconds, e.g. ${fallback}.`, message: `${key} must be a positive number`})
|
|
663
|
+
|
|
664
|
+
return fallback
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
return number
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
/**
|
|
671
|
+
* @param {string} image - Image reference.
|
|
672
|
+
* @returns {boolean} True when the reference is a usable Docker image name.
|
|
673
|
+
*/
|
|
674
|
+
function isComposeImageReference(image) {
|
|
675
|
+
if (/\s/.test(image) || image.startsWith("-")) return false
|
|
676
|
+
|
|
677
|
+
const [name, digest] = image.split("@")
|
|
678
|
+
|
|
679
|
+
if (name.length === 0) return false
|
|
680
|
+
if (digest === undefined) return true
|
|
681
|
+
|
|
682
|
+
return /^sha256:[a-f0-9]{64}$/.test(digest)
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
/**
|
|
686
|
+
* Validates cross-unit rules: unique unit ids, no collision with process ids, and the fields a
|
|
687
|
+
* drain-first roll needs.
|
|
688
|
+
* @param {ComposeUnitConfig[]} compose - Normalized Compose units.
|
|
689
|
+
* @param {ProcessConfig[]} processes - Normalized processes.
|
|
690
|
+
* @param {ConfigIssue[]} issues - Issue collector.
|
|
691
|
+
* @returns {void}
|
|
692
|
+
*/
|
|
693
|
+
function validateComposeUnits(compose, processes, issues) {
|
|
694
|
+
if (compose.length === 0) return
|
|
695
|
+
|
|
696
|
+
const processIds = new Set(processes.map((processConfig) => processConfig.id))
|
|
697
|
+
const seenIds = /** @type {Set<string>} */ (new Set())
|
|
698
|
+
|
|
699
|
+
for (const unit of compose) {
|
|
700
|
+
if (!unit.id) continue
|
|
701
|
+
|
|
702
|
+
if (seenIds.has(unit.id)) {
|
|
703
|
+
issues.push({fix: `Give each compose unit a unique id; "${unit.id}" is used more than once.`, message: `Duplicate compose unit id: ${unit.id}`})
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
seenIds.add(unit.id)
|
|
707
|
+
|
|
708
|
+
if (processIds.has(unit.id)) {
|
|
709
|
+
issues.push({fix: `Rename compose unit "${unit.id}" or the process with the same id; Rollbridge targets both by id.`, message: `Compose unit id "${unit.id}" collides with a process id`})
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
for (const [index, unit] of compose.entries()) {
|
|
714
|
+
if (!unit.service) issues.push({fix: `Set compose[${index}].service to the Compose service name to roll.`, message: `compose[${index}].service is required`})
|
|
715
|
+
if (!unit.projectDirectory) issues.push({fix: `Set compose[${index}].projectDirectory to the directory holding the Compose file.`, message: `compose[${index}].projectDirectory is required`})
|
|
716
|
+
if (!unit.image) issues.push({fix: `Set compose[${index}].image to the target image reference.`, message: `compose[${index}].image is required`})
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
|
|
521
720
|
/**
|
|
522
721
|
* @param {JsonValue} value - Raw stop signal.
|
|
523
722
|
* @param {string} key - Config key.
|
package/src/daemon.js
CHANGED
|
@@ -146,7 +146,7 @@ export default class RollbridgeDaemon {
|
|
|
146
146
|
this.logger("release generation transition recovery failed", {error: error instanceof Error ? error.message : String(error), releaseId: transition.candidateReleaseId})
|
|
147
147
|
}
|
|
148
148
|
return {}
|
|
149
|
-
})
|
|
149
|
+
}, {waitingFor: "the retained release generations to settle"})
|
|
150
150
|
} finally {
|
|
151
151
|
this.generationTransitionRecovery = false
|
|
152
152
|
}
|
|
@@ -1389,7 +1389,7 @@ export default class RollbridgeDaemon {
|
|
|
1389
1389
|
releaseId: stringOrUndefined(data.releaseId),
|
|
1390
1390
|
releasePath: requiredString(data.releasePath, "releasePath"),
|
|
1391
1391
|
revision: stringOrUndefined(data.revision)
|
|
1392
|
-
}))
|
|
1392
|
+
}), {waitingFor: "candidate activation and the retired generation handoff"})
|
|
1393
1393
|
}
|
|
1394
1394
|
|
|
1395
1395
|
if (commandName === "recover-generation-transition") {
|
|
@@ -1399,7 +1399,7 @@ export default class RollbridgeDaemon {
|
|
|
1399
1399
|
releaseId: requiredString(data.releaseId, "releaseId"),
|
|
1400
1400
|
releasePath: requiredString(data.releasePath, "releasePath"),
|
|
1401
1401
|
revision: requiredString(data.revision, "revision")
|
|
1402
|
-
}))
|
|
1402
|
+
}), {waitingFor: "the retained release generations to settle"})
|
|
1403
1403
|
}
|
|
1404
1404
|
|
|
1405
1405
|
if (commandName === "status") {
|
|
@@ -1414,18 +1414,18 @@ export default class RollbridgeDaemon {
|
|
|
1414
1414
|
return await this.executeOwnerMutation("stop", async () => {
|
|
1415
1415
|
await this.stopRelease(stringOrUndefined(data.releaseId))
|
|
1416
1416
|
return this.status()
|
|
1417
|
-
})
|
|
1417
|
+
}, {waitingFor: "the release's managed processes to stop"})
|
|
1418
1418
|
}
|
|
1419
1419
|
|
|
1420
1420
|
if (commandName === "restart") {
|
|
1421
1421
|
return await this.executeOwnerMutation("restart", async () => await this.restartProcesses({
|
|
1422
1422
|
policy: stringOrUndefined(data.policy),
|
|
1423
1423
|
processId: stringOrUndefined(data.processId)
|
|
1424
|
-
}))
|
|
1424
|
+
}), {waitingFor: "the managed process to stop and start again"})
|
|
1425
1425
|
}
|
|
1426
1426
|
|
|
1427
1427
|
if (commandName === "rollback") {
|
|
1428
|
-
return await this.executeOwnerMutation("rollback", async () => await this.rollback({releaseId: stringOrUndefined(data.releaseId)}))
|
|
1428
|
+
return await this.executeOwnerMutation("rollback", async () => await this.rollback({releaseId: stringOrUndefined(data.releaseId)}), {waitingFor: "the previous release to activate"})
|
|
1429
1429
|
}
|
|
1430
1430
|
|
|
1431
1431
|
if (commandName === "shutdown") {
|
|
@@ -1489,11 +1489,13 @@ export default class RollbridgeDaemon {
|
|
|
1489
1489
|
* Runs one owner mutation under the guardian's prepare/commit exclusion fence.
|
|
1490
1490
|
* @param {string} operation - Mutation diagnostic name.
|
|
1491
1491
|
* @param {() => Promise<Record<string, JsonValue>>} callback - Mutating operation.
|
|
1492
|
+
* @param {{waitingFor?: string}} [options] - What this mutation blocks on, so a refused
|
|
1493
|
+
* later mutation names the real blocker instead of just "in progress".
|
|
1492
1494
|
* @returns {Promise<Record<string, JsonValue>>} Operation result.
|
|
1493
1495
|
*/
|
|
1494
|
-
async executeOwnerMutation(operation, callback) {
|
|
1496
|
+
async executeOwnerMutation(operation, callback, {waitingFor} = {}) {
|
|
1495
1497
|
if (!this.guardian) return await callback()
|
|
1496
|
-
const mutationId = await this.guardian.beginOwnerMutation(operation)
|
|
1498
|
+
const mutationId = await this.guardian.beginOwnerMutation(operation, {ownerPid: process.pid, waitingFor})
|
|
1497
1499
|
let result
|
|
1498
1500
|
let operationError
|
|
1499
1501
|
|
|
@@ -1946,6 +1948,19 @@ export default class RollbridgeDaemon {
|
|
|
1946
1948
|
recoveryStatus: "already_recovered"
|
|
1947
1949
|
}
|
|
1948
1950
|
}
|
|
1951
|
+
// A committed journal describes the transition that already published the current
|
|
1952
|
+
// incumbent, not an in-flight transition to recover. When the recovery fence names
|
|
1953
|
+
// that live incumbent, the failed deploy never journaled its own transition, so the
|
|
1954
|
+
// only fence that describes the authoritative release is the completed one: recovery
|
|
1955
|
+
// is already done. In-flight phases keep the stale-fence guard failing closed.
|
|
1956
|
+
if (transition.phase === "committed" && transition.candidateReleaseId === identity.previousReleaseId && this.activeRelease?.releaseId === identity.previousReleaseId && this.releases.get(transition.candidateReleaseId) === this.activeRelease) {
|
|
1957
|
+
return {
|
|
1958
|
+
activeReleaseId: this.activeRelease.releaseId,
|
|
1959
|
+
candidateReleaseId: identity.releaseId,
|
|
1960
|
+
previousReleaseId: identity.previousReleaseId,
|
|
1961
|
+
recoveryStatus: "already_recovered"
|
|
1962
|
+
}
|
|
1963
|
+
}
|
|
1949
1964
|
if (transition.previousReleaseId !== identity.previousReleaseId) {
|
|
1950
1965
|
throw new Error(`Generation transition previous release is ${transition.previousReleaseId}; refusing stale recovery for ${identity.previousReleaseId}`)
|
|
1951
1966
|
}
|
package/src/doctor.js
CHANGED
|
@@ -5,6 +5,7 @@ import fs from "node:fs/promises"
|
|
|
5
5
|
import net from "node:net"
|
|
6
6
|
import path from "node:path"
|
|
7
7
|
import {inspectControlSocket} from "./daemon.js"
|
|
8
|
+
import {composeGlobalArgs, runComposeCommand} from "./compose-unit.js"
|
|
8
9
|
import {liveProcesses, readState} from "./state-store.js"
|
|
9
10
|
import {processTemplateContext, renderObject, renderTemplate} from "./template.js"
|
|
10
11
|
|
|
@@ -36,9 +37,75 @@ export async function runEnvironmentChecks(config) {
|
|
|
36
37
|
checks.push(await orphanCheck(config.statePath, daemonRunning))
|
|
37
38
|
}
|
|
38
39
|
|
|
40
|
+
if (config.compose !== undefined) checks.push(...await runComposeChecks(config))
|
|
41
|
+
|
|
42
|
+
return checks
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Pre-flights every declared Compose unit: the Docker Compose CLI, the unit's Compose file, and
|
|
47
|
+
* the service it names inside that file.
|
|
48
|
+
* @param {import("./config.js").RollbridgeConfig} config - Normalized config.
|
|
49
|
+
* @returns {Promise<DoctorCheck[]>} One check per probed aspect.
|
|
50
|
+
*/
|
|
51
|
+
async function runComposeChecks(config) {
|
|
52
|
+
/** @type {DoctorCheck[]} */
|
|
53
|
+
const checks = []
|
|
54
|
+
const version = await runComposeCommand({args: ["docker", "compose", "version"], timeoutMs: 15000})
|
|
55
|
+
|
|
56
|
+
if (version.code !== 0) {
|
|
57
|
+
return [{detail: `docker compose is not usable: ${version.stderr.trim() || `exit ${version.code}`}`, name: "docker compose", ok: false}]
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
checks.push({detail: version.stdout.trim() || "available", name: "docker compose", ok: true})
|
|
61
|
+
|
|
62
|
+
for (const unit of config.compose ?? []) {
|
|
63
|
+
const composeFile = path.resolve(unit.projectDirectory, unit.composeFile)
|
|
64
|
+
|
|
65
|
+
if (!await fileExists(composeFile)) {
|
|
66
|
+
checks.push({detail: `${composeFile} is missing; unit "${unit.id}" cannot be rolled`, name: `compose unit ${unit.id}`, ok: false})
|
|
67
|
+
continue
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
if (unit.envFile !== undefined && !await fileExists(path.resolve(unit.projectDirectory, unit.envFile))) {
|
|
71
|
+
checks.push({detail: `${path.resolve(unit.projectDirectory, unit.envFile)} is missing; unit "${unit.id}" cannot be rolled`, name: `compose unit ${unit.id}`, ok: false})
|
|
72
|
+
continue
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const services = await runComposeCommand({args: [...composeGlobalArgs(unit), "config", "--services"], cwd: unit.projectDirectory, timeoutMs: 30000})
|
|
76
|
+
|
|
77
|
+
if (services.code !== 0) {
|
|
78
|
+
checks.push({detail: `could not read ${composeFile}: ${services.stderr.trim() || `exit ${services.code}`}`, name: `compose unit ${unit.id}`, ok: false})
|
|
79
|
+
continue
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const listed = services.stdout.split("\n").map((line) => line.trim()).filter(Boolean)
|
|
83
|
+
|
|
84
|
+
if (!listed.includes(unit.service)) {
|
|
85
|
+
checks.push({detail: `service "${unit.service}" is not declared in ${composeFile}; found ${listed.join(", ") || "no services"}`, name: `compose unit ${unit.id}`, ok: false})
|
|
86
|
+
continue
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
checks.push({detail: `${unit.service} in ${composeFile}, target image ${unit.image}`, name: `compose unit ${unit.id}`, ok: true})
|
|
90
|
+
}
|
|
91
|
+
|
|
39
92
|
return checks
|
|
40
93
|
}
|
|
41
94
|
|
|
95
|
+
/**
|
|
96
|
+
* @param {string} filePath - Path to probe.
|
|
97
|
+
* @returns {Promise<boolean>} True when the path exists.
|
|
98
|
+
*/
|
|
99
|
+
async function fileExists(filePath) {
|
|
100
|
+
try {
|
|
101
|
+
await fs.access(filePath)
|
|
102
|
+
|
|
103
|
+
return true
|
|
104
|
+
} catch {
|
|
105
|
+
return false
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
42
109
|
/**
|
|
43
110
|
* @param {string} statePath - Configured state file path.
|
|
44
111
|
* @returns {Promise<DoctorCheck>} Whether the state file's directory is writable.
|
package/src/guardian-client.js
CHANGED
|
@@ -323,10 +323,11 @@ export default class GuardianClient {
|
|
|
323
323
|
/**
|
|
324
324
|
* Acquires the committed owner's mutation fence.
|
|
325
325
|
* @param {string} operation - Control mutation name.
|
|
326
|
+
* @param {{mutationTimeoutMs?: number, ownerPid?: number, waitingFor?: string}} [details] - Lease bound, holding PID and what the mutation blocks on.
|
|
326
327
|
* @returns {Promise<string>} Mutation lease id.
|
|
327
328
|
*/
|
|
328
|
-
async beginOwnerMutation(operation) {
|
|
329
|
-
const result = /** @type {{mutationId: string}} */ (await this.request({command: "begin-owner-mutation", operation}))
|
|
329
|
+
async beginOwnerMutation(operation, {mutationTimeoutMs, ownerPid, waitingFor} = {}) {
|
|
330
|
+
const result = /** @type {{mutationId: string}} */ (await this.request({command: "begin-owner-mutation", mutationTimeoutMs, operation, ownerPid, waitingFor}))
|
|
330
331
|
|
|
331
332
|
return result.mutationId
|
|
332
333
|
}
|
|
@@ -336,6 +337,16 @@ export default class GuardianClient {
|
|
|
336
337
|
await this.request({command: "end-owner-mutation", mutationId})
|
|
337
338
|
}
|
|
338
339
|
|
|
340
|
+
/**
|
|
341
|
+
* Refines what a held mutation is waiting for, so a later refusal names the real blocker.
|
|
342
|
+
* @param {string} mutationId - Mutation lease id.
|
|
343
|
+
* @param {string} waitingFor - Current blocker.
|
|
344
|
+
* @returns {Promise<void>} Report acknowledged.
|
|
345
|
+
*/
|
|
346
|
+
async reportOwnerMutationWait(mutationId, waitingFor) {
|
|
347
|
+
await this.request({command: "report-owner-mutation-wait", mutationId, waitingFor})
|
|
348
|
+
}
|
|
349
|
+
|
|
339
350
|
/** @returns {Promise<{committedReplacementId: string | null, ownerClaimed: boolean, retirementFailed?: boolean, retirementPending?: boolean, retirementReady?: boolean}>} Transaction status. */
|
|
340
351
|
async replacementStatus() {
|
|
341
352
|
return /** @type {{committedReplacementId: string | null, ownerClaimed: boolean, retirementFailed?: boolean, retirementPending?: boolean, retirementReady?: boolean}} */ (await this.request({command: "replacement-status"}))
|
package/src/process-guardian.js
CHANGED
|
@@ -12,6 +12,10 @@ import ManagedProcess from "./managed-process.js"
|
|
|
12
12
|
import {writeState} from "./state-store.js"
|
|
13
13
|
|
|
14
14
|
const OWNER_RESTART_RETRY_MS = 1000
|
|
15
|
+
// An owner mutation is deploy/stop/restart/rollback/recovery work. A retained
|
|
16
|
+
// generation may legitimately drain for hours, but a mutation that holds the
|
|
17
|
+
// fence that long is stuck, so the lease is bounded and released on expiry.
|
|
18
|
+
const OWNER_MUTATION_LEASE_MS = 900000
|
|
15
19
|
|
|
16
20
|
/** @typedef {import("node:child_process").ChildProcess["signalCode"]} ChildExitSignal */
|
|
17
21
|
|
|
@@ -32,7 +36,9 @@ const OWNER_RESTART_RETRY_MS = 1000
|
|
|
32
36
|
* @property {import("./json.js").JsonValue} [ownerState] - Private owner transfer state.
|
|
33
37
|
* @property {string} [replacementId] - Prepared replacement transaction id.
|
|
34
38
|
* @property {string} [mutationId] - Owner mutation lease id.
|
|
39
|
+
* @property {number} [mutationTimeoutMs] - Bound for the owner mutation lease.
|
|
35
40
|
* @property {string} [operation] - Owner mutation diagnostic name.
|
|
41
|
+
* @property {string} [waitingFor] - What the held owner mutation is blocked on.
|
|
36
42
|
* @property {number} [ownerPid] - Exact claimant process PID.
|
|
37
43
|
* @property {import("./json.js").JsonValue} [recoverySnapshot] - Public bridge recovery snapshot.
|
|
38
44
|
* @property {string} [statePath] - Exact public recovery state path.
|
|
@@ -103,6 +109,19 @@ const legacyKeys = legacyGuardian ? committedOwnerProcessKeys(ownerState) : new
|
|
|
103
109
|
let ownerMutationClient
|
|
104
110
|
/** @type {string | undefined} */
|
|
105
111
|
let ownerMutationId
|
|
112
|
+
/** @type {string | undefined} */
|
|
113
|
+
let ownerMutationOperation
|
|
114
|
+
/** @type {string | undefined} */
|
|
115
|
+
let ownerMutationWaitingFor
|
|
116
|
+
/** @type {number | undefined} */
|
|
117
|
+
let ownerMutationPid
|
|
118
|
+
/** @type {number | undefined} */
|
|
119
|
+
let ownerMutationStartedAt
|
|
120
|
+
let ownerMutationLeaseMs = OWNER_MUTATION_LEASE_MS
|
|
121
|
+
/** @type {ReturnType<typeof setTimeout> | undefined} */
|
|
122
|
+
let ownerMutationLeaseTimer
|
|
123
|
+
/** @type {{elapsedMs: number, leaseMs: number, mutationId: string, operation: string} | undefined} */
|
|
124
|
+
let ownerMutationExpired
|
|
106
125
|
/** @type {net.Socket | undefined} */
|
|
107
126
|
let retiringClient
|
|
108
127
|
/** @type {string | undefined} */
|
|
@@ -154,10 +173,7 @@ const server = net.createServer((socket) => {
|
|
|
154
173
|
clearTimeout(waiter.timer)
|
|
155
174
|
waiter.reject(new Error("Owner claimant disconnected"))
|
|
156
175
|
}
|
|
157
|
-
if (ownerMutationClient === socket && ownerClient !== socket)
|
|
158
|
-
ownerMutationClient = undefined
|
|
159
|
-
ownerMutationId = undefined
|
|
160
|
-
}
|
|
176
|
+
if (ownerMutationClient === socket && ownerClient !== socket) clearOwnerMutationLease()
|
|
161
177
|
if (controlLessRetirement) {
|
|
162
178
|
const preCommitHandoff = replacementClient && ownerClient === socket && retiringReplacementId === replacementId
|
|
163
179
|
|
|
@@ -301,8 +317,7 @@ async function execute(request, socket) {
|
|
|
301
317
|
rememberDisconnectedOwnerSuccessors(socket)
|
|
302
318
|
ownerClient = undefined
|
|
303
319
|
ownerClientPid = undefined
|
|
304
|
-
|
|
305
|
-
ownerMutationId = undefined
|
|
320
|
+
clearOwnerMutationLease()
|
|
306
321
|
ownerRevision += 1
|
|
307
322
|
grantNextOwner()
|
|
308
323
|
return {retired: true}
|
|
@@ -397,26 +412,48 @@ async function execute(request, socket) {
|
|
|
397
412
|
if (request.command === "begin-owner-mutation") {
|
|
398
413
|
requireOwner(socket, request.command)
|
|
399
414
|
if (replacementClient) throw new Error(`Owner mutation ${request.operation || "operation"} is fenced while an owner replacement is prepared`)
|
|
400
|
-
|
|
415
|
+
const leaseMs = request.mutationTimeoutMs === undefined ? OWNER_MUTATION_LEASE_MS : request.mutationTimeoutMs
|
|
416
|
+
|
|
417
|
+
if (!Number.isInteger(leaseMs) || leaseMs <= 0) throw new Error("Owner mutation lease bound must be a positive number of milliseconds")
|
|
418
|
+
if (ownerMutationClient) throw new Error(`Another owner mutation is already in progress: ${describeOwnerMutationLease()}; refused ${request.operation || "operation"}`)
|
|
401
419
|
ownerMutationClient = socket
|
|
402
420
|
ownerMutationId = crypto.randomUUID()
|
|
403
|
-
|
|
421
|
+
ownerMutationOperation = request.operation
|
|
422
|
+
ownerMutationWaitingFor = request.waitingFor
|
|
423
|
+
ownerMutationPid = request.ownerPid ?? ownerClientPid
|
|
424
|
+
ownerMutationStartedAt = Date.now()
|
|
425
|
+
ownerMutationLeaseMs = leaseMs
|
|
426
|
+
armOwnerMutationLease(ownerMutationId)
|
|
427
|
+
return {leaseMs, mutationId: ownerMutationId}
|
|
404
428
|
}
|
|
405
429
|
|
|
406
430
|
if (request.command === "end-owner-mutation") {
|
|
407
431
|
requireOwner(socket, request.command)
|
|
432
|
+
if (ownerMutationExpired && request.mutationId === ownerMutationExpired.mutationId) {
|
|
433
|
+
const expired = ownerMutationExpired
|
|
434
|
+
|
|
435
|
+
ownerMutationExpired = undefined
|
|
436
|
+
throw new Error(`Owner mutation ${expired.operation} exceeded its ${expired.leaseMs} ms lease and was released after ${expired.elapsedMs} ms; the mutation failed and its lease is gone`)
|
|
437
|
+
}
|
|
408
438
|
if (ownerMutationClient !== socket || request.mutationId !== ownerMutationId) throw new Error("Owner mutation lease does not match the active mutation")
|
|
409
|
-
|
|
410
|
-
ownerMutationId = undefined
|
|
439
|
+
clearOwnerMutationLease()
|
|
411
440
|
return {ended: true}
|
|
412
441
|
}
|
|
413
442
|
|
|
443
|
+
if (request.command === "report-owner-mutation-wait") {
|
|
444
|
+
requireOwner(socket, request.command)
|
|
445
|
+
if (ownerMutationClient !== socket || request.mutationId !== ownerMutationId) throw new Error("Owner mutation lease does not match the active mutation")
|
|
446
|
+
if (typeof request.waitingFor !== "string" || !request.waitingFor.trim()) throw new Error("Owner mutation wait report requires a waitingFor description")
|
|
447
|
+
ownerMutationWaitingFor = request.waitingFor.trim()
|
|
448
|
+
return {reported: true}
|
|
449
|
+
}
|
|
450
|
+
|
|
414
451
|
if (request.command === "prepare-owner-replacement") {
|
|
415
452
|
if (retirementFailed) throw new Error("Guardian owner replacement is fenced after incomplete retired-listener state transfer")
|
|
416
453
|
if (retiringClient) throw new Error("Guardian owner replacement cannot prepare while listener retirement is pending")
|
|
417
454
|
if (socket === ownerClient) throw new Error("Committed owner cannot prepare itself as a replacement")
|
|
418
455
|
if (replacementClient && replacementClient !== socket) throw new Error("Another owner replacement candidate is already prepared")
|
|
419
|
-
if (ownerMutationClient) throw new Error(
|
|
456
|
+
if (ownerMutationClient) throw new Error(`Owner replacement cannot prepare while an owner mutation is in progress: ${describeOwnerMutationLease()}`)
|
|
420
457
|
if (!ownerState) throw new Error("Committed owner has not published transferable state")
|
|
421
458
|
const currentAuthority = ownerAuthority(ownerState)
|
|
422
459
|
const resumesCommittedAuthority = !ownerClient && isDeepStrictEqual(currentAuthority, request.nextAuthority)
|
|
@@ -613,8 +650,18 @@ async function execute(request, socket) {
|
|
|
613
650
|
const existing = processes.get(request.key)
|
|
614
651
|
|
|
615
652
|
if (existing) {
|
|
616
|
-
if (existing.provenance
|
|
617
|
-
|
|
653
|
+
if (existing.provenance === request.provenance) return existing.process.status()
|
|
654
|
+
// A provenance mismatch only fences a process that still exists. When the registered
|
|
655
|
+
// child is conclusively gone (stopped or failed with no live pid), the old record is
|
|
656
|
+
// residue of a dead generation, and demanding its provenance wedges every later deploy
|
|
657
|
+
// against a process that no longer exists (TensorBuzz production, STC-695). The owner
|
|
658
|
+
// may re-register the key under the new definition; live processes stay fail-closed.
|
|
659
|
+
const existingStatus = existing.process.status()
|
|
660
|
+
const childIsGone = (existingStatus.state === "stopped" || existingStatus.state === "failed") && existingStatus.pid === undefined
|
|
661
|
+
|
|
662
|
+
if (!childIsGone) throw new Error(`Guardian provenance mismatch for ${request.key}`)
|
|
663
|
+
requireOwner(socket, request.command)
|
|
664
|
+
processes.delete(request.key)
|
|
618
665
|
}
|
|
619
666
|
|
|
620
667
|
const recoversLegacyProcess = Boolean(legacyGuardian && legacyKeys.has(request.key))
|
|
@@ -705,6 +752,82 @@ async function execute(request, socket) {
|
|
|
705
752
|
return status
|
|
706
753
|
}
|
|
707
754
|
|
|
755
|
+
/**
|
|
756
|
+
* Names the mutation holding the owner-mutation fence: which mutation, which process,
|
|
757
|
+
* since when, and what it is blocked on. Refusals must be diagnosable without a live
|
|
758
|
+
* inspection of the daemon.
|
|
759
|
+
* @returns {string} Holder description used in refusal messages.
|
|
760
|
+
*/
|
|
761
|
+
function describeOwnerMutationLease() {
|
|
762
|
+
const startedAt = ownerMutationStartedAt ?? 0
|
|
763
|
+
const elapsedSeconds = Math.max(0, Math.round((Date.now() - startedAt) / 1000))
|
|
764
|
+
const holderPid = ownerMutationPid ?? ownerClientPid
|
|
765
|
+
|
|
766
|
+
return [
|
|
767
|
+
`owner mutation "${ownerMutationOperation || "unnamed operation"}"`,
|
|
768
|
+
`mutation ${ownerMutationId || "unknown"}`,
|
|
769
|
+
`held by process ${holderPid ?? "unknown"}`,
|
|
770
|
+
`since ${new Date(startedAt).toISOString()}`,
|
|
771
|
+
`${elapsedSeconds}s elapsed`,
|
|
772
|
+
`waiting for ${ownerMutationWaitingFor || "the operation to report completion"}`
|
|
773
|
+
].join("; ")
|
|
774
|
+
}
|
|
775
|
+
|
|
776
|
+
/** @returns {void} */
|
|
777
|
+
function clearOwnerMutationLease() {
|
|
778
|
+
if (ownerMutationLeaseTimer) clearTimeout(ownerMutationLeaseTimer)
|
|
779
|
+
ownerMutationLeaseTimer = undefined
|
|
780
|
+
ownerMutationClient = undefined
|
|
781
|
+
ownerMutationId = undefined
|
|
782
|
+
ownerMutationOperation = undefined
|
|
783
|
+
ownerMutationWaitingFor = undefined
|
|
784
|
+
ownerMutationPid = undefined
|
|
785
|
+
ownerMutationStartedAt = undefined
|
|
786
|
+
ownerMutationLeaseMs = OWNER_MUTATION_LEASE_MS
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
/**
|
|
790
|
+
* Arms the lease bound for one mutation.
|
|
791
|
+
* @param {string} mutationId - Lease id the timer belongs to.
|
|
792
|
+
* @returns {void}
|
|
793
|
+
*/
|
|
794
|
+
function armOwnerMutationLease(mutationId) {
|
|
795
|
+
if (ownerMutationLeaseTimer) clearTimeout(ownerMutationLeaseTimer)
|
|
796
|
+
ownerMutationLeaseTimer = setTimeout(() => expireOwnerMutationLease(mutationId), ownerMutationLeaseMs)
|
|
797
|
+
ownerMutationLeaseTimer.unref()
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
/**
|
|
801
|
+
* A mutation that outruns its lease is reported and released instead of holding the fence
|
|
802
|
+
* forever, so a stuck mutation cannot lock out every later deploy. The holder learns the
|
|
803
|
+
* lease is gone when it tries to end it.
|
|
804
|
+
* @param {string} mutationId - Lease id the armed timer belongs to.
|
|
805
|
+
* @returns {void}
|
|
806
|
+
*/
|
|
807
|
+
function expireOwnerMutationLease(mutationId) {
|
|
808
|
+
ownerMutationLeaseTimer = undefined
|
|
809
|
+
if (!ownerMutationClient || ownerMutationId !== mutationId) return
|
|
810
|
+
const operation = ownerMutationOperation || "unnamed operation"
|
|
811
|
+
const leaseMs = ownerMutationLeaseMs
|
|
812
|
+
const elapsedMs = Math.max(0, Date.now() - (ownerMutationStartedAt ?? 0))
|
|
813
|
+
const diagnostic = {
|
|
814
|
+
code: "OWNER_MUTATION_LEASE_EXPIRED",
|
|
815
|
+
elapsedMs,
|
|
816
|
+
event: "owner-mutation-expired",
|
|
817
|
+
leaseMs,
|
|
818
|
+
mutationId,
|
|
819
|
+
operation,
|
|
820
|
+
pid: ownerMutationPid ?? ownerClientPid,
|
|
821
|
+
waitingFor: ownerMutationWaitingFor || "the operation to report completion"
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
process.stderr.write(`${JSON.stringify({at: new Date().toISOString(), ...diagnostic})}\n`)
|
|
825
|
+
if (!ownerMutationClient.destroyed) ownerMutationClient.write(`${JSON.stringify(diagnostic)}\n`)
|
|
826
|
+
broadcast(diagnostic)
|
|
827
|
+
ownerMutationExpired = {elapsedMs, leaseMs, mutationId, operation}
|
|
828
|
+
clearOwnerMutationLease()
|
|
829
|
+
}
|
|
830
|
+
|
|
708
831
|
/**
|
|
709
832
|
* @param {net.Socket} socket - Requesting client.
|
|
710
833
|
* @param {string} command - Command name.
|
|
@@ -725,10 +848,7 @@ function releaseClosedOwner(socket) {
|
|
|
725
848
|
rememberDisconnectedOwnerSuccessors(socket)
|
|
726
849
|
ownerClient = undefined
|
|
727
850
|
ownerClientPid = undefined
|
|
728
|
-
if (ownerMutationClient === socket)
|
|
729
|
-
ownerMutationClient = undefined
|
|
730
|
-
ownerMutationId = undefined
|
|
731
|
-
}
|
|
851
|
+
if (ownerMutationClient === socket) clearOwnerMutationLease()
|
|
732
852
|
if (replacementClient && replacementOwnerState) commitReplacement()
|
|
733
853
|
else {
|
|
734
854
|
if (incompleteRestartChild) {
|