querysub 0.568.0 → 0.569.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/deployManager/components/ServiceDetailPage.tsx +5 -25
- package/src/deployManager/components/deployButtons.tsx +2 -28
- package/src/deployManager/machineApplyMainCode.ts +31 -61
- package/src/deployManager/processLogs.ts +9 -0
- package/src/deployManager/processManager.ts +101 -14
package/package.json
CHANGED
|
@@ -82,28 +82,6 @@ export class ServiceDetailPage extends qreact.Component {
|
|
|
82
82
|
});
|
|
83
83
|
}
|
|
84
84
|
|
|
85
|
-
private confirmForceDeployNow(deployConfig: ServiceConfig) {
|
|
86
|
-
let modal: { close: () => void } | undefined;
|
|
87
|
-
modal = showModal({
|
|
88
|
-
content: <div className={css.fixed.pos(0, 0).size("100vw", "100vh").hsla(0, 0, 0, 0.5).display("flex").alignItems("center").justifyContent("center")}>
|
|
89
|
-
<div className={css.vbox(12).pad2(20).hsl(0, 0, 12).colorhsl(0, 0, 90).bord2(0, 0, 40) + " keepModalsOpen"}>
|
|
90
|
-
<div className={css.boldStyle.fontSize(16)}>🚀 Deploy?</div>
|
|
91
|
-
<div>{deployConfig.parameters.overlapTime
|
|
92
|
-
? `This config goes live now. The old instances keep running for ${formatTime(deployConfig.parameters.overlapTime)} before shutting down.`
|
|
93
|
-
: "This config goes live now. The old instances are shut down as soon as the new ones are up, so anything still connected to them is dropped."}</div>
|
|
94
|
-
<div className={css.hbox(10)}>
|
|
95
|
-
<Button hue={0} onClick={() => {
|
|
96
|
-
modal && modal.close();
|
|
97
|
-
deployConfig.parameters.releaseTime = Date.now();
|
|
98
|
-
this.deployConfig(deployConfig);
|
|
99
|
-
}}>🚀 Deploy</Button>
|
|
100
|
-
<Button onClick={() => modal && modal.close()}>Cancel</Button>
|
|
101
|
-
</div>
|
|
102
|
-
</div>
|
|
103
|
-
</div>,
|
|
104
|
-
});
|
|
105
|
-
}
|
|
106
|
-
|
|
107
85
|
// Shows the `${name}` variables found in the command, and lets each picked machine entry set its own values for them (which converts machineIds to the { machineId, variables } form).
|
|
108
86
|
private renderTemplateVariables(config: ServiceConfig) {
|
|
109
87
|
let templateVariables = getCommandTemplateVariables(config.parameters.command);
|
|
@@ -601,7 +579,9 @@ export class ServiceDetailPage extends qreact.Component {
|
|
|
601
579
|
disabled={this.state.isDeploying}
|
|
602
580
|
title="Changes the scheduled deploy to go live immediately"
|
|
603
581
|
onClick={() => {
|
|
604
|
-
|
|
582
|
+
let deployConfig = deepCloneJSON(stripReleaseState(originalConfig));
|
|
583
|
+
deployConfig.parameters.releaseTime = Date.now();
|
|
584
|
+
this.deployConfig(deployConfig);
|
|
605
585
|
}}>
|
|
606
586
|
⚡ Force Schedule Now
|
|
607
587
|
</button>
|
|
@@ -656,7 +636,7 @@ export class ServiceDetailPage extends qreact.Component {
|
|
|
656
636
|
disabled={this.state.isDeploying}
|
|
657
637
|
title="Deploys now; the old instances keep running for the overlap before shutting down"
|
|
658
638
|
onClick={() => {
|
|
659
|
-
this.
|
|
639
|
+
this.deployConfig(makeDeployConfig());
|
|
660
640
|
}}>
|
|
661
641
|
{this.state.isDeploying && "⏳ Deploying..." || "🚀 Deploy"}
|
|
662
642
|
</button>
|
|
@@ -665,7 +645,7 @@ export class ServiceDetailPage extends qreact.Component {
|
|
|
665
645
|
disabled={this.state.isDeploying}
|
|
666
646
|
title="Deploys now AND shuts the old instances down as soon as the new ones are up (overlap of 0)"
|
|
667
647
|
onClick={() => {
|
|
668
|
-
this.
|
|
648
|
+
this.deployConfig(makeDeployConfig(0));
|
|
669
649
|
}}>
|
|
670
650
|
Deploy, No Overlap
|
|
671
651
|
</button>
|
|
@@ -101,32 +101,6 @@ export class UpdateButtons extends qreact.Component<{
|
|
|
101
101
|
}
|
|
102
102
|
});
|
|
103
103
|
}
|
|
104
|
-
private confirmDeployAllNow(outdatedServices: ServiceConfig[], latestRef: string, noOverlap: boolean) {
|
|
105
|
-
let modal: { close: () => void } | undefined;
|
|
106
|
-
modal = showFullscreenModal({
|
|
107
|
-
content: <div className={css.vbox(12).pad2(20)}>
|
|
108
|
-
<div className={css.boldStyle.fontSize(16)}>⚡ Deploy all {outdatedServices.length} services now?</div>
|
|
109
|
-
<div>{noOverlap
|
|
110
|
-
? "The old instances are shut down as soon as the new ones are up (overlap of 0), so anything still connected to them is dropped."
|
|
111
|
-
: "The old instances keep running for their configured overlap before shutting down, so connections have somewhere to go while clients move across."}</div>
|
|
112
|
-
<div className={css.hbox(10)}>
|
|
113
|
-
<button
|
|
114
|
-
className={css.pad2(12, 8).button + (noOverlap && css.bord2(0, 90, 30).hsl(0, 85, 65).colorhsl(0, 0, 100).fontWeight("bold") || css.bord2(45, 80, 35).hsl(45, 85, 82))}
|
|
115
|
-
onClick={() => {
|
|
116
|
-
modal && modal.close();
|
|
117
|
-
this.deployAll(outdatedServices, latestRef, noOverlap);
|
|
118
|
-
}}>
|
|
119
|
-
⚡ Deploy Now
|
|
120
|
-
</button>
|
|
121
|
-
<button
|
|
122
|
-
className={css.pad2(12, 8).button.bord2(0, 0, 20).hsl(0, 0, 100)}
|
|
123
|
-
onClick={() => modal && modal.close()}>
|
|
124
|
-
Cancel
|
|
125
|
-
</button>
|
|
126
|
-
</div>
|
|
127
|
-
</div>
|
|
128
|
-
});
|
|
129
|
-
}
|
|
130
104
|
render() {
|
|
131
105
|
let controller = MachineServiceController(SocketFunction.browserNodeId());
|
|
132
106
|
const gitInfo = controller.getGitInfo();
|
|
@@ -185,7 +159,7 @@ export class UpdateButtons extends qreact.Component<{
|
|
|
185
159
|
disabled={this.state.isDeploying}
|
|
186
160
|
title="Deploys now; the old instances keep running for their configured overlap before shutting down"
|
|
187
161
|
onClick={() => {
|
|
188
|
-
this.
|
|
162
|
+
this.deployAll(outdatedServices, gitInfo.latestRef, false);
|
|
189
163
|
}}
|
|
190
164
|
>
|
|
191
165
|
<div>
|
|
@@ -205,7 +179,7 @@ export class UpdateButtons extends qreact.Component<{
|
|
|
205
179
|
disabled={this.state.isDeploying}
|
|
206
180
|
title="Deploys now AND shuts the old instances down as soon as the new ones are up (overlap of 0)"
|
|
207
181
|
onClick={() => {
|
|
208
|
-
this.
|
|
182
|
+
this.deployAll(outdatedServices, gitInfo.latestRef, true);
|
|
209
183
|
}}
|
|
210
184
|
>
|
|
211
185
|
<div>
|
|
@@ -26,8 +26,8 @@ import { PromiseObj } from "../promise";
|
|
|
26
26
|
import path from "path";
|
|
27
27
|
import { fsExistsAsync } from "../fs";
|
|
28
28
|
import { ALIVE_WINDOW_FOREVER, ParametersTimelineEntry, syncParametersTimelineFiles } from "./parametersTimeline";
|
|
29
|
-
import { getScreenName, getFutureScreenName,
|
|
30
|
-
import { ProcessRecord, syncProcessRecords, writeProcessRecord, listProcessRecords, getProcessLogPath } from "./processLogs";
|
|
29
|
+
import { getScreenName, getFutureScreenName, getTmuxPrefix, getScreenState, isScreenRunningProcess, getProcessStartTime, ensureProcessRecord, runScreenCommand, killScreen, streamProcessOutput, readServiceNodeId, removeServiceNode, takeoverScreen, scheduleTakeover } from "./processManager";
|
|
30
|
+
import { ProcessRecord, syncProcessRecords, writeProcessRecord, listProcessRecords, findRunningRecord, getProcessLogPath } from "./processLogs";
|
|
31
31
|
|
|
32
32
|
|
|
33
33
|
|
|
@@ -137,17 +137,6 @@ async function notifyNodeShutdown(screenName: string, nodeId: string, time: numb
|
|
|
137
137
|
console.warn(`Error notifying ${screenName} (${nodeId}) of its scheduled shutdown (will retry on the next resync): ${e.stack}`);
|
|
138
138
|
}
|
|
139
139
|
}
|
|
140
|
-
async function notifyServiceShutdown(screenName: string, time: number) {
|
|
141
|
-
if (notifiedShutdowns.get(screenName) === time) return;
|
|
142
|
-
let nodeIdFile = os.homedir() + "/" + SERVICE_FOLDER + screenName + "/" + SERVICE_NODE_FILE_NAME;
|
|
143
|
-
if (!await fsExistsAsync(nodeIdFile)) {
|
|
144
|
-
console.log(`No nodeId file found for ${screenName}, cannot notify it of its scheduled shutdown yet (will retry on the next resync)`);
|
|
145
|
-
return;
|
|
146
|
-
}
|
|
147
|
-
let nodeId = await fs.promises.readFile(nodeIdFile, "utf8");
|
|
148
|
-
await notifyNodeShutdown(screenName, nodeId, time);
|
|
149
|
-
}
|
|
150
|
-
|
|
151
140
|
|
|
152
141
|
|
|
153
142
|
|
|
@@ -233,14 +222,13 @@ let lastLaunchedTimePerService = new Map<string, number>();
|
|
|
233
222
|
|
|
234
223
|
const RELEASE_PREP_LEAD_TIME = timeInSecond * 30;
|
|
235
224
|
|
|
236
|
-
// In-memory only: the old process's nodeId, captured just before the future instance starts (the new process overwrites the shared folder's nodeId file), so the old node can be force-removed when its screen is finally killed. Lost on apply restart, which is fine — node discovery times dead nodes out anyway.
|
|
237
|
-
let takeoverOldNodeIds = new Map<string, string>();
|
|
238
|
-
|
|
239
225
|
// Syncs the canonical folder to the new parameters (git + yarn install + parameters.json) and creates the future screen, which just echos when it will start. Idempotent.
|
|
240
226
|
type FutureConfig = {
|
|
241
227
|
canonicalScreenName: string;
|
|
242
228
|
next: ServiceParameters;
|
|
243
229
|
releaseTime: number;
|
|
230
|
+
/** When the old process's overlap runs out, so the takeover can be scheduled the moment the new one starts */
|
|
231
|
+
killTime: number;
|
|
244
232
|
serviceId: string;
|
|
245
233
|
index: number;
|
|
246
234
|
machineId: string;
|
|
@@ -306,11 +294,6 @@ async function ensureFutureStarted(config: FutureConfig) {
|
|
|
306
294
|
}
|
|
307
295
|
await ensureFuturePrepared(config);
|
|
308
296
|
let folder = os.homedir() + "/" + SERVICE_FOLDER + canonicalScreenName + "/";
|
|
309
|
-
let nodeIdFile = folder + SERVICE_NODE_FILE_NAME;
|
|
310
|
-
// Only the FIRST start can trust the file to hold the old process's nodeId — on a future-instance crash + restart it may already hold the future's own nodeId
|
|
311
|
-
if (!takeoverOldNodeIds.has(canonicalScreenName) && await fsExistsAsync(nodeIdFile)) {
|
|
312
|
-
takeoverOldNodeIds.set(canonicalScreenName, await fs.promises.readFile(nodeIdFile, "utf8"));
|
|
313
|
-
}
|
|
314
297
|
console.log(green(`Starting future instance ${magenta(futureScreenName)} (release time reached)`));
|
|
315
298
|
await runScreenCommand({
|
|
316
299
|
screenName: futureScreenName,
|
|
@@ -324,9 +307,11 @@ async function ensureFutureStarted(config: FutureConfig) {
|
|
|
324
307
|
parameters: next,
|
|
325
308
|
},
|
|
326
309
|
});
|
|
310
|
+
// The old process's overlap is counted from here, not from whenever a resync next comes around
|
|
311
|
+
scheduleTakeover({ canonicalScreenName, folder, killTime: config.killTime });
|
|
327
312
|
}
|
|
328
313
|
|
|
329
|
-
|
|
314
|
+
/** The loop's route into the takeover: the timer scheduled at launch normally gets there first, and this is what catches up if we restarted and lost it. The takeover itself is shared, and locked, so the two can't interleave. */
|
|
330
315
|
async function takeoverFutureScreen(config: {
|
|
331
316
|
canonicalScreenName: string;
|
|
332
317
|
screenNamesUsed: Set<string>;
|
|
@@ -336,35 +321,15 @@ async function takeoverFutureScreen(config: {
|
|
|
336
321
|
let futureScreenName = getFutureScreenName(canonicalScreenName);
|
|
337
322
|
let future = screenStateMap.get(futureScreenName);
|
|
338
323
|
if (!future) return;
|
|
339
|
-
// The
|
|
324
|
+
// The renames make both the future and the retiring names vanish from the caller's screen list, so the unused-screen sweep must not try to kill them
|
|
340
325
|
config.screenNamesUsed.add(futureScreenName);
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
screenStateMap.delete(canonicalScreenName);
|
|
347
|
-
// Kept out of the unused-screen sweep, which would kill it a second time while it is already winding down
|
|
348
|
-
config.screenNamesUsed.add(retiringScreenName);
|
|
349
|
-
void killScreen({ screenName: retiringScreenName, skipNodeIdRemoval: true });
|
|
350
|
-
let oldNodeId = takeoverOldNodeIds.get(canonicalScreenName);
|
|
351
|
-
if (oldNodeId) {
|
|
352
|
-
void forceRemoveNode(oldNodeId);
|
|
353
|
-
}
|
|
354
|
-
}
|
|
355
|
-
takeoverOldNodeIds.delete(canonicalScreenName);
|
|
356
|
-
let prefix = getTmuxPrefix();
|
|
357
|
-
console.log(green(`Renaming future screen ${magenta(futureScreenName)} to ${magenta(canonicalScreenName)} (takeover complete)`));
|
|
358
|
-
await runPromise(`${prefix}tmux rename-session -t ${futureScreenName} ${canonicalScreenName}`);
|
|
326
|
+
let tookOver = await takeoverScreen({
|
|
327
|
+
canonicalScreenName,
|
|
328
|
+
folder: os.homedir() + "/" + SERVICE_FOLDER + canonicalScreenName + "/",
|
|
329
|
+
});
|
|
330
|
+
if (!tookOver) return;
|
|
359
331
|
screenStateMap.set(canonicalScreenName, { ...future, screenName: canonicalScreenName });
|
|
360
332
|
screenStateMap.delete(futureScreenName);
|
|
361
|
-
// Nothing to do to the logs: the future process has been writing to its own file all along, and keeps writing to it under the new session name. Its record still names the future session though, so point it at the name it now runs under.
|
|
362
|
-
let folder = os.homedir() + "/" + SERVICE_FOLDER + canonicalScreenName + "/";
|
|
363
|
-
for (let record of await listProcessRecords()) {
|
|
364
|
-
if (record.folder !== folder || record.screenName !== futureScreenName || record.deadTime !== undefined) continue;
|
|
365
|
-
record.screenName = canonicalScreenName;
|
|
366
|
-
await writeProcessRecord(record);
|
|
367
|
-
}
|
|
368
333
|
}
|
|
369
334
|
|
|
370
335
|
function sameRestartParameters(prevParametersJSON: string, next: ServiceParameters): boolean {
|
|
@@ -449,9 +414,9 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
|
|
|
449
414
|
let canonicalScreenName = getScreenName({ serviceKey: record.parameters.key, index: i });
|
|
450
415
|
try {
|
|
451
416
|
if (isOverlapTime) {
|
|
452
|
-
await ensureFutureStarted({ canonicalScreenName, next, releaseTime, serviceId: record.serviceId, index: i, machineId, screenNamesUsed, screenStateMap });
|
|
417
|
+
await ensureFutureStarted({ canonicalScreenName, next, releaseTime, killTime, serviceId: record.serviceId, index: i, machineId, screenNamesUsed, screenStateMap });
|
|
453
418
|
} else {
|
|
454
|
-
await ensureFuturePrepared({ canonicalScreenName, next, releaseTime, serviceId: record.serviceId, index: i, machineId, screenNamesUsed, screenStateMap });
|
|
419
|
+
await ensureFuturePrepared({ canonicalScreenName, next, releaseTime, killTime, serviceId: record.serviceId, index: i, machineId, screenNamesUsed, screenStateMap });
|
|
455
420
|
}
|
|
456
421
|
// The canonical loop below writes the timeline, but it is skipped entirely when the OLD parameters aren't deployed (a release that turns deploy on). The new instance would then run with no timeline file saying when it takes over, so it is written here instead. The old parameters aren't running, so the future entry is the whole timeline.
|
|
457
422
|
if (!config.parameters.deploy) {
|
|
@@ -519,15 +484,14 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
|
|
|
519
484
|
};
|
|
520
485
|
|
|
521
486
|
if (releaseInFlight) {
|
|
522
|
-
// Tell the running instance when it will be shut down, so it (and everything talking to it) can wind down gracefully. Fire and forget: it is a courtesy to the process, and nothing about taking it down may wait on a call to the process we are taking down.
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
void notifyNodeShutdown(screenName, oldNodeId, killTime);
|
|
487
|
+
// Tell the running instance when it will be shut down, so it (and everything talking to it) can wind down gracefully. Fire and forget: it is a courtesy to the process, and nothing about taking it down may wait on a call to the process we are taking down. The node comes off its own record, which was taken before any replacement could overwrite the folder's file.
|
|
488
|
+
void (async () => {
|
|
489
|
+
let running = await findRunningRecord(folder, screenName);
|
|
490
|
+
let nodeId = running?.nodeId || await readServiceNodeId(folder);
|
|
491
|
+
if (nodeId) {
|
|
492
|
+
await notifyNodeShutdown(screenName, nodeId, killTime);
|
|
529
493
|
}
|
|
530
|
-
}
|
|
494
|
+
})();
|
|
531
495
|
} else {
|
|
532
496
|
// A finished release's future screen takes over here: kill the old screen, rename the future one — the new process itself is untouched, so the normal logic below just verifies it
|
|
533
497
|
await takeoverFutureScreen({ canonicalScreenName: screenName, screenNamesUsed, screenStateMap });
|
|
@@ -702,9 +666,15 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
|
|
|
702
666
|
for (let { screenName } of screenState) {
|
|
703
667
|
if (screenNamesUsed.has(screenName)) continue;
|
|
704
668
|
// Detached, so a screen that takes its full grace period to go down doesn't hold up the end of the resync
|
|
705
|
-
void
|
|
706
|
-
|
|
707
|
-
|
|
669
|
+
void (async () => {
|
|
670
|
+
// Nothing is replacing this one, so its registration and the file it wrote both go
|
|
671
|
+
let folder = root + screenName + "/";
|
|
672
|
+
let nodeId = (await findRunningRecord(folder, screenName))?.nodeId || await readServiceNodeId(folder);
|
|
673
|
+
if (nodeId) {
|
|
674
|
+
await removeServiceNode({ folder, nodeId, ownsFile: true });
|
|
675
|
+
}
|
|
676
|
+
await killScreen({ screenName });
|
|
677
|
+
})();
|
|
708
678
|
}
|
|
709
679
|
|
|
710
680
|
scheduleReleaseResync(upcomingReleases);
|
|
@@ -30,6 +30,8 @@ export type ProcessRecord = {
|
|
|
30
30
|
machineId: string;
|
|
31
31
|
/** The start time the OS reports for the pid, which is also half of the launchId */
|
|
32
32
|
startTime: number;
|
|
33
|
+
/** The node this process registered as, read out of the folder before any replacement is started - after that the file belongs to the replacement. Kept here so taking this process down needs nothing but its own record. */
|
|
34
|
+
nodeId?: string;
|
|
33
35
|
/** When we first noticed the process was gone. Absent while it is running. */
|
|
34
36
|
deadTime?: number;
|
|
35
37
|
pid?: number;
|
|
@@ -99,6 +101,13 @@ while true; do
|
|
|
99
101
|
done`;
|
|
100
102
|
}
|
|
101
103
|
|
|
104
|
+
/** The record of whatever process is currently running in a screen, if we have one. */
|
|
105
|
+
export async function findRunningRecord(folder: string, screenName: string): Promise<ProcessRecord | undefined> {
|
|
106
|
+
return (await listProcessRecords()).find(x =>
|
|
107
|
+
x.folder === folder && x.screenName === screenName && x.deadTime === undefined
|
|
108
|
+
);
|
|
109
|
+
}
|
|
110
|
+
|
|
102
111
|
export async function writeProcessRecord(record: ProcessRecord): Promise<void> {
|
|
103
112
|
await fs.promises.mkdir(path.join(record.folder, PROCESS_FOLDER), { recursive: true });
|
|
104
113
|
await fs.promises.writeFile(getRecordPath(record.folder, record.launchId), JSON.stringify(record));
|
|
@@ -13,7 +13,8 @@ import { forceRemoveNode } from "../-f-node-discovery/NodeDiscovery";
|
|
|
13
13
|
import { fsExistsAsync } from "../fs";
|
|
14
14
|
import { PromiseObj } from "../promise";
|
|
15
15
|
import { SERVICE_FOLDER, SERVICE_NODE_FILE_NAME } from "./machineSchema";
|
|
16
|
-
import { ProcessRecord, createLaunchId, getLogPipeScript, getLogTailScript, getProcessLogPath, getPipeScriptPath, getTailScriptPath, listProcessRecords, writeProcessRecord } from "./processLogs";
|
|
16
|
+
import { ProcessRecord, createLaunchId, getLogPipeScript, getLogTailScript, getProcessLogPath, getPipeScriptPath, getTailScriptPath, listProcessRecords, findRunningRecord, writeProcessRecord } from "./processLogs";
|
|
17
|
+
import { setPreciseTimeout } from "../misc";
|
|
17
18
|
|
|
18
19
|
// Running, inspecting and killing the tmux screens services run in. This layer only knows "here is a configuration, run it" - which version should be running when is the deploy logic's problem.
|
|
19
20
|
|
|
@@ -34,14 +35,45 @@ export function getFutureScreenName(canonicalScreenName: string): string {
|
|
|
34
35
|
}
|
|
35
36
|
|
|
36
37
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
38
|
+
// Everything that touches a service's screens - the resync loop, and the timer that retires the old process at its kill time - runs under this, keyed by the canonical screen name. They fire at similar times by design, and a takeover is a rename dance that must not interleave with another one.
|
|
39
|
+
const screenLocks = new Map<string, Promise<unknown>>();
|
|
40
|
+
export function withScreenLock<T>(canonicalScreenName: string, run: () => Promise<T>): Promise<T> {
|
|
41
|
+
let previous = screenLocks.get(canonicalScreenName) || Promise.resolve();
|
|
42
|
+
let next = previous.then(run, run);
|
|
43
|
+
// Failures release the lock rather than poisoning every later operation on this screen
|
|
44
|
+
screenLocks.set(canonicalScreenName, next.then(() => { }, () => { }));
|
|
45
|
+
return next;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** Whether a session exists, without listing every session and its panes - the takeover already knows the exact names it cares about. */
|
|
49
|
+
async function screenExists(screenName: string): Promise<boolean> {
|
|
50
|
+
let prefix = getTmuxPrefix();
|
|
51
|
+
try {
|
|
52
|
+
await runPromise(`${prefix}tmux has-session -t ${screenName}`, { quiet: true });
|
|
53
|
+
return true;
|
|
54
|
+
} catch {
|
|
55
|
+
// has-session exits non-zero for "no such session", which is an answer, not a failure
|
|
56
|
+
return false;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** The node the process in this folder registered as. Read it BEFORE starting a replacement in the same folder - once the replacement starts, the file is the replacement's. */
|
|
61
|
+
export async function readServiceNodeId(folder: string): Promise<string | undefined> {
|
|
62
|
+
let nodeIdFile = path.join(folder, SERVICE_NODE_FILE_NAME);
|
|
63
|
+
if (!await fsExistsAsync(nodeIdFile)) return undefined;
|
|
64
|
+
return await fs.promises.readFile(nodeIdFile, "utf8");
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Drops a process's node registration. The file is only removed when it is still that process's - a replacement has its own registration in the same file. */
|
|
68
|
+
export async function removeServiceNode(config: { folder: string; nodeId: string; ownsFile: boolean }): Promise<void> {
|
|
69
|
+
console.log(green(`Removing the node registration ${config.nodeId} for the process in ${config.folder}`));
|
|
70
|
+
if (config.ownsFile) {
|
|
71
|
+
let nodeIdFile = path.join(config.folder, SERVICE_NODE_FILE_NAME);
|
|
72
|
+
if (await fsExistsAsync(nodeIdFile)) {
|
|
73
|
+
await fs.promises.unlink(nodeIdFile);
|
|
74
|
+
}
|
|
44
75
|
}
|
|
76
|
+
await forceRemoveNode(config.nodeId);
|
|
45
77
|
}
|
|
46
78
|
|
|
47
79
|
export const getTmuxPrefix = lazy(() => {
|
|
@@ -195,9 +227,17 @@ export const runScreenCommand = measureWrap(async function runScreenCommand(conf
|
|
|
195
227
|
}
|
|
196
228
|
await runPromise(`${prefix}tmux new -s ${screenName} -d`);
|
|
197
229
|
|
|
198
|
-
await removeOldNodeId(screenName);
|
|
199
230
|
let folder = config.folder || os.homedir() + "/" + SERVICE_FOLDER + screenName + "/";
|
|
200
231
|
|
|
232
|
+
// Read the outgoing process's node id onto its own record BEFORE this launch overwrites the file. Whoever takes that process down then has everything it needs on the record, instead of us having to remember not to touch a file that stopped being its own.
|
|
233
|
+
let outgoingNodeId = await readServiceNodeId(folder);
|
|
234
|
+
if (outgoingNodeId) {
|
|
235
|
+
for (let record of await listProcessRecords()) {
|
|
236
|
+
if (record.folder !== folder || record.deadTime !== undefined || record.nodeId) continue;
|
|
237
|
+
await writeProcessRecord({ ...record, nodeId: outgoingNodeId });
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
201
241
|
// The pane we are about to run the command in identifies the process: its pid, and the start time the OS reports for that pid. Both can be re-checked against the machine later, so nothing downstream has to trust our bookkeeping about what is still running. The session was just created, so this pid is always new.
|
|
202
242
|
let panePid = (await getScreenState(false)).find(x => x.screenName === screenName)?.pid;
|
|
203
243
|
if (!panePid) {
|
|
@@ -267,10 +307,60 @@ async function setupPipePane(config: { screenName: string; folder: string; launc
|
|
|
267
307
|
await runPromise(`${prefix}tmux pipe-pane -t ${config.screenName} 'bash ${pipeScript}'`);
|
|
268
308
|
}
|
|
269
309
|
|
|
310
|
+
/** Retires whatever is running under the canonical name and renames the future screen onto it. Everything it needs is derived from the two names, so it costs two `has-session` checks - the kill time is when this runs, not when a resync gets around to it. Returns whether there was a future screen to take over. */
|
|
311
|
+
export async function takeoverScreen(config: { canonicalScreenName: string; folder: string }): Promise<boolean> {
|
|
312
|
+
let { canonicalScreenName, folder } = config;
|
|
313
|
+
return await withScreenLock(canonicalScreenName, async () => {
|
|
314
|
+
let futureScreenName = getFutureScreenName(canonicalScreenName);
|
|
315
|
+
if (!await screenExists(futureScreenName)) return false;
|
|
316
|
+
let prefix = getTmuxPrefix();
|
|
317
|
+
if (await screenExists(canonicalScreenName)) {
|
|
318
|
+
let retiringScreenName = getRetiringScreenName(canonicalScreenName, Date.now());
|
|
319
|
+
// Its record carries the node it registered as, taken before the replacement started - the file in the folder belongs to the replacement now, so it is left alone
|
|
320
|
+
let outgoing = await findRunningRecord(folder, canonicalScreenName);
|
|
321
|
+
await runPromise(`${prefix}tmux rename-session -t ${canonicalScreenName} ${retiringScreenName}`);
|
|
322
|
+
if (outgoing) {
|
|
323
|
+
await writeProcessRecord({ ...outgoing, screenName: retiringScreenName });
|
|
324
|
+
if (outgoing.nodeId) {
|
|
325
|
+
void removeServiceNode({ folder, nodeId: outgoing.nodeId, ownsFile: false });
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
// Detached: the canonical name is already free, so nothing waits on the old process winding down
|
|
329
|
+
void killScreen({ screenName: retiringScreenName });
|
|
330
|
+
}
|
|
331
|
+
console.log(green(`Renaming future screen ${futureScreenName} to ${canonicalScreenName} (takeover complete)`));
|
|
332
|
+
await runPromise(`${prefix}tmux rename-session -t ${futureScreenName} ${canonicalScreenName}`);
|
|
333
|
+
let incoming = await findRunningRecord(folder, futureScreenName);
|
|
334
|
+
if (incoming) {
|
|
335
|
+
await writeProcessRecord({ ...incoming, screenName: canonicalScreenName });
|
|
336
|
+
}
|
|
337
|
+
return true;
|
|
338
|
+
});
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
// canonical screen name -> the takeover we already have scheduled for it
|
|
342
|
+
const scheduledTakeovers = new Map<string, { time: number; cancel: () => void }>();
|
|
343
|
+
|
|
344
|
+
/** Schedules the takeover for the moment the old process's overlap runs out. The resync loop still does the same takeover whenever it notices the time has passed, which is what recovers this if we restart and lose the timer. */
|
|
345
|
+
export function scheduleTakeover(config: { canonicalScreenName: string; folder: string; killTime: number }): void {
|
|
346
|
+
let { canonicalScreenName, folder, killTime } = config;
|
|
347
|
+
let existing = scheduledTakeovers.get(canonicalScreenName);
|
|
348
|
+
if (existing?.time === killTime) return;
|
|
349
|
+
existing?.cancel();
|
|
350
|
+
console.log(green(`Scheduling the takeover of ${canonicalScreenName} for ${new Date(killTime).toISOString()}`));
|
|
351
|
+
let cancel = setPreciseTimeout({
|
|
352
|
+
time: killTime,
|
|
353
|
+
callback: () => {
|
|
354
|
+
scheduledTakeovers.delete(canonicalScreenName);
|
|
355
|
+
void takeoverScreen({ canonicalScreenName, folder });
|
|
356
|
+
},
|
|
357
|
+
});
|
|
358
|
+
scheduledTakeovers.set(canonicalScreenName, { time: killTime, cancel });
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
// Kills a screen and nothing else. Which node the process registered as, and whether that registration should be dropped, is the deploy logic's business - it reads the node id off the process's record, which was taken before any replacement could overwrite the file.
|
|
270
362
|
export const killScreen = measureWrap(async function killScreen(config: {
|
|
271
363
|
screenName: string;
|
|
272
|
-
// During a takeover the folder's nodeId file already belongs to the NEW process, so the old screen's kill must not remove it
|
|
273
|
-
skipNodeIdRemoval?: boolean;
|
|
274
364
|
}) {
|
|
275
365
|
console.log(red(`Killing screen ${config.screenName} (Ctrl+C, then killing the session in ${formatTime(SHUTDOWN_GRACE_TIME)})`));
|
|
276
366
|
let prefix = getTmuxPrefix();
|
|
@@ -278,9 +368,6 @@ export const killScreen = measureWrap(async function killScreen(config: {
|
|
|
278
368
|
await runPromise(`${prefix}tmux send-keys -t ${config.screenName} 'C-c' Enter`);
|
|
279
369
|
await delay(SHUTDOWN_GRACE_TIME);
|
|
280
370
|
await runPromise(`${prefix}tmux kill-session -t ${config.screenName}`);
|
|
281
|
-
if (!config.skipNodeIdRemoval) {
|
|
282
|
-
await removeOldNodeId(config.screenName);
|
|
283
|
-
}
|
|
284
371
|
});
|
|
285
372
|
|
|
286
373
|
/** Streams one process's log: everything already in it, then everything appended after. One process, one file - nothing here has to reason about which process a byte came from. */
|