querysub 0.573.0 → 0.575.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/appSecrets.ts
CHANGED
|
@@ -3,6 +3,7 @@ import { lazy } from "socket-function/src/caching";
|
|
|
3
3
|
import { ArchivesBackblaze } from "sliftutils/storage/backblaze";
|
|
4
4
|
import { STORAGE_DIR, getDomain, getBackblazePath } from "./src/misc/appPaths";
|
|
5
5
|
import { Querysub } from "./src/4-querysub/Querysub";
|
|
6
|
+
import { formatDateTime } from "socket-function/src/formatting/format";
|
|
6
7
|
|
|
7
8
|
const keysArchives = lazy(() => new ArchivesBackblaze({ bucketName: getDomain() }));
|
|
8
9
|
|
|
@@ -55,7 +56,7 @@ let doImmediateShutdown = lazy(() => {
|
|
|
55
56
|
});
|
|
56
57
|
}
|
|
57
58
|
function doShutdown() {
|
|
58
|
-
console.log(
|
|
59
|
+
console.log(`SIGINT received, shutting down immediately (${formatDateTime(Date.now())})`);
|
|
59
60
|
process.exit();
|
|
60
61
|
}
|
|
61
62
|
process.on("SIGINT", doShutdown);
|
package/package.json
CHANGED
|
@@ -26,7 +26,7 @@ import { PromiseObj } from "../promise";
|
|
|
26
26
|
import path from "path";
|
|
27
27
|
import { fsExistsAsync } from "../fs";
|
|
28
28
|
import { ALIVE_WINDOW_FOREVER, ParametersTimelineEntry, syncParametersTimelineFiles } from "./parametersTimeline";
|
|
29
|
-
import { getScreenName,
|
|
29
|
+
import { getScreenName, getRetiringKillTime, retireScreen, retireCurrentScreen, getTmuxPrefix, getScreenState, isScreenRunningProcess, getProcessStartTime, getScreenPanePid, ensureProcessRecord, runScreenCommand, killScreen, readServiceNodeId, removeServiceNode, killScreenNow, withScreenLock } from "./processManager";
|
|
30
30
|
import { ProcessRecord, syncProcessRecords, listProcessRecords } from "./processLogs";
|
|
31
31
|
|
|
32
32
|
|
|
@@ -256,7 +256,7 @@ function getMachineInstances(configs: ServiceConfig[], machineId: string): { con
|
|
|
256
256
|
return instances;
|
|
257
257
|
}
|
|
258
258
|
|
|
259
|
-
/** Puts one instance of a service into the state its configuration describes: the folder synced, the process running what it should be, and the records and timeline saying so. Returns the
|
|
259
|
+
/** Puts one instance of a service into the state its configuration describes: the folder synced, the process running what it should be, and the records and timeline saying so. Returns the screen it is using, so anything nobody claimed can be killed. Everything about the screen happens under its lock, so two passes can never interleave. */
|
|
260
260
|
async function ensureScreen(config: {
|
|
261
261
|
config: ServiceConfig;
|
|
262
262
|
index: number;
|
|
@@ -267,22 +267,16 @@ async function ensureScreen(config: {
|
|
|
267
267
|
let serviceId = config.config.serviceId;
|
|
268
268
|
let serviceConfig = config.config;
|
|
269
269
|
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
//
|
|
274
|
-
let
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
let
|
|
278
|
-
if (!current?.deploy) current = undefined;
|
|
279
|
-
let incoming: ServiceParameters | undefined = releaseInFlight && Date.now() >= releaseTime && serviceConfig.parameters || undefined;
|
|
280
|
-
if (!incoming?.deploy) incoming = undefined;
|
|
281
|
-
let canonicalScreenName = getScreenName({ serviceKey: (current || incoming || serviceConfig.parameters).key, index });
|
|
282
|
-
let futureScreenName = getFutureScreenName(canonicalScreenName);
|
|
283
|
-
let folder = os.homedir() + "/" + SERVICE_FOLDER + canonicalScreenName + "/";
|
|
270
|
+
// A version being replaced is renamed aside and given this long before it is taken down, which is the only thing an overlap is
|
|
271
|
+
let killTime = Date.now() + (serviceConfig.parameters.overlapTime ?? DEFAULT_OVERLAP_TIME);
|
|
272
|
+
|
|
273
|
+
// The screen name means one thing: the version that should be running. There is no second name to reason about - anything it replaced is already renamed aside and carries its own deadline.
|
|
274
|
+
let parameters: ServiceParameters | undefined = getLiveServiceParameters(serviceConfig);
|
|
275
|
+
if (!parameters.deploy) parameters = undefined;
|
|
276
|
+
let screenName = getScreenName({ serviceKey: (parameters || serviceConfig.parameters).key, index });
|
|
277
|
+
let folder = os.homedir() + "/" + SERVICE_FOLDER + screenName + "/";
|
|
284
278
|
// Returned even when the work below throws, so a transient failure never makes the sweep kill a screen we still want
|
|
285
|
-
let usedScreenNames = [
|
|
279
|
+
let usedScreenNames = [screenName];
|
|
286
280
|
|
|
287
281
|
let launchCount = launchesPerService.get(serviceId) || 0;
|
|
288
282
|
let lastLaunchedTime = lastLaunchedTimePerService.get(serviceId) || 0;
|
|
@@ -294,108 +288,88 @@ async function ensureScreen(config: {
|
|
|
294
288
|
};
|
|
295
289
|
|
|
296
290
|
try {
|
|
297
|
-
await watchSlowPromise(`ensureScreen|${
|
|
291
|
+
await watchSlowPromise(`ensureScreen|${screenName}`, withScreenLock(screenName, async () => {
|
|
298
292
|
/* Three facts, each moving ? -> known -> correct as we go:
|
|
299
|
-
EXISTS -
|
|
300
|
-
VERSION - which parameters
|
|
301
|
-
RUNNING - whether a process is actually alive in
|
|
293
|
+
EXISTS - whether the screen is there
|
|
294
|
+
VERSION - which parameters it was started with
|
|
295
|
+
RUNNING - whether a process is actually alive in it
|
|
302
296
|
Every step below says where those three stand, so it is always clear what we are still assuming.
|
|
303
297
|
START: EXISTS ? VERSION ? RUNNING ?
|
|
304
298
|
*/
|
|
305
|
-
let parameters = incoming || current;
|
|
306
|
-
let screenName = incoming && futureScreenName || canonicalScreenName;
|
|
307
|
-
let takeoverAt = incoming && killTime || undefined;
|
|
308
299
|
|
|
309
|
-
// A screen we do not expect holds something we never asked for, so it goes at once - nothing here is worth winding down. Each name is judged on its own: a release that turns deploy on has an incoming version and nothing that should be running under the canonical name yet.
|
|
310
300
|
// AFTER: EXISTS correct in the "nothing extra" sense. A screen that SHOULD exist may still be missing, which is RUNNING's problem below.
|
|
311
|
-
if (!
|
|
312
|
-
await killScreenNow(
|
|
313
|
-
|
|
314
|
-
if (!current) {
|
|
315
|
-
await killScreenNow(canonicalScreenName, "nothing is supposed to be serving this name yet");
|
|
301
|
+
if (!parameters) {
|
|
302
|
+
await killScreenNow(screenName, "this instance deploys nothing here");
|
|
303
|
+
return;
|
|
316
304
|
}
|
|
317
|
-
if (!current && !incoming) return;
|
|
318
305
|
|
|
319
|
-
//
|
|
320
|
-
// AFTER: VERSION known
|
|
306
|
+
// The folder holds the version the screen was last started from
|
|
307
|
+
// AFTER: VERSION known
|
|
321
308
|
let parameterPath = folder + "parameters.json";
|
|
322
309
|
let prevParameters = "";
|
|
323
310
|
if (await fsExistsAsync(parameterPath)) {
|
|
324
311
|
prevParameters = await fs.promises.readFile(parameterPath, "utf8");
|
|
325
312
|
}
|
|
326
313
|
|
|
327
|
-
// AFTER: RUNNING known
|
|
328
|
-
let
|
|
329
|
-
let
|
|
330
|
-
let futurePid = incoming && await getScreenPanePid(futureScreenName) || undefined;
|
|
331
|
-
let futureUp = !!futurePid && await isScreenRunningProcess(futurePid);
|
|
332
|
-
|
|
333
|
-
// SMOOTH PATH: this is what a deploy looks like from the canonical name's side. The version being replaced is already correct - it keeps running untouched until its overlap ends, so this name is settled and only the future one is left to converge. It needs its record kept straight, and to hear when it goes down.
|
|
334
|
-
// AFTER (canonical, during a release): EXISTS correct VERSION correct RUNNING correct
|
|
335
|
-
if (incoming && current && !canonicalUp) {
|
|
336
|
-
console.log(red(`Nothing is running as ${canonicalScreenName}, which should be serving until ${new Date(killTime).toISOString()} while its replacement starts`));
|
|
337
|
-
}
|
|
338
|
-
if (incoming && current && canonicalUp && canonicalPid) {
|
|
339
|
-
await ensureProcessRecord({
|
|
340
|
-
screenName: canonicalScreenName,
|
|
341
|
-
folder,
|
|
342
|
-
panePid: canonicalPid,
|
|
343
|
-
record: { serviceId, serviceKey: current.key, index, machineId, parameters: current },
|
|
344
|
-
});
|
|
345
|
-
void (async () => {
|
|
346
|
-
let nodeId = await readServiceNodeId(folder);
|
|
347
|
-
if (nodeId) {
|
|
348
|
-
await notifyNodeShutdown(canonicalScreenName, nodeId, killTime);
|
|
349
|
-
}
|
|
350
|
-
})();
|
|
351
|
-
}
|
|
352
|
-
|
|
353
|
-
// From here on this is about ONE screen - the future one during a release, the canonical one otherwise. Everything else is already correct.
|
|
354
|
-
// STATE: EXISTS correct VERSION known RUNNING known
|
|
355
|
-
if (!parameters) return;
|
|
356
|
-
let panePid = incoming && futurePid || canonicalPid;
|
|
357
|
-
let isUp = incoming && futureUp || canonicalUp;
|
|
358
|
-
let aliveWindow: [number, number] = [parameters.releaseTime || 0, ALIVE_WINDOW_FOREVER];
|
|
359
|
-
let record = { serviceId, serviceKey: parameters.key, index, machineId, parameters };
|
|
314
|
+
// AFTER: RUNNING known. A screen can exist with nothing alive in it, and one that should exist can be missing entirely - both read as not up, and both are fixed the same way.
|
|
315
|
+
let panePid = await getScreenPanePid(screenName);
|
|
316
|
+
let isUp = !!panePid && await isScreenRunningProcess(panePid);
|
|
360
317
|
|
|
361
318
|
let newParametersString = JSON.stringify(parameters);
|
|
319
|
+
let record = { serviceId, serviceKey: parameters.key, index, machineId, parameters };
|
|
362
320
|
|
|
363
321
|
let nodePathId = folder + SERVICE_NODE_FILE_NAME;
|
|
364
322
|
if (await fsExistsAsync(nodePathId)) {
|
|
365
323
|
machineInfo.services[serviceId].nodeId = await fs.promises.readFile(nodePathId, "utf8");
|
|
366
324
|
}
|
|
367
325
|
|
|
368
|
-
//
|
|
369
|
-
|
|
370
|
-
|
|
326
|
+
// SMOOTH PATH: what is running is not what should be, so it is renamed aside with its own deadline and keeps serving out its overlap there. The name is free from here, which is what makes a replacement and a first start the same thing.
|
|
327
|
+
if (isUp && panePid && !sameRestartParameters(prevParameters, parameters)) {
|
|
328
|
+
console.log(green(`Replacing what is running as ${magenta(screenName)}: ${newParametersString}`));
|
|
329
|
+
// Recorded against the parameters it was actually started with, not the ones replacing it, so its history says what it really ran
|
|
330
|
+
let outgoingParameters: ServiceParameters | undefined;
|
|
331
|
+
try {
|
|
332
|
+
outgoingParameters = JSON.parse(prevParameters) as ServiceParameters;
|
|
333
|
+
} catch {
|
|
334
|
+
// Never started by us, or the file is unreadable - there is nothing to record it against
|
|
335
|
+
}
|
|
336
|
+
if (outgoingParameters) {
|
|
337
|
+
await ensureProcessRecord({
|
|
338
|
+
screenName,
|
|
339
|
+
folder,
|
|
340
|
+
panePid,
|
|
341
|
+
record: { serviceId, serviceKey: outgoingParameters.key, index, machineId, parameters: outgoingParameters },
|
|
342
|
+
});
|
|
343
|
+
}
|
|
344
|
+
let outgoingNodeId = await readServiceNodeId(folder);
|
|
345
|
+
if (outgoingNodeId) {
|
|
346
|
+
void notifyNodeShutdown(screenName, outgoingNodeId, killTime);
|
|
347
|
+
}
|
|
348
|
+
await retireCurrentScreen({ canonicalScreenName: screenName, folder, killTime, nodeId: outgoingNodeId });
|
|
349
|
+
isUp = false;
|
|
350
|
+
panePid = undefined;
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
// A screen with nothing alive in it is just debris in the way of starting one
|
|
354
|
+
if (!isUp && panePid) {
|
|
355
|
+
await killScreenNow(screenName, "the screen is there but nothing is running in it");
|
|
356
|
+
panePid = undefined;
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
// AFTER: EXISTS correct VERSION correct RUNNING correct - nothing to start
|
|
360
|
+
if (isUp && panePid) {
|
|
371
361
|
if (prevParameters !== newParametersString) {
|
|
372
362
|
await fs.promises.writeFile(parameterPath, newParametersString);
|
|
373
363
|
}
|
|
374
364
|
// We may not have started this one (we restarted, or it predates process records), so make sure it is recorded before leaving it alone
|
|
375
365
|
await ensureProcessRecord({ screenName, folder, panePid, record });
|
|
376
|
-
await syncParametersTimelineFiles(folder, [{ pid: parseInt(panePid) || undefined, aliveWindow, parameters }]);
|
|
377
|
-
if (takeoverAt !== undefined) {
|
|
378
|
-
if (!canonicalUp) {
|
|
379
|
-
// The version this one is waiting to replace is gone, so there is no overlap left to serve - take the name now instead of sitting under the future one
|
|
380
|
-
console.log(red(`Nothing is running as ${canonicalScreenName}, so ${screenName} takes it over now instead of waiting until ${new Date(takeoverAt).toISOString()}`));
|
|
381
|
-
await takeoverScreenLocked({ canonicalScreenName, folder });
|
|
382
|
-
} else {
|
|
383
|
-
// SMOOTH PATH: both versions are up, the old one serving and the new one waiting. A restart loses the timer that retires the old one, so it is put back every time we see it still waiting.
|
|
384
|
-
scheduleTakeover({ canonicalScreenName, folder, killTime: takeoverAt });
|
|
385
|
-
}
|
|
386
|
-
}
|
|
366
|
+
await syncParametersTimelineFiles(folder, [{ pid: parseInt(panePid) || undefined, aliveWindow: [parameters.releaseTime || 0, ALIVE_WINDOW_FOREVER], parameters }]);
|
|
387
367
|
console.log(green(`Verified ${magenta(screenName)} is running`));
|
|
388
368
|
return;
|
|
389
369
|
}
|
|
390
370
|
|
|
391
|
-
//
|
|
392
|
-
|
|
393
|
-
// Nothing should ever stop on its own: either it crashed, or it never came up
|
|
394
|
-
console.log(red(`Nothing is running as ${screenName}, starting it: ${newParametersString}`));
|
|
395
|
-
} else {
|
|
396
|
-
// SMOOTH PATH: the previous version is up and this replaces it with a new one, which is exactly what a deploy is
|
|
397
|
-
console.log(green(`Starting a new version as ${magenta(screenName)}, replacing the one running there: ${newParametersString}`));
|
|
398
|
-
}
|
|
371
|
+
// The name is free, so a first start and a replacement are the same from here
|
|
372
|
+
console.log(red(`Nothing is running as ${screenName}, starting it: ${newParametersString}`));
|
|
399
373
|
|
|
400
374
|
// Only now does the folder have to hold this version, which is why the git and yarn work sits behind the check above rather than running on every resync
|
|
401
375
|
let gitFolder = folder + "git/";
|
|
@@ -419,42 +393,24 @@ async function ensureScreen(config: {
|
|
|
419
393
|
void recordLaunch({
|
|
420
394
|
serviceId,
|
|
421
395
|
serviceKey: parameters.key,
|
|
422
|
-
screenName
|
|
396
|
+
screenName,
|
|
423
397
|
machineId,
|
|
424
|
-
reason:
|
|
398
|
+
reason: prevParameters && "update" || "crashed",
|
|
425
399
|
time: Date.now(),
|
|
426
400
|
});
|
|
427
401
|
|
|
428
|
-
|
|
429
|
-
let replacedNodeId = await readServiceNodeId(folder);
|
|
430
|
-
|
|
431
|
-
// Always started under the future name and renamed over the canonical one, releases and plain restarts alike: it is up before anything touches what it replaces, and it always gets a brand new pane - which is what makes it a distinct process with its own log.
|
|
432
|
-
let launchScreenName = futureScreenName;
|
|
433
|
-
let launchedPid = await runScreenCommand({
|
|
434
|
-
screenName: launchScreenName,
|
|
435
|
-
folder,
|
|
436
|
-
command: parameters.command,
|
|
437
|
-
record,
|
|
438
|
-
});
|
|
439
|
-
// SMOOTH PATH: the command is running as of the line above, so the overlap the version it replaces gets is measured from here
|
|
440
|
-
if (takeoverAt !== undefined) {
|
|
441
|
-
scheduleTakeover({ canonicalScreenName, folder, killTime: takeoverAt, replacedNodeId });
|
|
442
|
-
}
|
|
402
|
+
let startedPid = await runScreenCommand({ screenName, folder, command: parameters.command, record });
|
|
443
403
|
|
|
444
404
|
await delay(LAUNCH_VERIFY_DELAY);
|
|
445
|
-
let
|
|
446
|
-
if (!
|
|
405
|
+
let startedPanePid = await getScreenPanePid(screenName);
|
|
406
|
+
if (!startedPanePid || !await isScreenRunningProcess(startedPanePid)) {
|
|
447
407
|
let prefix = getTmuxPrefix();
|
|
448
|
-
let logs = await runPromise(`${prefix}tmux capture-pane -t ${
|
|
449
|
-
throw new Error(`${logs}\n\nService ${magenta(
|
|
450
|
-
}
|
|
451
|
-
console.log(green(`${magenta(launchScreenName)} is verified to be running (pid ${launchedPid})`));
|
|
452
|
-
// Take the canonical name now unless something is still serving under it - waiting out an overlap that nothing is serving would leave the name empty for no reason. SMOOTH PATH during a release is neither: the version being replaced is still up, so this one waits under the future name until its timer fires.
|
|
453
|
-
// AFTER: EXISTS correct VERSION correct RUNNING correct, under the canonical name, or under the future one with the version it is replacing still serving
|
|
454
|
-
if (takeoverAt === undefined || !canonicalUp) {
|
|
455
|
-
await takeoverScreenLocked({ canonicalScreenName, folder, replacedNodeId });
|
|
408
|
+
let logs = await runPromise(`${prefix}tmux capture-pane -t ${screenName} -p`);
|
|
409
|
+
throw new Error(`${logs}\n\nService ${magenta(screenName)} is not running after starting. Trying again in ${formatTime(MACHINE_RESYNC_INTERVAL)}, or on next change. Last logs above.`);
|
|
456
410
|
}
|
|
457
|
-
|
|
411
|
+
// AFTER: EXISTS correct VERSION correct RUNNING correct
|
|
412
|
+
console.log(green(`${magenta(screenName)} is verified to be running (pid ${startedPid})`));
|
|
413
|
+
await syncParametersTimelineFiles(folder, [{ pid: parseInt(startedPanePid) || undefined, aliveWindow: [parameters.releaseTime || 0, ALIVE_WINDOW_FOREVER], parameters }]);
|
|
458
414
|
|
|
459
415
|
launchCount++;
|
|
460
416
|
lastLaunchedTime = Date.now();
|
|
@@ -463,7 +419,7 @@ async function ensureScreen(config: {
|
|
|
463
419
|
// Counted on failure too, as a service that keeps crashing is only visible as a launch count climbing faster than everyone else's
|
|
464
420
|
launchCount++;
|
|
465
421
|
lastLaunchedTime = Date.now();
|
|
466
|
-
console.error(`Error resyncing ${magenta(
|
|
422
|
+
console.error(`Error resyncing ${magenta(screenName)} (${serviceId}): ${e.stack}`);
|
|
467
423
|
machineInfo.services[serviceId].errorFromLastRun = e.stack;
|
|
468
424
|
}
|
|
469
425
|
machineInfo.services[serviceId].totalTimesLaunched = launchCount;
|
|
@@ -516,8 +472,11 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
|
|
|
516
472
|
// Anything no instance claimed is not supposed to exist
|
|
517
473
|
for (let { screenName } of screenState) {
|
|
518
474
|
if (usedScreenNames.has(screenName)) continue;
|
|
519
|
-
//
|
|
520
|
-
if (
|
|
475
|
+
// A retiring screen is not unclaimed - it is a version that was replaced, and its name says when it is due to go. Cached per name, so seeing it on every resync only ever starts one, and a restart picks it up again from the name alone.
|
|
476
|
+
if (getRetiringKillTime(screenName) !== undefined) {
|
|
477
|
+
void retireScreen(screenName);
|
|
478
|
+
continue;
|
|
479
|
+
}
|
|
521
480
|
// Detached, so a screen that takes its full grace period to go down doesn't hold up the end of the resync
|
|
522
481
|
void (async () => {
|
|
523
482
|
// Nothing is replacing this one, so its registration and the file it wrote both go
|
|
@@ -25,17 +25,35 @@ const SCREEN_SUFFIX = "-dply";
|
|
|
25
25
|
export function getScreenName(config: { serviceKey: string; index: number }): string {
|
|
26
26
|
return `${config.serviceKey}-${config.index}${SCREEN_SUFFIX}`.replace(/[^a-zA-Z0-9\-_]/g, "_");
|
|
27
27
|
}
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
return canonicalScreenName.slice(0, -SCREEN_SUFFIX.length) + RETIRING_MARKER + at + SCREEN_SUFFIX;
|
|
28
|
+
const RETIRING_MARKER = "-retiring-";
|
|
29
|
+
/** A version being replaced is renamed to this, which frees the canonical name for its replacement at once. The name carries WHEN it is to be killed, so nothing has to remember: any pass that sees the screen can read its own deadline off it, and a restart loses nothing. */
|
|
30
|
+
export function getRetiringScreenName(canonicalScreenName: string, killTime: number): string {
|
|
31
|
+
return canonicalScreenName.slice(0, -SCREEN_SUFFIX.length) + RETIRING_MARKER + killTime + SCREEN_SUFFIX;
|
|
33
32
|
}
|
|
34
|
-
|
|
35
|
-
export function
|
|
36
|
-
|
|
33
|
+
/** When a retiring screen is due to be killed, or undefined when it is not a retiring screen at all. */
|
|
34
|
+
export function getRetiringKillTime(screenName: string): number | undefined {
|
|
35
|
+
let marker = screenName.lastIndexOf(RETIRING_MARKER);
|
|
36
|
+
if (marker === -1) return undefined;
|
|
37
|
+
return parseInt(screenName.slice(marker + RETIRING_MARKER.length, -SCREEN_SUFFIX.length)) || undefined;
|
|
37
38
|
}
|
|
38
39
|
|
|
40
|
+
/** Takes a retiring screen down at the time its name says. Cached per screen, so seeing the same screen on every resync only ever starts one of these, and it drops itself from the cache once the screen is gone. */
|
|
41
|
+
export const retireScreen = cache(async (screenName: string): Promise<void> => {
|
|
42
|
+
try {
|
|
43
|
+
let killTime = getRetiringKillTime(screenName);
|
|
44
|
+
if (killTime === undefined) return;
|
|
45
|
+
let dueAt = killTime;
|
|
46
|
+
if (dueAt > Date.now()) {
|
|
47
|
+
console.log(green(`${screenName} is retiring, and is due to be killed at ${new Date(dueAt).toISOString()}`));
|
|
48
|
+
await new Promise<void>(resolve => setPreciseTimeout({ time: dueAt, callback: () => resolve() }));
|
|
49
|
+
}
|
|
50
|
+
await killScreen({ screenName });
|
|
51
|
+
} finally {
|
|
52
|
+
// Dropped either way: a failure should be retried by the next resync rather than remembered as done
|
|
53
|
+
retireScreen.clear(screenName);
|
|
54
|
+
}
|
|
55
|
+
});
|
|
56
|
+
|
|
39
57
|
|
|
40
58
|
// Everything that touches a service's screens - the resync loop, and the timer that retires the old process at its kill time - runs under this, keyed by the canonical screen name. They fire at similar times by design, and a takeover is a rename dance that must not interleave with another one.
|
|
41
59
|
const getScreenLock = cache((canonicalScreenName: string) => runInSerial(async (run: () => Promise<unknown>) => await run()));
|
|
@@ -307,86 +325,30 @@ async function setupPipePane(config: { screenName: string; folder: string; pid:
|
|
|
307
325
|
await runPromise(`${prefix}tmux pipe-pane -t ${config.screenName} 'bash ${pipeScript}'`);
|
|
308
326
|
}
|
|
309
327
|
|
|
310
|
-
/**
|
|
311
|
-
export async function
|
|
312
|
-
canonicalScreenName: string;
|
|
313
|
-
folder: string;
|
|
314
|
-
/** The node the version being replaced registered as, read before the replacement overwrote the folder's file. */
|
|
315
|
-
replacedNodeId?: string;
|
|
316
|
-
}): Promise<boolean> {
|
|
317
|
-
return await withScreenLock(config.canonicalScreenName, () => takeoverScreenLocked(config));
|
|
318
|
-
}
|
|
319
|
-
|
|
320
|
-
/** The takeover itself. Callers already holding this screen's lock use it directly - taking the lock again from inside it would wait on themselves. */
|
|
321
|
-
export async function takeoverScreenLocked(config: {
|
|
328
|
+
/** Moves whatever is running under this name aside so its replacement can have the name at once, and gives it until killTime before it is taken down. Does nothing when the name is free. */
|
|
329
|
+
export async function retireCurrentScreen(config: {
|
|
322
330
|
canonicalScreenName: string;
|
|
323
331
|
folder: string;
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
let
|
|
329
|
-
if (!futurePid) return false;
|
|
330
|
-
let prefix = getTmuxPrefix();
|
|
331
|
-
|
|
332
|
+
killTime: number;
|
|
333
|
+
/** The node it registered as, read before its replacement overwrites the folder's file. */
|
|
334
|
+
nodeId?: string;
|
|
335
|
+
}): Promise<void> {
|
|
336
|
+
let { canonicalScreenName, folder, killTime, nodeId } = config;
|
|
332
337
|
let outgoingPid = await getScreenPanePid(canonicalScreenName);
|
|
333
|
-
if (outgoingPid)
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
void removeServiceNode({ folder, nodeId: replacedNodeId, ownsFile: false });
|
|
341
|
-
}
|
|
342
|
-
retiringScreens.add(retiringScreenName);
|
|
343
|
-
void killScreen({ screenName: retiringScreenName }).finally(() => {
|
|
344
|
-
retiringScreens.delete(retiringScreenName);
|
|
345
|
-
});
|
|
338
|
+
if (!outgoingPid) return;
|
|
339
|
+
let retiringScreenName = getRetiringScreenName(canonicalScreenName, killTime);
|
|
340
|
+
console.log(green(`Retiring ${canonicalScreenName} as ${retiringScreenName}`));
|
|
341
|
+
await runPromise(`${getTmuxPrefix()}tmux rename-session -t ${canonicalScreenName} ${retiringScreenName}`);
|
|
342
|
+
let record = await findProcessRecordByPid(folder, outgoingPid);
|
|
343
|
+
if (record) {
|
|
344
|
+
await writeProcessRecord({ ...record, screenName: retiringScreenName });
|
|
346
345
|
}
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
/** Points a process's record at the session name it now runs under. The pid is the key, so this reads exactly one file. */
|
|
354
|
-
async function renameProcessScreen(config: { folder: string; panePid: string; screenName: string }): Promise<void> {
|
|
355
|
-
let record = await findProcessRecordByPid(config.folder, config.panePid);
|
|
356
|
-
if (!record || record.screenName === config.screenName) return;
|
|
357
|
-
await writeProcessRecord({ ...record, screenName: config.screenName });
|
|
358
|
-
}
|
|
359
|
-
|
|
360
|
-
// canonical screen name -> the takeover we already have scheduled for it
|
|
361
|
-
const scheduledTakeovers = new Map<string, { time: number; cancel: () => void }>();
|
|
362
|
-
|
|
363
|
-
/** Schedules the takeover for the moment the old process's overlap runs out. The resync loop still does the same takeover whenever it notices the time has passed, which is what recovers this if we restart and lose the timer. */
|
|
364
|
-
export function scheduleTakeover(config: {
|
|
365
|
-
canonicalScreenName: string;
|
|
366
|
-
folder: string;
|
|
367
|
-
killTime: number;
|
|
368
|
-
/** The node the version being replaced registered as, read before the replacement overwrote the folder's file. */
|
|
369
|
-
replacedNodeId?: string;
|
|
370
|
-
}): void {
|
|
371
|
-
let { canonicalScreenName, folder, killTime, replacedNodeId } = config;
|
|
372
|
-
let existing = scheduledTakeovers.get(canonicalScreenName);
|
|
373
|
-
if (existing?.time === killTime) return;
|
|
374
|
-
existing?.cancel();
|
|
375
|
-
console.log(green(`Scheduling the takeover of ${canonicalScreenName} for ${new Date(killTime).toISOString()}`));
|
|
376
|
-
let cancel = setPreciseTimeout({
|
|
377
|
-
time: killTime,
|
|
378
|
-
callback: () => {
|
|
379
|
-
scheduledTakeovers.delete(canonicalScreenName);
|
|
380
|
-
void takeoverScreen({ canonicalScreenName, folder, replacedNodeId });
|
|
381
|
-
},
|
|
382
|
-
});
|
|
383
|
-
scheduledTakeovers.set(canonicalScreenName, { time: killTime, cancel });
|
|
384
|
-
}
|
|
385
|
-
|
|
386
|
-
// The screens whose shutdown is already running. Nothing else should start a second one, but a screen left behind by a restart is NOT in here, so the unused-screen sweep still cleans it up.
|
|
387
|
-
const retiringScreens = new Set<string>();
|
|
388
|
-
export function isRetiring(screenName: string): boolean {
|
|
389
|
-
return retiringScreens.has(screenName);
|
|
346
|
+
if (nodeId) {
|
|
347
|
+
// The file in the folder belongs to its replacement now, so only the registration goes
|
|
348
|
+
void removeServiceNode({ folder, nodeId, ownsFile: false });
|
|
349
|
+
}
|
|
350
|
+
// Started here rather than waiting for a resync to notice the screen: with a killTime already past - a zero overlap - this takes it down right now
|
|
351
|
+
void retireScreen(retiringScreenName);
|
|
390
352
|
}
|
|
391
353
|
|
|
392
354
|
/** Ends a session at once, with no notice - for a screen that should not exist at all, so there is nothing to wind down gracefully. Finding one is always a surprise, so the reason it was not supposed to be there is logged. */
|
|
@@ -403,7 +365,7 @@ export const killScreen = measureWrap(async function killScreen(config: {
|
|
|
403
365
|
console.log(red(`Killing screen ${config.screenName} (Ctrl+C, then killing the session in ${formatTime(SHUTDOWN_GRACE_TIME)})`));
|
|
404
366
|
let prefix = getTmuxPrefix();
|
|
405
367
|
// Purely a notice to the process that it is going away, so it can wind down. We never check whether it worked and we never reuse the screen - the session is killed either way.
|
|
406
|
-
|
|
368
|
+
void runPromise(`${prefix}tmux send-keys -t ${config.screenName} 'C-c' Enter`);
|
|
407
369
|
await delay(SHUTDOWN_GRACE_TIME);
|
|
408
370
|
await runPromise(`${prefix}tmux kill-session -t ${config.screenName}`);
|
|
409
371
|
});
|