querysub 0.529.0 → 0.531.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -2
- package/src/-a-archives/archivesBackBlaze.ts +1 -1
- package/src/deployManager/components/MachineDetailPage.tsx +2 -10
- package/src/deployManager/components/MachinesListPage.tsx +2 -11
- package/src/deployManager/components/ServiceDetailPage.tsx +108 -46
- package/src/deployManager/components/ServicesListPage.tsx +13 -1
- package/src/deployManager/components/deployButtons.tsx +52 -11
- package/src/deployManager/machineApplyMainCode.ts +77 -42
- package/src/deployManager/machineSchema.ts +3 -14
- package/src/deployManager/parametersTimeline.ts +62 -0
- package/src/deployManager/parametersTimelineMain.ts +123 -0
- package/src/diagnostics/periodic.ts +2 -2
- package/src/library-components/UsageBar.tsx +58 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "querysub",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.531.0",
|
|
4
4
|
"main": "index.js",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
|
|
@@ -25,6 +25,7 @@
|
|
|
25
25
|
"mcp2": "yarn typenode ./src/diagnostics/debugger/mcp-server.ts",
|
|
26
26
|
"mc2": "yarn typenode ./src/diagnostics/debugger/mcp-server.ts --cwd D:/repos/qs-cyoa/",
|
|
27
27
|
"autofix": "yarn typenode ./src/diagnostics/logs/errorTickets/autoFixerEntry.ts",
|
|
28
|
+
"parameters-timeline": "yarn typenode ./src/deployManager/parametersTimelineMain.ts",
|
|
28
29
|
"ssh-a-claude": "ssh root@a.querysubtest.com"
|
|
29
30
|
},
|
|
30
31
|
"bin": {
|
|
@@ -70,7 +71,7 @@
|
|
|
70
71
|
"node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
|
|
71
72
|
"pako": "^2.1.0",
|
|
72
73
|
"peggy": "^5.0.6",
|
|
73
|
-
"sliftutils": "^1.7.
|
|
74
|
+
"sliftutils": "^1.7.14",
|
|
74
75
|
"socket-function": "^1.2.21",
|
|
75
76
|
"terser": "^5.31.0",
|
|
76
77
|
"typenode": "^6.6.1",
|
|
@@ -59,7 +59,7 @@ export class ArchivesBackblaze {
|
|
|
59
59
|
public async get2(fileName: string, config?: { range?: { start: number; end: number; } }): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
|
|
60
60
|
return this.archives.get2(fileName, config);
|
|
61
61
|
}
|
|
62
|
-
public async set(fileName: string, data: Buffer, config?: { lastModified?: number }): Promise<
|
|
62
|
+
public async set(fileName: string, data: Buffer, config?: { lastModified?: number }): Promise<string> {
|
|
63
63
|
return this.archives.set(fileName, data, config);
|
|
64
64
|
}
|
|
65
65
|
public async append(fileName: string, data: Buffer): Promise<void> {
|
|
@@ -11,6 +11,7 @@ import { managementPageURL } from "../../diagnostics/managementPages";
|
|
|
11
11
|
import { t } from "../../2-proxy/schema2";
|
|
12
12
|
import { Querysub } from "../../4-querysub/Querysub";
|
|
13
13
|
import { getLogViewerParams } from "../../diagnostics/logs/IndexedLogs/LogViewerParams";
|
|
14
|
+
import { UsageBar, getUsageThresholds } from "../../library-components/UsageBar";
|
|
14
15
|
|
|
15
16
|
export class MachineDetailPage extends qreact.Component {
|
|
16
17
|
render() {
|
|
@@ -119,16 +120,7 @@ export class MachineDetailPage extends qreact.Component {
|
|
|
119
120
|
<ShowMore className={css.whiteSpace("pre-wrap")} maxHeight={80}>
|
|
120
121
|
{(() => {
|
|
121
122
|
if (typeof value === "object") {
|
|
122
|
-
return <
|
|
123
|
-
<div className={
|
|
124
|
-
css.absolute.pos(0, 0)
|
|
125
|
-
.size(`${value.value / value.max * 100}%`, "100%")
|
|
126
|
-
.hsl(0, 0, 70)
|
|
127
|
-
} />
|
|
128
|
-
<div className={css.relative}>
|
|
129
|
-
{value.type} ({formatNumber(value.value)} / {formatNumber(value.max)})
|
|
130
|
-
</div>
|
|
131
|
-
</div>;
|
|
123
|
+
return <UsageBar label={value.type} value={value.value} max={value.max} {...getUsageThresholds(value.type)} />;
|
|
132
124
|
}
|
|
133
125
|
return value;
|
|
134
126
|
})()}
|
|
@@ -11,6 +11,7 @@ import { formatDateJSX } from "../../misc/formatJSX";
|
|
|
11
11
|
import { DeployMachineButtons, RenderGitRefInfo } from "./deployButtons";
|
|
12
12
|
import { isDefined } from "../../misc";
|
|
13
13
|
import { Anchor, createLink } from "../../library-components/ATag";
|
|
14
|
+
import { UsageBar, getUsageThresholds } from "../../library-components/UsageBar";
|
|
14
15
|
|
|
15
16
|
export class MachinesListPage extends qreact.Component {
|
|
16
17
|
state = t.state({
|
|
@@ -182,17 +183,7 @@ export class MachinesListPage extends qreact.Component {
|
|
|
182
183
|
<RenderGitRefInfo gitRef={machineInfo.gitRef} />
|
|
183
184
|
{Object.values(machineInfo.info).map(value => {
|
|
184
185
|
if (typeof value === "string") return undefined;
|
|
185
|
-
return <
|
|
186
|
-
<div className={
|
|
187
|
-
css.absolute.pos(0, 0)
|
|
188
|
-
.size(`${value.value / value.max * 100}%`, "100%")
|
|
189
|
-
.hsl(0, 0, 70)
|
|
190
|
-
} />
|
|
191
|
-
<div className={css.relative}>
|
|
192
|
-
{value.type} ({formatNumber(value.value)} / {formatNumber(value.max)})
|
|
193
|
-
</div>
|
|
194
|
-
</div>;
|
|
195
|
-
|
|
186
|
+
return <UsageBar label={value.type} value={value.value} max={value.max} {...getUsageThresholds(value.type)} />;
|
|
196
187
|
})}
|
|
197
188
|
</div>
|
|
198
189
|
<div className={css.vbox(4).flexGrow(1)}>
|
|
@@ -5,7 +5,7 @@ import { css } from "typesafecss";
|
|
|
5
5
|
import { t } from "../../2-proxy/schema2";
|
|
6
6
|
import { Querysub } from "../../4-querysub/Querysub";
|
|
7
7
|
import { currentViewParam, selectedServiceIdParam, selectedMachineIdParam } from "../urlParams";
|
|
8
|
-
import { formatDateTimeDetailed, formatTime, formatVeryNiceDateTime } from "socket-function/src/formatting/format";
|
|
8
|
+
import { formatDateTime, formatDateTimeDetailed, formatTime, formatVeryNiceDateTime } from "socket-function/src/formatting/format";
|
|
9
9
|
import { InputPicker } from "../../library-components/InputPicker";
|
|
10
10
|
import { MachinePicker } from "./MachinePicker";
|
|
11
11
|
import { deepCloneJSON, nextId, sort, timeInSecond } from "socket-function/src/misc";
|
|
@@ -46,7 +46,9 @@ export class ServiceDetailPage extends qreact.Component {
|
|
|
46
46
|
}),
|
|
47
47
|
// Milliseconds; 0 means no scheduled time picked yet (use Deploy Now instead)
|
|
48
48
|
switchTime: t.number(0),
|
|
49
|
-
// Seconds; 0 means use the
|
|
49
|
+
// Seconds until the release goes live; 0 means use the default (DEFAULT_OVERLAP_TIME)
|
|
50
|
+
releaseDelayOverride: t.number(0),
|
|
51
|
+
// Seconds the old instances outlive the release; 0 means use the default (DEFAULT_OVERLAP_TIME). The no-overlap deploy buttons are how you deploy with an actual 0.
|
|
50
52
|
overlapOverride: t.number(0),
|
|
51
53
|
});
|
|
52
54
|
|
|
@@ -123,6 +125,7 @@ export class ServiceDetailPage extends qreact.Component {
|
|
|
123
125
|
Querysub.commit(() => {
|
|
124
126
|
this.state.editorState = stripReleaseState(deployConfig);
|
|
125
127
|
this.state.switchTime = 0;
|
|
128
|
+
this.state.releaseDelayOverride = 0;
|
|
126
129
|
this.state.overlapOverride = 0;
|
|
127
130
|
});
|
|
128
131
|
} finally {
|
|
@@ -151,11 +154,13 @@ export class ServiceDetailPage extends qreact.Component {
|
|
|
151
154
|
content: <div className={css.fixed.pos(0, 0).size("100vw", "100vh").hsla(0, 0, 0, 0.5).display("flex").alignItems("center").justifyContent("center")}>
|
|
152
155
|
<div className={css.vbox(12).pad2(20).hsl(0, 0, 12).colorhsl(0, 0, 90).bord2(0, 0, 40) + " keepModalsOpen"}>
|
|
153
156
|
<div className={css.boldStyle.fontSize(16)}>⚡ Force deploy now?</div>
|
|
154
|
-
<div>
|
|
157
|
+
<div>{deployConfig.parameters.overlapTime
|
|
158
|
+
? `This config becomes live immediately. The old instances keep running for ${formatTime(deployConfig.parameters.overlapTime)} before shutting down.`
|
|
159
|
+
: "This config becomes live immediately. The old instances are shut down right away, with no overlap."}</div>
|
|
155
160
|
<div className={css.hbox(10)}>
|
|
156
161
|
<Button hue={0} onClick={() => {
|
|
157
162
|
modal && modal.close();
|
|
158
|
-
deployConfig.parameters.releaseTime =
|
|
163
|
+
deployConfig.parameters.releaseTime = Date.now();
|
|
159
164
|
this.deployConfig(deployConfig);
|
|
160
165
|
}}>⚡ Force Deploy Now</Button>
|
|
161
166
|
<Button onClick={() => modal && modal.close()}>Cancel</Button>
|
|
@@ -251,7 +256,18 @@ export class ServiceDetailPage extends qreact.Component {
|
|
|
251
256
|
if (!releaseTime || Date.now() >= releaseTime || !originalConfig.oldParameters) return undefined;
|
|
252
257
|
return {
|
|
253
258
|
switchTime: releaseTime,
|
|
254
|
-
overlapTime: originalConfig.parameters.overlapTime
|
|
259
|
+
overlapTime: originalConfig.parameters.overlapTime ?? DEFAULT_OVERLAP_TIME,
|
|
260
|
+
};
|
|
261
|
+
})();
|
|
262
|
+
// By our calculation both the old and new parameters' instances are running (the release passed, but the old instances haven't been shut down yet). Purely time-based — we don't verify the instances are actually running.
|
|
263
|
+
const overlappingDeploy = (() => {
|
|
264
|
+
let releaseTime = originalConfig.parameters.releaseTime || 0;
|
|
265
|
+
let overlapTime = originalConfig.parameters.overlapTime ?? DEFAULT_OVERLAP_TIME;
|
|
266
|
+
if (!releaseTime || !originalConfig.oldParameters) return undefined;
|
|
267
|
+
if (Date.now() < releaseTime || Date.now() >= releaseTime + overlapTime) return undefined;
|
|
268
|
+
return {
|
|
269
|
+
oldParameters: originalConfig.oldParameters,
|
|
270
|
+
shutdownTime: releaseTime + overlapTime,
|
|
255
271
|
};
|
|
256
272
|
})();
|
|
257
273
|
|
|
@@ -378,16 +394,15 @@ export class ServiceDetailPage extends qreact.Component {
|
|
|
378
394
|
this.updateEditorState(updated);
|
|
379
395
|
}}
|
|
380
396
|
/>
|
|
381
|
-
<
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
/>
|
|
397
|
+
<div className={css.hbox(12)}>
|
|
398
|
+
{(() => {
|
|
399
|
+
let lastDeployTime = getLiveServiceParameters(originalConfig).releaseTime;
|
|
400
|
+
if (!lastDeployTime || lastDeployTime > now) return undefined;
|
|
401
|
+
return <span>
|
|
402
|
+
Last deploy at {formatDateTime(lastDeployTime)} ({formatTime(now - lastDeployTime)} ago)
|
|
403
|
+
</span>;
|
|
404
|
+
})()}
|
|
405
|
+
</div>
|
|
391
406
|
{/* Machine Status */}
|
|
392
407
|
{config.parameters.deploy && <div className={css.vbox(8).fillWidth}>
|
|
393
408
|
<h3>Deployed Machines ({machineIds.length})</h3>
|
|
@@ -668,7 +683,7 @@ export class ServiceDetailPage extends qreact.Component {
|
|
|
668
683
|
<ParametersDiff base={getLiveServiceParameters(originalConfig)} next={originalConfig.parameters} />
|
|
669
684
|
<div className={css.vbox(4)}>
|
|
670
685
|
<div>
|
|
671
|
-
|
|
686
|
+
Old instances shut down in {formatTime(scheduledDeploy.switchTime + scheduledDeploy.overlapTime - now)} ({formatDateTimeDetailed(scheduledDeploy.switchTime + scheduledDeploy.overlapTime)})
|
|
672
687
|
</div>
|
|
673
688
|
<div className={css.hbox(8)}>
|
|
674
689
|
<button
|
|
@@ -684,10 +699,27 @@ export class ServiceDetailPage extends qreact.Component {
|
|
|
684
699
|
}}>
|
|
685
700
|
Cancel Scheduled Deploy
|
|
686
701
|
</button>
|
|
702
|
+
<button
|
|
703
|
+
className={css.pad2(12, 8).button.bord2(0, 70, 40).hsl(0, 70, 90)}
|
|
704
|
+
disabled={this.state.isDeploying}
|
|
705
|
+
title="Changes the scheduled deploy to go live immediately"
|
|
706
|
+
onClick={() => {
|
|
707
|
+
this.confirmForceDeployNow(deepCloneJSON(stripReleaseState(originalConfig)));
|
|
708
|
+
}}>
|
|
709
|
+
⚡ Force Schedule Now
|
|
710
|
+
</button>
|
|
687
711
|
</div>
|
|
688
712
|
</div>
|
|
689
713
|
</div>}
|
|
690
714
|
|
|
715
|
+
{/* The release went live, but the old instances are still running out their overlap */}
|
|
716
|
+
{overlappingDeploy && <div className={css.vbox(10).pad2(12).fillWidth.bord2(280, 60, 50).hsl(280, 40, 95)}>
|
|
717
|
+
<div className={css.boldStyle.fontSize(15)}>
|
|
718
|
+
🔀 Overlapping configs — the new config is live, and the old instances shut down in {formatTime(overlappingDeploy.shutdownTime - now)} ({formatDateTimeDetailed(overlappingDeploy.shutdownTime)})
|
|
719
|
+
</div>
|
|
720
|
+
<ParametersDiff base={overlappingDeploy.oldParameters} next={originalConfig.parameters} />
|
|
721
|
+
</div>}
|
|
722
|
+
|
|
691
723
|
{/* The EDITOR's changes (browser-only) — deploying them schedules a (new) deploy */}
|
|
692
724
|
{(() => {
|
|
693
725
|
if (!hasUnsavedChanges) return undefined;
|
|
@@ -707,18 +739,19 @@ export class ServiceDetailPage extends qreact.Component {
|
|
|
707
739
|
<span title={`${configCompareKey(config)} !== ${configCompareKey(deployedConfig)}`}>Only the title / notes changed, so this saves immediately</span>
|
|
708
740
|
</div>;
|
|
709
741
|
}
|
|
710
|
-
let
|
|
742
|
+
let releaseDelaySeconds = this.state.releaseDelayOverride || Math.round(DEFAULT_OVERLAP_TIME / timeInSecond);
|
|
743
|
+
let releaseDelay = releaseDelaySeconds * timeInSecond;
|
|
744
|
+
let overlapSeconds = this.state.overlapOverride || Math.round(DEFAULT_OVERLAP_TIME / timeInSecond);
|
|
711
745
|
let overlap = overlapSeconds * timeInSecond;
|
|
712
746
|
let scheduledTime = this.state.switchTime;
|
|
713
|
-
|
|
714
|
-
const makeDeployConfig = (releaseTime: number) => {
|
|
747
|
+
const makeDeployConfig = (releaseTime: number, overlapTime = overlap) => {
|
|
715
748
|
let deployConfig = deepCloneJSON(config);
|
|
716
749
|
deployConfig.parameters.releaseTime = releaseTime;
|
|
717
|
-
deployConfig.parameters.overlapTime =
|
|
750
|
+
deployConfig.parameters.overlapTime = overlapTime;
|
|
718
751
|
return deployConfig;
|
|
719
752
|
};
|
|
720
|
-
// The switch time is inferred from the
|
|
721
|
-
let effectiveSwitchTime = scheduledTime || now +
|
|
753
|
+
// The switch time is inferred from the release delay, unless one is explicitly set
|
|
754
|
+
let effectiveSwitchTime = scheduledTime || now + releaseDelay;
|
|
722
755
|
return <div className={css.vbox(10).pad2(12).fillWidth.bord2(45, 70, 45).hsl(45, 60, 95)}>
|
|
723
756
|
<div className={css.boldStyle.fontSize(15)}>
|
|
724
757
|
✏️ Edited (only in this browser, until deployed)
|
|
@@ -729,10 +762,19 @@ export class ServiceDetailPage extends qreact.Component {
|
|
|
729
762
|
className={css.pad2(12, 8).button.bord2(0, 0, 20).hsl(120, 70, 85).fontWeight("bold")}
|
|
730
763
|
disabled={this.state.isDeploying}
|
|
731
764
|
onClick={() => {
|
|
732
|
-
this.deployConfig(makeDeployConfig(scheduledTime || Date.now() +
|
|
765
|
+
this.deployConfig(makeDeployConfig(scheduledTime || Date.now() + releaseDelay));
|
|
733
766
|
}}>
|
|
734
767
|
{this.state.isDeploying && "⏳ Scheduling..." || scheduledDeploy && "🔁 Replace Scheduled Deploy" || "🚀 Schedule Deploy"}
|
|
735
768
|
</button>
|
|
769
|
+
<button
|
|
770
|
+
className={css.pad2(12, 8).button.bord2(0, 0, 20).hsl(35, 70, 85)}
|
|
771
|
+
disabled={this.state.isDeploying}
|
|
772
|
+
title="Deploys with the configured release delay, but the old instances shut down immediately at release time (overlap of 0)"
|
|
773
|
+
onClick={() => {
|
|
774
|
+
this.deployConfig(makeDeployConfig(scheduledTime || Date.now() + releaseDelay, 0));
|
|
775
|
+
}}>
|
|
776
|
+
Deploy, No Overlap
|
|
777
|
+
</button>
|
|
736
778
|
<div className={css.opacity(scheduledTime && 1 || 0.5)}>
|
|
737
779
|
<InputLabel
|
|
738
780
|
label="Switch at"
|
|
@@ -749,34 +791,48 @@ export class ServiceDetailPage extends qreact.Component {
|
|
|
749
791
|
onClick={() => {
|
|
750
792
|
this.state.switchTime = 0;
|
|
751
793
|
}}>
|
|
752
|
-
↺ Reset to
|
|
794
|
+
↺ Reset to delay-based time
|
|
753
795
|
</button>}
|
|
754
796
|
<div className={css.opacity(scheduledTime && 0.5 || 1)}>
|
|
755
797
|
<InputLabel
|
|
756
|
-
label="
|
|
798
|
+
label="Release in (seconds)"
|
|
757
799
|
integer
|
|
758
|
-
value={
|
|
800
|
+
value={releaseDelaySeconds}
|
|
759
801
|
onChangeValue={value => {
|
|
760
|
-
this.state.
|
|
802
|
+
this.state.releaseDelayOverride = Number(value) || 0;
|
|
761
803
|
}}
|
|
762
804
|
/>
|
|
763
805
|
</div>
|
|
806
|
+
<InputLabel
|
|
807
|
+
label="Overlap (seconds)"
|
|
808
|
+
integer
|
|
809
|
+
value={overlapSeconds}
|
|
810
|
+
onChangeValue={value => {
|
|
811
|
+
this.state.overlapOverride = Number(value) || 0;
|
|
812
|
+
}}
|
|
813
|
+
/>
|
|
764
814
|
</div>
|
|
765
815
|
{!!scheduledTime && scheduledTime <= now && <span className={css.colorhsl(0, 85, 45).fontWeight("bold")}>
|
|
766
816
|
⚠️ The switch time is in the past — deploying switches immediately
|
|
767
817
|
</span>}
|
|
768
|
-
{notEnoughTime && <span className={css.colorhsl(35, 85, 40).fontWeight("bold")}>
|
|
769
|
-
⚠️ The switch time is sooner than the overlap ({formatTime(overlap)}) — the new instances won't be fully started before the old ones shut down
|
|
770
|
-
</span>}
|
|
771
818
|
<div className={css.hbox(8)}>
|
|
772
819
|
<button
|
|
773
820
|
className={css.pad2(12, 8).button.bord2(0, 70, 40).hsl(0, 70, 90)}
|
|
774
821
|
disabled={this.state.isDeploying}
|
|
775
822
|
onClick={() => {
|
|
776
|
-
this.confirmForceDeployNow(makeDeployConfig(
|
|
823
|
+
this.confirmForceDeployNow(makeDeployConfig(Date.now()));
|
|
777
824
|
}}>
|
|
778
825
|
⚡ Force Deploy Now
|
|
779
826
|
</button>
|
|
827
|
+
<button
|
|
828
|
+
className={css.pad2(12, 8).button.bord2(0, 90, 30).hsl(0, 85, 65).colorhsl(0, 0, 100).fontWeight("bold")}
|
|
829
|
+
disabled={this.state.isDeploying}
|
|
830
|
+
title="Deploys immediately AND shuts down the old instances immediately (overlap of 0)"
|
|
831
|
+
onClick={() => {
|
|
832
|
+
this.confirmForceDeployNow(makeDeployConfig(Date.now(), 0));
|
|
833
|
+
}}>
|
|
834
|
+
⚡ Force Deploy Now, No Overlap
|
|
835
|
+
</button>
|
|
780
836
|
</div>
|
|
781
837
|
</div>;
|
|
782
838
|
})()}
|
|
@@ -790,21 +846,27 @@ export class ServiceDetailPage extends qreact.Component {
|
|
|
790
846
|
)}
|
|
791
847
|
|
|
792
848
|
{/* Config Editor */}
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
849
|
+
{(() => {
|
|
850
|
+
const discardButton = hasUnsavedChanges && <button
|
|
851
|
+
className={css.pad2(12, 8).button.bord2(0, 0, 20).hsl(50, 80, 50)}
|
|
852
|
+
onClick={() => {
|
|
853
|
+
this.state.editorState = undefined;
|
|
854
|
+
}}>
|
|
855
|
+
Discard Changes
|
|
856
|
+
</button>;
|
|
857
|
+
return <>
|
|
858
|
+
{discardButton}
|
|
859
|
+
<TypedConfigEditor
|
|
860
|
+
value={config}
|
|
861
|
+
onValueChange={(newValue) => {
|
|
862
|
+
this.updateEditorState(newValue as ServiceConfig);
|
|
863
|
+
}}
|
|
864
|
+
typeDefinition={serviceConfigType}
|
|
865
|
+
sizeClassName={css.size(1000, 600)}
|
|
866
|
+
/>
|
|
867
|
+
{discardButton}
|
|
868
|
+
</>;
|
|
869
|
+
})()}
|
|
808
870
|
</div>;
|
|
809
871
|
}
|
|
810
872
|
}
|
|
@@ -141,7 +141,19 @@ export class ServicesListPage extends qreact.Component {
|
|
|
141
141
|
</div>
|
|
142
142
|
<div className={css.vbox(4)}>
|
|
143
143
|
<div>Updated {formatDateJSX(config.info.lastUpdatedTime)} AGO</div>
|
|
144
|
-
{
|
|
144
|
+
{(() => {
|
|
145
|
+
// The old instances probably still run out their overlap (purely time-based, we don't verify they're actually running)
|
|
146
|
+
let releaseTime = config.parameters.releaseTime || 0;
|
|
147
|
+
let overlapTime = config.parameters.overlapTime ?? DEFAULT_OVERLAP_TIME;
|
|
148
|
+
if (!releaseTime || !config.oldParameters) return undefined;
|
|
149
|
+
if (now < releaseTime || now >= releaseTime + overlapTime) return undefined;
|
|
150
|
+
return <div
|
|
151
|
+
className={css.colorhsl(280, 60, 45).fontSize(14).fontWeight("bold")}
|
|
152
|
+
title={`Old instances shut down at ${formatDateTimeDetailed(releaseTime + overlapTime)}`}
|
|
153
|
+
>
|
|
154
|
+
🔀 Old version still running for {formatTime(releaseTime + overlapTime - now)}
|
|
155
|
+
</div>;
|
|
156
|
+
})()}
|
|
145
157
|
{(() => {
|
|
146
158
|
let releaseTime = config.parameters.releaseTime || 0;
|
|
147
159
|
if (!releaseTime || releaseTime <= now || !config.oldParameters) return undefined;
|
|
@@ -46,18 +46,21 @@ export class UpdateButtons extends qreact.Component<{
|
|
|
46
46
|
state = t.state({
|
|
47
47
|
isDeploying: t.type(false),
|
|
48
48
|
});
|
|
49
|
-
// Deploys all outdated services to latestRef.
|
|
50
|
-
private deployAll(outdatedServices: ServiceConfig[], latestRef: string,
|
|
49
|
+
// Deploys all outdated services to latestRef. "scheduled" delays the release by the configured overlap (an unrelated value, but a reasonable delay); "now" releases immediately but the old instances still run out their configured overlap; "nowNoOverlap" releases immediately AND shuts the old instances down immediately.
|
|
50
|
+
private deployAll(outdatedServices: ServiceConfig[], latestRef: string, mode: "scheduled" | "now" | "nowNoOverlap") {
|
|
51
51
|
this.state.isDeploying = true;
|
|
52
52
|
// Props are synchronized state, so everything the async work needs is cloned into plain locals HERE, in the synced part
|
|
53
53
|
let toDeploy = outdatedServices.map(service => {
|
|
54
54
|
let updated = deepCloneJSON(service);
|
|
55
55
|
updated.parameters.gitRef = latestRef;
|
|
56
|
-
if (
|
|
57
|
-
updated.parameters.
|
|
58
|
-
} else {
|
|
59
|
-
let overlap = updated.parameters.overlapTime || DEFAULT_OVERLAP_TIME;
|
|
56
|
+
if (mode === "scheduled") {
|
|
57
|
+
let overlap = updated.parameters.overlapTime ?? DEFAULT_OVERLAP_TIME;
|
|
60
58
|
updated.parameters.releaseTime = Date.now() + overlap;
|
|
59
|
+
} else {
|
|
60
|
+
updated.parameters.releaseTime = Date.now();
|
|
61
|
+
if (mode === "nowNoOverlap") {
|
|
62
|
+
updated.parameters.overlapTime = 0;
|
|
63
|
+
}
|
|
61
64
|
}
|
|
62
65
|
return updated;
|
|
63
66
|
});
|
|
@@ -72,6 +75,32 @@ export class UpdateButtons extends qreact.Component<{
|
|
|
72
75
|
}
|
|
73
76
|
});
|
|
74
77
|
}
|
|
78
|
+
private confirmDeployAllNow(outdatedServices: ServiceConfig[], latestRef: string, noOverlap: boolean) {
|
|
79
|
+
let modal: { close: () => void } | undefined;
|
|
80
|
+
modal = showFullscreenModal({
|
|
81
|
+
content: <div className={css.vbox(12).pad2(20)}>
|
|
82
|
+
<div className={css.boldStyle.fontSize(16)}>⚡ Deploy all {outdatedServices.length} services now?</div>
|
|
83
|
+
<div>{noOverlap
|
|
84
|
+
? "All services release immediately AND their old instances shut down immediately (overlap of 0)."
|
|
85
|
+
: "All services release immediately. Their old instances keep running for their configured overlap before shutting down."}</div>
|
|
86
|
+
<div className={css.hbox(10)}>
|
|
87
|
+
<button
|
|
88
|
+
className={css.pad2(12, 8).button.bord2(0, 70, 40).hsl(0, 70, 90)}
|
|
89
|
+
onClick={() => {
|
|
90
|
+
modal && modal.close();
|
|
91
|
+
this.deployAll(outdatedServices, latestRef, noOverlap ? "nowNoOverlap" : "now");
|
|
92
|
+
}}>
|
|
93
|
+
⚡ Deploy Now
|
|
94
|
+
</button>
|
|
95
|
+
<button
|
|
96
|
+
className={css.pad2(12, 8).button.bord2(0, 0, 20).hsl(0, 0, 100)}
|
|
97
|
+
onClick={() => modal && modal.close()}>
|
|
98
|
+
Cancel
|
|
99
|
+
</button>
|
|
100
|
+
</div>
|
|
101
|
+
</div>
|
|
102
|
+
});
|
|
103
|
+
}
|
|
75
104
|
render() {
|
|
76
105
|
let controller = MachineServiceController(SocketFunction.browserNodeId());
|
|
77
106
|
const gitInfo = controller.getGitInfo();
|
|
@@ -129,7 +158,7 @@ export class UpdateButtons extends qreact.Component<{
|
|
|
129
158
|
className={buttonStyle.hsl(120, 70, 90)}
|
|
130
159
|
disabled={this.state.isDeploying}
|
|
131
160
|
onClick={() => {
|
|
132
|
-
this.deployAll(outdatedServices, gitInfo.latestRef,
|
|
161
|
+
this.deployAll(outdatedServices, gitInfo.latestRef, "scheduled");
|
|
133
162
|
}}
|
|
134
163
|
>
|
|
135
164
|
<div>
|
|
@@ -147,15 +176,27 @@ export class UpdateButtons extends qreact.Component<{
|
|
|
147
176
|
<button
|
|
148
177
|
className={buttonStyle.hsl(0, 70, 90)}
|
|
149
178
|
disabled={this.state.isDeploying}
|
|
150
|
-
title="Deploys immediately
|
|
179
|
+
title="Deploys immediately; the old instances keep running for their configured overlap before shutting down"
|
|
151
180
|
onClick={() => {
|
|
152
|
-
this.
|
|
181
|
+
this.confirmDeployAllNow(outdatedServices, gitInfo.latestRef, false);
|
|
153
182
|
}}
|
|
154
183
|
>
|
|
155
184
|
<div>
|
|
156
185
|
{bigEmoji("⚡")} <span>{this.state.isDeploying && "⏳ Deploying..." || `Deploy All (${outdatedServices.length}) Now (no delay)`}</span>
|
|
157
186
|
</div>
|
|
158
187
|
</button>
|
|
188
|
+
<button
|
|
189
|
+
className={buttonStyle.hsl(0, 85, 70)}
|
|
190
|
+
disabled={this.state.isDeploying}
|
|
191
|
+
title="Deploys immediately AND shuts down the old instances immediately (overlap of 0)"
|
|
192
|
+
onClick={() => {
|
|
193
|
+
this.confirmDeployAllNow(outdatedServices, gitInfo.latestRef, true);
|
|
194
|
+
}}
|
|
195
|
+
>
|
|
196
|
+
<div>
|
|
197
|
+
{bigEmoji("⚡")} <span>{this.state.isDeploying && "⏳ Deploying..." || `Deploy All (${outdatedServices.length}) Now (no delay, no overlap)`}</span>
|
|
198
|
+
</div>
|
|
199
|
+
</button>
|
|
159
200
|
</>}
|
|
160
201
|
</>;
|
|
161
202
|
}
|
|
@@ -183,8 +224,8 @@ export class UpdateServiceButtons extends qreact.Component<{
|
|
|
183
224
|
// Props are synchronized state, so everything the async work needs is cloned into plain locals HERE, in the synced part
|
|
184
225
|
let service = deepCloneJSON(this.props.service);
|
|
185
226
|
service.parameters.gitRef = gitInfo.latestRef;
|
|
186
|
-
//
|
|
187
|
-
let overlap = service.parameters.overlapTime
|
|
227
|
+
// Delay the release by the configured overlap (an unrelated value, but a reasonable delay)
|
|
228
|
+
let overlap = service.parameters.overlapTime ?? DEFAULT_OVERLAP_TIME;
|
|
188
229
|
service.parameters.releaseTime = Date.now() + overlap;
|
|
189
230
|
Querysub.onCommitFinished(async () => {
|
|
190
231
|
try {
|
|
@@ -25,6 +25,7 @@ import { onServiceConfigChange, scheduleNodeShutdown } from "./machineController
|
|
|
25
25
|
import { PromiseObj } from "../promise";
|
|
26
26
|
import path from "path";
|
|
27
27
|
import { fsExistsAsync } from "../fs";
|
|
28
|
+
import { ALIVE_WINDOW_FOREVER, ParametersTimelineEntry, syncParametersTimelineFiles } from "./parametersTimeline";
|
|
28
29
|
|
|
29
30
|
// The deployed-service stdout log (pipe.txt) is a rolling buffer: once it grows
|
|
30
31
|
// to PIPE_FILE_LINE_LIMIT lines it is truncated down to PIPE_FILE_LINE_KEEP. The
|
|
@@ -132,9 +133,9 @@ const SCREEN_SUFFIX = "-dply";
|
|
|
132
133
|
export function getScreenName(config: { serviceKey: string; index: number }): string {
|
|
133
134
|
return `${config.serviceKey}-${config.index}${SCREEN_SUFFIX}`.replace(/[^a-zA-Z0-9\-_]/g, "_");
|
|
134
135
|
}
|
|
135
|
-
// The
|
|
136
|
-
export function
|
|
137
|
-
return getScreenName({ serviceKey: config.serviceKey + "-
|
|
136
|
+
// The old instances kept alive during a release overlap. The canonical screens restart with the new config at release time; these keep the OLD version serving until releaseTime + overlapTime, and are killed shortly after.
|
|
137
|
+
export function getOldScreenName(config: { serviceKey: string; index: number }): string {
|
|
138
|
+
return getScreenName({ serviceKey: config.serviceKey + "-old", index: config.index });
|
|
138
139
|
}
|
|
139
140
|
|
|
140
141
|
export async function streamScreenOutput(config: {
|
|
@@ -585,35 +586,41 @@ const ensureGitSynced = measureWrap(async function ensureGitSynced(config: {
|
|
|
585
586
|
let launchesPerService = new Map<string, number>();
|
|
586
587
|
let lastLaunchedTimePerService = new Map<string, number>();
|
|
587
588
|
|
|
588
|
-
// Starts (or verifies) the "-
|
|
589
|
-
async function
|
|
590
|
-
|
|
589
|
+
// Starts (or verifies) the "-old" instance of a service during a release overlap, running the old (no longer live) parameters. It keeps the old version serving while the new canonical instances start up, and is killed by the unused-screen cleanup afterwards (once releaseTime + overlapTime passes we stop ensuring it).
|
|
590
|
+
async function ensureOldInstance(config: {
|
|
591
|
+
old: ServiceParameters;
|
|
592
|
+
aliveWindow: [number, number];
|
|
591
593
|
index: number;
|
|
592
594
|
screenNamesUsed: Set<string>;
|
|
593
595
|
screenStateMap: Map<string, { screenName: string; isProcessRunning: boolean; pid: string }>;
|
|
594
596
|
}) {
|
|
595
|
-
let {
|
|
596
|
-
let
|
|
597
|
-
config.screenNamesUsed.add(
|
|
597
|
+
let { old, index } = config;
|
|
598
|
+
let oldScreenName = getOldScreenName({ serviceKey: old.key, index });
|
|
599
|
+
config.screenNamesUsed.add(oldScreenName);
|
|
600
|
+
let shutdownTime = config.aliveWindow[1];
|
|
601
|
+
const syncTimeline = async () => {
|
|
602
|
+
let pid = parseInt(config.screenStateMap.get(oldScreenName)?.pid || "") || undefined;
|
|
603
|
+
await syncParametersTimelineFiles(os.homedir() + "/" + SERVICE_FOLDER + oldScreenName + "/", [{ pid, aliveWindow: config.aliveWindow, parameters: old }]);
|
|
604
|
+
};
|
|
598
605
|
|
|
599
606
|
let root = os.homedir() + "/" + SERVICE_FOLDER;
|
|
600
|
-
let folder = root +
|
|
607
|
+
let folder = root + oldScreenName + "/";
|
|
601
608
|
let gitFolder = folder + "git/";
|
|
602
609
|
await fs.promises.mkdir(gitFolder, { recursive: true });
|
|
603
|
-
if (
|
|
610
|
+
if (old.gitRef) {
|
|
604
611
|
let prevGitRef = "";
|
|
605
612
|
try {
|
|
606
613
|
prevGitRef = await getGitRefLive(gitFolder);
|
|
607
614
|
} catch { }
|
|
608
615
|
await ensureGitSynced({
|
|
609
616
|
gitFolder,
|
|
610
|
-
repoUrl:
|
|
611
|
-
gitRef:
|
|
617
|
+
repoUrl: old.repoUrl,
|
|
618
|
+
gitRef: old.gitRef,
|
|
612
619
|
});
|
|
613
620
|
let afterGitRef = await getGitRefLive(gitFolder);
|
|
614
621
|
let nodeModulesMissing = !await fsExistsAsync(gitFolder + "node_modules");
|
|
615
622
|
if (afterGitRef !== prevGitRef || nodeModulesMissing) {
|
|
616
|
-
console.log(green(`Yarn installing for ${magenta(
|
|
623
|
+
console.log(green(`Yarn installing for ${magenta(oldScreenName)} (ref ${prevGitRef} -> ${afterGitRef}, nodeModulesMissing=${nodeModulesMissing})`));
|
|
617
624
|
await runPromise(`yarn install --mutex network`, { cwd: gitFolder });
|
|
618
625
|
}
|
|
619
626
|
}
|
|
@@ -623,23 +630,27 @@ async function ensureNextInstance(config: {
|
|
|
623
630
|
if (await fsExistsAsync(parameterPath)) {
|
|
624
631
|
prevParameters = await fs.promises.readFile(parameterPath, "utf8");
|
|
625
632
|
}
|
|
626
|
-
let newParametersString = JSON.stringify(
|
|
627
|
-
let sameParameters = sameRestartParameters(prevParameters,
|
|
628
|
-
let screenIsRunning = config.screenStateMap.get(
|
|
633
|
+
let newParametersString = JSON.stringify(old);
|
|
634
|
+
let sameParameters = sameRestartParameters(prevParameters, old);
|
|
635
|
+
let screenIsRunning = config.screenStateMap.get(oldScreenName)?.isProcessRunning;
|
|
629
636
|
if (sameParameters && screenIsRunning) {
|
|
630
637
|
if (prevParameters !== newParametersString) {
|
|
631
638
|
await fs.promises.writeFile(parameterPath, newParametersString);
|
|
632
639
|
}
|
|
633
|
-
|
|
640
|
+
await notifyServiceShutdown(oldScreenName, shutdownTime);
|
|
641
|
+
await syncTimeline();
|
|
642
|
+
console.log(green(`Verified release-overlap old instance ${magenta(oldScreenName)} is running`));
|
|
634
643
|
return;
|
|
635
644
|
}
|
|
636
645
|
|
|
637
|
-
console.log(`Starting release-overlap instance ${magenta(
|
|
646
|
+
console.log(`Starting release-overlap old instance ${magenta(oldScreenName)}, with ${newParametersString}, isRunning = ${screenIsRunning}, sameParameters = ${sameParameters}`);
|
|
638
647
|
await fs.promises.writeFile(parameterPath, newParametersString);
|
|
639
648
|
await runScreenCommand({
|
|
640
|
-
screenName:
|
|
641
|
-
command:
|
|
649
|
+
screenName: oldScreenName,
|
|
650
|
+
command: old.command,
|
|
642
651
|
});
|
|
652
|
+
await notifyServiceShutdown(oldScreenName, shutdownTime);
|
|
653
|
+
await syncTimeline();
|
|
643
654
|
}
|
|
644
655
|
|
|
645
656
|
function sameRestartParameters(prevParametersJSON: string, next: ServiceParameters): boolean {
|
|
@@ -656,8 +667,8 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
|
|
|
656
667
|
let machineId = getOwnMachineId(getDomain());
|
|
657
668
|
let allConfigs = await getEffectiveServiceConfigs();
|
|
658
669
|
let relevantConfigs = allConfigs
|
|
659
|
-
.filter(config => getMachineIdList(getLiveServiceParameters(config)).includes(machineId) || getMachineIdList(config.parameters).includes(machineId))
|
|
660
|
-
.filter(x => getLiveServiceParameters(x).deploy || x.parameters.deploy);
|
|
670
|
+
.filter(config => getMachineIdList(getLiveServiceParameters(config)).includes(machineId) || getMachineIdList(config.parameters).includes(machineId) || config.oldParameters && getMachineIdList(config.oldParameters).includes(machineId))
|
|
671
|
+
.filter(x => getLiveServiceParameters(x).deploy || x.parameters.deploy || x.oldParameters?.deploy);
|
|
661
672
|
|
|
662
673
|
let machineInfo = await getLiveMachineInfo();
|
|
663
674
|
|
|
@@ -669,25 +680,29 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
|
|
|
669
680
|
for (let record of relevantConfigs) {
|
|
670
681
|
let config = { ...record, parameters: getLiveServiceParameters(record) };
|
|
671
682
|
let releaseTime = record.parameters.releaseTime || 0;
|
|
683
|
+
// The overlap length always comes from the NEWEST parameters, even though it governs how long the OLD instances outlive the release
|
|
684
|
+
let overlapTime = record.parameters.overlapTime ?? DEFAULT_OVERLAP_TIME;
|
|
672
685
|
let isUpcomingRelease = !!releaseTime && Date.now() < releaseTime && !!record.oldParameters;
|
|
673
|
-
let
|
|
674
|
-
if (isUpcomingRelease) {
|
|
686
|
+
let isOverlapRunning = !!releaseTime && Date.now() >= releaseTime && Date.now() < releaseTime + overlapTime && !!record.oldParameters;
|
|
687
|
+
if (isUpcomingRelease || isOverlapRunning) {
|
|
675
688
|
upcomingReleases.push({ releaseTime, overlapTime });
|
|
676
|
-
console.log(magenta(`Release in flight for ${record.serviceId} (${record.parameters.key}): new instances start at ${new Date(releaseTime
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
}
|
|
689
|
+
console.log(magenta(`Release in flight for ${record.serviceId} (${record.parameters.key}): new instances start at ${new Date(releaseTime).toLocaleString()}${isOverlapRunning && " (overlap running)" || ""}, old instances shut down at ${new Date(releaseTime + overlapTime).toLocaleString()}`));
|
|
690
|
+
}
|
|
691
|
+
// The old-overlap instances follow the OLD parameters' machineIds, so machines being removed by the release still keep the old version alive through the overlap (their canonical screens die at release time)
|
|
692
|
+
if (isOverlapRunning && record.oldParameters?.deploy) {
|
|
693
|
+
let oldParameters = record.oldParameters;
|
|
694
|
+
let oldTargets = getMachineTargets(oldParameters).filter(target => target.machineId === machineId);
|
|
695
|
+
for (let i = 0; i < oldTargets.length; i++) {
|
|
696
|
+
try {
|
|
697
|
+
await ensureOldInstance({
|
|
698
|
+
old: { ...oldParameters, command: applyCommandTemplate(oldParameters.command, oldTargets[i].variables) },
|
|
699
|
+
aliveWindow: [releaseTime, releaseTime + overlapTime],
|
|
700
|
+
index: i,
|
|
701
|
+
screenNamesUsed,
|
|
702
|
+
screenStateMap,
|
|
703
|
+
});
|
|
704
|
+
} catch (e: any) {
|
|
705
|
+
console.error(`Error starting release-overlap old instance for ${record.serviceId} (${record.parameters.key}): ${e.stack}`);
|
|
691
706
|
}
|
|
692
707
|
}
|
|
693
708
|
}
|
|
@@ -748,6 +763,24 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
|
|
|
748
763
|
let sameParameters = sameRestartParameters(prevParameters, instanceParameters);
|
|
749
764
|
let screenIsRunning = screenStateMap.get(screenName)?.isProcessRunning;
|
|
750
765
|
|
|
766
|
+
const syncTimeline = async (pid: string | undefined) => {
|
|
767
|
+
let entries: ParametersTimelineEntry[] = [{
|
|
768
|
+
pid: parseInt(pid || "") || undefined,
|
|
769
|
+
aliveWindow: [config.parameters.releaseTime || 0, isUpcomingRelease ? releaseTime : ALIVE_WINDOW_FOREVER],
|
|
770
|
+
parameters: instanceParameters,
|
|
771
|
+
}];
|
|
772
|
+
if (isUpcomingRelease) {
|
|
773
|
+
let futureTargets = getMachineTargets(record.parameters).filter(target => target.machineId === machineId);
|
|
774
|
+
if (i < futureTargets.length) {
|
|
775
|
+
entries.push({
|
|
776
|
+
aliveWindow: [releaseTime, ALIVE_WINDOW_FOREVER],
|
|
777
|
+
parameters: { ...record.parameters, command: applyCommandTemplate(record.parameters.command, futureTargets[i].variables) },
|
|
778
|
+
});
|
|
779
|
+
}
|
|
780
|
+
}
|
|
781
|
+
await syncParametersTimelineFiles(folder, entries);
|
|
782
|
+
};
|
|
783
|
+
|
|
751
784
|
let nodePathId = folder + SERVICE_NODE_FILE_NAME;
|
|
752
785
|
if (await fsExistsAsync(nodePathId)) {
|
|
753
786
|
let nodeId = await fs.promises.readFile(nodePathId, "utf8");
|
|
@@ -758,6 +791,7 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
|
|
|
758
791
|
if (prevParameters !== newParametersString) {
|
|
759
792
|
await fs.promises.writeFile(parameterPath, newParametersString);
|
|
760
793
|
}
|
|
794
|
+
await syncTimeline(screenStateMap.get(screenName)?.pid);
|
|
761
795
|
console.log(green(`Verified ${magenta(screenName)} is running`));
|
|
762
796
|
continue;
|
|
763
797
|
}
|
|
@@ -798,6 +832,7 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
|
|
|
798
832
|
throw new Error(`${logs}\n\nService ${magenta(screenName)} is not running after starting. Trying again in ${formatTime(MACHINE_RESYNC_INTERVAL)}, or on next change. Last logs above.`);
|
|
799
833
|
}
|
|
800
834
|
console.log(green(`Service ${magenta(screenName)} is verified to be running after starting.`));
|
|
835
|
+
await syncTimeline(foundScreen?.pid);
|
|
801
836
|
|
|
802
837
|
launchCount++;
|
|
803
838
|
lastLaunchedTime = Date.now();
|
|
@@ -838,13 +873,13 @@ async function resyncServices() {
|
|
|
838
873
|
}
|
|
839
874
|
}
|
|
840
875
|
|
|
841
|
-
// The regular resync poll is too coarse for release boundaries (the
|
|
876
|
+
// The regular resync poll is too coarse for release boundaries (the release itself and the overlap end), so schedule an extra resync at the next boundary.
|
|
842
877
|
let cancelReleaseResync: (() => void) | undefined;
|
|
843
878
|
function scheduleReleaseResync(upcomingReleases: { releaseTime: number; overlapTime: number }[]) {
|
|
844
879
|
let now = Date.now();
|
|
845
880
|
let next: number | undefined;
|
|
846
881
|
for (let release of upcomingReleases) {
|
|
847
|
-
for (let boundary of [release.releaseTime
|
|
882
|
+
for (let boundary of [release.releaseTime, release.releaseTime + release.overlapTime]) {
|
|
848
883
|
if (boundary <= now) continue;
|
|
849
884
|
if (next === undefined || boundary < next) {
|
|
850
885
|
next = boundary;
|
|
@@ -89,7 +89,7 @@ export type ServiceParameters = {
|
|
|
89
89
|
/** Not set by default, so we can setup the configuration before deploying it (or so we can undeploy easily without deleting it) */
|
|
90
90
|
deploy?: boolean;
|
|
91
91
|
|
|
92
|
-
/** How long the
|
|
92
|
+
/** How long the old instances keep running after a release. The new instances start at releaseTime, and the old instances are shut down at releaseTime + overlapTime (using the NEWEST parameters' overlapTime). */
|
|
93
93
|
overlapTime?: number;
|
|
94
94
|
|
|
95
95
|
/** When these parameters go live. Until then machines keep running oldParameters. Absent (or in the past) means live now. */
|
|
@@ -378,19 +378,8 @@ export class MachineServiceControllerBase {
|
|
|
378
378
|
}
|
|
379
379
|
normalizeServiceConfig(serviceConfig);
|
|
380
380
|
|
|
381
|
-
//
|
|
382
|
-
|
|
383
|
-
let liveParameters = getLiveServiceParameters(serviceConfig);
|
|
384
|
-
if (JSON.stringify({ ...liveParameters, releaseTime: undefined }) === JSON.stringify({ ...config.parameters, releaseTime: undefined })) {
|
|
385
|
-
// The parameters aren't changing, so there is nothing to release (machineIds / info changes apply immediately, only parameters wait for the release time)
|
|
386
|
-
config.parameters.releaseTime = undefined;
|
|
387
|
-
config.oldParameters = undefined;
|
|
388
|
-
} else {
|
|
389
|
-
config.oldParameters = liveParameters;
|
|
390
|
-
}
|
|
391
|
-
} else {
|
|
392
|
-
config.oldParameters = undefined;
|
|
393
|
-
}
|
|
381
|
+
// NOTE: This is correct. This is obviously correct. For some reason, the AI wrote some really big, complicated thing here. I don't know why. This is the only way it should be done, obviously.
|
|
382
|
+
config.oldParameters = getLiveServiceParameters(serviceConfig);
|
|
394
383
|
|
|
395
384
|
await serviceConfigs.set(serviceId, config);
|
|
396
385
|
// Only notify we were or are deployed. If it's not deployed, this will be ignored anyways.
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import type { ServiceParameters } from "./machineSchema";
|
|
3
|
+
|
|
4
|
+
/*
|
|
5
|
+
Each service instance folder holds one <n>-parameters.json per parameters version that is running or scheduled to run there, written by the machine apply code and cleaned up once a version is neither. A process can find which version it is by matching pid against its own ancestor pids (the recorded pid is the screen's shell, an ancestor of the service process), and read the other files to see what deploys next and when it will be shut down. See parametersTimelineMain.ts for a template reader.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
export const PARAMETERS_TIMELINE_FILE_REGEX = /^(\d+)-parameters\.json$/;
|
|
9
|
+
export const ALIVE_WINDOW_FOREVER = Number.MAX_SAFE_INTEGER;
|
|
10
|
+
|
|
11
|
+
export type ParametersTimelineEntry = {
|
|
12
|
+
/** Absent until the instance has actually been started (future versions have no pid yet). */
|
|
13
|
+
pid?: number;
|
|
14
|
+
/** [start, end] of when these parameters run. end is ALIVE_WINDOW_FOREVER when no shutdown is scheduled. */
|
|
15
|
+
aliveWindow: [number, number];
|
|
16
|
+
parameters: ServiceParameters;
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
/** Makes the folder's <n>-parameters.json files exactly match `entries`: versions keep their file (and number) across updates, matched by their parameters JSON, and files for versions no longer present are deleted. */
|
|
20
|
+
export async function syncParametersTimelineFiles(folder: string, entries: ParametersTimelineEntry[]): Promise<void> {
|
|
21
|
+
let files: string[] = [];
|
|
22
|
+
try {
|
|
23
|
+
files = await fs.promises.readdir(folder);
|
|
24
|
+
} catch {
|
|
25
|
+
return;
|
|
26
|
+
}
|
|
27
|
+
let existing = new Map<string, { n: number; entry: ParametersTimelineEntry }>();
|
|
28
|
+
let timelineFiles: { file: string; n: number }[] = [];
|
|
29
|
+
let maxN = -1;
|
|
30
|
+
for (let file of files) {
|
|
31
|
+
let match = PARAMETERS_TIMELINE_FILE_REGEX.exec(file);
|
|
32
|
+
if (!match) continue;
|
|
33
|
+
let n = parseInt(match[1]);
|
|
34
|
+
timelineFiles.push({ file, n });
|
|
35
|
+
maxN = Math.max(maxN, n);
|
|
36
|
+
try {
|
|
37
|
+
let entry = JSON.parse(await fs.promises.readFile(folder + file, "utf8")) as ParametersTimelineEntry;
|
|
38
|
+
existing.set(JSON.stringify(entry.parameters), { n, entry });
|
|
39
|
+
} catch {
|
|
40
|
+
// Unreadable file stays unmatched, so it is cleaned up below
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
let usedNs = new Set<number>();
|
|
44
|
+
for (let entry of entries) {
|
|
45
|
+
let prev = existing.get(JSON.stringify(entry.parameters));
|
|
46
|
+
let n = prev ? prev.n : ++maxN;
|
|
47
|
+
usedNs.add(n);
|
|
48
|
+
// Keep a previously discovered pid, as the caller's screen state may lag behind a recent start
|
|
49
|
+
let newEntry: ParametersTimelineEntry = { pid: entry.pid ?? prev?.entry.pid, aliveWindow: entry.aliveWindow, parameters: entry.parameters };
|
|
50
|
+
if (!prev || JSON.stringify(prev.entry) !== JSON.stringify(newEntry)) {
|
|
51
|
+
await fs.promises.writeFile(folder + n + "-parameters.json", JSON.stringify(newEntry));
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
for (let { file, n } of timelineFiles) {
|
|
55
|
+
if (usedNs.has(n)) continue;
|
|
56
|
+
try {
|
|
57
|
+
await fs.promises.unlink(folder + file);
|
|
58
|
+
} catch {
|
|
59
|
+
// Concurrent removal is fine
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/*
|
|
2
|
+
Debug utility that shows the deployment timeline for the service instance it runs inside. It is also the TEMPLATE for how services themselves can read the <n>-parameters.json files (written by the machine apply code into each instance folder) to learn which parameters version they are, what deploys next, and when they will be shut down.
|
|
3
|
+
|
|
4
|
+
The discovery steps a service should copy:
|
|
5
|
+
1. Walk up from cwd until a folder contains <n>-parameters.json files (services run in <instance folder>/git/, so the instance folder is an ancestor).
|
|
6
|
+
2. Walk up your own pid chain (ps -o ppid=) — the entry whose pid is one of your ancestors is YOU (the recorded pid is the screen's shell, which spawned your process).
|
|
7
|
+
3. Your entry's aliveWindow[1] is your scheduled shutdown time (ALIVE_WINDOW_FOREVER means none); an entry with a future aliveWindow[0] and no pid is the next deploy.
|
|
8
|
+
|
|
9
|
+
Run from inside a service folder (or as a service command): yarn parameters-timeline
|
|
10
|
+
*/
|
|
11
|
+
import fs from "fs";
|
|
12
|
+
import path from "path";
|
|
13
|
+
import { runPromise } from "../functional/runCommand";
|
|
14
|
+
import { delay } from "socket-function/src/batching";
|
|
15
|
+
import { formatDateTimeDetailed, formatTime } from "socket-function/src/formatting/format";
|
|
16
|
+
import { ALIVE_WINDOW_FOREVER, PARAMETERS_TIMELINE_FILE_REGEX, ParametersTimelineEntry } from "./parametersTimeline";
|
|
17
|
+
|
|
18
|
+
const POLL_INTERVAL = 30_000;
|
|
19
|
+
|
|
20
|
+
async function getAncestorPids(): Promise<Set<number>> {
|
|
21
|
+
let pids = new Set<number>([process.pid]);
|
|
22
|
+
let current = process.pid;
|
|
23
|
+
while (true) {
|
|
24
|
+
let ppid: number;
|
|
25
|
+
if (current === process.pid && process.ppid) {
|
|
26
|
+
ppid = process.ppid;
|
|
27
|
+
} else {
|
|
28
|
+
try {
|
|
29
|
+
ppid = parseInt((await runPromise(`ps -o ppid= -p ${current}`)).trim());
|
|
30
|
+
} catch {
|
|
31
|
+
break;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
if (!ppid || Number.isNaN(ppid) || ppid <= 1 || pids.has(ppid)) break;
|
|
35
|
+
pids.add(ppid);
|
|
36
|
+
current = ppid;
|
|
37
|
+
}
|
|
38
|
+
return pids;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
async function findTimelineFolder(): Promise<string | undefined> {
|
|
42
|
+
let dir = process.cwd();
|
|
43
|
+
while (true) {
|
|
44
|
+
let files: string[] = [];
|
|
45
|
+
try {
|
|
46
|
+
files = await fs.promises.readdir(dir);
|
|
47
|
+
} catch { }
|
|
48
|
+
if (files.some(file => PARAMETERS_TIMELINE_FILE_REGEX.test(file))) {
|
|
49
|
+
return dir.replaceAll("\\", "/") + "/";
|
|
50
|
+
}
|
|
51
|
+
let parent = path.dirname(dir);
|
|
52
|
+
if (parent === dir) return undefined;
|
|
53
|
+
dir = parent;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function formatBoundary(time: number): string {
|
|
58
|
+
if (time === ALIVE_WINDOW_FOREVER) return "∞";
|
|
59
|
+
if (!time) return "start";
|
|
60
|
+
return formatDateTimeDetailed(time);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async function showTimeline() {
|
|
64
|
+
let folder = await findTimelineFolder();
|
|
65
|
+
if (!folder) {
|
|
66
|
+
console.log(`No <n>-parameters.json files found in ${process.cwd()} or any ancestor folder`);
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
let ancestorPids = await getAncestorPids();
|
|
70
|
+
let entries: { n: number; entry: ParametersTimelineEntry }[] = [];
|
|
71
|
+
for (let file of await fs.promises.readdir(folder)) {
|
|
72
|
+
let match = PARAMETERS_TIMELINE_FILE_REGEX.exec(file);
|
|
73
|
+
if (!match) continue;
|
|
74
|
+
try {
|
|
75
|
+
entries.push({ n: parseInt(match[1]), entry: JSON.parse(await fs.promises.readFile(folder + file, "utf8")) as ParametersTimelineEntry });
|
|
76
|
+
} catch (e) {
|
|
77
|
+
console.warn(`Could not read ${folder + file}: ${(e as Error).stack ?? e}`);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
entries.sort((a, b) => a.entry.aliveWindow[0] - b.entry.aliveWindow[0] || a.n - b.n);
|
|
81
|
+
|
|
82
|
+
let now = Date.now();
|
|
83
|
+
const isUs = (e: { entry: ParametersTimelineEntry }) => !!e.entry.pid && ancestorPids.has(e.entry.pid);
|
|
84
|
+
|
|
85
|
+
console.log(`\n${formatDateTimeDetailed(now)} — deployment timeline in ${folder} (our pid chain: ${[...ancestorPids].join(" → ")})`);
|
|
86
|
+
console.log(entries.map(({ n, entry }) => {
|
|
87
|
+
let [start, end] = entry.aliveWindow;
|
|
88
|
+
let marker = end <= now && "✓ done" || start > now && "… future" || "▶ live";
|
|
89
|
+
return `[#${n} ${marker} ${formatBoundary(start)} → ${formatBoundary(end)}${isUs({ entry }) && " (us)" || ""}]`;
|
|
90
|
+
}).join(" ") || "(no entries)");
|
|
91
|
+
for (let { n, entry } of entries) {
|
|
92
|
+
console.log(`#${n}${isUs({ entry }) && " (us)" || ""} pid=${entry.pid ?? "not started"} parameters=${JSON.stringify(entry.parameters)}`);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
let us = entries.find(isUs);
|
|
96
|
+
if (us) {
|
|
97
|
+
let end = us.entry.aliveWindow[1];
|
|
98
|
+
if (end === ALIVE_WINDOW_FOREVER) {
|
|
99
|
+
console.log(`We are #${us.n}, with no scheduled shutdown`);
|
|
100
|
+
} else {
|
|
101
|
+
console.log(`We are #${us.n}, scheduled to shut down at ${formatDateTimeDetailed(end)} (in ${formatTime(end - now)})`);
|
|
102
|
+
}
|
|
103
|
+
} else {
|
|
104
|
+
console.log(`No entry's pid matches our pid chain — we are either a future version that hasn't been assigned a pid yet, or not a descendant of a deployed instance`);
|
|
105
|
+
}
|
|
106
|
+
let nextDeploy = entries.find(e => e.entry.aliveWindow[0] > now);
|
|
107
|
+
if (nextDeploy) {
|
|
108
|
+
console.log(`Next deploy (#${nextDeploy.n}) releases at ${formatDateTimeDetailed(nextDeploy.entry.aliveWindow[0])} (in ${formatTime(nextDeploy.entry.aliveWindow[0] - now)})`);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
async function main() {
|
|
113
|
+
while (true) {
|
|
114
|
+
try {
|
|
115
|
+
await showTimeline();
|
|
116
|
+
} catch (e) {
|
|
117
|
+
console.error((e as Error).stack ?? e);
|
|
118
|
+
}
|
|
119
|
+
await delay(POLL_INTERVAL);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
main().catch(e => console.error((e as Error).stack ?? e));
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { qreact } from "../4-dom/qreact";
|
|
2
|
+
import { css } from "typesafecss";
|
|
3
|
+
import { formatNumber } from "socket-function/src/formatting/format";
|
|
4
|
+
|
|
5
|
+
const FLASH_CLASS_NAME = "UsageBar-flash";
|
|
6
|
+
|
|
7
|
+
export const MEMORY_WARNING_THRESHOLD = 0.7;
|
|
8
|
+
export const MEMORY_ERROR_THRESHOLD = 0.85;
|
|
9
|
+
export const DISK_WARNING_THRESHOLD = 0.8;
|
|
10
|
+
export const DISK_ERROR_THRESHOLD = 0.92;
|
|
11
|
+
|
|
12
|
+
/** The standard thresholds for the machine resource bar types ("MEMORY" / "DISK"), so every page showing these bars warns identically. */
|
|
13
|
+
export function getUsageThresholds(type: string): { warningThreshold?: number; errorThreshold?: number } {
|
|
14
|
+
if (type === "MEMORY") {
|
|
15
|
+
return { warningThreshold: MEMORY_WARNING_THRESHOLD, errorThreshold: MEMORY_ERROR_THRESHOLD };
|
|
16
|
+
}
|
|
17
|
+
if (type === "DISK") {
|
|
18
|
+
return { warningThreshold: DISK_WARNING_THRESHOLD, errorThreshold: DISK_ERROR_THRESHOLD };
|
|
19
|
+
}
|
|
20
|
+
return {};
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export class UsageBar extends qreact.Component<{
|
|
24
|
+
label: string;
|
|
25
|
+
value: number;
|
|
26
|
+
max: number;
|
|
27
|
+
/** Fraction of max at which the bar fill turns yellow. */
|
|
28
|
+
warningThreshold?: number;
|
|
29
|
+
/** Fraction of max at which the bar fill turns red and flashes. */
|
|
30
|
+
errorThreshold?: number;
|
|
31
|
+
}> {
|
|
32
|
+
render() {
|
|
33
|
+
let { label, value, max, warningThreshold, errorThreshold } = this.props;
|
|
34
|
+
let fraction = max && value / max || 0;
|
|
35
|
+
let isError = errorThreshold !== undefined && fraction >= errorThreshold;
|
|
36
|
+
let isWarning = !isError && warningThreshold !== undefined && fraction >= warningThreshold;
|
|
37
|
+
let fillClass = isError && css.hsl(0, 80, 60) || isWarning && css.hsl(45, 90, 60) || css.hsl(0, 0, 70);
|
|
38
|
+
return <div className={css.pad2(6, 2).hsla(0, 0, 0, 0.1).relative}>
|
|
39
|
+
<div className={
|
|
40
|
+
css.absolute.pos(0, 0).size(`${Math.min(fraction, 1) * 100}%`, "100%")
|
|
41
|
+
+ fillClass
|
|
42
|
+
+ (isError && (" " + FLASH_CLASS_NAME) || "")
|
|
43
|
+
} />
|
|
44
|
+
<div className={css.relative}>
|
|
45
|
+
{label} ({formatNumber(value)} / {formatNumber(max)})
|
|
46
|
+
</div>
|
|
47
|
+
{isError && <style>{`
|
|
48
|
+
@keyframes ${FLASH_CLASS_NAME}-anim {
|
|
49
|
+
0%, 100% { background-color: hsl(0, 80%, 60%); }
|
|
50
|
+
50% { background-color: hsl(0, 80%, 30%); }
|
|
51
|
+
}
|
|
52
|
+
.${FLASH_CLASS_NAME} {
|
|
53
|
+
animation: ${FLASH_CLASS_NAME}-anim 0.6s infinite;
|
|
54
|
+
}
|
|
55
|
+
`}</style>}
|
|
56
|
+
</div>;
|
|
57
|
+
}
|
|
58
|
+
}
|