querysub 0.546.0 → 0.548.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/-a-archives/archives2.ts +9 -9
- package/src/deployManager/MachinesPage.tsx +3 -0
- package/src/deployManager/components/RouteConfigView.tsx +538 -0
- package/src/deployManager/components/StoragePage.tsx +454 -0
- package/src/deployManager/urlParams.ts +1 -1
- package/src/misc/random.ts +1 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "querysub",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.548.0",
|
|
4
4
|
"main": "index.js",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
|
|
@@ -71,7 +71,7 @@
|
|
|
71
71
|
"node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
|
|
72
72
|
"pako": "^2.1.0",
|
|
73
73
|
"peggy": "^5.0.6",
|
|
74
|
-
"sliftutils": "^1.7.
|
|
74
|
+
"sliftutils": "^1.7.34",
|
|
75
75
|
"socket-function": "^1.2.26",
|
|
76
76
|
"terser": "^5.31.0",
|
|
77
77
|
"typenode": "^6.6.1",
|
|
@@ -5,6 +5,8 @@ import { formatDateTime } from "socket-function/src/formatting/format";
|
|
|
5
5
|
import { getDomain } from "../config";
|
|
6
6
|
import { timeInMinute } from "socket-function/src/misc";
|
|
7
7
|
|
|
8
|
+
export const STORAGE_ACCOUNT = "root";
|
|
9
|
+
|
|
8
10
|
function createSourceWindows(
|
|
9
11
|
overrides: Partial<RemoteConfigBase>,
|
|
10
12
|
sourceWindows: {
|
|
@@ -33,8 +35,8 @@ function createSourceWindows(
|
|
|
33
35
|
}
|
|
34
36
|
|
|
35
37
|
function archiveBuilder(bucket: string, overrides: Partial<RemoteConfigBase>) {
|
|
36
|
-
const HETZNER = `https://65-109-93-113.querysubtest.com:5233/file
|
|
37
|
-
const ONTARIO = `https://99-250-124-91.querysubtest.com:5234/file
|
|
38
|
+
const HETZNER = `https://65-109-93-113.querysubtest.com:5233/file/${STORAGE_ACCOUNT}/${bucket}/storage/storagerouting.json`;
|
|
39
|
+
const ONTARIO = `https://99-250-124-91.querysubtest.com:5234/file/${STORAGE_ACCOUNT}/${bucket}/storage/storagerouting.json`;
|
|
38
40
|
const BACKBLAZE = `https://f002.backblazeb2.com/file/${bucket}/storage/storagerouting.json`;
|
|
39
41
|
let sources = createSourceWindows(overrides, [
|
|
40
42
|
{
|
|
@@ -47,22 +49,20 @@ function archiveBuilder(bucket: string, overrides: Partial<RemoteConfigBase>) {
|
|
|
47
49
|
{ type: "backblaze", url: BACKBLAZE, },
|
|
48
50
|
]
|
|
49
51
|
},
|
|
50
|
-
/*
|
|
51
52
|
{
|
|
52
|
-
startTime: +new Date("2026-07-20
|
|
53
|
+
startTime: +new Date("2026-07-20 18:00:00-04:00"),
|
|
53
54
|
sources: [
|
|
54
|
-
{ type: "remote", url: ONTARIO, route: [0,
|
|
55
|
-
{ type: "remote", url: HETZNER, route: [0.5
|
|
55
|
+
{ type: "remote", url: ONTARIO, route: [0.5, 1], },
|
|
56
|
+
{ type: "remote", url: HETZNER, route: [0, 0.5], },
|
|
56
57
|
{ type: "remote", url: ONTARIO, },
|
|
57
58
|
{ type: "remote", url: HETZNER, },
|
|
58
59
|
{ type: "backblaze", url: BACKBLAZE, },
|
|
59
60
|
]
|
|
60
61
|
}
|
|
61
|
-
*/
|
|
62
62
|
]);
|
|
63
63
|
|
|
64
64
|
return createArchives({
|
|
65
|
-
version:
|
|
65
|
+
version: 14,
|
|
66
66
|
sources,
|
|
67
67
|
});
|
|
68
68
|
}
|
|
@@ -120,5 +120,5 @@ export function getArchives2PublicImmutable(folder: string) {
|
|
|
120
120
|
}
|
|
121
121
|
export function getArchives2Public(folder: string) {
|
|
122
122
|
let domain = getSafeDomain();
|
|
123
|
-
return nestBucket(folder, archivesBuilderCache(domain + "-public", {
|
|
123
|
+
return nestBucket(folder, archivesBuilderCache(domain + "-public", { public: true, allowedOrigins: getAllowedOrigins(getDomain()) }));
|
|
124
124
|
}
|
|
@@ -9,6 +9,7 @@ import { ServiceDetailPage } from "./components/ServiceDetailPage";
|
|
|
9
9
|
import { MachineDetailPage } from "./components/MachineDetailPage";
|
|
10
10
|
import { Anchor } from "../library-components/ATag";
|
|
11
11
|
import { DeployPage } from "./components/DeployPage";
|
|
12
|
+
import { StoragePage } from "./components/StoragePage";
|
|
12
13
|
|
|
13
14
|
export class MachinesPage extends qreact.Component {
|
|
14
15
|
private renderTabs() {
|
|
@@ -20,6 +21,7 @@ export class MachinesPage extends qreact.Component {
|
|
|
20
21
|
{ key: "machines", label: "Machines", otherKeys: ["machine-detail"] },
|
|
21
22
|
{ key: "services", label: "Services", otherKeys: ["service-detail"] },
|
|
22
23
|
{ key: "deploy", label: "Deploy", otherKeys: [] },
|
|
24
|
+
{ key: "storage", label: "Storage", otherKeys: [] },
|
|
23
25
|
].map(tab => {
|
|
24
26
|
let isActive = currentViewParam.value === tab.key || tab.otherKeys.includes(currentViewParam.value);
|
|
25
27
|
return <Anchor noStyles key={tab.key}
|
|
@@ -47,6 +49,7 @@ export class MachinesPage extends qreact.Component {
|
|
|
47
49
|
{currentViewParam.value === "service-detail" && <ServiceDetailPage />}
|
|
48
50
|
{currentViewParam.value === "machine-detail" && <MachineDetailPage />}
|
|
49
51
|
{currentViewParam.value === "deploy" && <DeployPage />}
|
|
52
|
+
{currentViewParam.value === "storage" && <StoragePage />}
|
|
50
53
|
</div>
|
|
51
54
|
</div>;
|
|
52
55
|
}
|
|
@@ -0,0 +1,538 @@
|
|
|
1
|
+
import { qreact } from "../../4-dom/qreact";
|
|
2
|
+
import { Querysub } from "../../4-querysub/Querysub";
|
|
3
|
+
import { css } from "typesafecss";
|
|
4
|
+
import { sort } from "socket-function/src/misc";
|
|
5
|
+
import { formatNumber, formatDateTime, formatTime } from "socket-function/src/formatting/format";
|
|
6
|
+
import { parseHostedUrl, parseBackblazeUrl } from "sliftutils/storage/remoteStorage/remoteConfig";
|
|
7
|
+
import { getServerActiveBucket } from "sliftutils/storage/remoteStorage/createArchives";
|
|
8
|
+
import { STORAGE_ACCOUNT } from "../../-a-archives/archives2";
|
|
9
|
+
import { FULL_VALID_WINDOW, FULL_ROUTE } from "sliftutils/storage/IArchives";
|
|
10
|
+
import type { RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig } from "sliftutils/storage/IArchives";
|
|
11
|
+
import { showModal } from "../../5-diagnostics/Modal";
|
|
12
|
+
import { FullscreenModal } from "../../5-diagnostics/FullscreenModal";
|
|
13
|
+
import { Button } from "../../library-components/Button";
|
|
14
|
+
import type { StorageServerBuckets } from "./StoragePage";
|
|
15
|
+
|
|
16
|
+
module.hotreload = true;
|
|
17
|
+
|
|
18
|
+
type Source = HostedConfig | BackblazeConfig;
|
|
19
|
+
|
|
20
|
+
const DEFAULT_HTTPS_PORT = 443;
|
|
21
|
+
const TAG_COLOR = { h: 210, s: 45, l: 88 };
|
|
22
|
+
const WARNING_COLOR = { h: 35, s: 90, l: 85 };
|
|
23
|
+
const SOURCE_BORDER_COLOR = { h: 0, s: 0, l: 75 };
|
|
24
|
+
const ACTIVE_COLOR = { h: 130, s: 55, l: 85 };
|
|
25
|
+
const PAST_COLOR = { h: 0, s: 0, l: 90 };
|
|
26
|
+
const FUTURE_COLOR = { h: 45, s: 80, l: 85 };
|
|
27
|
+
// Windows are days or hours wide, so the countdown only has to be roughly right
|
|
28
|
+
const TIME_REFRESH_INTERVAL = 60 * 1000;
|
|
29
|
+
const WATCH_POLL_INTERVAL = 60 * 1000;
|
|
30
|
+
const WATCH_ERROR_LIMIT = 500;
|
|
31
|
+
const WATCH_ACTIVE_COLOR = { h: 195, s: 60, l: 85 };
|
|
32
|
+
const WINDOW_HEADER_SIZE = 13;
|
|
33
|
+
const BUCKET_TITLE_SIZE = 15;
|
|
34
|
+
const GROUP_BODY_INDENT = 16;
|
|
35
|
+
const TAG_FONT_SIZE = 11;
|
|
36
|
+
const JSON_INDENT = 4;
|
|
37
|
+
|
|
38
|
+
/** String sources are the shorthand for an always-valid, unsharded backblaze bucket. */
|
|
39
|
+
function normalizeSource(source: RemoteConfigBase): Source {
|
|
40
|
+
if (typeof source !== "string") return source;
|
|
41
|
+
return { type: "backblaze", url: source, validWindow: FULL_VALID_WINDOW };
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Where a source lives, with the bucket the config belongs to masked out so buckets that differ only by their own name group together. */
|
|
45
|
+
function getUrlIdentity(source: Source, ownBucketName: string): unknown {
|
|
46
|
+
let maskBucket = (bucketName: string) => bucketName === ownBucketName && "" || bucketName;
|
|
47
|
+
try {
|
|
48
|
+
if (source.type === "backblaze") {
|
|
49
|
+
return { type: "backblaze", bucketName: maskBucket(parseBackblazeUrl(source.url).bucketName) };
|
|
50
|
+
}
|
|
51
|
+
let { address, port, account, bucketName } = parseHostedUrl(source.url);
|
|
52
|
+
return { type: "remote", address, port, account, bucketName: maskBucket(bucketName) };
|
|
53
|
+
} catch {
|
|
54
|
+
return source.url;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Just the host - the account and the source's own bucket name are already in the table above, so repeating them here is noise. A source pointing at a different bucket keeps that name, since it's the one part the table doesn't already say. */
|
|
59
|
+
function describeUrl(source: Source, ownBucketName: string): string {
|
|
60
|
+
try {
|
|
61
|
+
if (source.type === "backblaze") {
|
|
62
|
+
let { bucketName } = parseBackblazeUrl(source.url);
|
|
63
|
+
return bucketName === ownBucketName && "b2" || `b2 ${bucketName}`;
|
|
64
|
+
}
|
|
65
|
+
let { address, port, bucketName } = parseHostedUrl(source.url);
|
|
66
|
+
let host = port === DEFAULT_HTTPS_PORT && address || `${address}:${port}`;
|
|
67
|
+
return bucketName === ownBucketName && host || `${host} ${bucketName}`;
|
|
68
|
+
} catch {
|
|
69
|
+
return source.url;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function getSourceKey(source: Source, ownBucketName: string): string {
|
|
74
|
+
return JSON.stringify({ ...source, url: getUrlIdentity(source, ownBucketName) });
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Identifies a config independently of which bucket it configures, so identical configs on different buckets collapse into one display. */
|
|
78
|
+
function getVariantKey(variant: ConfigVariant, ownBucketName: string): string {
|
|
79
|
+
return JSON.stringify({ version: variant.version, sources: variant.sources.map(x => getSourceKey(x, ownBucketName)) });
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function formatWindowTime(time: number): string {
|
|
83
|
+
if (time <= FULL_VALID_WINDOW[0]) return "the beginning";
|
|
84
|
+
if (time >= FULL_VALID_WINDOW[1]) return "forever";
|
|
85
|
+
return formatDateTime(time);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// A missing version sorts below any explicit one, matching getConfigVersion
|
|
89
|
+
function getSortVersion(variant: ConfigVariant): number {
|
|
90
|
+
return variant.version ?? -1;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
type WindowStatus = {
|
|
94
|
+
icon: string;
|
|
95
|
+
text: string;
|
|
96
|
+
color: { h: number; s: number; l: number };
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
function getWindowStatus(window: [number, number], now: number): WindowStatus {
|
|
100
|
+
let [start, end] = window;
|
|
101
|
+
if (now < start) {
|
|
102
|
+
return { icon: "⏳", text: `starts in ${formatTime(start - now)}`, color: FUTURE_COLOR };
|
|
103
|
+
}
|
|
104
|
+
if (now >= end) {
|
|
105
|
+
return { icon: "✔", text: `ended ${formatTime(now - end)} ago`, color: PAST_COLOR };
|
|
106
|
+
}
|
|
107
|
+
if (end >= FULL_VALID_WINDOW[1]) {
|
|
108
|
+
return { icon: "●", text: "active", color: ACTIVE_COLOR };
|
|
109
|
+
}
|
|
110
|
+
return { icon: "●", text: `active, ends in ${formatTime(end - now)}`, color: ACTIVE_COLOR };
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function windowsOverlap(a: [number, number], b: [number, number]): boolean {
|
|
114
|
+
return a[0] < b[1] && b[0] < a[1];
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
type WindowCluster = {
|
|
118
|
+
window: [number, number];
|
|
119
|
+
sources: Source[];
|
|
120
|
+
};
|
|
121
|
+
|
|
122
|
+
/** Sources whose valid windows overlap describe one configuration of the bucket, so they merge into a single displayed window. Sources keep their config order within a cluster - the order sources are listed in decides which one shadows which. */
|
|
123
|
+
function getWindowClusters(sources: Source[]): WindowCluster[] {
|
|
124
|
+
let byStart = [...sources];
|
|
125
|
+
sort(byStart, x => x.validWindow[0]);
|
|
126
|
+
let windows: [number, number][] = [];
|
|
127
|
+
for (let source of byStart) {
|
|
128
|
+
let last = windows[windows.length - 1];
|
|
129
|
+
if (last && windowsOverlap(last, source.validWindow)) {
|
|
130
|
+
last[0] = Math.min(last[0], source.validWindow[0]);
|
|
131
|
+
last[1] = Math.max(last[1], source.validWindow[1]);
|
|
132
|
+
continue;
|
|
133
|
+
}
|
|
134
|
+
windows.push([...source.validWindow]);
|
|
135
|
+
}
|
|
136
|
+
let clusters: WindowCluster[] = windows.map(window => ({ window, sources: [] }));
|
|
137
|
+
for (let source of sources) {
|
|
138
|
+
let cluster = clusters.find(x => windowsOverlap(x.window, source.validWindow));
|
|
139
|
+
if (!cluster) continue;
|
|
140
|
+
cluster.sources.push(source);
|
|
141
|
+
}
|
|
142
|
+
return clusters;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export type ConfigVariant = {
|
|
146
|
+
servers: string[];
|
|
147
|
+
sources: Source[];
|
|
148
|
+
version?: number;
|
|
149
|
+
rawConfig: RemoteConfig;
|
|
150
|
+
};
|
|
151
|
+
|
|
152
|
+
export type ConfigConflict = {
|
|
153
|
+
bucketName: string;
|
|
154
|
+
variant: ConfigVariant;
|
|
155
|
+
};
|
|
156
|
+
|
|
157
|
+
export type RouteConfigGroup = {
|
|
158
|
+
buckets: { bucketName: string; rawConfig: RemoteConfig }[];
|
|
159
|
+
consensus: ConfigVariant;
|
|
160
|
+
conflicts: ConfigConflict[];
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
/** Groups every server's view of each bucket's routing config, taking the most widely reported one as the truth, surfacing the rest as conflicts, then collapsing buckets that share a config. */
|
|
164
|
+
export function getRouteConfigGroups(servers: StorageServerBuckets[]): RouteConfigGroup[] {
|
|
165
|
+
let byBucket = new Map<string, Map<string, ConfigVariant>>();
|
|
166
|
+
for (let server of servers) {
|
|
167
|
+
for (let bucket of server.buckets || []) {
|
|
168
|
+
let remoteConfig: RemoteConfig | undefined = bucket.config?.remoteConfig;
|
|
169
|
+
if (!remoteConfig) continue;
|
|
170
|
+
let sources = remoteConfig.sources.map(normalizeSource);
|
|
171
|
+
let variants = byBucket.get(bucket.bucketName);
|
|
172
|
+
if (!variants) {
|
|
173
|
+
variants = new Map();
|
|
174
|
+
byBucket.set(bucket.bucketName, variants);
|
|
175
|
+
}
|
|
176
|
+
let key = JSON.stringify({ version: remoteConfig.version, sources });
|
|
177
|
+
let variant = variants.get(key);
|
|
178
|
+
if (!variant) {
|
|
179
|
+
variant = { servers: [], sources, version: remoteConfig.version, rawConfig: remoteConfig };
|
|
180
|
+
variants.set(key, variant);
|
|
181
|
+
}
|
|
182
|
+
variant.servers.push(server.url);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
let groups = new Map<string, RouteConfigGroup>();
|
|
187
|
+
for (let [bucketName, variants] of byBucket) {
|
|
188
|
+
let all = [...variants.values()];
|
|
189
|
+
// Ties keep insertion order, so the first server to report a config wins
|
|
190
|
+
let consensus = all.reduce((best, x) => x.servers.length > best.servers.length && x || best);
|
|
191
|
+
let conflicts = all.filter(x => x !== consensus).map(variant => ({ bucketName, variant }));
|
|
192
|
+
let key = getVariantKey(consensus, bucketName);
|
|
193
|
+
let group = groups.get(key);
|
|
194
|
+
if (!group) {
|
|
195
|
+
group = { buckets: [], consensus, conflicts: [] };
|
|
196
|
+
groups.set(key, group);
|
|
197
|
+
}
|
|
198
|
+
group.buckets.push({ bucketName, rawConfig: consensus.rawConfig });
|
|
199
|
+
for (let server of consensus.servers) {
|
|
200
|
+
if (group.consensus.servers.includes(server)) continue;
|
|
201
|
+
group.consensus.servers.push(server);
|
|
202
|
+
}
|
|
203
|
+
group.conflicts.push(...conflicts);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
let results = [...groups.values()];
|
|
207
|
+
for (let group of results) {
|
|
208
|
+
sort(group.buckets, x => x.bucketName);
|
|
209
|
+
}
|
|
210
|
+
sort(results, x => -getSortVersion(x.consensus));
|
|
211
|
+
return results;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
type LiveWatch = {
|
|
215
|
+
serverUrl: string;
|
|
216
|
+
bucketName: string;
|
|
217
|
+
loading: boolean;
|
|
218
|
+
routing?: RemoteConfig;
|
|
219
|
+
error?: string;
|
|
220
|
+
updatedTime?: number;
|
|
221
|
+
};
|
|
222
|
+
|
|
223
|
+
// Watches only live as long as the page is open - nothing about them is persisted
|
|
224
|
+
const liveWatchData = Querysub.createLocalSchema<{ watches: { [key: string]: LiveWatch } }>("storageLiveWatches");
|
|
225
|
+
|
|
226
|
+
function getWatchKey(serverUrl: string, bucketName: string): string {
|
|
227
|
+
return `${serverUrl}|${bucketName}`;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/** The storage server behind a source URL, which is what the live-state calls connect to (the source URL itself points at the routing file inside the bucket). */
|
|
231
|
+
function getServerUrl(source: Source): string | undefined {
|
|
232
|
+
if (source.type !== "remote") return undefined;
|
|
233
|
+
try {
|
|
234
|
+
let { address, port } = parseHostedUrl(source.url);
|
|
235
|
+
return `https://${address}:${port}`;
|
|
236
|
+
} catch {
|
|
237
|
+
return undefined;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
async function refreshWatch(key: string): Promise<void> {
|
|
242
|
+
let watch = liveWatchData().watches[key];
|
|
243
|
+
if (!watch) return;
|
|
244
|
+
let { serverUrl, bucketName } = watch;
|
|
245
|
+
Querysub.localCommit(() => {
|
|
246
|
+
let current = liveWatchData().watches[key];
|
|
247
|
+
if (current) current.loading = true;
|
|
248
|
+
});
|
|
249
|
+
let result: RemoteConfig | undefined;
|
|
250
|
+
let error: string | undefined;
|
|
251
|
+
try {
|
|
252
|
+
let live = await getServerActiveBucket({ url: serverUrl, account: STORAGE_ACCOUNT, bucketName });
|
|
253
|
+
// The server reports "not loaded here" as a string rather than throwing
|
|
254
|
+
if (typeof live === "string") {
|
|
255
|
+
error = live;
|
|
256
|
+
} else {
|
|
257
|
+
result = live.routing;
|
|
258
|
+
}
|
|
259
|
+
} catch (e) {
|
|
260
|
+
error = String((e as Error).stack ?? e).slice(0, WATCH_ERROR_LIMIT);
|
|
261
|
+
console.error(`Reading the live config of ${bucketName} on ${serverUrl} failed:`, (e as Error).stack ?? e);
|
|
262
|
+
}
|
|
263
|
+
Querysub.localCommit(() => {
|
|
264
|
+
let current = liveWatchData().watches[key];
|
|
265
|
+
// The watch may have been removed while the call was in flight
|
|
266
|
+
if (!current) return;
|
|
267
|
+
current.loading = false;
|
|
268
|
+
current.routing = result;
|
|
269
|
+
current.error = error;
|
|
270
|
+
current.updatedTime = Date.now();
|
|
271
|
+
});
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
function isWatched(serverUrl: string, bucketName: string): boolean {
|
|
275
|
+
return !!liveWatchData().watches[getWatchKey(serverUrl, bucketName)];
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
function toggleWatch(serverUrl: string, bucketName: string): void {
|
|
279
|
+
let key = getWatchKey(serverUrl, bucketName);
|
|
280
|
+
let existed = !!liveWatchData().watches[key];
|
|
281
|
+
Querysub.localCommit(() => {
|
|
282
|
+
if (existed) {
|
|
283
|
+
delete liveWatchData().watches[key];
|
|
284
|
+
return;
|
|
285
|
+
}
|
|
286
|
+
liveWatchData().watches[key] = { serverUrl, bucketName, loading: true };
|
|
287
|
+
});
|
|
288
|
+
if (existed) return;
|
|
289
|
+
void refreshWatch(key);
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
function refreshAllWatches(): void {
|
|
293
|
+
for (let key of Object.keys(liveWatchData().watches)) {
|
|
294
|
+
void refreshWatch(key);
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
function clearWatches(): void {
|
|
299
|
+
Querysub.localCommit(() => {
|
|
300
|
+
for (let key of Object.keys(liveWatchData().watches)) {
|
|
301
|
+
delete liveWatchData().watches[key];
|
|
302
|
+
}
|
|
303
|
+
});
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
function showConfigModal(bucketName: string, rawConfig: RemoteConfig): void {
|
|
307
|
+
let close = showModal({
|
|
308
|
+
content: <FullscreenModal onCancel={() => close.close()}>
|
|
309
|
+
<div className={css.vbox(10)}>
|
|
310
|
+
<div className={css.boldStyle}>{bucketName}</div>
|
|
311
|
+
<div className={css.fontFamily("monospace").whiteSpace("pre").overflow("auto")}>
|
|
312
|
+
{JSON.stringify(rawConfig, undefined, JSON_INDENT)}
|
|
313
|
+
</div>
|
|
314
|
+
</div>
|
|
315
|
+
</FullscreenModal>
|
|
316
|
+
});
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
class Tag extends qreact.Component<{ icon: string; text: string; warning?: boolean; color?: { h: number; s: number; l: number } }> {
|
|
320
|
+
render() {
|
|
321
|
+
let color = this.props.color || this.props.warning && WARNING_COLOR || TAG_COLOR;
|
|
322
|
+
return <div className={
|
|
323
|
+
css.hbox(4).pad2(6, 1).fontSize(TAG_FONT_SIZE).whiteSpace("nowrap")
|
|
324
|
+
.hsl(color.h, color.s, color.l).bord2(color.h, color.s, color.l - 20)
|
|
325
|
+
}>
|
|
326
|
+
<span>{this.props.icon}</span>
|
|
327
|
+
<span>{this.props.text}</span>
|
|
328
|
+
</div>;
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
function getSourceTags(source: Source): { icon: string; text: string }[] {
|
|
333
|
+
let tags: { icon: string; text: string }[] = [];
|
|
334
|
+
if (source.public) tags.push({ icon: "🌐", text: "public" });
|
|
335
|
+
if (source.immutable) tags.push({ icon: "🔒", text: "immutable" });
|
|
336
|
+
if (source.noFullSync) tags.push({ icon: "⇣", text: "noFullSync" });
|
|
337
|
+
if (source.intermediate) tags.push({ icon: "⏳", text: "intermediate" });
|
|
338
|
+
if (source.type === "remote") {
|
|
339
|
+
if (source.fast) tags.push({ icon: "⚡", text: "fast" });
|
|
340
|
+
if (source.rawDisk) tags.push({ icon: "💽", text: "rawDisk" });
|
|
341
|
+
if (source.writeDelay !== undefined) tags.push({ icon: "⏱", text: `writeDelay ${formatTime(source.writeDelay)}` });
|
|
342
|
+
}
|
|
343
|
+
if (source.readerDiskLimit !== undefined) tags.push({ icon: "📦", text: `readCache ${formatNumber(source.readerDiskLimit)}B` });
|
|
344
|
+
return tags;
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
/** Watching a source pins that server's own live view of the bucket to the top of the page, so disagreements between servers are visible directly. */
|
|
348
|
+
class WatchButton extends qreact.Component<{ source: Source; bucketName: string }> {
|
|
349
|
+
render() {
|
|
350
|
+
let serverUrl = getServerUrl(this.props.source);
|
|
351
|
+
// Only storage servers have a live in-memory state to read
|
|
352
|
+
if (!serverUrl) return undefined;
|
|
353
|
+
let watching = isWatched(serverUrl, this.props.bucketName);
|
|
354
|
+
return <Button
|
|
355
|
+
flavor="tiny"
|
|
356
|
+
className={watching && css.hsl(WATCH_ACTIVE_COLOR.h, WATCH_ACTIVE_COLOR.s, WATCH_ACTIVE_COLOR.l) || ""}
|
|
357
|
+
title={watching && "Stop watching this server's live config" || "Watch this server's live config"}
|
|
358
|
+
onClick={() => toggleWatch(serverUrl, this.props.bucketName)}
|
|
359
|
+
>
|
|
360
|
+
👁
|
|
361
|
+
</Button>;
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
class SourceRow extends qreact.Component<{ source: Source; window: [number, number]; ownBucketName: string }> {
|
|
366
|
+
render() {
|
|
367
|
+
let { source, window, ownBucketName } = this.props;
|
|
368
|
+
let [routeStart, routeEnd] = source.route || FULL_ROUTE;
|
|
369
|
+
let [start, end] = source.validWindow;
|
|
370
|
+
// A source that stops short of the window it's displayed under has a gap where the bucket has no configuration
|
|
371
|
+
let partial = start > window[0] || end < window[1];
|
|
372
|
+
return <div className={css.fillWidth}>
|
|
373
|
+
<div className={
|
|
374
|
+
css.marginLeft(`${routeStart * 100}%`).width(`${(routeEnd - routeStart) * 100}%`)
|
|
375
|
+
.hbox(6).pad2(8, 4).boxSizing("border-box").whiteSpace("nowrap")
|
|
376
|
+
.hsl(0, 0, 100).bord2(SOURCE_BORDER_COLOR.h, SOURCE_BORDER_COLOR.s, SOURCE_BORDER_COLOR.l)
|
|
377
|
+
}>
|
|
378
|
+
<WatchButton source={source} bucketName={ownBucketName} />
|
|
379
|
+
<div className={css.boldStyle}>{describeUrl(source, ownBucketName)}</div>
|
|
380
|
+
{source.route && <div className={css.fontSize(TAG_FONT_SIZE).colorhsl(0, 0, 40)}>
|
|
381
|
+
route {routeStart} – {routeEnd}
|
|
382
|
+
</div>}
|
|
383
|
+
{getSourceTags(source).map(tag => <Tag key={tag.text} icon={tag.icon} text={tag.text} />)}
|
|
384
|
+
{partial && <Tag
|
|
385
|
+
icon="⚠"
|
|
386
|
+
warning={true}
|
|
387
|
+
text={`only valid ${formatWindowTime(start)} → ${formatWindowTime(end)}`}
|
|
388
|
+
/>}
|
|
389
|
+
</div>
|
|
390
|
+
</div>;
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
class ConflictWarning extends qreact.Component<{ consensus: ConfigVariant; conflict: ConfigConflict }> {
|
|
395
|
+
render() {
|
|
396
|
+
let { consensus, conflict } = this.props;
|
|
397
|
+
let { bucketName, variant } = conflict;
|
|
398
|
+
let consensusKeys = new Set(consensus.sources.map(x => getSourceKey(x, bucketName)));
|
|
399
|
+
let conflictKeys = new Set(variant.sources.map(x => getSourceKey(x, bucketName)));
|
|
400
|
+
let extra = variant.sources.filter(x => !consensusKeys.has(getSourceKey(x, bucketName)));
|
|
401
|
+
let missing = consensus.sources.filter(x => !conflictKeys.has(getSourceKey(x, bucketName)));
|
|
402
|
+
let describeDifference = (source: Source, label: string) => <div key={label + getSourceKey(source, bucketName)} className={css.hbox(6).wrap}>
|
|
403
|
+
<div className={css.boldStyle}>{label}</div>
|
|
404
|
+
<div>{describeUrl(source, bucketName)}</div>
|
|
405
|
+
<div className={css.fontSize(TAG_FONT_SIZE)}>
|
|
406
|
+
route {(source.route || FULL_ROUTE).join(" – ")}, valid {formatWindowTime(source.validWindow[0])} → {formatWindowTime(source.validWindow[1])}
|
|
407
|
+
</div>
|
|
408
|
+
</div>;
|
|
409
|
+
return <div className={css.vbox(6).pad2(10, 8).hsl(WARNING_COLOR.h, WARNING_COLOR.s, WARNING_COLOR.l).bord2(WARNING_COLOR.h, WARNING_COLOR.s, WARNING_COLOR.l - 25)}>
|
|
410
|
+
<div className={css.boldStyle}>
|
|
411
|
+
⚠ {bucketName}: {variant.servers.length} server{variant.servers.length === 1 && "" || "s"} report a different routing config (version {String(variant.version ?? "none")}, consensus is version {String(consensus.version ?? "none")})
|
|
412
|
+
</div>
|
|
413
|
+
<div className={css.vbox(2).fontSize(TAG_FONT_SIZE)}>
|
|
414
|
+
{variant.servers.map(server => <div key={server}>{server}</div>)}
|
|
415
|
+
</div>
|
|
416
|
+
{extra.map(source => describeDifference(source, "only there:"))}
|
|
417
|
+
{missing.map(source => describeDifference(source, "missing there:"))}
|
|
418
|
+
</div>;
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
class RouteConfigGroupView extends qreact.Component<{ group: RouteConfigGroup }> {
|
|
423
|
+
render() {
|
|
424
|
+
let { buckets, consensus, conflicts } = this.props.group;
|
|
425
|
+
let ownBucketName = buckets[0].bucketName;
|
|
426
|
+
let clusters = getWindowClusters(consensus.sources);
|
|
427
|
+
let now = Querysub.timeDelayed(TIME_REFRESH_INTERVAL);
|
|
428
|
+
return <div className={css.vbox(10).fillWidth}>
|
|
429
|
+
<div className={css.hbox(8).wrap.fontSize(WINDOW_HEADER_SIZE)}>
|
|
430
|
+
<div className={css.hbox(4).wrap}>
|
|
431
|
+
{buckets.map((bucket, index) => <div key={bucket.bucketName} className={css.hbox(4)}>
|
|
432
|
+
{index > 0 && <div className={css.colorhsl(0, 0, 60)}>|</div>}
|
|
433
|
+
<Button
|
|
434
|
+
flavor="tiny"
|
|
435
|
+
className={css.fontSize(BUCKET_TITLE_SIZE)}
|
|
436
|
+
onClick={() => showConfigModal(bucket.bucketName, bucket.rawConfig)}
|
|
437
|
+
>
|
|
438
|
+
{bucket.bucketName}
|
|
439
|
+
</Button>
|
|
440
|
+
</div>)}
|
|
441
|
+
</div>
|
|
442
|
+
<div className={css.colorhsl(0, 0, 45)}>
|
|
443
|
+
version {String(consensus.version ?? "none")}, from {consensus.servers.length} server{consensus.servers.length === 1 && "" || "s"}
|
|
444
|
+
</div>
|
|
445
|
+
</div>
|
|
446
|
+
<div className={css.vbox(10).fillWidth.paddingLeft(GROUP_BODY_INDENT).boxSizing("border-box")}>
|
|
447
|
+
{conflicts.map((conflict, index) => <ConflictWarning key={index} consensus={consensus} conflict={conflict} />)}
|
|
448
|
+
{clusters.map((cluster, index) => {
|
|
449
|
+
let status = getWindowStatus(cluster.window, now);
|
|
450
|
+
return <div key={index} className={css.vbox(6).fillWidth}>
|
|
451
|
+
<div className={css.hbox(6).wrap.colorhsl(0, 0, 35).fontSize(TAG_FONT_SIZE)}>
|
|
452
|
+
<div>{formatWindowTime(cluster.window[0])} → {formatWindowTime(cluster.window[1])}</div>
|
|
453
|
+
<Tag icon={status.icon} text={status.text} color={status.color} />
|
|
454
|
+
</div>
|
|
455
|
+
{cluster.sources.map(source => <SourceRow
|
|
456
|
+
key={getSourceKey(source, ownBucketName)}
|
|
457
|
+
source={source}
|
|
458
|
+
window={cluster.window}
|
|
459
|
+
ownBucketName={ownBucketName}
|
|
460
|
+
/>)}
|
|
461
|
+
</div>;
|
|
462
|
+
})}
|
|
463
|
+
</div>
|
|
464
|
+
</div>;
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
/** One watched server's own live, in-memory view of a bucket - which is what shows a server disagreeing with everyone else. */
|
|
469
|
+
class LiveWatchView extends qreact.Component<{ watchKey: string; watch: LiveWatch }> {
|
|
470
|
+
render() {
|
|
471
|
+
let { watchKey, watch } = this.props;
|
|
472
|
+
let now = Querysub.timeDelayed(TIME_REFRESH_INTERVAL);
|
|
473
|
+
let sources = (watch.routing?.sources || []).map(normalizeSource);
|
|
474
|
+
let clusters = getWindowClusters(sources);
|
|
475
|
+
return <div className={css.vbox(10).fillWidth.pad2(10, 8).bord2(WATCH_ACTIVE_COLOR.h, WATCH_ACTIVE_COLOR.s, WATCH_ACTIVE_COLOR.l - 25)}>
|
|
476
|
+
<div className={css.hbox(8).wrap.fontSize(WINDOW_HEADER_SIZE)}>
|
|
477
|
+
<div className={css.boldStyle}>👁 {watch.bucketName}</div>
|
|
478
|
+
<div className={css.colorhsl(0, 0, 45)}>live on {watch.serverUrl}</div>
|
|
479
|
+
<div className={css.colorhsl(0, 0, 45)}>
|
|
480
|
+
{watch.loading && "refreshing..."
|
|
481
|
+
|| watch.updatedTime && `updated ${formatTime(now - watch.updatedTime)} ago`
|
|
482
|
+
|| ""}
|
|
483
|
+
</div>
|
|
484
|
+
<div className={css.flexGrow(1)} />
|
|
485
|
+
<Button flavor="tiny" onClick={() => void refreshWatch(watchKey)}>Refresh</Button>
|
|
486
|
+
<Button flavor="tiny" onClick={() => toggleWatch(watch.serverUrl, watch.bucketName)}>Stop watching</Button>
|
|
487
|
+
</div>
|
|
488
|
+
{watch.error && <div className={css.pad2(10, 8).hsl(WARNING_COLOR.h, WARNING_COLOR.s, WARNING_COLOR.l).bord2(WARNING_COLOR.h, WARNING_COLOR.s, WARNING_COLOR.l - 25)}>
|
|
489
|
+
⚠ {watch.error}
|
|
490
|
+
</div>}
|
|
491
|
+
<div className={css.vbox(10).fillWidth.paddingLeft(GROUP_BODY_INDENT).boxSizing("border-box")}>
|
|
492
|
+
{clusters.map((cluster, index) => {
|
|
493
|
+
let status = getWindowStatus(cluster.window, now);
|
|
494
|
+
return <div key={index} className={css.vbox(6).fillWidth}>
|
|
495
|
+
<div className={css.hbox(6).wrap.colorhsl(0, 0, 35).fontSize(TAG_FONT_SIZE)}>
|
|
496
|
+
<div>{formatWindowTime(cluster.window[0])} → {formatWindowTime(cluster.window[1])}</div>
|
|
497
|
+
<Tag icon={status.icon} text={status.text} color={status.color} />
|
|
498
|
+
</div>
|
|
499
|
+
{cluster.sources.map(source => <SourceRow
|
|
500
|
+
key={getSourceKey(source, watch.bucketName)}
|
|
501
|
+
source={source}
|
|
502
|
+
window={cluster.window}
|
|
503
|
+
ownBucketName={watch.bucketName}
|
|
504
|
+
/>)}
|
|
505
|
+
</div>;
|
|
506
|
+
})}
|
|
507
|
+
</div>
|
|
508
|
+
</div>;
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
export class RouteConfigView extends qreact.Component<{ servers: StorageServerBuckets[] }> {
|
|
513
|
+
private pollTimer: ReturnType<typeof setInterval> | undefined;
|
|
514
|
+
componentDidMount() {
|
|
515
|
+
this.pollTimer = setInterval(refreshAllWatches, WATCH_POLL_INTERVAL);
|
|
516
|
+
}
|
|
517
|
+
componentWillUnmount() {
|
|
518
|
+
if (this.pollTimer !== undefined) {
|
|
519
|
+
clearInterval(this.pollTimer);
|
|
520
|
+
this.pollTimer = undefined;
|
|
521
|
+
}
|
|
522
|
+
// Watches are only meaningful while the page is open, and leaving them would keep polling on the next mount
|
|
523
|
+
clearWatches();
|
|
524
|
+
}
|
|
525
|
+
render() {
|
|
526
|
+
let groups = getRouteConfigGroups(this.props.servers);
|
|
527
|
+
let watches = Object.entries(liveWatchData().watches);
|
|
528
|
+
if (!groups.length && !watches.length) return undefined;
|
|
529
|
+
return <div className={css.vbox(20).fillWidth}>
|
|
530
|
+
{watches.length > 0 && <div className={css.vbox(10).fillWidth}>
|
|
531
|
+
<h3>Live configs</h3>
|
|
532
|
+
{watches.map(([key, watch]) => <LiveWatchView key={key} watchKey={key} watch={watch} />)}
|
|
533
|
+
</div>}
|
|
534
|
+
<h3>Routing configs</h3>
|
|
535
|
+
{groups.map((group, index) => <RouteConfigGroupView key={index} group={group} />)}
|
|
536
|
+
</div>;
|
|
537
|
+
}
|
|
538
|
+
}
|
|
@@ -0,0 +1,454 @@
|
|
|
1
|
+
import { SocketFunction } from "socket-function/SocketFunction";
|
|
2
|
+
import { qreact } from "../../4-dom/qreact";
|
|
3
|
+
import { css } from "typesafecss";
|
|
4
|
+
import { formatNumber, formatTime, formatDateTimeDetailed } from "socket-function/src/formatting/format";
|
|
5
|
+
import { sort, timeInSecond } from "socket-function/src/misc";
|
|
6
|
+
import preact from "preact";
|
|
7
|
+
import { Querysub } from "../../4-querysub/Querysub";
|
|
8
|
+
import { listServerBuckets, clearServerWriteStats, activateServerBucket } from "sliftutils/storage/remoteStorage/createArchives";
|
|
9
|
+
import type { ServerBucketInfo, BucketDiskInfo, BucketWriteStats } from "sliftutils/storage/remoteStorage/storageServerState";
|
|
10
|
+
import type { ArchivesConfig } from "sliftutils/storage/IArchives";
|
|
11
|
+
import { UsageBar, getUsageThresholds } from "../../library-components/UsageBar";
|
|
12
|
+
import { getSyncedController } from "../../library-components/SyncedController";
|
|
13
|
+
import { assertIsManagementUser } from "../../diagnostics/managementPages";
|
|
14
|
+
import { getEffectiveServiceConfigs, getLiveServiceParameters, getMachineTargets, applyCommandTemplate, ServiceParameters, DEFAULT_OVERLAP_TIME } from "../machineSchema";
|
|
15
|
+
import { STORAGE_ACCOUNT } from "../../-a-archives/archives2";
|
|
16
|
+
import { Table } from "../../5-diagnostics/Table";
|
|
17
|
+
import { RouteConfigView } from "./RouteConfigView";
|
|
18
|
+
|
|
19
|
+
module.hotreload = true;
|
|
20
|
+
|
|
21
|
+
const STORAGE_COMMAND_PREFIX = "yarn storageserve";
|
|
22
|
+
const URL_ARG_REGEX = /--url\s+(\S+)/g;
|
|
23
|
+
const ERROR_TEXT_LIMIT = 500;
|
|
24
|
+
|
|
25
|
+
export type StorageServerBuckets = {
|
|
26
|
+
serviceKey: string;
|
|
27
|
+
serviceTitle: string;
|
|
28
|
+
/** The release the service is in the middle of, so the page can warn that these servers are about to move (or just did) */
|
|
29
|
+
releaseTime?: number;
|
|
30
|
+
overlapTime: number;
|
|
31
|
+
hasOldParameters: boolean;
|
|
32
|
+
url: string;
|
|
33
|
+
buckets?: ServerBucketInfo[];
|
|
34
|
+
error?: string;
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
function getStorageUrls(parameters: ServiceParameters): string[] {
|
|
38
|
+
if (!parameters.command.trimStart().startsWith(STORAGE_COMMAND_PREFIX)) return [];
|
|
39
|
+
let urls: string[] = [];
|
|
40
|
+
for (let target of getMachineTargets(parameters)) {
|
|
41
|
+
let command = applyCommandTemplate(parameters.command, target.variables);
|
|
42
|
+
for (let match of command.matchAll(URL_ARG_REGEX)) {
|
|
43
|
+
urls.push(match[1]);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return urls;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
class StoragePageControllerBase {
|
|
50
|
+
public async getStorageBuckets(): Promise<StorageServerBuckets[]> {
|
|
51
|
+
let configs = await getEffectiveServiceConfigs();
|
|
52
|
+
type ServiceForUrl = Omit<StorageServerBuckets, "url" | "buckets" | "error">;
|
|
53
|
+
let serviceByUrl = new Map<string, ServiceForUrl>();
|
|
54
|
+
for (let config of configs) {
|
|
55
|
+
let allParameters = [getLiveServiceParameters(config), config.parameters, config.oldParameters];
|
|
56
|
+
for (let parameters of allParameters) {
|
|
57
|
+
if (!parameters) continue;
|
|
58
|
+
for (let url of getStorageUrls(parameters)) {
|
|
59
|
+
if (serviceByUrl.has(url)) continue;
|
|
60
|
+
serviceByUrl.set(url, {
|
|
61
|
+
serviceKey: parameters.key,
|
|
62
|
+
serviceTitle: config.info.title,
|
|
63
|
+
releaseTime: config.parameters.releaseTime,
|
|
64
|
+
// The overlap always comes from the newest parameters, even though it governs how long the old instances outlive the release
|
|
65
|
+
overlapTime: config.parameters.overlapTime ?? DEFAULT_OVERLAP_TIME,
|
|
66
|
+
hasOldParameters: !!config.oldParameters,
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return await Promise.all([...serviceByUrl].map(async ([url, service]) => {
|
|
72
|
+
try {
|
|
73
|
+
return { ...service, url, buckets: await listServerBuckets({ url, account: STORAGE_ACCOUNT }) };
|
|
74
|
+
} catch (e) {
|
|
75
|
+
return { ...service, url, error: String((e as Error).stack ?? e).slice(0, ERROR_TEXT_LIMIT) };
|
|
76
|
+
}
|
|
77
|
+
}));
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export const StoragePageController = SocketFunction.register(
|
|
82
|
+
"StoragePageController-4f1c93ab-77e2-4d15-9a30-1c6b8f5d2e04",
|
|
83
|
+
new StoragePageControllerBase(),
|
|
84
|
+
() => ({
|
|
85
|
+
getStorageBuckets: {},
|
|
86
|
+
}),
|
|
87
|
+
() => ({
|
|
88
|
+
hooks: [assertIsManagementUser],
|
|
89
|
+
}),
|
|
90
|
+
{
|
|
91
|
+
}
|
|
92
|
+
);
|
|
93
|
+
|
|
94
|
+
const StorageSynced = getSyncedController(StoragePageController, {
|
|
95
|
+
reads: {},
|
|
96
|
+
writes: {},
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
type BucketRow = {
|
|
100
|
+
server: string;
|
|
101
|
+
/** Kept even on the rows that blank out the server column, so the row's buttons know which server to call */
|
|
102
|
+
serverUrl: string;
|
|
103
|
+
bucket: string;
|
|
104
|
+
state: string;
|
|
105
|
+
files: string;
|
|
106
|
+
bytes: string;
|
|
107
|
+
indexSources: ArchivesConfig["indexSources"];
|
|
108
|
+
readerDiskLimit: string;
|
|
109
|
+
writes: string;
|
|
110
|
+
written: string;
|
|
111
|
+
writeGain: string;
|
|
112
|
+
disk?: BucketDiskInfo;
|
|
113
|
+
diskError: string;
|
|
114
|
+
syncing: ArchivesConfig["syncing"];
|
|
115
|
+
error: string;
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
const GAIN_DECIMALS = 1;
|
|
119
|
+
const DISK_BAR_TYPE = "DISK";
|
|
120
|
+
const ACTIVE_STATE = "active";
|
|
121
|
+
const INACTIVE_STATE = "inactive";
|
|
122
|
+
|
|
123
|
+
/** How much fast-mode coalescing saved: every accepted write over the ones that actually reached a source. */
|
|
124
|
+
function getWriteGain(stats: BucketWriteStats | undefined): string {
|
|
125
|
+
if (!stats) return "";
|
|
126
|
+
if (!stats.flushedWrites || !stats.flushedBytes) return "";
|
|
127
|
+
let writeGain = stats.originalWrites / stats.flushedWrites;
|
|
128
|
+
let byteGain = stats.originalBytes / stats.flushedBytes;
|
|
129
|
+
return `${writeGain.toFixed(GAIN_DECIMALS)}X / ${byteGain.toFixed(GAIN_DECIMALS)}X B`;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function getBucketRows(servers: StorageServerBuckets[]): BucketRow[] {
|
|
133
|
+
let rows: BucketRow[] = [];
|
|
134
|
+
for (let server of servers) {
|
|
135
|
+
let baseRow: BucketRow = {
|
|
136
|
+
server: server.url,
|
|
137
|
+
serverUrl: server.url,
|
|
138
|
+
bucket: "",
|
|
139
|
+
state: "",
|
|
140
|
+
files: "",
|
|
141
|
+
bytes: "",
|
|
142
|
+
indexSources: undefined,
|
|
143
|
+
readerDiskLimit: "",
|
|
144
|
+
writes: "",
|
|
145
|
+
written: "",
|
|
146
|
+
writeGain: "",
|
|
147
|
+
disk: undefined,
|
|
148
|
+
diskError: "",
|
|
149
|
+
syncing: undefined,
|
|
150
|
+
error: "",
|
|
151
|
+
};
|
|
152
|
+
if (server.error) {
|
|
153
|
+
rows.push({ ...baseRow, error: server.error });
|
|
154
|
+
continue;
|
|
155
|
+
}
|
|
156
|
+
let buckets = [...server.buckets || []];
|
|
157
|
+
sort(buckets, x => x.bucketName);
|
|
158
|
+
if (!buckets.length) {
|
|
159
|
+
rows.push({ ...baseRow, bucket: "No buckets" });
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
162
|
+
for (let [index, bucket] of buckets.entries()) {
|
|
163
|
+
let config = bucket.config;
|
|
164
|
+
rows.push({
|
|
165
|
+
...baseRow,
|
|
166
|
+
server: index === 0 && server.url || "",
|
|
167
|
+
bucket: bucket.bucketName,
|
|
168
|
+
state: bucket.active && ACTIVE_STATE || INACTIVE_STATE,
|
|
169
|
+
files: config?.index && formatNumber(config.index.fileCount) || "",
|
|
170
|
+
bytes: config?.index && formatNumber(config.index.byteCount) + "B" || "",
|
|
171
|
+
indexSources: config?.indexSources,
|
|
172
|
+
readerDiskLimit: config?.readerDiskLimit && formatNumber(config.readerDiskLimit) + "B" || "",
|
|
173
|
+
writes: bucket.writeStats && formatNumber(bucket.writeStats.originalWrites) || "",
|
|
174
|
+
written: bucket.writeStats && formatNumber(bucket.writeStats.originalBytes) + "B" || "",
|
|
175
|
+
writeGain: getWriteGain(bucket.writeStats),
|
|
176
|
+
disk: bucket.disk,
|
|
177
|
+
diskError: bucket.diskError || "",
|
|
178
|
+
syncing: config?.syncing,
|
|
179
|
+
error: bucket.error || "",
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
return rows;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
const ACTIVATE_ERROR_COLOR = { h: 0, s: 60, l: 40 };
|
|
187
|
+
|
|
188
|
+
/** An inactive bucket exists on the server's disk but isn't loaded, so it isn't synchronizing. Activating loads it, which starts synchronization. */
|
|
189
|
+
class ActivateBucketButton extends qreact.Component<{ serverUrl: string; bucketName: string }> {
|
|
190
|
+
state = {
|
|
191
|
+
activating: false,
|
|
192
|
+
error: "",
|
|
193
|
+
};
|
|
194
|
+
private async activate() {
|
|
195
|
+
this.state.activating = true;
|
|
196
|
+
this.state.error = "";
|
|
197
|
+
try {
|
|
198
|
+
let result = await activateServerBucket({
|
|
199
|
+
url: this.props.serverUrl,
|
|
200
|
+
account: STORAGE_ACCOUNT,
|
|
201
|
+
bucketName: this.props.bucketName,
|
|
202
|
+
});
|
|
203
|
+
// The server reports a refusal as a string instead of throwing
|
|
204
|
+
if (typeof result === "string") {
|
|
205
|
+
this.state.error = result;
|
|
206
|
+
console.error(`Activating bucket ${this.props.bucketName} on ${this.props.serverUrl} was refused: ${result}`);
|
|
207
|
+
return;
|
|
208
|
+
}
|
|
209
|
+
console.log(`Activated bucket ${this.props.bucketName} on ${this.props.serverUrl}`);
|
|
210
|
+
StorageSynced(SocketFunction.browserNodeId()).getStorageBuckets.resetAll();
|
|
211
|
+
} catch (e) {
|
|
212
|
+
this.state.error = String((e as Error).stack ?? e).slice(0, ERROR_TEXT_LIMIT);
|
|
213
|
+
console.error(`Activating bucket ${this.props.bucketName} on ${this.props.serverUrl} failed:`, (e as Error).stack ?? e);
|
|
214
|
+
} finally {
|
|
215
|
+
this.state.activating = false;
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
render() {
|
|
219
|
+
return <div className={css.hbox(6).wrap}>
|
|
220
|
+
<div className={css.whiteSpace("nowrap")}>{INACTIVE_STATE}</div>
|
|
221
|
+
<button
|
|
222
|
+
className={css.pad2(6, 1).button.bord2(0, 0, 20).hsl(0, 0, 100).whiteSpace("nowrap")}
|
|
223
|
+
disabled={this.state.activating}
|
|
224
|
+
onClick={() => {
|
|
225
|
+
void this.activate();
|
|
226
|
+
}}
|
|
227
|
+
>
|
|
228
|
+
{this.state.activating && "Activating..." || "Activate"}
|
|
229
|
+
</button>
|
|
230
|
+
{this.state.error && <div
|
|
231
|
+
className={css.colorhsl(ACTIVATE_ERROR_COLOR.h, ACTIVATE_ERROR_COLOR.s, ACTIVATE_ERROR_COLOR.l).ellipsis}
|
|
232
|
+
title={this.state.error}
|
|
233
|
+
>
|
|
234
|
+
{this.state.error}
|
|
235
|
+
</div>}
|
|
236
|
+
</div>;
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
const SOURCE_BAR_COLOR = { h: 210, s: 65, l: 55 };
|
|
241
|
+
const REMOTE_SOURCE_BAR_COLOR = { h: 0, s: 75, l: 55 };
|
|
242
|
+
const SOURCE_BAR_OPACITY = 0.3;
|
|
243
|
+
const SOURCE_ROW_PADDING = { horizontal: 4, vertical: 3 };
|
|
244
|
+
const DISK_SOURCE_TYPE = "disk";
|
|
245
|
+
const REMOTE_SOURCE_TOOLTIP = "Values stored on another server. If this other server goes down, we'll lose access to the data.";
|
|
246
|
+
const SOURCE_TYPE_COLOR = { h: 265, s: 40, l: 88 };
|
|
247
|
+
const SOURCE_PART_COLOR = { h: 0, s: 0, l: 96 };
|
|
248
|
+
const SOURCE_PART_FONT_SIZE = 11;
|
|
249
|
+
|
|
250
|
+
const DEBUG_NAME_PATTERNS: { regex: RegExp; parts: string[] }[] = [
|
|
251
|
+
{ regex: /^remoteStorage (\S+) account (\S+) bucket (.+)$/, parts: ["remoteStorage"] },
|
|
252
|
+
{ regex: /^localBucket account (\S+) bucket (.+)$/, parts: ["localBucket"] },
|
|
253
|
+
{ regex: /^backblaze (.+)$/, parts: ["backblaze"] },
|
|
254
|
+
{ regex: /^disk (.+)$/, parts: ["disk"] },
|
|
255
|
+
];
|
|
256
|
+
|
|
257
|
+
/** Splits an IArchives debug name into its type and the identifying pieces after it, so each piece can be boxed separately. */
|
|
258
|
+
function parseSourceDebugName(debugName: string): { type: string; parts: string[] } {
|
|
259
|
+
for (let pattern of DEBUG_NAME_PATTERNS) {
|
|
260
|
+
let match = pattern.regex.exec(debugName);
|
|
261
|
+
if (!match) continue;
|
|
262
|
+
return { type: pattern.parts[0], parts: match.slice(1) };
|
|
263
|
+
}
|
|
264
|
+
return { type: debugName, parts: [] };
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
class SourceNameParts extends qreact.Component<{ debugName: string }> {
|
|
268
|
+
render() {
|
|
269
|
+
let { type, parts } = parseSourceDebugName(this.props.debugName);
|
|
270
|
+
let partStyle = css.pad2(5, 0).fontSize(SOURCE_PART_FONT_SIZE).whiteSpace("nowrap");
|
|
271
|
+
return <div className={css.hbox(3).wrap}>
|
|
272
|
+
<div className={partStyle.hsl(SOURCE_TYPE_COLOR.h, SOURCE_TYPE_COLOR.s, SOURCE_TYPE_COLOR.l).bord2(SOURCE_TYPE_COLOR.h, SOURCE_TYPE_COLOR.s, SOURCE_TYPE_COLOR.l - 20)}>
|
|
273
|
+
{type}
|
|
274
|
+
</div>
|
|
275
|
+
{parts.map(part => <div
|
|
276
|
+
key={part}
|
|
277
|
+
className={partStyle.hsl(SOURCE_PART_COLOR.h, SOURCE_PART_COLOR.s, SOURCE_PART_COLOR.l).bord2(SOURCE_PART_COLOR.h, SOURCE_PART_COLOR.s, SOURCE_PART_COLOR.l - 20)}
|
|
278
|
+
>
|
|
279
|
+
{part}
|
|
280
|
+
</div>)}
|
|
281
|
+
</div>;
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
class IndexSourcesCell extends qreact.Component<{ sources: ArchivesConfig["indexSources"] }> {
|
|
286
|
+
render() {
|
|
287
|
+
let sources = [...this.props.sources || []];
|
|
288
|
+
if (!sources.length) return undefined;
|
|
289
|
+
sort(sources, x => -x.byteCount);
|
|
290
|
+
let totalBytes = sources.reduce((sum, x) => sum + x.byteCount, 0);
|
|
291
|
+
return <div className={css.vbox(2).fillWidth}>
|
|
292
|
+
{sources.map(source => {
|
|
293
|
+
let fraction = totalBytes && source.byteCount / totalBytes || 0;
|
|
294
|
+
// Only the server's own disk holds bytes we can't lose to another machine going down
|
|
295
|
+
let remote = parseSourceDebugName(source.debugName).type !== DISK_SOURCE_TYPE;
|
|
296
|
+
let barColor = remote && REMOTE_SOURCE_BAR_COLOR || SOURCE_BAR_COLOR;
|
|
297
|
+
return <div
|
|
298
|
+
key={source.debugName}
|
|
299
|
+
className={css.relative.fillWidth.pad2(SOURCE_ROW_PADDING.horizontal, SOURCE_ROW_PADDING.vertical)}
|
|
300
|
+
title={remote && REMOTE_SOURCE_TOOLTIP || undefined}
|
|
301
|
+
>
|
|
302
|
+
<div className={
|
|
303
|
+
css.absolute.top(0).left(0).size(`${fraction * 100}%`, "100%")
|
|
304
|
+
.hsla(barColor.h, barColor.s, barColor.l, SOURCE_BAR_OPACITY)
|
|
305
|
+
} />
|
|
306
|
+
<div className={css.relative.hbox(6).wrap}>
|
|
307
|
+
<div className={css.whiteSpace("nowrap")}>
|
|
308
|
+
{formatNumber(source.byteCount)}B, {formatNumber(source.fileCount)} files
|
|
309
|
+
</div>
|
|
310
|
+
<SourceNameParts debugName={source.debugName} />
|
|
311
|
+
</div>
|
|
312
|
+
</div>;
|
|
313
|
+
})}
|
|
314
|
+
</div>;
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
const DEPLOY_PENDING_COLOR = { h: 35, s: 80, l: 45 };
|
|
319
|
+
const DEPLOY_OVERLAP_COLOR = { h: 280, s: 60, l: 45 };
|
|
320
|
+
const DEPLOY_NOTICE_FONT_SIZE = 14;
|
|
321
|
+
|
|
322
|
+
/** The services behind these storage servers may be mid-release, which moves (or duplicates) the servers the page just read from. */
|
|
323
|
+
class DeployNotices extends qreact.Component<{ servers: StorageServerBuckets[] }> {
|
|
324
|
+
render() {
|
|
325
|
+
let now = Querysub.nowDelayed(timeInSecond);
|
|
326
|
+
let byService = new Map<string, StorageServerBuckets>();
|
|
327
|
+
for (let server of this.props.servers) {
|
|
328
|
+
if (byService.has(server.serviceKey)) continue;
|
|
329
|
+
byService.set(server.serviceKey, server);
|
|
330
|
+
}
|
|
331
|
+
let notices: preact.ComponentChild[] = [];
|
|
332
|
+
for (let [serviceKey, service] of byService) {
|
|
333
|
+
let releaseTime = service.releaseTime || 0;
|
|
334
|
+
// The old instances run out their overlap purely on time - we don't verify they're actually still up
|
|
335
|
+
if (!releaseTime || !service.hasOldParameters) continue;
|
|
336
|
+
let killTime = releaseTime + service.overlapTime;
|
|
337
|
+
if (now < releaseTime) {
|
|
338
|
+
notices.push(<div
|
|
339
|
+
key={serviceKey + "pending"}
|
|
340
|
+
className={css.pad2(10, 6).fontSize(DEPLOY_NOTICE_FONT_SIZE).boldStyle.colorhsl(DEPLOY_PENDING_COLOR.h, DEPLOY_PENDING_COLOR.s, DEPLOY_PENDING_COLOR.l).bord2(DEPLOY_PENDING_COLOR.h, DEPLOY_PENDING_COLOR.s, DEPLOY_PENDING_COLOR.l + 35)}
|
|
341
|
+
title={`Deploys at ${formatDateTimeDetailed(releaseTime)}`}
|
|
342
|
+
>
|
|
343
|
+
🕐 {service.serviceTitle} ({serviceKey}) deploys in {formatTime(releaseTime - now)}
|
|
344
|
+
</div>);
|
|
345
|
+
continue;
|
|
346
|
+
}
|
|
347
|
+
if (now < killTime) {
|
|
348
|
+
notices.push(<div
|
|
349
|
+
key={serviceKey + "overlap"}
|
|
350
|
+
className={css.pad2(10, 6).fontSize(DEPLOY_NOTICE_FONT_SIZE).boldStyle.colorhsl(DEPLOY_OVERLAP_COLOR.h, DEPLOY_OVERLAP_COLOR.s, DEPLOY_OVERLAP_COLOR.l).bord2(DEPLOY_OVERLAP_COLOR.h, DEPLOY_OVERLAP_COLOR.s, DEPLOY_OVERLAP_COLOR.l + 35)}
|
|
351
|
+
title={`Old instances shut down at ${formatDateTimeDetailed(killTime)}`}
|
|
352
|
+
>
|
|
353
|
+
🔀 {service.serviceTitle} ({serviceKey}) still has an old version running for {formatTime(killTime - now)}
|
|
354
|
+
</div>);
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
if (!notices.length) return undefined;
|
|
358
|
+
return <div className={css.hbox(8).wrap}>{notices}</div>;
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
async function resetWriteStats(servers: StorageServerBuckets[]): Promise<void> {
|
|
363
|
+
await Promise.all(servers.map(async server => {
|
|
364
|
+
if (server.error) return;
|
|
365
|
+
try {
|
|
366
|
+
let { clearedBuckets } = await clearServerWriteStats({ url: server.url, account: STORAGE_ACCOUNT });
|
|
367
|
+
console.log(`Cleared write stats for ${clearedBuckets} buckets on ${server.url}`);
|
|
368
|
+
} catch (e) {
|
|
369
|
+
console.error(`Clearing write stats on ${server.url} failed:`, (e as Error).stack ?? e);
|
|
370
|
+
}
|
|
371
|
+
}));
|
|
372
|
+
StorageSynced(SocketFunction.browserNodeId()).getStorageBuckets.resetAll();
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
export class StoragePage extends qreact.Component {
|
|
376
|
+
render() {
|
|
377
|
+
const servers = StorageSynced(SocketFunction.browserNodeId()).getStorageBuckets();
|
|
378
|
+
if (!servers) return <div>Loading storage servers...</div>;
|
|
379
|
+
if (!servers.length) {
|
|
380
|
+
return <div className={css.vbox(16)}>
|
|
381
|
+
<h2>Storage</h2>
|
|
382
|
+
<div>No services with a command starting with {JSON.stringify(STORAGE_COMMAND_PREFIX)} were found.</div>
|
|
383
|
+
</div>;
|
|
384
|
+
}
|
|
385
|
+
return <div className={css.vbox(16)}>
|
|
386
|
+
<div className={css.hbox(12)}>
|
|
387
|
+
<h2 className={css.flexGrow(1)}>Storage</h2>
|
|
388
|
+
<button className={css.pad2(12, 8).button.bord2(0, 0, 20).hsl(0, 0, 100)}
|
|
389
|
+
onClick={() => {
|
|
390
|
+
void resetWriteStats(servers);
|
|
391
|
+
}}>
|
|
392
|
+
Reset write stats
|
|
393
|
+
</button>
|
|
394
|
+
<button className={css.pad2(12, 8).button.bord2(0, 0, 20).hsl(0, 0, 100)}
|
|
395
|
+
onClick={() => {
|
|
396
|
+
StorageSynced(SocketFunction.browserNodeId()).getStorageBuckets.resetAll();
|
|
397
|
+
}}>
|
|
398
|
+
Refresh
|
|
399
|
+
</button>
|
|
400
|
+
</div>
|
|
401
|
+
<DeployNotices servers={servers} />
|
|
402
|
+
<Table
|
|
403
|
+
rows={getBucketRows(servers)}
|
|
404
|
+
columns={{
|
|
405
|
+
server: { title: "Server" },
|
|
406
|
+
bucket: { title: "Bucket" },
|
|
407
|
+
serverUrl: null,
|
|
408
|
+
state: {
|
|
409
|
+
title: "State",
|
|
410
|
+
formatter: (state, context) => {
|
|
411
|
+
let row = context?.row;
|
|
412
|
+
if (!row || !row.bucket || state !== INACTIVE_STATE) return state;
|
|
413
|
+
return <ActivateBucketButton serverUrl={row.serverUrl} bucketName={row.bucket} />;
|
|
414
|
+
}
|
|
415
|
+
},
|
|
416
|
+
files: { title: "Files" },
|
|
417
|
+
bytes: { title: "Bytes" },
|
|
418
|
+
indexSources: {
|
|
419
|
+
title: "Index sources",
|
|
420
|
+
formatter: sources => <IndexSourcesCell sources={sources} />
|
|
421
|
+
},
|
|
422
|
+
readerDiskLimit: { title: "Read cache limit" },
|
|
423
|
+
writes: { title: "Writes" },
|
|
424
|
+
written: { title: "Written" },
|
|
425
|
+
writeGain: { title: "Coalescing gain" },
|
|
426
|
+
disk: {
|
|
427
|
+
title: "Drive",
|
|
428
|
+
formatter: (disk, context) => {
|
|
429
|
+
if (!disk) return context?.row?.diskError || "";
|
|
430
|
+
return <UsageBar
|
|
431
|
+
label={DISK_BAR_TYPE}
|
|
432
|
+
value={disk.usedBytes}
|
|
433
|
+
max={disk.totalBytes}
|
|
434
|
+
{...getUsageThresholds(DISK_BAR_TYPE)}
|
|
435
|
+
/>;
|
|
436
|
+
}
|
|
437
|
+
},
|
|
438
|
+
diskError: null,
|
|
439
|
+
syncing: {
|
|
440
|
+
title: "Syncing",
|
|
441
|
+
formatter: syncing => <div className={css.vbox(2)}>
|
|
442
|
+
{(syncing || []).map((activity, activityIndex) => <div key={activityIndex}>
|
|
443
|
+
{activity.type} {activity.sourceDebugName}
|
|
444
|
+
{activity.totalFiles !== undefined && ` (${formatNumber(activity.doneFiles || 0)}/${formatNumber(activity.totalFiles)} files)`}
|
|
445
|
+
</div>)}
|
|
446
|
+
</div>
|
|
447
|
+
},
|
|
448
|
+
error: { title: "Error" },
|
|
449
|
+
}}
|
|
450
|
+
/>
|
|
451
|
+
<RouteConfigView servers={servers} />
|
|
452
|
+
</div>;
|
|
453
|
+
}
|
|
454
|
+
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { URLParam } from "../library-components/URLParam";
|
|
2
2
|
|
|
3
|
-
export type ViewType = "services" | "machines" | "service-detail" | "machine-detail" | "deploy";
|
|
3
|
+
export type ViewType = "services" | "machines" | "service-detail" | "machine-detail" | "deploy" | "storage";
|
|
4
4
|
|
|
5
5
|
export const currentViewParam = new URLParam<ViewType>("machineview", "machines");
|
|
6
6
|
export const selectedServiceIdParam = new URLParam("serviceId", "");
|
package/src/misc/random.ts
CHANGED