querysub 0.725.0 → 0.727.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/bin/backfill-log-categories.js +7 -0
  2. package/package.json +8 -2
  3. package/src/-a-archives/archiveCache.ts +7 -7
  4. package/src/-a-archives/archivesDisk.ts +20 -7
  5. package/src/0-path-value-core/AuthorityLookup.ts +113 -19
  6. package/src/deployManager/components/DeployPage.tsx +2 -2
  7. package/src/deployManager/components/ServiceDetailPage.tsx +17 -4
  8. package/src/deployManager/components/ServicesListPage.tsx +61 -8
  9. package/src/deployManager/components/deployButtons.tsx +166 -101
  10. package/src/deployManager/machineSchema.ts +13 -2
  11. package/src/deployManager/publishQuerysub.ts +121 -49
  12. package/src/deployManager/serviceCategories.ts +90 -1
  13. package/src/diagnostics/logs/IndexedLogs/BufferIndexLogsOptimizationConstants.ts +2 -0
  14. package/src/diagnostics/logs/IndexedLogs/FindProgressTracker.ts +2 -1
  15. package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts +40 -1
  16. package/src/diagnostics/logs/IndexedLogs/LogStreamer.ts +6 -2
  17. package/src/diagnostics/logs/IndexedLogs/MCPIndexedLogs.ts +2 -6
  18. package/src/diagnostics/logs/IndexedLogs/TimeFileTree.ts +34 -0
  19. package/src/diagnostics/logs/IndexedLogs/limitPublicLogsSize.ts +119 -0
  20. package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts +13 -5
  21. package/src/diagnostics/logs/IndexedLogs/moveLogsEntry.ts +2 -0
  22. package/src/diagnostics/logs/errorNotifications2/ErrorNotificationPage.tsx +93 -21
  23. package/src/diagnostics/logs/errorNotifications2/openRouterHelper.ts +4 -2
  24. package/src/diagnostics/logs/errorNotifications2/suppressionPatternMatcher.ts +1 -1
  25. package/src/diagnostics/logs/logCategories/LogBreakdownPage.tsx +421 -0
  26. package/src/diagnostics/logs/logCategories/logCategories.ts +273 -0
  27. package/src/diagnostics/managementPages.tsx +6 -0
  28. package/src/diagnostics/network/backfillLogCategoriesMain.ts +128 -0
  29. package/src/diagnostics/network/deleteOldLogsMain.ts +57 -0
  30. package/src/diagnostics/network/listLogFilesMain.ts +60 -0
  31. package/src/diagnostics/network/logFilesMain.ts +13 -15
  32. package/src/diagnostics/network/probeLogDeleteMain.ts +71 -0
  33. package/src/diagnostics/network/watchServiceMain.ts +106 -0
  34. package/src/fs.ts +29 -0
  35. package/src/misc/moveFolderContents.ts +67 -0
@@ -13,6 +13,7 @@ import { deepCloneJSON } from "socket-function/src/misc";
13
13
  import { unique } from "../../misc";
14
14
  import { CommitModal } from "./CommitModal";
15
15
  import { closeAllModals } from "../../5-diagnostics/Modal";
16
+ import { ServiceDeployGroup, getDeployGroupInfo, getServiceDeployGroup } from "../serviceCategories";
16
17
 
17
18
  module.hotreload = true;
18
19
 
@@ -72,62 +73,48 @@ export class RenderGitRefInfo extends qreact.Component<{
72
73
 
73
74
  export const buttonStyle = css.pad2(12, 8).button.bord2(0, 0, 20).fontWeight("bold").vbox(2).alignItems("center");
74
75
 
75
- /** Confirms a no-overlap deploy. It is worth interrupting for because the thing it buys is smaller than it looks: the old instances still get their shutdown while the new ones are up, so it trades minutes of overlap for seconds of it rather than removing overlap at all. */
76
- class NoOverlapConfirmModal extends qreact.Component<{ actionLabel: string; onConfirm: () => void }> {
77
- render() {
78
- return <div className={css.vbox(14).maxWidth(560)}>
79
- <div className={css.fontSize(18).boldStyle}>Shut the old instances down immediately?</div>
80
- <div>
81
- This does not give you a clean cut over. The new instances are already up before the old ones are told to stop, and stopping is not instant, so there is still a few seconds of overlap either way.
82
- </div>
83
- <div>
84
- Since the overlap cannot be avoided, deploying normally — and letting the old instances run for {formatTime(DEFAULT_OVERLAP_TIME)} — is usually the better choice.
85
- </div>
86
- <div className={css.hbox(10)}>
87
- <button
88
- className={buttonStyle.hsl(0, 85, 70)}
89
- onClick={() => {
90
- closeAllModals();
91
- this.props.onConfirm();
92
- }}
93
- >
94
- {this.props.actionLabel}
95
- </button>
96
- <button className={buttonStyle.hsl(0, 0, 90)} onClick={() => closeAllModals()}>
97
- Cancel
98
- </button>
99
- </div>
100
- </div>;
101
- }
102
- }
103
- export function confirmNoOverlapDeploy(actionLabel: string, onConfirm: () => void) {
104
- showFullscreenModal({
105
- content: <NoOverlapConfirmModal actionLabel={actionLabel} onConfirm={onConfirm} />,
106
- });
107
- }
108
-
109
76
  const CONFIRM_LIST_MAX_HEIGHT = 220;
110
77
 
111
- class DeployAllConfirmModal extends qreact.Component<{ titles: string[]; latestRef: string; onConfirm: () => void }> {
78
+ class DeployConfirmModal extends qreact.Component<{
79
+ actionLabel: string;
80
+ group: ServiceDeployGroup;
81
+ noOverlap: boolean;
82
+ services: { title: string; gitRef: string }[];
83
+ latestRef: string;
84
+ onConfirm: () => void;
85
+ }> {
112
86
  render() {
113
- let count = this.props.titles.length;
114
- return <div className={css.vbox(14).maxWidth(560)}>
115
- <div className={css.fontSize(18).boldStyle}>Deploy {count} service{count === 1 && "" || "s"}?</div>
116
- <div>
117
- Each one restarts onto {this.props.latestRef.slice(0, 6)}. The old instances keep running for {formatTime(DEFAULT_OVERLAP_TIME)} before shutting down, so there is somewhere for connections to go while clients move across.
118
- </div>
87
+ let { actionLabel, group, noOverlap, services, latestRef } = this.props;
88
+ let groupInfo = getDeployGroupInfo(group);
89
+ let warning = groupInfo.deployWarning;
90
+ let count = services.length;
91
+ return <div className={css.vbox(14).maxWidth(620)}>
92
+ {warning && <div className={css.vbox(8).pad2(12).fillWidth.bord2(groupInfo.hue, 80, 40, 2).hsl(groupInfo.hue, 80, 92)}>
93
+ <div className={css.fontSize(18).boldStyle}>⚠️ {groupInfo.emoji} {warning.title}</div>
94
+ {warning.paragraphs.map(paragraph => <div>{paragraph}</div>)}
95
+ </div>}
96
+ <div className={css.fontSize(18).boldStyle}>Deploy {count} {groupInfo.label} service{count === 1 ? "" : "s"} onto {latestRef.slice(0, 6)}?</div>
97
+ {noOverlap && <div>
98
+ The old instances are shut down as soon as the new ones are up. This does not give you a clean cut over: the new instances are already up before the old ones are told to stop, and stopping is not instant, so there is still a few seconds of overlap either way. Deploying normally, letting the old instances run for {formatTime(DEFAULT_OVERLAP_TIME)}, is usually the better choice.
99
+ </div>}
100
+ {!noOverlap && <div>
101
+ The old instances keep running for {formatTime(DEFAULT_OVERLAP_TIME)} before shutting down, so there is somewhere for connections to go while clients move across.
102
+ </div>}
119
103
  <div className={css.vbox(2).fillWidth.maxHeight(CONFIRM_LIST_MAX_HEIGHT).overflowAuto}>
120
- {this.props.titles.map(title => <div>{title}</div>)}
104
+ {services.map(service => <div className={css.hbox(8)}>
105
+ <span className={css.boldStyle}>{service.title}</span>
106
+ <span className={css.colorhsl(0, 0, 40)}>currently {service.gitRef.slice(0, 6)}</span>
107
+ </div>)}
121
108
  </div>
122
109
  <div className={css.hbox(10)}>
123
110
  <button
124
- className={buttonStyle.hsl(120, 70, 90)}
111
+ className={buttonStyle.hsl(noOverlap ? 0 : 120, 70, noOverlap ? 70 : 90)}
125
112
  onClick={() => {
126
113
  closeAllModals();
127
114
  this.props.onConfirm();
128
115
  }}
129
116
  >
130
- Deploy & Restart All ({count})
117
+ {actionLabel}
131
118
  </button>
132
119
  <button className={buttonStyle.hsl(0, 0, 90)} onClick={() => closeAllModals()}>
133
120
  Cancel
@@ -136,21 +123,44 @@ class DeployAllConfirmModal extends qreact.Component<{ titles: string[]; latestR
136
123
  </div>;
137
124
  }
138
125
  }
139
- export function confirmDeployAll(config: { titles: string[]; latestRef: string; onConfirm: () => void }) {
126
+
127
+ /** Every deploy goes through here, so the storage core and front-end proxy warnings can never be skipped. A plain overlapped deploy of a group with no warning only asks first when `alwaysConfirm` is set (the per category buttons, which deploy many services at once). */
128
+ export function confirmDeploy(config: {
129
+ actionLabel: string;
130
+ group: ServiceDeployGroup;
131
+ noOverlap: boolean;
132
+ services: ServiceConfig[];
133
+ latestRef: string;
134
+ alwaysConfirm?: boolean;
135
+ onConfirm: () => void;
136
+ }) {
137
+ let groupInfo = getDeployGroupInfo(config.group);
138
+ if (!groupInfo.deployWarning && (groupInfo.noOverlap || !config.noOverlap && !config.alwaysConfirm)) {
139
+ config.onConfirm();
140
+ return;
141
+ }
140
142
  showFullscreenModal({
141
- content: <DeployAllConfirmModal titles={config.titles} latestRef={config.latestRef} onConfirm={config.onConfirm} />,
143
+ content: <DeployConfirmModal
144
+ actionLabel={config.actionLabel}
145
+ group={config.group}
146
+ noOverlap={config.noOverlap}
147
+ services={config.services.map(service => ({ title: service.info.title, gitRef: service.parameters.gitRef }))}
148
+ latestRef={config.latestRef}
149
+ onConfirm={config.onConfirm}
150
+ />,
142
151
  });
143
152
  }
144
153
 
145
- export class UpdateButtons extends qreact.Component<{
154
+ /** The deploy buttons for one category, which deploy every outdated service in it. The services can be on different commits, so every distinct current commit is shown, with how many services are on it. */
155
+ export class DeployGroupButtons extends qreact.Component<{
156
+ group: ServiceDeployGroup;
146
157
  services: ServiceConfig[];
147
158
  }> {
148
159
  state = t.state({
149
160
  isDeploying: t.type(false),
150
161
  });
151
- private deployAll(outdatedServices: ServiceConfig[], latestRef: string, noOverlap: boolean) {
162
+ private deployGroup(outdatedServices: ServiceConfig[], latestRef: string, noOverlap: boolean) {
152
163
  this.state.isDeploying = true;
153
- // Props are synchronized state, so everything the async work needs is cloned into plain locals HERE, in the synced part
154
164
  let toDeploy = outdatedServices.map(service => {
155
165
  let updated = deepCloneJSON(service);
156
166
  updated.parameters.gitRef = latestRef;
@@ -172,12 +182,91 @@ export class UpdateButtons extends qreact.Component<{
172
182
  render() {
173
183
  let controller = MachineServiceController(SocketFunction.browserNodeId());
174
184
  const gitInfo = controller.getGitInfo();
175
-
176
- let outdatedServices = this.props.services.filter(service => {
185
+ let group = this.props.group;
186
+ let groupInfo = getDeployGroupInfo(group);
187
+ let allServices = this.props.services;
188
+ let outdatedServices = allServices.filter(service => {
177
189
  let { gitRef, repoUrl } = service.parameters;
178
190
  return repoUrl === gitInfo?.repoUrl && gitRef !== gitInfo?.latestRef;
179
191
  });
180
- let outdatedRefs = unique(outdatedServices.map(service => service.parameters.gitRef));
192
+ let refCounts = new Map<string, number>();
193
+ for (let service of outdatedServices) {
194
+ refCounts.set(service.parameters.gitRef, (refCounts.get(service.parameters.gitRef) || 0) + 1);
195
+ }
196
+ let isDeploying = this.state.isDeploying;
197
+ let label = <div className={css.vbox(2).minWidth(160)} title={groupInfo.description}>
198
+ <div className={css.fontSize(16).boldStyle}>{groupInfo.emoji} {groupInfo.label}</div>
199
+ <div className={css.fontSize(12).colorhsl(0, 0, 40)}>
200
+ {outdatedServices.length} of {allServices.length} service{allServices.length === 1 ? "" : "s"} outdated
201
+ </div>
202
+ </div>;
203
+ if (!gitInfo || outdatedServices.length === 0) {
204
+ return <div className={css.hbox(10).pad2(8).hsl(groupInfo.hue, 40, 95).bord2(groupInfo.hue, 40, 70)}>
205
+ {label}
206
+ {gitInfo && <span className={css.colorhsl(120, 50, 30).boldStyle}>✓ Up to date</span>}
207
+ </div>;
208
+ }
209
+ const latestRef = gitInfo.latestRef;
210
+ let noOverlapOnly = !!groupInfo.noOverlap;
211
+ return <div className={css.hbox(10).pad2(8).hsl(groupInfo.hue, 40, 95).bord2(groupInfo.hue, 40, 70)}>
212
+ {label}
213
+ <button
214
+ className={buttonStyle.hsl(120, 70, 90) + (isDeploying && css.opacity(0.6))}
215
+ title={noOverlapOnly
216
+ ? `Deploys every outdated ${groupInfo.label} service now; the old instances are shut down as soon as the new ones are up, so two copies never run at once`
217
+ : `Asks first, then deploys every outdated ${groupInfo.label} service; the old instances keep running for ${formatTime(DEFAULT_OVERLAP_TIME)} before shutting down`}
218
+ onClick={() => {
219
+ if (isDeploying) return;
220
+ confirmDeploy({
221
+ actionLabel: `Update ${groupInfo.label} to New & Restart (${outdatedServices.length})`,
222
+ group,
223
+ noOverlap: noOverlapOnly,
224
+ services: outdatedServices,
225
+ latestRef,
226
+ alwaysConfirm: !noOverlapOnly,
227
+ onConfirm: () => this.deployGroup(outdatedServices, latestRef, noOverlapOnly),
228
+ });
229
+ }}
230
+ >
231
+ <div>
232
+ {bigEmoji("🎯", -4)} <span>{isDeploying && "⏳ Deploying..." || `Update ${outdatedServices.length} to New${noOverlapOnly ? "" : " & Restart"}`}</span>
233
+ </div>
234
+ <div className={css.hbox(5)}>
235
+ <b>New</b>
236
+ <RenderGitRefInfo gitRef={latestRef} />
237
+ </div>
238
+ {[...refCounts.entries()].map(([ref, count]) => <div className={css.hbox(5)}>
239
+ <b>Current ({count} service{count === 1 ? "" : "s"})</b>
240
+ <RenderGitRefInfo gitRef={ref} />
241
+ </div>)}
242
+ </button>
243
+ {!noOverlapOnly && <button
244
+ className={buttonStyle.hsl(0, 85, 70).alignSelf("stretch") + (isDeploying && css.opacity(0.6))}
245
+ title={`Deploys every outdated ${groupInfo.label} service now AND shuts the old instances down as soon as the new ones are up (overlap of 0)`}
246
+ onClick={() => {
247
+ if (isDeploying) return;
248
+ confirmDeploy({
249
+ actionLabel: `Update ${groupInfo.label} to New, No Overlap (${outdatedServices.length})`,
250
+ group,
251
+ noOverlap: true,
252
+ services: outdatedServices,
253
+ latestRef,
254
+ onConfirm: () => this.deployGroup(outdatedServices, latestRef, true),
255
+ });
256
+ }}
257
+ >
258
+ <div>
259
+ {bigEmoji("⚡", -4)} <span>{isDeploying && "⏳ Deploying..." || `Update ${outdatedServices.length} to New, No Overlap`}</span>
260
+ </div>
261
+ </button>}
262
+ </div>;
263
+ }
264
+ }
265
+
266
+ export class CommitPushButton extends qreact.Component {
267
+ render() {
268
+ let controller = MachineServiceController(SocketFunction.browserNodeId());
269
+ const gitInfo = controller.getGitInfo();
181
270
 
182
271
  let querysubChanges = gitInfo?.querysubUncommitted ?? [];
183
272
  let serviceChanges = gitInfo?.uncommitted ?? [];
@@ -219,47 +308,6 @@ export class UpdateButtons extends qreact.Component<{
219
308
  <RenderGitRefInfo gitRef={gitInfo.querysubRef} isQuerysub />
220
309
  </div>}
221
310
  </button>}
222
-
223
- {outdatedServices.length > 0 && gitInfo && <>
224
- <button
225
- className={buttonStyle.hsl(120, 70, 90)}
226
- disabled={this.state.isDeploying}
227
- title="Asks first, then deploys; the old instances keep running for their configured overlap before shutting down"
228
- onClick={() => {
229
- confirmDeployAll({
230
- titles: outdatedServices.map(service => service.info.title),
231
- latestRef: gitInfo.latestRef,
232
- onConfirm: () => this.deployAll(outdatedServices, gitInfo.latestRef, false),
233
- });
234
- }}
235
- >
236
- <div>
237
- {bigEmoji("🎯")} <span>{this.state.isDeploying && "⏳ Deploying..." || `Deploy & Restart All (${outdatedServices.length}) Services`}</span>
238
- </div>
239
- <div className={css.hbox(5)}>
240
- <b>New</b>
241
- <RenderGitRefInfo gitRef={gitInfo.latestRef} />
242
- </div>
243
- {outdatedRefs.map(ref => <div className={css.hbox(5)}>
244
- <b>Current</b>
245
- <RenderGitRefInfo gitRef={ref} />
246
- </div>)}
247
- </button>
248
- <button
249
- className={buttonStyle.hsl(0, 85, 70)}
250
- disabled={this.state.isDeploying}
251
- title="Deploys now AND shuts the old instances down as soon as the new ones are up (overlap of 0)"
252
- onClick={() => {
253
- confirmNoOverlapDeploy(`Deploy All (${outdatedServices.length}), No Overlap`, () => {
254
- this.deployAll(outdatedServices, gitInfo.latestRef, true);
255
- });
256
- }}
257
- >
258
- <div>
259
- {bigEmoji("⚡")} <span>{this.state.isDeploying && "⏳ Deploying..." || `Deploy All (${outdatedServices.length}), No Overlap`}</span>
260
- </div>
261
- </button>
262
- </>}
263
311
  </>;
264
312
  }
265
313
  }
@@ -277,7 +325,6 @@ export class UpdateServiceButtons extends qreact.Component<{
277
325
  let service = deepCloneJSON(this.props.service);
278
326
  service.parameters.gitRef = latestRef;
279
327
  service.parameters.releaseTime = Date.now();
280
- // Always set, never inherited - see deployAll. A ternary, as 0 is the value we actually want here and it is falsy.
281
328
  service.parameters.overlapTime = noOverlap ? 0 : DEFAULT_OVERLAP_TIME;
282
329
  let controller = MachineServiceController(SocketFunction.browserNodeId());
283
330
  Querysub.onCommitFinished(async () => {
@@ -297,18 +344,31 @@ export class UpdateServiceButtons extends qreact.Component<{
297
344
  let repoUrl = this.props.service.parameters.repoUrl;
298
345
  if (gitInfo?.repoUrl !== repoUrl) return undefined;
299
346
  if (gitRef === gitInfo.latestRef) return undefined;
347
+ const service = this.props.service;
348
+ const group = getServiceDeployGroup(service);
349
+ const latestRef = gitInfo.latestRef;
350
+ let noOverlapOnly = !!getDeployGroupInfo(group).noOverlap;
300
351
 
301
352
  return <>
302
353
  <button
303
354
  className={buttonStyle.hsl(120, 70, 90).alignSelf("stretch")}
304
355
  disabled={this.state.isDeploying}
305
- title={`Deploys now; the old instances keep running for ${formatTime(DEFAULT_OVERLAP_TIME)} before shutting down`}
356
+ title={noOverlapOnly
357
+ ? "Deploys now; the old instance is shut down as soon as the new one is up, so two copies never run at once"
358
+ : `Deploys now; the old instances keep running for ${formatTime(DEFAULT_OVERLAP_TIME)} before shutting down`}
306
359
  onClick={() => {
307
- this.deploy(gitInfo.latestRef, false);
360
+ confirmDeploy({
361
+ actionLabel: "Update to New & Restart",
362
+ group,
363
+ noOverlap: noOverlapOnly,
364
+ services: [service],
365
+ latestRef,
366
+ onConfirm: () => this.deploy(latestRef, noOverlapOnly),
367
+ });
308
368
  }}
309
369
  >
310
370
  <div>
311
- {bigEmoji("🎯", -4)} <span>{this.state.isDeploying && "⏳ Deploying..." || "Update to New & Restart"}</span>
371
+ {bigEmoji("🎯", -4)} <span>{this.state.isDeploying && "⏳ Deploying..." || `Update to New${noOverlapOnly ? "" : " & Restart"}`}</span>
312
372
  </div>
313
373
  <div className={css.hbox(5)}>
314
374
  <b>New</b>
@@ -319,20 +379,25 @@ export class UpdateServiceButtons extends qreact.Component<{
319
379
  <RenderGitRefInfo gitRef={gitRef} />
320
380
  </div>
321
381
  </button>
322
- <button
382
+ {!noOverlapOnly && <button
323
383
  className={buttonStyle.hsl(0, 85, 70).alignSelf("stretch")}
324
384
  disabled={this.state.isDeploying}
325
385
  title="Deploys now AND shuts the old instances down as soon as the new ones are up (overlap of 0)"
326
386
  onClick={() => {
327
- confirmNoOverlapDeploy("Update to New, No Overlap", () => {
328
- this.deploy(gitInfo.latestRef, true);
387
+ confirmDeploy({
388
+ actionLabel: "Update to New, No Overlap",
389
+ group,
390
+ noOverlap: true,
391
+ services: [service],
392
+ latestRef,
393
+ onConfirm: () => this.deploy(latestRef, true),
329
394
  });
330
395
  }}
331
396
  >
332
397
  <div>
333
398
  {bigEmoji("⚡", -4)} <span>{this.state.isDeploying && "⏳ Deploying..." || "Update to New, No Overlap"}</span>
334
399
  </div>
335
- </button>
400
+ </button>}
336
401
  </>;
337
402
  }
338
403
  }
@@ -23,7 +23,8 @@ import { FunctionSpec, functionSchema } from "../3-path-functions/PathFunctionRu
23
23
  import { Querysub } from "../4-querysub/Querysub";
24
24
  import { green, red } from "socket-function/src/formatting/logColors";
25
25
  import { fsExistsAsync } from "../fs";
26
- import { publishQuerysubAsync, waitForPublishOtpAsync } from "./publishQuerysub";
26
+ import { publishQuerysubAsync } from "./publishQuerysub";
27
+ import type { ServiceDeployGroup } from "./serviceCategories";
27
28
 
28
29
  const SERVICE_FOLDER_NAME = "machine-services";
29
30
  export const SERVICE_FOLDER = `${SERVICE_FOLDER_NAME}/`;
@@ -127,6 +128,7 @@ export type ServiceConfig = {
127
128
  title: string;
128
129
  notes: string;
129
130
  lastUpdatedTime: number;
131
+ deployGroup?: ServiceDeployGroup;
130
132
  };
131
133
  };
132
134
 
@@ -427,6 +429,13 @@ export class MachineServiceControllerBase {
427
429
  void this.notifyMachines([...newMachines], [...oldMachines]);
428
430
  }
429
431
 
432
+ public async setServiceDeployGroup(config: { serviceId: string; deployGroup: ServiceDeployGroup }) {
433
+ let serviceConfig = await serviceConfigs.get(config.serviceId);
434
+ if (!serviceConfig) throw new Error(`Service ${config.serviceId} does not exist`);
435
+ serviceConfig.info.deployGroup = config.deployGroup;
436
+ await serviceConfigs.set(config.serviceId, serviceConfig);
437
+ }
438
+
430
439
  public async deleteServiceConfig(serviceId: string) {
431
440
  let serviceConfig = await serviceConfigs.get(serviceId);
432
441
 
@@ -522,7 +531,7 @@ export class MachineServiceControllerBase {
522
531
  throw new Error(`Querysub folder does not exist at ${querysubFolder}`);
523
532
  }
524
533
 
525
- let newVersion = await publishQuerysubAsync({ querysubFolder, getOtp: waitForPublishOtpAsync });
534
+ let newVersion = await publishQuerysubAsync({ querysubFolder });
526
535
 
527
536
  // Update package.json reference in our actual repo (version only, no install)
528
537
 
@@ -666,6 +675,7 @@ export const MachineServiceController = getSyncedController(
666
675
  setMachineName: {},
667
676
  addServiceConfig: {},
668
677
  setServiceConfigs: {},
678
+ setServiceDeployGroup: {},
669
679
  getServiceConfigType: {},
670
680
  deleteServiceConfig: {},
671
681
  getGitInfo: {},
@@ -691,6 +701,7 @@ export const MachineServiceController = getSyncedController(
691
701
  setMachineConfig: ["MachineConfig", "MachineConfigList"],
692
702
  addServiceConfig: ["ServiceConfig", "ServiceConfigList"],
693
703
  setServiceConfigs: ["ServiceConfig"],
704
+ setServiceDeployGroup: ["ServiceConfig"],
694
705
  deleteServiceConfig: ["ServiceConfig", "ServiceConfigList"],
695
706
  commitPushService: ["gitInfo", "ServiceConfig"],
696
707
  commitPushAndPublishQuerysub: ["gitInfo", "ServiceConfig"],
@@ -2,22 +2,22 @@ import child_process from "child_process";
2
2
  import fs from "fs";
3
3
  import os from "os";
4
4
  import path from "path";
5
- import readline from "readline";
6
5
  import { runPromise } from "socket-function/src/runPromise";
7
- import { magenta, yellow } from "socket-function/src/formatting/logColors";
6
+ import { blue, magenta, yellow } from "socket-function/src/formatting/logColors";
8
7
  import { delay } from "socket-function/src/batching";
9
- import { lazy } from "socket-function/src/caching";
8
+ import { httpsRequest, HttpsResponseInfo } from "socket-function/src/https";
9
+ import { formatTime } from "socket-function/src/formatting/format";
10
10
 
11
- export const getPublishOtpFile = lazy(() => path.join(os.tmpdir(), "querysub-publish-otp.txt"));
12
- const OTP_FILE_POLL_INTERVAL = 1000;
13
- const LOGIN_URL_PATTERN = /Open (https:\/\/www\.npmjs\.com\/login\/\S+)/;
11
+ const PUBLISH_REGISTRY = "https://registry.npmjs.org";
12
+ const NPMRC_TOKEN_PATTERN = /^\/\/registry\.npmjs\.org\/:_authToken=(.+)$/m;
13
+ const LOGIN_URL_PATTERN = /https:\/\/www\.npmjs\.com\/login\S*/;
14
+ const OTP_REQUIRED_PATTERN = /two.factor|one-time|\botp\b|user TTY/i;
14
15
  const PUBLISH_SUCCESS_TEXT = "success Published";
16
+ const WEB_OTP_POLL_FALLBACK = 2000;
17
+ const WEB_OTP_TIMEOUT = 1000 * 60 * 10;
18
+ const ERROR_BODY_LIMIT = 500;
15
19
 
16
- /** Bumps the minor version (unless the current one was never published), then publishes with yarn. npm requires a one-time code for every publish: yarn prints a login link, and the code that page hands out has to come back through getOtp. */
17
- export async function publishQuerysubAsync(config: {
18
- querysubFolder: string;
19
- getOtp: (loginUrl: string) => Promise<string>;
20
- }): Promise<string> {
20
+ export async function publishQuerysubAsync(config: { querysubFolder: string }): Promise<string> {
21
21
  let { querysubFolder } = config;
22
22
  let version = readVersion(querysubFolder);
23
23
  let published = (await runPromise(`npm view querysub@${version} version`, { quiet: true, nothrow: true })).trim();
@@ -25,19 +25,29 @@ export async function publishQuerysubAsync(config: {
25
25
  await runPromise("yarn version --minor --no-git-tag-version", { cwd: querysubFolder });
26
26
  version = readVersion(querysubFolder);
27
27
  }
28
- console.log(magenta(`Publishing querysub ${version}`));
29
28
 
30
- let output = await runPromise("yarn publish --non-interactive", { cwd: querysubFolder, nothrow: true, includeStderrInOutput: true });
29
+ let user = (await runPromise(`npm whoami --registry ${PUBLISH_REGISTRY}`, { quiet: true, nothrow: true })).trim();
30
+ if (!user) {
31
+ console.log(yellow(`Not logged in to npm (npm whoami against ${PUBLISH_REGISTRY} failed), logging in through the browser`));
32
+ await npmWebLoginAsync();
33
+ user = (await runPromise(`npm whoami --registry ${PUBLISH_REGISTRY}`, { quiet: true, nothrow: true })).trim();
34
+ if (!user) {
35
+ throw new Error(`npm login finished, but npm whoami against ${PUBLISH_REGISTRY} still fails`);
36
+ }
37
+ }
38
+ console.log(magenta(`Publishing querysub ${version} to ${PUBLISH_REGISTRY} as ${user}`));
39
+
40
+ let publishCommand = `yarn publish --non-interactive --registry ${PUBLISH_REGISTRY}`;
41
+ let output = await runPromise(publishCommand, { cwd: querysubFolder, nothrow: true, includeStderrInOutput: true });
31
42
  if (output.includes(PUBLISH_SUCCESS_TEXT)) return version;
32
- let loginUrl = LOGIN_URL_PATTERN.exec(output)?.[1];
33
- if (!loginUrl) {
34
- throw new Error(`yarn publish of querysub ${version} failed without giving a login link:\n${output}`);
43
+ if (!OTP_REQUIRED_PATTERN.test(output)) {
44
+ throw new Error(`yarn publish of querysub ${version} as ${user} failed, and didn't ask for a one-time code:\n${output}`);
35
45
  }
36
46
 
37
- let otp = await config.getOtp(loginUrl);
38
- output = await runPromise(`yarn publish --non-interactive --otp ${otp}`, { cwd: querysubFolder, nothrow: true, includeStderrInOutput: true });
47
+ let otp = await getWebOtpAsync();
48
+ output = await runPromise(`${publishCommand} --otp ${otp}`, { cwd: querysubFolder, nothrow: true, includeStderrInOutput: true });
39
49
  if (!output.includes(PUBLISH_SUCCESS_TEXT)) {
40
- throw new Error(`yarn publish of querysub ${version} with the one-time code failed:\n${output}`);
50
+ throw new Error(`yarn publish of querysub ${version} as ${user} with the one-time code from the browser failed:\n${output}`);
41
51
  }
42
52
  return version;
43
53
  }
@@ -46,39 +56,101 @@ function readVersion(querysubFolder: string): string {
46
56
  return String(JSON.parse(fs.readFileSync(path.join(querysubFolder, "package.json"), "utf8")).version);
47
57
  }
48
58
 
49
- /** Opens the login link, explains the LastPass passkey dance, and waits for the code: typed into this terminal, or written to getPublishOtpFile() (for when something other than a person is running the publish). */
50
- export async function waitForPublishOtpAsync(loginUrl: string): Promise<string> {
51
- let otpFile = getPublishOtpFile();
52
- await fs.promises.rm(otpFile, { force: true });
53
- openInBrowser(loginUrl);
54
- console.log(yellow(`npm needs a one-time code to publish. Opened ${loginUrl}`));
55
- console.log(yellow(`LastPass is NOT enabled on npmjs.com by default: click the LastPass icon, enable it for this site, then sign in with the passkey from LastPass. Copy the code the page gives you.`));
56
- console.log(yellow(`Type the code here and press enter, or write it to ${otpFile}`));
59
+ async function getWebOtpAsync(): Promise<string> {
60
+ let npmrcPath = path.join(os.homedir(), ".npmrc");
61
+ let token = NPMRC_TOKEN_PATTERN.exec(await fs.promises.readFile(npmrcPath, "utf8"))?.[1]?.trim();
62
+ if (!token) {
63
+ throw new Error(`Expected an auth token for ${PUBLISH_REGISTRY} in ${npmrcPath} after logging in, but there wasn't one`);
64
+ }
65
+ let latest = (await runPromise(`npm view querysub dist-tags.latest`, { quiet: true })).trim();
66
+ if (!latest) {
67
+ throw new Error(`Expected npm view querysub dist-tags.latest to give a version, was empty`);
68
+ }
69
+
70
+ let npmVersion = (await runPromise(`npm --version`, { quiet: true })).trim();
71
+ let userAgent = `npm/${npmVersion} node/${process.version} ${process.platform} ${process.arch} workspaces/false`;
57
72
 
58
- let done = false;
59
- let fromTerminal = new Promise<string>(resolve => {
60
- if (!process.stdin.isTTY) return;
61
- let reader = readline.createInterface({ input: process.stdin, output: process.stdout });
62
- reader.question("One-time code: ", answer => {
63
- reader.close();
64
- resolve(answer);
65
- });
66
- });
67
- let fromFile = (async () => {
68
- while (!done) {
69
- let text = await fs.promises.readFile(otpFile, "utf8").catch(() => "");
70
- if (text.trim()) return text;
71
- await delay(OTP_FILE_POLL_INTERVAL);
73
+ let probeUrl = `${PUBLISH_REGISTRY}/-/package/querysub/dist-tags/latest`;
74
+ let probe = await registryRequestAsync({ url: probeUrl, method: "PUT", token, userAgent, payload: JSON.stringify(latest) });
75
+ if (probe.statusCode !== 401) {
76
+ throw new Error(`Expected setting dist-tag latest to its current ${latest} to require two-factor (401), was ${probe.statusCode}: ${probe.body.slice(0, ERROR_BODY_LIMIT)}`);
77
+ }
78
+ let { authUrl, doneUrl } = JSON.parse(probe.body) as { authUrl?: string; doneUrl?: string };
79
+ if (!authUrl || !doneUrl) {
80
+ throw new Error(`Expected the two-factor response from ${probeUrl} to have authUrl and doneUrl, was: ${probe.body.slice(0, ERROR_BODY_LIMIT)}`);
81
+ }
82
+
83
+ openInBrowser(authUrl);
84
+ console.log(yellow(`Opened ${authUrl} to approve the publish`));
85
+ let deadline = Date.now() + WEB_OTP_TIMEOUT;
86
+ while (Date.now() < deadline) {
87
+ let done = await registryRequestAsync({ url: doneUrl, method: "GET", token, userAgent });
88
+ if (done.statusCode === 200) {
89
+ let otp = (JSON.parse(done.body) as { token?: string }).token;
90
+ if (!otp) {
91
+ throw new Error(`Expected ${doneUrl} to return a token once approved, was: ${done.body.slice(0, ERROR_BODY_LIMIT)}`);
92
+ }
93
+ console.log(magenta(`Publish approved in the browser`));
94
+ return otp;
72
95
  }
73
- return "";
74
- })();
75
- let code = (await Promise.race([fromTerminal, fromFile])).trim();
76
- done = true;
77
- await fs.promises.rm(otpFile, { force: true });
78
- if (!/^\w+$/.test(code)) {
79
- throw new Error(`Expected a one-time code of letters and digits, was ${JSON.stringify(code)}`);
96
+ if (done.statusCode !== 202) {
97
+ throw new Error(`Expected 200 or 202 polling ${doneUrl}, was ${done.statusCode}: ${done.body.slice(0, ERROR_BODY_LIMIT)}`);
98
+ }
99
+ let retryAfter = Number(done.headers["retry-after"]) * 1000;
100
+ await delay(retryAfter > 0 ? retryAfter : WEB_OTP_POLL_FALLBACK);
101
+ }
102
+ throw new Error(`The publish wasn't approved at ${authUrl} within ${formatTime(WEB_OTP_TIMEOUT)}`);
103
+ }
104
+
105
+ async function registryRequestAsync(config: { url: string; method: string; token: string; userAgent: string; payload?: string }) {
106
+ let response: HttpsResponseInfo = { headers: {} };
107
+ let headers = {
108
+ authorization: `Bearer ${config.token}`,
109
+ "content-type": "application/json",
110
+ "user-agent": config.userAgent,
111
+ "npm-auth-type": "web",
112
+ "npm-command": "dist-tag",
113
+ };
114
+ try {
115
+ let body = await httpsRequest(config.url, config.payload ? Buffer.from(config.payload) : undefined, config.method, false, { headers, outResponse: response });
116
+ return { statusCode: response.statusCode || 0, headers: response.headers, body: body.toString() };
117
+ } catch (e) {
118
+ if (!response.statusCode) throw e;
119
+ let message = String((e as Error).message);
120
+ return { statusCode: response.statusCode, headers: response.headers, body: message.slice(message.indexOf("\n") + 1) };
121
+ }
122
+ }
123
+
124
+ async function npmWebLoginAsync() {
125
+ let command = `npm login --auth-type=web --registry ${PUBLISH_REGISTRY}`;
126
+ console.log(">" + blue(command));
127
+ let child = child_process.spawn(command, { shell: true, stdio: ["pipe", "pipe", "pipe"] });
128
+ let output = "";
129
+ let opened = false;
130
+ let onData = (data: Buffer) => {
131
+ let chunk = data.toString();
132
+ output += chunk;
133
+ process.stdout.write(chunk);
134
+ if (opened) return;
135
+ let loginUrl = LOGIN_URL_PATTERN.exec(output)?.[0];
136
+ if (!loginUrl) return;
137
+ opened = true;
138
+ openInBrowser(loginUrl);
139
+ console.log(yellow(`Opened ${loginUrl}`));
140
+ console.log(yellow(`LastPass is NOT enabled on npmjs.com by default: click the LastPass icon, enable it for this site, then sign in with the passkey from LastPass. The login finishes on its own once the page says you're logged in.`));
141
+ };
142
+ child.stdout.on("data", onData);
143
+ child.stderr.on("data", onData);
144
+ let code = await new Promise<number | null>((resolve, reject) => {
145
+ child.on("error", reject);
146
+ child.on("close", resolve);
147
+ });
148
+ if (code !== 0) {
149
+ throw new Error(`${command} exited with code ${code}:\n${output}`);
150
+ }
151
+ if (!opened) {
152
+ throw new Error(`${command} succeeded without ever printing a login link matching ${LOGIN_URL_PATTERN}:\n${output}`);
80
153
  }
81
- return code;
82
154
  }
83
155
 
84
156
  function openInBrowser(url: string) {