querysub 0.727.0 → 0.729.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/debugging.md CHANGED
@@ -4,6 +4,19 @@ When the site won't load, the debug pages are unreachable, or nodes keep dying o
4
4
 
5
5
  Every script takes `--cwd <project root>` so it runs with that project's domain, keys, and certs (for the cyoa network: `--cwd D:/repos/qs-cyoa/`). Both scripts are read-only, except that, like every server-mode querysub process, they run the node-discovery audit, which deletes node list entries whose port has provably been taken by a newer thread.
6
6
 
7
+ ## The processes
8
+
9
+ Every service runs as its own process in its own tmux session, `<service key>-<instance index>-dply`, on the machines its config lists. `overview.md` explains the roles; these are the processes that implement them, in the order a request flows through them:
10
+
11
+ - **bootstrapper** (`bootstrapper-0-dply`, port 443 behind Cloudflare): the front-end proxy. It serves the JavaScript payload that boots the site and nothing else, so it rarely needs updating. It binds a fixed port, so it must be deployed with no overlap: if the old instance still holds 443 when the new one starts, the new one fails with `Port 443 is already in use`, never retries, and the site is down once the old one exits.
12
+ - **cyoa** (`cyoa-N-dply`): the client-facing querysub server. The browser connects to it; it handles the front end, security and login, and adds the client's function calls as writes. Several instances run per machine and clients pick the closest.
13
+ - **serverall** (`serverall-N-dply`): the path value servers. This is where the data lives. Each instance owns a slice of the paths (see sharding in `overview.md`), and every other process reads and writes data through them. If these are unhealthy, everything reads as missing or stale.
14
+ - **shardall** (`shardall-N-dply`): the function runners. They read the function call writes and run the functions against the data in serverall. "The function runner isn't receiving data" means shardall cannot reach or route to serverall.
15
+ - **storageserve** (`storageserve-0-dply`, ports 5234 and 5235): the storage core, the archive store behind the node list, service configs, trust, logs and everything else that is not a live path value. Everything depends on it and it is deployed separately and rarely.
16
+ - **machine-alwaysup**: each machine's deploy process. It re-reads the service configs, launches and verifies the instances, and writes the machine heartbeat. A stale machine heartbeat with services still running means this process is stuck, not the machine.
17
+
18
+ The rest are background tasks: `movelogs` (moves pending logs to public storage and categorizes them), `errorwatch`, `gc`, `thin`, `join`, `ssh-trust`, `ping`, and the app's `queue` and `gpuworker` workers. They can go down without taking the site down.
19
+
7
20
  ## 1. What nodes exist
8
21
 
9
22
  ```
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "querysub",
3
- "version": "0.727.0",
3
+ "version": "0.729.0",
4
4
  "main": "index.js",
5
5
  "license": "MIT",
6
6
  "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -40,6 +40,7 @@
40
40
  "watch-service": "yarn typenode ./src/diagnostics/network/watchServiceMain.ts",
41
41
  "remote-flushlogs": "yarn typenode ./src/diagnostics/network/remoteFlushLogsMain.ts",
42
42
  "publish-querysub": "yarn typenode ./src/deployManager/publishQuerysubMain.ts",
43
+ "web-otp-claude": "yarn typenode ./src/deployManager/webOtpMain.ts",
43
44
  "parameters-timeline": "yarn typenode ./src/deployManager/parametersTimelineMain.ts",
44
45
  "setup-machine": "yarn typenode ./src/deployManager/setupMachineMain.ts",
45
46
  "stop-machine": "yarn typenode ./src/deployManager/stopMachineMain.ts",
@@ -95,8 +96,8 @@
95
96
  "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
96
97
  "pako": "^2.1.0",
97
98
  "peggy": "^5.0.6",
98
- "sliftutils": "^1.9.20",
99
- "socket-function": "^1.3.1",
99
+ "sliftutils": "1.9.22",
100
+ "socket-function": "1.3.13",
100
101
  "terser": "^5.31.0",
101
102
  "typenode": "^6.6.3",
102
103
  "typesafecss": "^0.32.0",
@@ -65,6 +65,33 @@ export function getArchiveCachePath(archives: CacheKeySource, key: string): stri
65
65
  return getCacheDir() + hash.slice(0, 16) + "." + name.slice(0, 128) + CACHE_SUFFIX;
66
66
  }
67
67
 
68
+ /** Serves an immutable cache hit straight from disk, without waiting for the cache folder scan that getDiskMetrics does on first use. Misses (and empty files, which are deletions) return undefined so the caller falls back to the full path. */
69
+ export async function readCacheFileFast(
70
+ archives: CacheKeySource,
71
+ key: string,
72
+ config?: { range?: { start: number; end: number } }
73
+ ): Promise<Buffer | undefined> {
74
+ let path = getArchiveCachePath(archives, key);
75
+ let handle: fs.promises.FileHandle | undefined;
76
+ try {
77
+ handle = await fs.promises.open(path, "r");
78
+ let stats = await handle.stat();
79
+ if (stats.size === 0) return undefined;
80
+ let start = config?.range?.start ?? 0;
81
+ let end = config?.range?.end ?? stats.size;
82
+ let readBuffer = Buffer.alloc(end - start);
83
+ let read = await handle.read(readBuffer, 0, end - start, start);
84
+ bumpAtimeIfStale(path);
85
+ return readBuffer.slice(0, read.bytesRead);
86
+ } catch {
87
+ return undefined;
88
+ } finally {
89
+ if (handle) {
90
+ await handle.close();
91
+ }
92
+ }
93
+ }
94
+
68
95
  const getDiskMetricsBase = async () => {
69
96
  await moveCacheDir();
70
97
  await fs.promises.mkdir(getCacheDir(), { recursive: true });
@@ -1,6 +1,6 @@
1
1
  import fs from "fs";
2
2
  import { GetConfig, IArchives, SetConfig } from "sliftutils/storage/IArchives";
3
- import { getArchiveCachePath, getDiskMetrics, getTempFilePath, LARGE_FILE_CHUNK } from "./archiveCache";
3
+ import { getArchiveCachePath, getDiskMetrics, getTempFilePath, LARGE_FILE_CHUNK, readCacheFileFast } from "./archiveCache";
4
4
 
5
5
  let cacheArchives2Symbol = Symbol("cacheArchives2");
6
6
 
@@ -89,6 +89,10 @@ export function wrapArchivesWithCache2(archives: IArchives, rootConfig?: {
89
89
  getSyncStatus: archives.getSyncStatus && (() => archives.getSyncStatus!()),
90
90
 
91
91
  get: async (fileName: string, config?: GetConfig) => {
92
+ if (rootConfig?.immutable) {
93
+ let fast = await readCacheFileFast(archives, fileName, config);
94
+ if (fast) return fast;
95
+ }
92
96
  let metrics = await getDiskMetrics();
93
97
  // Get info to at least see if it is removed or not. This makes sure we don't get in
94
98
  // a bad state, at least for immutable files (for mutable files we don't have an easy
@@ -140,7 +140,5 @@ export const LatencyController = SocketFunction.register(
140
140
  );
141
141
 
142
142
  if (isServer()) {
143
- setImmediate(() => {
144
- logErrors(startLatencyTracking());
145
- });
143
+ void SocketFunction.mountPromise.then(() => logErrors(startLatencyTracking()));
146
144
  }
@@ -17,7 +17,8 @@ import { getScheduledShutdownTime, watchScheduledShutdown } from "../-g-core-val
17
17
  setImmediate(() => import("../3-path-functions/syncSchema"));
18
18
 
19
19
  let NETWORK_POLL_INTERVAL = timeInMinute * 1;
20
- let CALL_TIMEOUT = timeInSecond * 5;
20
+ let INITIAL_CALL_TIMEOUT = timeInSecond * 5;
21
+ const CONNECTED_CALL_TIMEOUT = timeInMinute;
21
22
 
22
23
  // Every node's per-node poll is created together at discovery, so without jitter all of their
23
24
  // networkSyncPaths calls fire in lockstep each NETWORK_POLL_INTERVAL — and when many nodes are
@@ -162,6 +163,7 @@ class AuthorityLookup {
162
163
  scheduledShutdownTime?: number;
163
164
  }) {
164
165
  let { nodeId, spec, specHash, isReady, scheduledShutdownTime } = config;
166
+ this.syncedNodeIds.add(nodeId);
165
167
  let prevEntry = this.topology.nodes.get(nodeId);
166
168
  let specUnchanged = !!prevEntry && (prevEntry.authoritySpec === spec || JSON.stringify(prevEntry.authoritySpec) === JSON.stringify(spec));
167
169
  let entry: AuthorityEntry = {
@@ -248,7 +250,7 @@ class AuthorityLookup {
248
250
  return;
249
251
  }
250
252
  let full = await timeoutToError(
251
- CALL_TIMEOUT,
253
+ this.getCallTimeout(nodeId),
252
254
  AuthorityLookupController.nodes[nodeId].getAuthoritySpec(),
253
255
  () => new Error(`Timeout calling getAuthoritySpec for ${nodeId}`)
254
256
  );
@@ -256,6 +258,11 @@ class AuthorityLookup {
256
258
  }
257
259
 
258
260
  private legacySyncNodeIds = new Set<string>();
261
+ private syncedNodeIds = new Set<string>();
262
+
263
+ private getCallTimeout(nodeId: string): number {
264
+ return this.syncedNodeIds.has(nodeId) ? CONNECTED_CALL_TIMEOUT : INITIAL_CALL_TIMEOUT;
265
+ }
259
266
 
260
267
  private connectTo = cache((nodeId: string) => {
261
268
  let stopObj = { stop: false };
@@ -272,6 +279,7 @@ class AuthorityLookup {
272
279
  this.removeNode(nodeId, "node removed from node discovery");
273
280
  stopObj.stop = true;
274
281
  this.legacySyncNodeIds.delete(nodeId);
282
+ this.syncedNodeIds.delete(nodeId);
275
283
  this.connectTo.clear(nodeId);
276
284
  this.disconnectWatch.clear(nodeId);
277
285
  };
@@ -339,7 +347,7 @@ class AuthorityLookup {
339
347
  let other: AuthorityHashSync;
340
348
  try {
341
349
  other = await timeoutToError(
342
- CALL_TIMEOUT,
350
+ this.getCallTimeout(nodeId),
343
351
  AuthorityLookupController.nodes[nodeId].getSetNetworkSyncHash(this.getOurHashSync()),
344
352
  () => new Error(`Timeout calling getSetNetworkSyncHash for ${nodeId}`)
345
353
  );
@@ -356,7 +364,7 @@ class AuthorityLookup {
356
364
 
357
365
  private async syncNodeLegacy(nodeId: string) {
358
366
  let otherPaths = await timeoutToError(
359
- CALL_TIMEOUT,
367
+ this.getCallTimeout(nodeId),
360
368
  AuthorityLookupController.nodes[nodeId].networkSyncPaths({
361
369
  spec: this.ourSpec,
362
370
  isReady: this.ourIsReady,
@@ -462,6 +462,7 @@ class PathValueCommitter {
462
462
  }
463
463
 
464
464
  export const pathValueCommitter = new PathValueCommitter();
465
+ (globalThis as any).pathValueCommitter = pathValueCommitter;
465
466
 
466
467
  registerShutdownHandler(async () => {
467
468
  await pathValueCommitter.waitForValuesToCommit();
@@ -89,12 +89,15 @@ class DeployConfirmModal extends qreact.Component<{
89
89
  let warning = groupInfo.deployWarning;
90
90
  let count = services.length;
91
91
  return <div className={css.vbox(14).maxWidth(620)}>
92
- {warning && <div className={css.vbox(8).pad2(12).fillWidth.bord2(groupInfo.hue, 80, 40, 2).hsl(groupInfo.hue, 80, 92)}>
93
- <div className={css.fontSize(18).boldStyle}>⚠️ {groupInfo.emoji} {warning.title}</div>
92
+ {warning && <div className={css.vbox(8).pad2(12).fillWidth.bord2(groupInfo.hue, 70, 50, 2).hsl(groupInfo.hue, 50, 18)}>
93
+ <div className={css.fontSize(18).boldStyle.colorhsl(groupInfo.hue, 90, 75)}>⚠️ {groupInfo.emoji} {warning.title}</div>
94
94
  {warning.paragraphs.map(paragraph => <div>{paragraph}</div>)}
95
95
  </div>}
96
96
  <div className={css.fontSize(18).boldStyle}>Deploy {count} {groupInfo.label} service{count === 1 ? "" : "s"} onto {latestRef.slice(0, 6)}?</div>
97
- {noOverlap && <div>
97
+ {groupInfo.noOverlap && <div>
98
+ {groupInfo.label} services are always deployed without overlap: the old instances are shut down as soon as the new ones are up, because two copies must never run at once.
99
+ </div>}
100
+ {!groupInfo.noOverlap && noOverlap && <div>
98
101
  The old instances are shut down as soon as the new ones are up. This does not give you a clean cut over: the new instances are already up before the old ones are told to stop, and stopping is not instant, so there is still a few seconds of overlap either way. Deploying normally, letting the old instances run for {formatTime(DEFAULT_OVERLAP_TIME)}, is usually the better choice.
99
102
  </div>}
100
103
  {!noOverlap && <div>
@@ -103,12 +106,12 @@ class DeployConfirmModal extends qreact.Component<{
103
106
  <div className={css.vbox(2).fillWidth.maxHeight(CONFIRM_LIST_MAX_HEIGHT).overflowAuto}>
104
107
  {services.map(service => <div className={css.hbox(8)}>
105
108
  <span className={css.boldStyle}>{service.title}</span>
106
- <span className={css.colorhsl(0, 0, 40)}>currently {service.gitRef.slice(0, 6)}</span>
109
+ <span className={css.colorhsl(0, 0, 65)}>currently {service.gitRef.slice(0, 6)}</span>
107
110
  </div>)}
108
111
  </div>
109
112
  <div className={css.hbox(10)}>
110
113
  <button
111
- className={buttonStyle.hsl(noOverlap ? 0 : 120, 70, noOverlap ? 70 : 90)}
114
+ className={buttonStyle.hsl(noOverlap && !groupInfo.noOverlap ? 0 : 120, 70, noOverlap && !groupInfo.noOverlap ? 70 : 90)}
112
115
  onClick={() => {
113
116
  closeAllModals();
114
117
  this.props.onConfirm();
@@ -211,14 +214,14 @@ export class DeployGroupButtons extends qreact.Component<{
211
214
  return <div className={css.hbox(10).pad2(8).hsl(groupInfo.hue, 40, 95).bord2(groupInfo.hue, 40, 70)}>
212
215
  {label}
213
216
  <button
214
- className={buttonStyle.hsl(120, 70, 90) + (isDeploying && css.opacity(0.6))}
217
+ className={buttonStyle.hsl(noOverlapOnly ? 210 : 120, 70, 90) + (isDeploying && css.opacity(0.6))}
215
218
  title={noOverlapOnly
216
219
  ? `Deploys every outdated ${groupInfo.label} service now; the old instances are shut down as soon as the new ones are up, so two copies never run at once`
217
220
  : `Asks first, then deploys every outdated ${groupInfo.label} service; the old instances keep running for ${formatTime(DEFAULT_OVERLAP_TIME)} before shutting down`}
218
221
  onClick={() => {
219
222
  if (isDeploying) return;
220
223
  confirmDeploy({
221
- actionLabel: `Update ${groupInfo.label} to New & Restart (${outdatedServices.length})`,
224
+ actionLabel: `Update ${groupInfo.label} to New${noOverlapOnly ? ", No Overlap" : " & Restart"} (${outdatedServices.length})`,
222
225
  group,
223
226
  noOverlap: noOverlapOnly,
224
227
  services: outdatedServices,
@@ -229,7 +232,7 @@ export class DeployGroupButtons extends qreact.Component<{
229
232
  }}
230
233
  >
231
234
  <div>
232
- {bigEmoji("🎯", -4)} <span>{isDeploying && "⏳ Deploying..." || `Update ${outdatedServices.length} to New${noOverlapOnly ? "" : " & Restart"}`}</span>
235
+ {bigEmoji(noOverlapOnly ? "⚡" : "🎯", -4)} <span>{isDeploying && "⏳ Deploying..." || `Update ${outdatedServices.length} to New${noOverlapOnly ? ", No Overlap" : " & Restart"}`}</span>
233
236
  </div>
234
237
  <div className={css.hbox(5)}>
235
238
  <b>New</b>
@@ -351,14 +354,14 @@ export class UpdateServiceButtons extends qreact.Component<{
351
354
 
352
355
  return <>
353
356
  <button
354
- className={buttonStyle.hsl(120, 70, 90).alignSelf("stretch")}
357
+ className={buttonStyle.hsl(noOverlapOnly ? 210 : 120, 70, 90).alignSelf("stretch")}
355
358
  disabled={this.state.isDeploying}
356
359
  title={noOverlapOnly
357
360
  ? "Deploys now; the old instance is shut down as soon as the new one is up, so two copies never run at once"
358
361
  : `Deploys now; the old instances keep running for ${formatTime(DEFAULT_OVERLAP_TIME)} before shutting down`}
359
362
  onClick={() => {
360
363
  confirmDeploy({
361
- actionLabel: "Update to New & Restart",
364
+ actionLabel: noOverlapOnly ? "Update to New, No Overlap" : "Update to New & Restart",
362
365
  group,
363
366
  noOverlap: noOverlapOnly,
364
367
  services: [service],
@@ -368,7 +371,7 @@ export class UpdateServiceButtons extends qreact.Component<{
368
371
  }}
369
372
  >
370
373
  <div>
371
- {bigEmoji("🎯", -4)} <span>{this.state.isDeploying && "⏳ Deploying..." || `Update to New${noOverlapOnly ? "" : " & Restart"}`}</span>
374
+ {bigEmoji(noOverlapOnly ? "⚡" : "🎯", -4)} <span>{this.state.isDeploying && "⏳ Deploying..." || `Update to New${noOverlapOnly ? ", No Overlap" : " & Restart"}`}</span>
372
375
  </div>
373
376
  <div className={css.hbox(5)}>
374
377
  <b>New</b>
@@ -13,7 +13,7 @@ import { forceRemoveNode } from "../-f-node-discovery/NodeDiscovery";
13
13
  import { fsExistsAsync } from "../fs";
14
14
  import { PromiseObj } from "../promise";
15
15
  import { SERVICE_FOLDER, SERVICE_NODE_FILE_NAME } from "./machineSchema";
16
- import { ProcessRecord, getLogPipeScript, getLogTailScript, getProcessLogPath, getPipeScriptPath, getTailScriptPath, findProcessRecordByPid, writeProcessRecord, markProcessDead } from "./processLogs";
16
+ import { ProcessRecord, getLogPipeScript, getLogTailScript, getProcessLogPath, getPipeScriptPath, getTailScriptPath, findProcessRecordByPid, writeProcessRecord, markProcessDead, listProcessRecords } from "./processLogs";
17
17
  import { setPreciseTimeout } from "../misc";
18
18
 
19
19
  // Running, inspecting and killing the tmux screens services run in. This layer only knows "here is a configuration, run it" - which version should be running when is the deploy logic's problem.
@@ -48,7 +48,13 @@ export const retireScreen = cache(async (screenName: string): Promise<void> => {
48
48
  console.log(green(`${screenName} is retiring, and is due to be killed at ${new Date(dueAt).toISOString()}`));
49
49
  await new Promise<void>(resolve => setPreciseTimeout({ time: dueAt, callback: () => resolve() }));
50
50
  }
51
+ let records = (await listProcessRecords()).filter(record => record.screenName === screenName && record.deadTime === undefined);
51
52
  await killScreen({ screenName });
53
+ for (let record of records) {
54
+ if (!record.nodeId) continue;
55
+ console.log(green(`Removing the node registration ${record.nodeId} of the killed retiring screen ${screenName}`));
56
+ await forceRemoveNode(record.nodeId);
57
+ }
52
58
  } finally {
53
59
  // Dropped either way: a failure should be retried by the next resync rather than remembered as done
54
60
  retireScreen.clear(screenName);
@@ -340,17 +346,11 @@ export async function retireCurrentScreen(config: {
340
346
  let outgoingPid = await getScreenPanePid(canonicalScreenName);
341
347
  if (!outgoingPid) return;
342
348
  let retiringScreenName = getRetiringScreenName(canonicalScreenName, killTime);
343
- console.log(green(`Retiring ${canonicalScreenName} as ${retiringScreenName}`));
349
+ console.log(green(`Retiring ${canonicalScreenName} as ${retiringScreenName}, it keeps its node registration ${nodeId || "(unknown)"} until it is killed at ${new Date(killTime).toISOString()}`));
344
350
  await runPromise(`${getTmuxPrefix()}tmux rename-session -t ${canonicalScreenName} ${retiringScreenName}`);
345
- void (async () => {
346
- let record = await findProcessRecordByPid(folder, outgoingPid);
347
- if (record) {
348
- await writeProcessRecord({ ...record, screenName: retiringScreenName });
349
- }
350
- })();
351
- if (nodeId) {
352
- // The file in the folder belongs to its replacement now, so only the registration goes
353
- void removeServiceNode({ folder, nodeId, ownsFile: false });
351
+ let record = await findProcessRecordByPid(folder, outgoingPid);
352
+ if (record) {
353
+ await writeProcessRecord({ ...record, screenName: retiringScreenName, nodeId: nodeId || record.nodeId });
354
354
  }
355
355
  // Started here rather than waiting for a resync to notice the screen: with a killTime already past - a zero overlap - this takes it down right now
356
356
  void retireScreen(retiringScreenName);
@@ -44,7 +44,7 @@ export async function publishQuerysubAsync(config: { querysubFolder: string }):
44
44
  throw new Error(`yarn publish of querysub ${version} as ${user} failed, and didn't ask for a one-time code:\n${output}`);
45
45
  }
46
46
 
47
- let otp = await getWebOtpAsync();
47
+ let otp = await getWebOtpAsync("querysub");
48
48
  output = await runPromise(`${publishCommand} --otp ${otp}`, { cwd: querysubFolder, nothrow: true, includeStderrInOutput: true });
49
49
  if (!output.includes(PUBLISH_SUCCESS_TEXT)) {
50
50
  throw new Error(`yarn publish of querysub ${version} as ${user} with the one-time code from the browser failed:\n${output}`);
@@ -56,21 +56,21 @@ function readVersion(querysubFolder: string): string {
56
56
  return String(JSON.parse(fs.readFileSync(path.join(querysubFolder, "package.json"), "utf8")).version);
57
57
  }
58
58
 
59
- async function getWebOtpAsync(): Promise<string> {
59
+ export async function getWebOtpAsync(packageName: string): Promise<string> {
60
60
  let npmrcPath = path.join(os.homedir(), ".npmrc");
61
61
  let token = NPMRC_TOKEN_PATTERN.exec(await fs.promises.readFile(npmrcPath, "utf8"))?.[1]?.trim();
62
62
  if (!token) {
63
63
  throw new Error(`Expected an auth token for ${PUBLISH_REGISTRY} in ${npmrcPath} after logging in, but there wasn't one`);
64
64
  }
65
- let latest = (await runPromise(`npm view querysub dist-tags.latest`, { quiet: true })).trim();
65
+ let latest = (await runPromise(`npm view ${packageName} dist-tags.latest`, { quiet: true })).trim();
66
66
  if (!latest) {
67
- throw new Error(`Expected npm view querysub dist-tags.latest to give a version, was empty`);
67
+ throw new Error(`Expected npm view ${packageName} dist-tags.latest to give a version, was empty`);
68
68
  }
69
69
 
70
70
  let npmVersion = (await runPromise(`npm --version`, { quiet: true })).trim();
71
71
  let userAgent = `npm/${npmVersion} node/${process.version} ${process.platform} ${process.arch} workspaces/false`;
72
72
 
73
- let probeUrl = `${PUBLISH_REGISTRY}/-/package/querysub/dist-tags/latest`;
73
+ let probeUrl = `${PUBLISH_REGISTRY}/-/package/${packageName}/dist-tags/latest`;
74
74
  let probe = await registryRequestAsync({ url: probeUrl, method: "PUT", token, userAgent, payload: JSON.stringify(latest) });
75
75
  if (probe.statusCode !== 401) {
76
76
  throw new Error(`Expected setting dist-tag latest to its current ${latest} to require two-factor (401), was ${probe.statusCode}: ${probe.body.slice(0, ERROR_BODY_LIMIT)}`);
@@ -60,7 +60,7 @@ export const DEPLOY_GROUPS: DeployGroupInfo[] = [
60
60
  label: "Front-end Proxy",
61
61
  emoji: "🚪",
62
62
  hue: 35,
63
- description: "The HTTP bootstrapper. If it goes down users can't load the site, but it does very little, so it rarely needs updating.",
63
+ description: "The HTTP bootstrapper. If it goes down users can't load the site, but it does very little, so it rarely needs updating. It binds port 443, so it is always deployed without overlap: an overlapped deploy leaves the new instance unable to bind the port, and the site goes down when the old one exits.",
64
64
  deployWarning: {
65
65
  title: "You probably don't need to update the front-end proxy",
66
66
  paragraphs: [
@@ -69,6 +69,7 @@ export const DEPLOY_GROUPS: DeployGroupInfo[] = [
69
69
  "It's usually best not to touch it.",
70
70
  ],
71
71
  },
72
+ noOverlap: true,
72
73
  },
73
74
  {
74
75
  group: "storageCore",
@@ -0,0 +1,25 @@
1
+ module.hotreload = false;
2
+
3
+ import fs from "fs";
4
+ import yargs from "yargs";
5
+ import { getWebOtpAsync } from "./publishQuerysub";
6
+
7
+ let args = yargs(process.argv.slice(2))
8
+ .option("package", { type: "string", demandOption: true, describe: "npm package to get a publish one-time code for" })
9
+ .option("out", { type: "string", demandOption: true, describe: "file to write the one-time code to" })
10
+ .strict()
11
+ .argv;
12
+
13
+ async function main() {
14
+ let otp = await getWebOtpAsync(args.package);
15
+ await fs.promises.writeFile(args.out, otp);
16
+ console.log(`Wrote the one-time code for ${args.package} to ${args.out}`);
17
+ }
18
+
19
+ main().then(
20
+ () => process.exit(),
21
+ e => {
22
+ console.error((e as Error).stack || e);
23
+ process.exit(1);
24
+ },
25
+ );
@@ -1,5 +1,9 @@
1
- import { sort, timeInHour, timeInMinute, timeInSecond, timeoutToError, timeoutToUndefined, timeoutToUndefinedSilent } from "socket-function/src/misc";
1
+ import fs from "fs";
2
+ import path from "path";
3
+ import { sort, timeInDay, timeInHour, timeInMinute, timeInSecond, timeoutToError, timeoutToUndefined, timeoutToUndefinedSilent } from "socket-function/src/misc";
2
4
  import { lazy } from "socket-function/src/caching";
5
+ import { runInSerial } from "socket-function/src/batching";
6
+ import { getSharedFolderPath } from "../../../fs";
3
7
  import { getMachineId } from "sliftutils/misc/https/certs";
4
8
  import { getDomain } from "../../../config";
5
9
  import { getAllNodeIds, getOwnMachineId, isOwnNodeId } from "../../../-f-node-discovery/NodeDiscovery";
@@ -22,6 +26,9 @@ const END_TIME_MIN_AGE = timeInMinute;
22
26
  const MOVE_GRACE = timeInMinute;
23
27
  // TTL for the path-cache (the slow TimeFileTree.findAllPaths walk).
24
28
  const PATHS_CACHE_TTL = timeInMinute * 15;
29
+ const PATHS_STATE_MAX_AGE = timeInDay * 30;
30
+ export const SEARCH_STATE_FOLDER = "log-search-cache";
31
+ const SEARCH_STATE_FILE = "search-state.json";
25
32
  // During the read/scan loops, emit a progress line at most this often.
26
33
  const PROGRESS_LOG_INTERVAL = timeInSecond * 5;
27
34
  const FLUSH_CALL_TIMEOUT = timeInSecond * 30;
@@ -101,6 +108,44 @@ export type SearchResult = {
101
108
 
102
109
  type Direction = "fromStart" | "fromEnd";
103
110
 
111
+ type SearchState = {
112
+ movedThrough: { [machineId: string]: { through: number; at: number } };
113
+ paths: { [key: string]: { at: number; paths: TimeFilePath[] } };
114
+ };
115
+
116
+ const getSearchStatePath = lazy(() => getSharedFolderPath(SEARCH_STATE_FOLDER) + SEARCH_STATE_FILE);
117
+
118
+ const loadSearchState = lazy(async (): Promise<SearchState> => {
119
+ let statePath = getSearchStatePath();
120
+ try {
121
+ let state: SearchState = JSON.parse(await fs.promises.readFile(statePath, "utf8"));
122
+ return { movedThrough: state.movedThrough || {}, paths: state.paths || {} };
123
+ } catch (e: any) {
124
+ if (e?.code !== "ENOENT") {
125
+ console.warn(`Ignoring unreadable log search state ${statePath}, starting with an empty one: ${String(e?.stack ?? e)}`);
126
+ }
127
+ return { movedThrough: {}, paths: {} };
128
+ }
129
+ });
130
+
131
+ const saveSearchState = runInSerial(async (state: SearchState) => {
132
+ let statePath = getSearchStatePath();
133
+ await fs.promises.mkdir(path.dirname(statePath), { recursive: true });
134
+ let tempPath = statePath + ".tmp";
135
+ await fs.promises.writeFile(tempPath, JSON.stringify(state));
136
+ await fs.promises.rename(tempPath, statePath);
137
+ });
138
+
139
+ export function assertSearchTimeRange(startTime: number, endTime: number) {
140
+ let now = Date.now();
141
+ if (endTime > now - END_TIME_MIN_AGE) {
142
+ throw new Error(`endTime must be at least ${formatTime(END_TIME_MIN_AGE)} in the past (got ${formatTime(now - endTime)} ago)`);
143
+ }
144
+ if (startTime >= endTime) {
145
+ throw new Error(`startTime (${formatDateTime(startTime)}) must be < endTime (${formatDateTime(endTime)})`);
146
+ }
147
+ }
148
+
104
149
  // One log file within a specific logger's archive, as gathered for searching.
105
150
  type FileEntry = {
106
151
  path: TimeFilePath;
@@ -144,13 +189,6 @@ function createProgressLogger(): (message: string) => void {
144
189
  }
145
190
 
146
191
  export class MCPIndexedLogs {
147
- // machineId -> latest timestamp guaranteed to already be moved-to-public.
148
- private movedThroughByMachine = new Map<string, number>();
149
-
150
- // Cache: `${type}|${loggerName}|${startBucket}|${endBucket}` -> { time, paths }.
151
- // Buckets are hour-aligned start/end so adjacent searches reuse work.
152
- private pathsCache = new Map<string, { time: number; paths: TimeFilePath[] }>();
153
-
154
192
  public async search(config: {
155
193
  query: string;
156
194
  machine: string | "local";
@@ -168,19 +206,16 @@ export class MCPIndexedLogs {
168
206
  // meaningfully read a log row without knowing when it happened.
169
207
  let columns = config.columns.includes("time") ? config.columns : ["time", ...config.columns];
170
208
  console.log(`[search] query=${JSON.stringify(config.query)} | machine=${config.machine} | startTime=${formatDateTime(startTime)} | endTime=${formatDateTime(endTime)} | direction=${config.direction} | columns=[${config.columns.join(",")}] | limit=${config.limit ?? "(default)"}`);
171
- let now = Date.now();
172
- if (endTime > now - END_TIME_MIN_AGE) {
173
- throw new Error(`endTime must be at least ${formatTime(END_TIME_MIN_AGE)} in the past (got ${formatTime(now - endTime)} ago)`);
174
- }
175
- if (startTime >= endTime) {
176
- throw new Error(`startTime (${formatDateTime(startTime)}) must be < endTime (${formatDateTime(endTime)})`);
177
- }
209
+ assertSearchTimeRange(startTime, endTime);
178
210
 
179
211
  let machineId = config.machine === "local" ? getOwnMachineId() : config.machine;
180
212
 
181
213
  let moveStart = Date.now();
182
214
  let moveOutcome = await this.ensureMovedThrough(machineId, endTime, config.forceRefresh);
183
215
  console.log(`[search] ensureMovedThrough ${moveOutcome} in ${formatTime(Date.now() - moveStart)}`);
216
+ let state = await loadSearchState();
217
+ let flushedAt = state.movedThrough[machineId]?.at;
218
+ let minListingTime = flushedAt ?? Date.now() - PATHS_CACHE_TTL;
184
219
 
185
220
  let loggers = await getLoggers2Async();
186
221
  let useType: "local" | "public" = config.machine === "local" ? "local" : "public";
@@ -207,6 +242,7 @@ export class MCPIndexedLogs {
207
242
  loggerName,
208
243
  startTime,
209
244
  endTime,
245
+ minListingTime,
210
246
  forceRefresh: config.forceRefresh,
211
247
  });
212
248
  totalPathsSeen += paths.length;
@@ -485,8 +521,9 @@ export class MCPIndexedLogs {
485
521
  // now - MOVE_GRACE so we skip this on subsequent calls covering the same
486
522
  // window.
487
523
  private async ensureMovedThrough(machineId: string, endTime: number, forceRefresh?: boolean): Promise<"cached" | "no-node" | "unreachable" | "moved"> {
488
- let lastMoved = this.movedThroughByMachine.get(machineId) ?? 0;
489
- if (!forceRefresh && lastMoved >= endTime) return "cached";
524
+ let state = await loadSearchState();
525
+ let moved = state.movedThrough[machineId];
526
+ if (!forceRefresh && moved && moved.through >= endTime) return "cached";
490
527
 
491
528
  let nodeIds = await this.findRemoteNodesOnMachine(machineId);
492
529
  if (nodeIds.length === 0) {
@@ -537,7 +574,8 @@ export class MCPIndexedLogs {
537
574
  console.warn(`MCPIndexedLogs: machine ${machineId} could not be flushed for ${unflushed.join(", ")}. Results only include what it had already moved to public storage on its own.`);
538
575
  return "unreachable";
539
576
  }
540
- this.movedThroughByMachine.set(machineId, Date.now() - MOVE_GRACE);
577
+ state.movedThrough[machineId] = { through: Date.now() - MOVE_GRACE, at: Date.now() };
578
+ await saveSearchState(state);
541
579
  return "moved";
542
580
  }
543
581
 
@@ -576,32 +614,27 @@ export class MCPIndexedLogs {
576
614
  throw new Error(`Unknown logger ${name}`);
577
615
  }
578
616
 
579
- // Caches TimeFileTree.findAllPaths by hour-aligned bucket so repeated
580
- // searches over similar windows reuse the slow folder walk.
581
617
  private async getCachedPaths(config: {
582
618
  archives: LogArchives;
583
619
  type: "local" | "public";
584
620
  loggerName: LoggerName;
585
621
  startTime: number;
586
622
  endTime: number;
623
+ minListingTime: number;
587
624
  forceRefresh?: boolean;
588
625
  }): Promise<TimeFilePath[]> {
589
626
  let bucketStart = Math.floor(config.startTime / timeInHour) * timeInHour;
590
627
  let bucketEnd = Math.ceil(config.endTime / timeInHour) * timeInHour;
591
628
  let key = `${config.type}|${config.loggerName}|${bucketStart}|${bucketEnd}`;
592
629
  let now = Date.now();
630
+ let state = await loadSearchState();
593
631
 
594
- // Prune stale entries on read.
595
- for (let [k, v] of Array.from(this.pathsCache.entries())) {
596
- if (now - v.time > PATHS_CACHE_TTL) this.pathsCache.delete(k);
597
- }
598
-
599
- if (config.forceRefresh) {
600
- this.pathsCache.delete(key);
632
+ for (let [k, v] of Object.entries(state.paths)) {
633
+ if (now - v.at > PATHS_STATE_MAX_AGE) delete state.paths[k];
601
634
  }
602
635
 
603
- let cached = this.pathsCache.get(key);
604
- if (cached && now - cached.time <= PATHS_CACHE_TTL) {
636
+ let cached = state.paths[key];
637
+ if (!config.forceRefresh && cached && cached.at >= config.minListingTime) {
605
638
  return cached.paths;
606
639
  }
607
640
 
@@ -609,7 +642,8 @@ export class MCPIndexedLogs {
609
642
  startTime: bucketStart,
610
643
  endTime: bucketEnd,
611
644
  });
612
- this.pathsCache.set(key, { time: now, paths });
645
+ state.paths[key] = { at: now, paths };
646
+ await saveSearchState(state);
613
647
  return paths;
614
648
  }
615
649
 
@@ -11,17 +11,22 @@ module.hotreload = false;
11
11
 
12
12
  import "../../inject";
13
13
 
14
+ import fs from "fs";
15
+ import path from "path";
14
16
  import yargs from "yargs";
15
17
  import { formatDateTime } from "socket-function/src/formatting/format";
16
18
  import { blue, magenta, yellow } from "socket-function/src/formatting/logColors";
17
- import { timeInMinute } from "socket-function/src/misc";
18
- import { MCPIndexedLogs, normalizeTime } from "../logs/IndexedLogs/MCPIndexedLogs";
19
- import { machineNames } from "../../deployManager/machineSchema";
19
+ import { timeInDay, timeInMinute } from "socket-function/src/misc";
20
+ import { MCPIndexedLogs, SEARCH_STATE_FOLDER, assertSearchTimeRange, normalizeTime } from "../logs/IndexedLogs/MCPIndexedLogs";
21
+ import { MachineName, machineNames } from "../../deployManager/machineSchema";
22
+ import { getSharedFolderPath } from "../../fs";
20
23
  import { disablePathAuditer } from "../pathAuditerCallback";
21
24
 
22
25
  const DEFAULT_LIMIT = 200;
23
26
  const DEFAULT_COLUMNS = ["__entry", "__threadId", "param0", "param1", "param2", "param3"];
24
27
  const DEFAULT_END_AGE = timeInMinute * 2;
28
+ const MACHINE_NAMES_CACHE_TTL = timeInDay;
29
+ const MACHINE_NAMES_CACHE_FILE = "machine-names.json";
25
30
 
26
31
  let args = yargs(process.argv.slice(2))
27
32
  .option("machine", { type: "string", demandOption: true, describe: "machine id, a substring of a machine name (ex \"5600x\"), or \"local\"" })
@@ -31,14 +36,34 @@ let args = yargs(process.argv.slice(2))
31
36
  .option("direction", { choices: ["fromStart", "fromEnd"] as const, default: "fromEnd" as const })
32
37
  .option("columns", { type: "array", string: true, describe: `fields to print (default ${DEFAULT_COLUMNS.join(", ")})` })
33
38
  .option("limit", { type: "number", default: DEFAULT_LIMIT })
34
- .option("refresh", { type: "boolean", default: false, describe: "re-walk the log folders instead of using the cached file listing" })
39
+ .option("refresh", { type: "boolean", default: false, describe: "re-walk the log folders and re-flush the machine instead of using the cached state" })
35
40
  .option("every", { type: "number", default: 1, describe: "print only every Nth row (oldest first), to see a trend over many rows" })
36
41
  .strict()
37
42
  .argv;
38
43
 
39
- async function resolveMachineId(machine: string): Promise<string> {
40
- if (machine === "local") return machine;
44
+ type MachineNamesCache = { at: number; names: MachineName[] };
45
+
46
+ async function getMachineNames(): Promise<MachineName[]> {
47
+ let cachePath = getSharedFolderPath(SEARCH_STATE_FOLDER) + MACHINE_NAMES_CACHE_FILE;
48
+ let cached: MachineNamesCache | undefined;
49
+ try {
50
+ cached = JSON.parse(await fs.promises.readFile(cachePath, "utf8"));
51
+ } catch (e: any) {
52
+ if (e?.code !== "ENOENT") {
53
+ console.warn(yellow(`Ignoring unreadable machine name cache ${cachePath}: ${String(e?.stack ?? e)}`));
54
+ }
55
+ }
56
+ if (cached && !args.refresh && Date.now() - cached.at < MACHINE_NAMES_CACHE_TTL) {
57
+ return cached.names;
58
+ }
41
59
  let names = await machineNames.values();
60
+ let fresh: MachineNamesCache = { at: Date.now(), names };
61
+ await fs.promises.mkdir(path.dirname(cachePath), { recursive: true });
62
+ await fs.promises.writeFile(cachePath, JSON.stringify(fresh));
63
+ return names;
64
+ }
65
+
66
+ function findMachine(names: MachineName[], machine: string): string | undefined {
42
67
  if (names.some(x => x.machineId === machine)) return machine;
43
68
  let matches = names.filter(x => x.name.toLowerCase().includes(machine.toLowerCase()));
44
69
  if (matches.length === 1) {
@@ -48,19 +73,29 @@ async function resolveMachineId(machine: string): Promise<string> {
48
73
  if (matches.length > 1) {
49
74
  throw new Error(`Machine "${machine}" matches several machines: ${matches.map(x => `${x.name} (${x.machineId})`).join(", ")}`);
50
75
  }
76
+ return undefined;
77
+ }
78
+
79
+ async function resolveMachineId(machine: string): Promise<string> {
80
+ if (machine === "local") return machine;
81
+ let names = await getMachineNames();
82
+ let found = findMachine(names, machine);
83
+ if (found) return found;
51
84
  console.warn(yellow(`No machine is named like "${machine}", treating it as a machine id`));
52
85
  return machine;
53
86
  }
54
87
 
55
88
  async function main() {
56
89
  disablePathAuditer();
90
+ let startTime = normalizeTime(args.start, "start");
91
+ let endTime = args.end && normalizeTime(args.end, "end") || Date.now() - DEFAULT_END_AGE;
92
+ assertSearchTimeRange(startTime, endTime);
57
93
  let machine = await resolveMachineId(args.machine);
58
94
  let columns = (args.columns || DEFAULT_COLUMNS).map(String);
59
- let endTime = args.end && normalizeTime(args.end, "end") || Date.now() - DEFAULT_END_AGE;
60
95
  let result = await new MCPIndexedLogs().search({
61
96
  query: args.query,
62
97
  machine,
63
- startTime: normalizeTime(args.start, "start"),
98
+ startTime,
64
99
  endTime,
65
100
  direction: args.direction,
66
101
  columns,
@@ -1172,5 +1172,38 @@ export async function nodeJSLoginToUser(userId: string) {
1172
1172
  });
1173
1173
  }
1174
1174
 
1175
+ export const TEST_USER_ID_PREFIX = "test-";
1176
+ export async function scriptAddTrust(config: { userId: string; machineId: string; }) {
1177
+ const { userId, machineId } = config;
1178
+ if (!userId.startsWith(TEST_USER_ID_PREFIX)) {
1179
+ throw new Error(`Trust can only be added to test accounts, whose user ids start with "${TEST_USER_ID_PREFIX}", was ${userId}`);
1180
+ }
1181
+ const ips = [await getExternalIP(), "127.0.0.1"];
1182
+ let now = Date.now();
1183
+ await Querysub.commitSynced(() => {
1184
+ if (!(userId in data().users)) {
1185
+ data().users[userId] = {
1186
+ userId,
1187
+ email: `${userId} is a test account without email login`,
1188
+ settings: { displayName: userId },
1189
+ userType: "user",
1190
+ createTime: now,
1191
+ machineIds: {},
1192
+ allowedIPs: {},
1193
+ loginTokens: {},
1194
+ lastPageLoadsIPs: {},
1195
+ invitedUsers2: {},
1196
+ invitesRemaining: 0,
1197
+ };
1198
+ }
1199
+ let userObj = data().users[userId];
1200
+ for (let ip of ips) {
1201
+ userObj.allowedIPs[ip] = atomicObjectWrite({ time: now, machineId });
1202
+ }
1203
+ userObj.machineIds[machineId] = atomicObjectWrite({ time: now, ip: ips[0] });
1204
+ data().machineSecure[machineId].userId = userId;
1205
+ });
1206
+ }
1207
+
1175
1208
  export const user_data = data;
1176
1209
  export const user_functions = functions;