querysub 0.723.0 → 0.725.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "querysub",
3
- "version": "0.723.0",
3
+ "version": "0.725.0",
4
4
  "main": "index.js",
5
5
  "license": "MIT",
6
6
  "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -33,6 +33,7 @@
33
33
  "port-mappings": "yarn typenode ./src/diagnostics/network/portMappingsMain.ts",
34
34
  "log-search": "yarn typenode ./src/diagnostics/network/logSearchMain.ts",
35
35
  "log-files": "yarn typenode ./src/diagnostics/network/logFilesMain.ts",
36
+ "remote-flushlogs": "yarn typenode ./src/diagnostics/network/remoteFlushLogsMain.ts",
36
37
  "publish-querysub": "yarn typenode ./src/deployManager/publishQuerysubMain.ts",
37
38
  "parameters-timeline": "yarn typenode ./src/deployManager/parametersTimelineMain.ts",
38
39
  "setup-machine": "yarn typenode ./src/deployManager/setupMachineMain.ts",
@@ -6,8 +6,9 @@ import readline from "readline";
6
6
  import { runPromise } from "socket-function/src/runPromise";
7
7
  import { magenta, yellow } from "socket-function/src/formatting/logColors";
8
8
  import { delay } from "socket-function/src/batching";
9
+ import { lazy } from "socket-function/src/caching";
9
10
 
10
- export const PUBLISH_OTP_FILE = path.join(os.tmpdir(), "querysub-publish-otp.txt");
11
+ export const getPublishOtpFile = lazy(() => path.join(os.tmpdir(), "querysub-publish-otp.txt"));
11
12
  const OTP_FILE_POLL_INTERVAL = 1000;
12
13
  const LOGIN_URL_PATTERN = /Open (https:\/\/www\.npmjs\.com\/login\/\S+)/;
13
14
  const PUBLISH_SUCCESS_TEXT = "success Published";
@@ -45,13 +46,14 @@ function readVersion(querysubFolder: string): string {
45
46
  return String(JSON.parse(fs.readFileSync(path.join(querysubFolder, "package.json"), "utf8")).version);
46
47
  }
47
48
 
48
- /** Opens the login link, explains the LastPass passkey dance, and waits for the code: typed into this terminal, or written to PUBLISH_OTP_FILE (for when something other than a person is running the publish). */
49
+ /** Opens the login link, explains the LastPass passkey dance, and waits for the code: typed into this terminal, or written to getPublishOtpFile() (for when something other than a person is running the publish). */
49
50
  export async function waitForPublishOtpAsync(loginUrl: string): Promise<string> {
50
- await fs.promises.rm(PUBLISH_OTP_FILE, { force: true });
51
+ let otpFile = getPublishOtpFile();
52
+ await fs.promises.rm(otpFile, { force: true });
51
53
  openInBrowser(loginUrl);
52
54
  console.log(yellow(`npm needs a one-time code to publish. Opened ${loginUrl}`));
53
55
  console.log(yellow(`LastPass is NOT enabled on npmjs.com by default: click the LastPass icon, enable it for this site, then sign in with the passkey from LastPass. Copy the code the page gives you.`));
54
- console.log(yellow(`Type the code here and press enter, or write it to ${PUBLISH_OTP_FILE}`));
56
+ console.log(yellow(`Type the code here and press enter, or write it to ${otpFile}`));
55
57
 
56
58
  let done = false;
57
59
  let fromTerminal = new Promise<string>(resolve => {
@@ -64,7 +66,7 @@ export async function waitForPublishOtpAsync(loginUrl: string): Promise<string>
64
66
  });
65
67
  let fromFile = (async () => {
66
68
  while (!done) {
67
- let text = await fs.promises.readFile(PUBLISH_OTP_FILE, "utf8").catch(() => "");
69
+ let text = await fs.promises.readFile(otpFile, "utf8").catch(() => "");
68
70
  if (text.trim()) return text;
69
71
  await delay(OTP_FILE_POLL_INTERVAL);
70
72
  }
@@ -72,7 +74,7 @@ export async function waitForPublishOtpAsync(loginUrl: string): Promise<string>
72
74
  })();
73
75
  let code = (await Promise.race([fromTerminal, fromFile])).trim();
74
76
  done = true;
75
- await fs.promises.rm(PUBLISH_OTP_FILE, { force: true });
77
+ await fs.promises.rm(otpFile, { force: true });
76
78
  if (!/^\w+$/.test(code)) {
77
79
  throw new Error(`Expected a one-time code of letters and digits, was ${JSON.stringify(code)}`);
78
80
  }
@@ -6,6 +6,9 @@ import { LogStreamer } from "./LogStreamer";
6
6
  import { LogArchives, TimeFileTree } from "./TimeFileTree";
7
7
  import { blue, green, magenta } from "socket-function/src/formatting/logColors";
8
8
  import { formatDateTime, formatNumber, formatTime } from "socket-function/src/formatting/format";
9
+ import { timeInSecond } from "socket-function/src/misc";
10
+
11
+ const PROGRESS_LOG_INTERVAL = timeInSecond * 5;
9
12
 
10
13
 
11
14
 
@@ -144,22 +147,32 @@ export async function moveLogsToPublic(config: {
144
147
  // Get compressed file sizes for all paths in parallel to avoid loading too much data at once
145
148
  let sizeThreshold = maxSingleFileData / 10;
146
149
  let allPathsFlat = initialGroups.flat();
150
+ let moveStart = Date.now();
151
+ let sizesRead = 0;
152
+ let lastProgressLog = Date.now();
147
153
  let pathSizes = await Promise.all(allPathsFlat.map(async path => {
148
154
  let info = await localLogs.getInfo(path.fullPath);
155
+ sizesRead++;
156
+ if (Date.now() - lastProgressLog >= PROGRESS_LOG_INTERVAL) {
157
+ lastProgressLog = Date.now();
158
+ console.log(magenta(`Reading log file sizes ${sizesRead}/${allPathsFlat.length} (${blue(config.loggerName)})`));
159
+ }
149
160
  return { path, size: info?.size || 0 };
150
161
  }));
151
162
  let sizeMap = new Map(pathSizes.map(x => [x.path, x.size]));
163
+ let totalBytes = pathSizes.reduce((acc, x) => acc + x.size, 0);
152
164
 
153
165
  // Split individual groups if their compressed file sizes exceed threshold
154
166
  // but keep groups separate (don't mix startTimes)
155
167
  let groups: typeof initialGroups = [];
156
168
  for (let group of initialGroups) {
157
169
  let currentSize = 0;
170
+ groups.push([]);
158
171
 
159
172
  for (let path of group) {
160
173
  let size = sizeMap.get(path) || 0;
161
174
 
162
- if (groups.length === 0 || currentSize + size > sizeThreshold) {
175
+ if (groups[groups.length - 1].length > 0 && currentSize + size > sizeThreshold) {
163
176
  groups.push([]);
164
177
  currentSize = 0;
165
178
  }
@@ -169,6 +182,11 @@ export async function moveLogsToPublic(config: {
169
182
  }
170
183
  }
171
184
 
185
+ console.log(magenta(`Read ${allPathsFlat.length} log file sizes (${formatNumber(totalBytes)}B) in ${formatTime(Date.now() - moveStart)}, moving them in ${groups.length} groups (${blue(config.loggerName)})`));
186
+
187
+ let filesDone = 0;
188
+ let bytesDone = 0;
189
+ let groupsStart = Date.now();
172
190
  for (let i = 0; i < groups.length; i++) {
173
191
  let group = groups[i];
174
192
  let time = Date.now();
@@ -261,6 +279,11 @@ export async function moveLogsToPublic(config: {
261
279
  await localLogs.del(path.fullPath + indexExtension);
262
280
  }));
263
281
 
282
+ filesDone += group.length;
283
+ bytesDone += group.reduce((acc, x) => acc + (sizeMap.get(x) || 0), 0);
284
+ let elapsed = Date.now() - groupsStart;
285
+ let remaining = bytesDone && elapsed * (totalBytes - bytesDone) / bytesDone || 0;
286
+ console.log(magenta(`Moved ${filesDone}/${allPathsFlat.length} files, ${formatNumber(bytesDone)}B/${formatNumber(totalBytes)}B, ${formatTime(elapsed)} so far, about ${formatTime(remaining)} left (${blue(config.loggerName)})`));
264
287
  console.log(green(`(${i + 1}/${groups.length}) Wrote ${encoded.length} log files to public from ${group.length} input files, ${formatDateTime(startTime)} to ${formatDateTime(endTime)} (${formatNumber(encoded.reduce((acc, x) => acc + x.uncompressedSize, 0))}B compressed to ${formatNumber(encoded.reduce((acc, x) => acc + x.compressedSize, 0))}B + ${formatNumber(encoded.reduce((acc, x) => acc + x.index.length, 0))}B index) in ${formatTime(Date.now() - time)}`) + ` (${blue(config.loggerName)})`);
265
288
  }
266
289
 
@@ -32,6 +32,7 @@ let args = yargs(process.argv.slice(2))
32
32
  .option("columns", { type: "array", string: true, describe: `fields to print (default ${DEFAULT_COLUMNS.join(", ")})` })
33
33
  .option("limit", { type: "number", default: DEFAULT_LIMIT })
34
34
  .option("refresh", { type: "boolean", default: false, describe: "re-walk the log folders instead of using the cached file listing" })
35
+ .option("every", { type: "number", default: 1, describe: "print only every Nth row (oldest first), to see a trend over many rows" })
35
36
  .strict()
36
37
  .argv;
37
38
 
@@ -68,7 +69,9 @@ async function main() {
68
69
  });
69
70
  let rows = result.results.slice();
70
71
  rows.sort((a, b) => Number(a.time) - Number(b.time));
71
- for (let row of rows) {
72
+ for (let index = 0; index < rows.length; index++) {
73
+ if (index % args.every && index !== rows.length - 1) continue;
74
+ let row = rows[index];
72
75
  let fields = columns.filter(x => row[x] !== undefined).map(x => x.startsWith("param") ? row[x] : `${x}=${row[x]}`);
73
76
  console.log(blue(formatDateTime(Number(row.time))), ...fields);
74
77
  }
@@ -0,0 +1,61 @@
1
+ import yargs from "yargs";
2
+ import { runPromise } from "socket-function/src/runPromise";
3
+ import { green, magenta, yellow } from "socket-function/src/formatting/logColors";
4
+ import { formatTime } from "socket-function/src/formatting/format";
5
+ import { delay } from "socket-function/src/batching";
6
+ import { timeInSecond } from "socket-function/src/misc";
7
+
8
+ const REMOTE_TOOLS_FOLDER = "~/querysub-tools";
9
+ const NPM_NOT_YET_VISIBLE_TEXT = "Couldn't find any versions for";
10
+ const INSTALL_MAX_ATTEMPTS = 20;
11
+ const INSTALL_RETRY_DELAY = timeInSecond * 30;
12
+ const SAFE_VALUE = /^[\w.@:/~-]+$/;
13
+
14
+ let args = yargs(process.argv.slice(2))
15
+ .option("host", { type: "string", demandOption: true, describe: "machine to flush, ex 10.0.0.188 (or root@10.0.0.188)" })
16
+ .option("repo", { type: "string", describe: "git url of the service repo to run flushlogs from (default: the origin of --cwd / the current folder)" })
17
+ .option("cwd", { type: "string", describe: "local service repo whose origin is used as --repo" })
18
+ .strict()
19
+ .argv;
20
+
21
+ function assertSafe(value: string, label: string): string {
22
+ if (!SAFE_VALUE.test(value)) {
23
+ throw new Error(`Expected ${label} to match ${SAFE_VALUE}, was ${JSON.stringify(value)}`);
24
+ }
25
+ return value;
26
+ }
27
+
28
+ async function main() {
29
+ let host = assertSafe(args.host.includes("@") ? args.host : `root@${args.host}`, "host");
30
+ let repoUrl = assertSafe(args.repo || (await runPromise("git remote get-url origin", { cwd: args.cwd, quiet: true })).trim(), "repo");
31
+ let repoName = assertSafe(repoUrl.split("/").pop()?.replace(/\.git$/, "") || "", "repo name");
32
+ let folder = `${REMOTE_TOOLS_FOLDER}/${repoName}`;
33
+ let ssh = async (command: string) => {
34
+ console.log(magenta(`${host}: ${command}`));
35
+ await runPromise(`ssh -o BatchMode=yes -o ConnectTimeout=20 ${host} "${command}"`);
36
+ };
37
+
38
+ await ssh(`mkdir -p ${REMOTE_TOOLS_FOLDER} && (test -d ${folder}/.git || git clone ${repoUrl} ${folder})`);
39
+ await ssh(`cd ${folder} && git pull --ff-only`);
40
+ for (let attempt = 1; ; attempt++) {
41
+ try {
42
+ await ssh(`cd ${folder} && yarn install`);
43
+ break;
44
+ } catch (e) {
45
+ let stack = String((e as Error).stack || e);
46
+ if (!stack.includes(NPM_NOT_YET_VISIBLE_TEXT) || attempt >= INSTALL_MAX_ATTEMPTS) throw e;
47
+ console.warn(yellow(`yarn install on ${host} failed because npm doesn't show a just-published version yet (attempt ${attempt}/${INSTALL_MAX_ATTEMPTS}), retrying in ${formatTime(INSTALL_RETRY_DELAY)}`));
48
+ await delay(INSTALL_RETRY_DELAY);
49
+ }
50
+ }
51
+ await ssh(`cd ${folder} && yarn flushlogs`);
52
+ console.log(green(`Flushed the logs on ${host}`));
53
+ }
54
+
55
+ main().then(
56
+ () => process.exit(),
57
+ e => {
58
+ console.error((e as Error).stack || e);
59
+ process.exit(1);
60
+ },
61
+ );