querysub 0.722.0 → 0.724.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "querysub",
3
- "version": "0.722.0",
3
+ "version": "0.724.0",
4
4
  "main": "index.js",
5
5
  "license": "MIT",
6
6
  "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -33,6 +33,8 @@
33
33
  "port-mappings": "yarn typenode ./src/diagnostics/network/portMappingsMain.ts",
34
34
  "log-search": "yarn typenode ./src/diagnostics/network/logSearchMain.ts",
35
35
  "log-files": "yarn typenode ./src/diagnostics/network/logFilesMain.ts",
36
+ "remote-flushlogs": "yarn typenode ./src/diagnostics/network/remoteFlushLogsMain.ts",
37
+ "publish-querysub": "yarn typenode ./src/deployManager/publishQuerysubMain.ts",
36
38
  "parameters-timeline": "yarn typenode ./src/deployManager/parametersTimelineMain.ts",
37
39
  "setup-machine": "yarn typenode ./src/deployManager/setupMachineMain.ts",
38
40
  "stop-machine": "yarn typenode ./src/deployManager/stopMachineMain.ts",
@@ -23,6 +23,7 @@ import { FunctionSpec, functionSchema } from "../3-path-functions/PathFunctionRu
23
23
  import { Querysub } from "../4-querysub/Querysub";
24
24
  import { green, red } from "socket-function/src/formatting/logColors";
25
25
  import { fsExistsAsync } from "../fs";
26
+ import { publishQuerysubAsync, waitForPublishOtpAsync } from "./publishQuerysub";
26
27
 
27
28
  const SERVICE_FOLDER_NAME = "machine-services";
28
29
  export const SERVICE_FOLDER = `${SERVICE_FOLDER_NAME}/`;
@@ -521,13 +522,9 @@ export class MachineServiceControllerBase {
521
522
  throw new Error(`Querysub folder does not exist at ${querysubFolder}`);
522
523
  }
523
524
 
524
- // Increment minor version and publish to npm with yarn
525
- await runPromise("yarn version --minor --no-git-tag-version", { cwd: querysubFolder });
526
- await runPromise("yarn publish --non-interactive", { cwd: querysubFolder });
525
+ let newVersion = await publishQuerysubAsync({ querysubFolder, getOtp: waitForPublishOtpAsync });
527
526
 
528
527
  // Update package.json reference in our actual repo (version only, no install)
529
- let querysubPackageJson = JSON.parse(fs.readFileSync(path.join(querysubFolder, "package.json"), "utf8"));
530
- let newVersion = querysubPackageJson.version;
531
528
 
532
529
  let currentPackageJsonPath = path.join(".", "package.json");
533
530
  let currentPackageJson = JSON.parse(fs.readFileSync(currentPackageJsonPath, "utf8"));
@@ -0,0 +1,89 @@
1
+ import child_process from "child_process";
2
+ import fs from "fs";
3
+ import os from "os";
4
+ import path from "path";
5
+ import readline from "readline";
6
+ import { runPromise } from "socket-function/src/runPromise";
7
+ import { magenta, yellow } from "socket-function/src/formatting/logColors";
8
+ import { delay } from "socket-function/src/batching";
9
+ import { lazy } from "socket-function/src/caching";
10
+
11
+ export const getPublishOtpFile = lazy(() => path.join(os.tmpdir(), "querysub-publish-otp.txt"));
12
+ const OTP_FILE_POLL_INTERVAL = 1000;
13
+ const LOGIN_URL_PATTERN = /Open (https:\/\/www\.npmjs\.com\/login\/\S+)/;
14
+ const PUBLISH_SUCCESS_TEXT = "success Published";
15
+
16
+ /** Bumps the minor version (unless the current one was never published), then publishes with yarn. npm requires a one-time code for every publish: yarn prints a login link, and the code that page hands out has to come back through getOtp. */
17
+ export async function publishQuerysubAsync(config: {
18
+ querysubFolder: string;
19
+ getOtp: (loginUrl: string) => Promise<string>;
20
+ }): Promise<string> {
21
+ let { querysubFolder } = config;
22
+ let version = readVersion(querysubFolder);
23
+ let published = (await runPromise(`npm view querysub@${version} version`, { quiet: true, nothrow: true })).trim();
24
+ if (published === version) {
25
+ await runPromise("yarn version --minor --no-git-tag-version", { cwd: querysubFolder });
26
+ version = readVersion(querysubFolder);
27
+ }
28
+ console.log(magenta(`Publishing querysub ${version}`));
29
+
30
+ let output = await runPromise("yarn publish --non-interactive", { cwd: querysubFolder, nothrow: true, includeStderrInOutput: true });
31
+ if (output.includes(PUBLISH_SUCCESS_TEXT)) return version;
32
+ let loginUrl = LOGIN_URL_PATTERN.exec(output)?.[1];
33
+ if (!loginUrl) {
34
+ throw new Error(`yarn publish of querysub ${version} failed without giving a login link:\n${output}`);
35
+ }
36
+
37
+ let otp = await config.getOtp(loginUrl);
38
+ output = await runPromise(`yarn publish --non-interactive --otp ${otp}`, { cwd: querysubFolder, nothrow: true, includeStderrInOutput: true });
39
+ if (!output.includes(PUBLISH_SUCCESS_TEXT)) {
40
+ throw new Error(`yarn publish of querysub ${version} with the one-time code failed:\n${output}`);
41
+ }
42
+ return version;
43
+ }
44
+
45
+ function readVersion(querysubFolder: string): string {
46
+ return String(JSON.parse(fs.readFileSync(path.join(querysubFolder, "package.json"), "utf8")).version);
47
+ }
48
+
49
+ /** Opens the login link, explains the LastPass passkey dance, and waits for the code: typed into this terminal, or written to getPublishOtpFile() (for when something other than a person is running the publish). */
50
+ export async function waitForPublishOtpAsync(loginUrl: string): Promise<string> {
51
+ let otpFile = getPublishOtpFile();
52
+ await fs.promises.rm(otpFile, { force: true });
53
+ openInBrowser(loginUrl);
54
+ console.log(yellow(`npm needs a one-time code to publish. Opened ${loginUrl}`));
55
+ console.log(yellow(`LastPass is NOT enabled on npmjs.com by default: click the LastPass icon, enable it for this site, then sign in with the passkey from LastPass. Copy the code the page gives you.`));
56
+ console.log(yellow(`Type the code here and press enter, or write it to ${otpFile}`));
57
+
58
+ let done = false;
59
+ let fromTerminal = new Promise<string>(resolve => {
60
+ if (!process.stdin.isTTY) return;
61
+ let reader = readline.createInterface({ input: process.stdin, output: process.stdout });
62
+ reader.question("One-time code: ", answer => {
63
+ reader.close();
64
+ resolve(answer);
65
+ });
66
+ });
67
+ let fromFile = (async () => {
68
+ while (!done) {
69
+ let text = await fs.promises.readFile(otpFile, "utf8").catch(() => "");
70
+ if (text.trim()) return text;
71
+ await delay(OTP_FILE_POLL_INTERVAL);
72
+ }
73
+ return "";
74
+ })();
75
+ let code = (await Promise.race([fromTerminal, fromFile])).trim();
76
+ done = true;
77
+ await fs.promises.rm(otpFile, { force: true });
78
+ if (!/^\w+$/.test(code)) {
79
+ throw new Error(`Expected a one-time code of letters and digits, was ${JSON.stringify(code)}`);
80
+ }
81
+ return code;
82
+ }
83
+
84
+ function openInBrowser(url: string) {
85
+ let child = process.platform === "win32"
86
+ ? child_process.spawn("cmd", ["/c", "start", "", url], { detached: true, stdio: "ignore" })
87
+ : child_process.spawn("xdg-open", [url], { detached: true, stdio: "ignore" });
88
+ child.unref();
89
+ }
@@ -0,0 +1,38 @@
1
+ module.hotreload = false;
2
+
3
+ {
4
+ let argv = process.argv;
5
+ let cwdIdx = argv.indexOf("--cwd");
6
+ if (cwdIdx >= 0 && argv[cwdIdx + 1]) {
7
+ process.chdir(argv[cwdIdx + 1]);
8
+ argv.splice(cwdIdx, 2);
9
+ }
10
+ }
11
+
12
+ import "../inject";
13
+
14
+ import yargs from "yargs";
15
+ import { green } from "socket-function/src/formatting/logColors";
16
+ import { disablePathAuditer } from "../diagnostics/pathAuditerCallback";
17
+ import { MachineServiceControllerBase } from "./machineSchema";
18
+
19
+ let args = yargs(process.argv.slice(2))
20
+ .option("message", { type: "string", demandOption: true, describe: "commit message for both querysub and the service repo" })
21
+ .strict()
22
+ .argv;
23
+
24
+ async function main() {
25
+ disablePathAuditer();
26
+ let controller = new MachineServiceControllerBase();
27
+ await controller.commitPushAndPublishQuerysub(args.message);
28
+ await controller.commitPushService(args.message);
29
+ console.log(green(`Published querysub, pointed ${process.cwd()} at it, and pushed both repos`));
30
+ }
31
+
32
+ main().then(
33
+ () => process.exit(),
34
+ e => {
35
+ console.error((e as Error).stack || e);
36
+ process.exit(1);
37
+ },
38
+ );
@@ -6,6 +6,9 @@ import { LogStreamer } from "./LogStreamer";
6
6
  import { LogArchives, TimeFileTree } from "./TimeFileTree";
7
7
  import { blue, green, magenta } from "socket-function/src/formatting/logColors";
8
8
  import { formatDateTime, formatNumber, formatTime } from "socket-function/src/formatting/format";
9
+ import { timeInSecond } from "socket-function/src/misc";
10
+
11
+ const PROGRESS_LOG_INTERVAL = timeInSecond * 5;
9
12
 
10
13
 
11
14
 
@@ -144,11 +147,20 @@ export async function moveLogsToPublic(config: {
144
147
  // Get compressed file sizes for all paths in parallel to avoid loading too much data at once
145
148
  let sizeThreshold = maxSingleFileData / 10;
146
149
  let allPathsFlat = initialGroups.flat();
150
+ let moveStart = Date.now();
151
+ let sizesRead = 0;
152
+ let lastProgressLog = Date.now();
147
153
  let pathSizes = await Promise.all(allPathsFlat.map(async path => {
148
154
  let info = await localLogs.getInfo(path.fullPath);
155
+ sizesRead++;
156
+ if (Date.now() - lastProgressLog >= PROGRESS_LOG_INTERVAL) {
157
+ lastProgressLog = Date.now();
158
+ console.log(magenta(`Reading log file sizes ${sizesRead}/${allPathsFlat.length} (${blue(config.loggerName)})`));
159
+ }
149
160
  return { path, size: info?.size || 0 };
150
161
  }));
151
162
  let sizeMap = new Map(pathSizes.map(x => [x.path, x.size]));
163
+ let totalBytes = pathSizes.reduce((acc, x) => acc + x.size, 0);
152
164
 
153
165
  // Split individual groups if their compressed file sizes exceed threshold
154
166
  // but keep groups separate (don't mix startTimes)
@@ -169,6 +181,11 @@ export async function moveLogsToPublic(config: {
169
181
  }
170
182
  }
171
183
 
184
+ console.log(magenta(`Read ${allPathsFlat.length} log file sizes (${formatNumber(totalBytes)}B) in ${formatTime(Date.now() - moveStart)}, moving them in ${groups.length} groups (${blue(config.loggerName)})`));
185
+
186
+ let filesDone = 0;
187
+ let bytesDone = 0;
188
+ let groupsStart = Date.now();
172
189
  for (let i = 0; i < groups.length; i++) {
173
190
  let group = groups[i];
174
191
  let time = Date.now();
@@ -261,6 +278,11 @@ export async function moveLogsToPublic(config: {
261
278
  await localLogs.del(path.fullPath + indexExtension);
262
279
  }));
263
280
 
281
+ filesDone += group.length;
282
+ bytesDone += group.reduce((acc, x) => acc + (sizeMap.get(x) || 0), 0);
283
+ let elapsed = Date.now() - groupsStart;
284
+ let remaining = bytesDone && elapsed * (totalBytes - bytesDone) / bytesDone || 0;
285
+ console.log(magenta(`Moved ${filesDone}/${allPathsFlat.length} files, ${formatNumber(bytesDone)}B/${formatNumber(totalBytes)}B, ${formatTime(elapsed)} so far, about ${formatTime(remaining)} left (${blue(config.loggerName)})`));
264
286
  console.log(green(`(${i + 1}/${groups.length}) Wrote ${encoded.length} log files to public from ${group.length} input files, ${formatDateTime(startTime)} to ${formatDateTime(endTime)} (${formatNumber(encoded.reduce((acc, x) => acc + x.uncompressedSize, 0))}B compressed to ${formatNumber(encoded.reduce((acc, x) => acc + x.compressedSize, 0))}B + ${formatNumber(encoded.reduce((acc, x) => acc + x.index.length, 0))}B index) in ${formatTime(Date.now() - time)}`) + ` (${blue(config.loggerName)})`);
265
287
  }
266
288
 
@@ -32,6 +32,7 @@ let args = yargs(process.argv.slice(2))
32
32
  .option("columns", { type: "array", string: true, describe: `fields to print (default ${DEFAULT_COLUMNS.join(", ")})` })
33
33
  .option("limit", { type: "number", default: DEFAULT_LIMIT })
34
34
  .option("refresh", { type: "boolean", default: false, describe: "re-walk the log folders instead of using the cached file listing" })
35
+ .option("every", { type: "number", default: 1, describe: "print only every Nth row (oldest first), to see a trend over many rows" })
35
36
  .strict()
36
37
  .argv;
37
38
 
@@ -68,7 +69,9 @@ async function main() {
68
69
  });
69
70
  let rows = result.results.slice();
70
71
  rows.sort((a, b) => Number(a.time) - Number(b.time));
71
- for (let row of rows) {
72
+ for (let index = 0; index < rows.length; index++) {
73
+ if (index % args.every && index !== rows.length - 1) continue;
74
+ let row = rows[index];
72
75
  let fields = columns.filter(x => row[x] !== undefined).map(x => x.startsWith("param") ? row[x] : `${x}=${row[x]}`);
73
76
  console.log(blue(formatDateTime(Number(row.time))), ...fields);
74
77
  }
@@ -0,0 +1,45 @@
1
+ import yargs from "yargs";
2
+ import { runPromise } from "socket-function/src/runPromise";
3
+ import { green, magenta } from "socket-function/src/formatting/logColors";
4
+
5
+ const REMOTE_TOOLS_FOLDER = "~/querysub-tools";
6
+ const SAFE_VALUE = /^[\w.@:/~-]+$/;
7
+
8
+ let args = yargs(process.argv.slice(2))
9
+ .option("host", { type: "string", demandOption: true, describe: "machine to flush, ex 10.0.0.188 (or root@10.0.0.188)" })
10
+ .option("repo", { type: "string", describe: "git url of the service repo to run flushlogs from (default: the origin of --cwd / the current folder)" })
11
+ .option("cwd", { type: "string", describe: "local service repo whose origin is used as --repo" })
12
+ .strict()
13
+ .argv;
14
+
15
+ function assertSafe(value: string, label: string): string {
16
+ if (!SAFE_VALUE.test(value)) {
17
+ throw new Error(`Expected ${label} to match ${SAFE_VALUE}, was ${JSON.stringify(value)}`);
18
+ }
19
+ return value;
20
+ }
21
+
22
+ async function main() {
23
+ let host = assertSafe(args.host.includes("@") ? args.host : `root@${args.host}`, "host");
24
+ let repoUrl = assertSafe(args.repo || (await runPromise("git remote get-url origin", { cwd: args.cwd, quiet: true })).trim(), "repo");
25
+ let repoName = assertSafe(repoUrl.split("/").pop()?.replace(/\.git$/, "") || "", "repo name");
26
+ let folder = `${REMOTE_TOOLS_FOLDER}/${repoName}`;
27
+ let ssh = async (command: string) => {
28
+ console.log(magenta(`${host}: ${command}`));
29
+ await runPromise(`ssh -o BatchMode=yes -o ConnectTimeout=20 ${host} "${command}"`);
30
+ };
31
+
32
+ await ssh(`mkdir -p ${REMOTE_TOOLS_FOLDER} && (test -d ${folder}/.git || git clone ${repoUrl} ${folder})`);
33
+ await ssh(`cd ${folder} && git pull --ff-only`);
34
+ await ssh(`cd ${folder} && yarn install`);
35
+ await ssh(`cd ${folder} && yarn flushlogs`);
36
+ console.log(green(`Flushed the logs on ${host}`));
37
+ }
38
+
39
+ main().then(
40
+ () => process.exit(),
41
+ e => {
42
+ console.error((e as Error).stack || e);
43
+ process.exit(1);
44
+ },
45
+ );