querysub 0.723.0 → 0.725.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "querysub",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.725.0",
|
|
4
4
|
"main": "index.js",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
|
|
@@ -33,6 +33,7 @@
|
|
|
33
33
|
"port-mappings": "yarn typenode ./src/diagnostics/network/portMappingsMain.ts",
|
|
34
34
|
"log-search": "yarn typenode ./src/diagnostics/network/logSearchMain.ts",
|
|
35
35
|
"log-files": "yarn typenode ./src/diagnostics/network/logFilesMain.ts",
|
|
36
|
+
"remote-flushlogs": "yarn typenode ./src/diagnostics/network/remoteFlushLogsMain.ts",
|
|
36
37
|
"publish-querysub": "yarn typenode ./src/deployManager/publishQuerysubMain.ts",
|
|
37
38
|
"parameters-timeline": "yarn typenode ./src/deployManager/parametersTimelineMain.ts",
|
|
38
39
|
"setup-machine": "yarn typenode ./src/deployManager/setupMachineMain.ts",
|
|
@@ -6,8 +6,9 @@ import readline from "readline";
|
|
|
6
6
|
import { runPromise } from "socket-function/src/runPromise";
|
|
7
7
|
import { magenta, yellow } from "socket-function/src/formatting/logColors";
|
|
8
8
|
import { delay } from "socket-function/src/batching";
|
|
9
|
+
import { lazy } from "socket-function/src/caching";
|
|
9
10
|
|
|
10
|
-
export const
|
|
11
|
+
export const getPublishOtpFile = lazy(() => path.join(os.tmpdir(), "querysub-publish-otp.txt"));
|
|
11
12
|
const OTP_FILE_POLL_INTERVAL = 1000;
|
|
12
13
|
const LOGIN_URL_PATTERN = /Open (https:\/\/www\.npmjs\.com\/login\/\S+)/;
|
|
13
14
|
const PUBLISH_SUCCESS_TEXT = "success Published";
|
|
@@ -45,13 +46,14 @@ function readVersion(querysubFolder: string): string {
|
|
|
45
46
|
return String(JSON.parse(fs.readFileSync(path.join(querysubFolder, "package.json"), "utf8")).version);
|
|
46
47
|
}
|
|
47
48
|
|
|
48
|
-
/** Opens the login link, explains the LastPass passkey dance, and waits for the code: typed into this terminal, or written to
|
|
49
|
+
/** Opens the login link, explains the LastPass passkey dance, and waits for the code: typed into this terminal, or written to getPublishOtpFile() (for when something other than a person is running the publish). */
|
|
49
50
|
export async function waitForPublishOtpAsync(loginUrl: string): Promise<string> {
|
|
50
|
-
|
|
51
|
+
let otpFile = getPublishOtpFile();
|
|
52
|
+
await fs.promises.rm(otpFile, { force: true });
|
|
51
53
|
openInBrowser(loginUrl);
|
|
52
54
|
console.log(yellow(`npm needs a one-time code to publish. Opened ${loginUrl}`));
|
|
53
55
|
console.log(yellow(`LastPass is NOT enabled on npmjs.com by default: click the LastPass icon, enable it for this site, then sign in with the passkey from LastPass. Copy the code the page gives you.`));
|
|
54
|
-
console.log(yellow(`Type the code here and press enter, or write it to ${
|
|
56
|
+
console.log(yellow(`Type the code here and press enter, or write it to ${otpFile}`));
|
|
55
57
|
|
|
56
58
|
let done = false;
|
|
57
59
|
let fromTerminal = new Promise<string>(resolve => {
|
|
@@ -64,7 +66,7 @@ export async function waitForPublishOtpAsync(loginUrl: string): Promise<string>
|
|
|
64
66
|
});
|
|
65
67
|
let fromFile = (async () => {
|
|
66
68
|
while (!done) {
|
|
67
|
-
let text = await fs.promises.readFile(
|
|
69
|
+
let text = await fs.promises.readFile(otpFile, "utf8").catch(() => "");
|
|
68
70
|
if (text.trim()) return text;
|
|
69
71
|
await delay(OTP_FILE_POLL_INTERVAL);
|
|
70
72
|
}
|
|
@@ -72,7 +74,7 @@ export async function waitForPublishOtpAsync(loginUrl: string): Promise<string>
|
|
|
72
74
|
})();
|
|
73
75
|
let code = (await Promise.race([fromTerminal, fromFile])).trim();
|
|
74
76
|
done = true;
|
|
75
|
-
await fs.promises.rm(
|
|
77
|
+
await fs.promises.rm(otpFile, { force: true });
|
|
76
78
|
if (!/^\w+$/.test(code)) {
|
|
77
79
|
throw new Error(`Expected a one-time code of letters and digits, was ${JSON.stringify(code)}`);
|
|
78
80
|
}
|
|
@@ -6,6 +6,9 @@ import { LogStreamer } from "./LogStreamer";
|
|
|
6
6
|
import { LogArchives, TimeFileTree } from "./TimeFileTree";
|
|
7
7
|
import { blue, green, magenta } from "socket-function/src/formatting/logColors";
|
|
8
8
|
import { formatDateTime, formatNumber, formatTime } from "socket-function/src/formatting/format";
|
|
9
|
+
import { timeInSecond } from "socket-function/src/misc";
|
|
10
|
+
|
|
11
|
+
const PROGRESS_LOG_INTERVAL = timeInSecond * 5;
|
|
9
12
|
|
|
10
13
|
|
|
11
14
|
|
|
@@ -144,22 +147,32 @@ export async function moveLogsToPublic(config: {
|
|
|
144
147
|
// Get compressed file sizes for all paths in parallel to avoid loading too much data at once
|
|
145
148
|
let sizeThreshold = maxSingleFileData / 10;
|
|
146
149
|
let allPathsFlat = initialGroups.flat();
|
|
150
|
+
let moveStart = Date.now();
|
|
151
|
+
let sizesRead = 0;
|
|
152
|
+
let lastProgressLog = Date.now();
|
|
147
153
|
let pathSizes = await Promise.all(allPathsFlat.map(async path => {
|
|
148
154
|
let info = await localLogs.getInfo(path.fullPath);
|
|
155
|
+
sizesRead++;
|
|
156
|
+
if (Date.now() - lastProgressLog >= PROGRESS_LOG_INTERVAL) {
|
|
157
|
+
lastProgressLog = Date.now();
|
|
158
|
+
console.log(magenta(`Reading log file sizes ${sizesRead}/${allPathsFlat.length} (${blue(config.loggerName)})`));
|
|
159
|
+
}
|
|
149
160
|
return { path, size: info?.size || 0 };
|
|
150
161
|
}));
|
|
151
162
|
let sizeMap = new Map(pathSizes.map(x => [x.path, x.size]));
|
|
163
|
+
let totalBytes = pathSizes.reduce((acc, x) => acc + x.size, 0);
|
|
152
164
|
|
|
153
165
|
// Split individual groups if their compressed file sizes exceed threshold
|
|
154
166
|
// but keep groups separate (don't mix startTimes)
|
|
155
167
|
let groups: typeof initialGroups = [];
|
|
156
168
|
for (let group of initialGroups) {
|
|
157
169
|
let currentSize = 0;
|
|
170
|
+
groups.push([]);
|
|
158
171
|
|
|
159
172
|
for (let path of group) {
|
|
160
173
|
let size = sizeMap.get(path) || 0;
|
|
161
174
|
|
|
162
|
-
if (groups.length
|
|
175
|
+
if (groups[groups.length - 1].length > 0 && currentSize + size > sizeThreshold) {
|
|
163
176
|
groups.push([]);
|
|
164
177
|
currentSize = 0;
|
|
165
178
|
}
|
|
@@ -169,6 +182,11 @@ export async function moveLogsToPublic(config: {
|
|
|
169
182
|
}
|
|
170
183
|
}
|
|
171
184
|
|
|
185
|
+
console.log(magenta(`Read ${allPathsFlat.length} log file sizes (${formatNumber(totalBytes)}B) in ${formatTime(Date.now() - moveStart)}, moving them in ${groups.length} groups (${blue(config.loggerName)})`));
|
|
186
|
+
|
|
187
|
+
let filesDone = 0;
|
|
188
|
+
let bytesDone = 0;
|
|
189
|
+
let groupsStart = Date.now();
|
|
172
190
|
for (let i = 0; i < groups.length; i++) {
|
|
173
191
|
let group = groups[i];
|
|
174
192
|
let time = Date.now();
|
|
@@ -261,6 +279,11 @@ export async function moveLogsToPublic(config: {
|
|
|
261
279
|
await localLogs.del(path.fullPath + indexExtension);
|
|
262
280
|
}));
|
|
263
281
|
|
|
282
|
+
filesDone += group.length;
|
|
283
|
+
bytesDone += group.reduce((acc, x) => acc + (sizeMap.get(x) || 0), 0);
|
|
284
|
+
let elapsed = Date.now() - groupsStart;
|
|
285
|
+
let remaining = bytesDone && elapsed * (totalBytes - bytesDone) / bytesDone || 0;
|
|
286
|
+
console.log(magenta(`Moved ${filesDone}/${allPathsFlat.length} files, ${formatNumber(bytesDone)}B/${formatNumber(totalBytes)}B, ${formatTime(elapsed)} so far, about ${formatTime(remaining)} left (${blue(config.loggerName)})`));
|
|
264
287
|
console.log(green(`(${i + 1}/${groups.length}) Wrote ${encoded.length} log files to public from ${group.length} input files, ${formatDateTime(startTime)} to ${formatDateTime(endTime)} (${formatNumber(encoded.reduce((acc, x) => acc + x.uncompressedSize, 0))}B compressed to ${formatNumber(encoded.reduce((acc, x) => acc + x.compressedSize, 0))}B + ${formatNumber(encoded.reduce((acc, x) => acc + x.index.length, 0))}B index) in ${formatTime(Date.now() - time)}`) + ` (${blue(config.loggerName)})`);
|
|
265
288
|
}
|
|
266
289
|
|
|
@@ -32,6 +32,7 @@ let args = yargs(process.argv.slice(2))
|
|
|
32
32
|
.option("columns", { type: "array", string: true, describe: `fields to print (default ${DEFAULT_COLUMNS.join(", ")})` })
|
|
33
33
|
.option("limit", { type: "number", default: DEFAULT_LIMIT })
|
|
34
34
|
.option("refresh", { type: "boolean", default: false, describe: "re-walk the log folders instead of using the cached file listing" })
|
|
35
|
+
.option("every", { type: "number", default: 1, describe: "print only every Nth row (oldest first), to see a trend over many rows" })
|
|
35
36
|
.strict()
|
|
36
37
|
.argv;
|
|
37
38
|
|
|
@@ -68,7 +69,9 @@ async function main() {
|
|
|
68
69
|
});
|
|
69
70
|
let rows = result.results.slice();
|
|
70
71
|
rows.sort((a, b) => Number(a.time) - Number(b.time));
|
|
71
|
-
for (let
|
|
72
|
+
for (let index = 0; index < rows.length; index++) {
|
|
73
|
+
if (index % args.every && index !== rows.length - 1) continue;
|
|
74
|
+
let row = rows[index];
|
|
72
75
|
let fields = columns.filter(x => row[x] !== undefined).map(x => x.startsWith("param") ? row[x] : `${x}=${row[x]}`);
|
|
73
76
|
console.log(blue(formatDateTime(Number(row.time))), ...fields);
|
|
74
77
|
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import yargs from "yargs";
|
|
2
|
+
import { runPromise } from "socket-function/src/runPromise";
|
|
3
|
+
import { green, magenta, yellow } from "socket-function/src/formatting/logColors";
|
|
4
|
+
import { formatTime } from "socket-function/src/formatting/format";
|
|
5
|
+
import { delay } from "socket-function/src/batching";
|
|
6
|
+
import { timeInSecond } from "socket-function/src/misc";
|
|
7
|
+
|
|
8
|
+
const REMOTE_TOOLS_FOLDER = "~/querysub-tools";
|
|
9
|
+
const NPM_NOT_YET_VISIBLE_TEXT = "Couldn't find any versions for";
|
|
10
|
+
const INSTALL_MAX_ATTEMPTS = 20;
|
|
11
|
+
const INSTALL_RETRY_DELAY = timeInSecond * 30;
|
|
12
|
+
const SAFE_VALUE = /^[\w.@:/~-]+$/;
|
|
13
|
+
|
|
14
|
+
let args = yargs(process.argv.slice(2))
|
|
15
|
+
.option("host", { type: "string", demandOption: true, describe: "machine to flush, ex 10.0.0.188 (or root@10.0.0.188)" })
|
|
16
|
+
.option("repo", { type: "string", describe: "git url of the service repo to run flushlogs from (default: the origin of --cwd / the current folder)" })
|
|
17
|
+
.option("cwd", { type: "string", describe: "local service repo whose origin is used as --repo" })
|
|
18
|
+
.strict()
|
|
19
|
+
.argv;
|
|
20
|
+
|
|
21
|
+
function assertSafe(value: string, label: string): string {
|
|
22
|
+
if (!SAFE_VALUE.test(value)) {
|
|
23
|
+
throw new Error(`Expected ${label} to match ${SAFE_VALUE}, was ${JSON.stringify(value)}`);
|
|
24
|
+
}
|
|
25
|
+
return value;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
async function main() {
|
|
29
|
+
let host = assertSafe(args.host.includes("@") ? args.host : `root@${args.host}`, "host");
|
|
30
|
+
let repoUrl = assertSafe(args.repo || (await runPromise("git remote get-url origin", { cwd: args.cwd, quiet: true })).trim(), "repo");
|
|
31
|
+
let repoName = assertSafe(repoUrl.split("/").pop()?.replace(/\.git$/, "") || "", "repo name");
|
|
32
|
+
let folder = `${REMOTE_TOOLS_FOLDER}/${repoName}`;
|
|
33
|
+
let ssh = async (command: string) => {
|
|
34
|
+
console.log(magenta(`${host}: ${command}`));
|
|
35
|
+
await runPromise(`ssh -o BatchMode=yes -o ConnectTimeout=20 ${host} "${command}"`);
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
await ssh(`mkdir -p ${REMOTE_TOOLS_FOLDER} && (test -d ${folder}/.git || git clone ${repoUrl} ${folder})`);
|
|
39
|
+
await ssh(`cd ${folder} && git pull --ff-only`);
|
|
40
|
+
for (let attempt = 1; ; attempt++) {
|
|
41
|
+
try {
|
|
42
|
+
await ssh(`cd ${folder} && yarn install`);
|
|
43
|
+
break;
|
|
44
|
+
} catch (e) {
|
|
45
|
+
let stack = String((e as Error).stack || e);
|
|
46
|
+
if (!stack.includes(NPM_NOT_YET_VISIBLE_TEXT) || attempt >= INSTALL_MAX_ATTEMPTS) throw e;
|
|
47
|
+
console.warn(yellow(`yarn install on ${host} failed because npm doesn't show a just-published version yet (attempt ${attempt}/${INSTALL_MAX_ATTEMPTS}), retrying in ${formatTime(INSTALL_RETRY_DELAY)}`));
|
|
48
|
+
await delay(INSTALL_RETRY_DELAY);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
await ssh(`cd ${folder} && yarn flushlogs`);
|
|
52
|
+
console.log(green(`Flushed the logs on ${host}`));
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
main().then(
|
|
56
|
+
() => process.exit(),
|
|
57
|
+
e => {
|
|
58
|
+
console.error((e as Error).stack || e);
|
|
59
|
+
process.exit(1);
|
|
60
|
+
},
|
|
61
|
+
);
|