querysub 0.722.0 → 0.724.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -1
- package/src/deployManager/machineSchema.ts +2 -5
- package/src/deployManager/publishQuerysub.ts +89 -0
- package/src/deployManager/publishQuerysubMain.ts +38 -0
- package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts +22 -0
- package/src/diagnostics/network/logSearchMain.ts +4 -1
- package/src/diagnostics/network/remoteFlushLogsMain.ts +45 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "querysub",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.724.0",
|
|
4
4
|
"main": "index.js",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
|
|
@@ -33,6 +33,8 @@
|
|
|
33
33
|
"port-mappings": "yarn typenode ./src/diagnostics/network/portMappingsMain.ts",
|
|
34
34
|
"log-search": "yarn typenode ./src/diagnostics/network/logSearchMain.ts",
|
|
35
35
|
"log-files": "yarn typenode ./src/diagnostics/network/logFilesMain.ts",
|
|
36
|
+
"remote-flushlogs": "yarn typenode ./src/diagnostics/network/remoteFlushLogsMain.ts",
|
|
37
|
+
"publish-querysub": "yarn typenode ./src/deployManager/publishQuerysubMain.ts",
|
|
36
38
|
"parameters-timeline": "yarn typenode ./src/deployManager/parametersTimelineMain.ts",
|
|
37
39
|
"setup-machine": "yarn typenode ./src/deployManager/setupMachineMain.ts",
|
|
38
40
|
"stop-machine": "yarn typenode ./src/deployManager/stopMachineMain.ts",
|
|
@@ -23,6 +23,7 @@ import { FunctionSpec, functionSchema } from "../3-path-functions/PathFunctionRu
|
|
|
23
23
|
import { Querysub } from "../4-querysub/Querysub";
|
|
24
24
|
import { green, red } from "socket-function/src/formatting/logColors";
|
|
25
25
|
import { fsExistsAsync } from "../fs";
|
|
26
|
+
import { publishQuerysubAsync, waitForPublishOtpAsync } from "./publishQuerysub";
|
|
26
27
|
|
|
27
28
|
const SERVICE_FOLDER_NAME = "machine-services";
|
|
28
29
|
export const SERVICE_FOLDER = `${SERVICE_FOLDER_NAME}/`;
|
|
@@ -521,13 +522,9 @@ export class MachineServiceControllerBase {
|
|
|
521
522
|
throw new Error(`Querysub folder does not exist at ${querysubFolder}`);
|
|
522
523
|
}
|
|
523
524
|
|
|
524
|
-
|
|
525
|
-
await runPromise("yarn version --minor --no-git-tag-version", { cwd: querysubFolder });
|
|
526
|
-
await runPromise("yarn publish --non-interactive", { cwd: querysubFolder });
|
|
525
|
+
let newVersion = await publishQuerysubAsync({ querysubFolder, getOtp: waitForPublishOtpAsync });
|
|
527
526
|
|
|
528
527
|
// Update package.json reference in our actual repo (version only, no install)
|
|
529
|
-
let querysubPackageJson = JSON.parse(fs.readFileSync(path.join(querysubFolder, "package.json"), "utf8"));
|
|
530
|
-
let newVersion = querysubPackageJson.version;
|
|
531
528
|
|
|
532
529
|
let currentPackageJsonPath = path.join(".", "package.json");
|
|
533
530
|
let currentPackageJson = JSON.parse(fs.readFileSync(currentPackageJsonPath, "utf8"));
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import child_process from "child_process";
|
|
2
|
+
import fs from "fs";
|
|
3
|
+
import os from "os";
|
|
4
|
+
import path from "path";
|
|
5
|
+
import readline from "readline";
|
|
6
|
+
import { runPromise } from "socket-function/src/runPromise";
|
|
7
|
+
import { magenta, yellow } from "socket-function/src/formatting/logColors";
|
|
8
|
+
import { delay } from "socket-function/src/batching";
|
|
9
|
+
import { lazy } from "socket-function/src/caching";
|
|
10
|
+
|
|
11
|
+
export const getPublishOtpFile = lazy(() => path.join(os.tmpdir(), "querysub-publish-otp.txt"));
|
|
12
|
+
const OTP_FILE_POLL_INTERVAL = 1000;
|
|
13
|
+
const LOGIN_URL_PATTERN = /Open (https:\/\/www\.npmjs\.com\/login\/\S+)/;
|
|
14
|
+
const PUBLISH_SUCCESS_TEXT = "success Published";
|
|
15
|
+
|
|
16
|
+
/** Bumps the minor version (unless the current one was never published), then publishes with yarn. npm requires a one-time code for every publish: yarn prints a login link, and the code that page hands out has to come back through getOtp. */
|
|
17
|
+
export async function publishQuerysubAsync(config: {
|
|
18
|
+
querysubFolder: string;
|
|
19
|
+
getOtp: (loginUrl: string) => Promise<string>;
|
|
20
|
+
}): Promise<string> {
|
|
21
|
+
let { querysubFolder } = config;
|
|
22
|
+
let version = readVersion(querysubFolder);
|
|
23
|
+
let published = (await runPromise(`npm view querysub@${version} version`, { quiet: true, nothrow: true })).trim();
|
|
24
|
+
if (published === version) {
|
|
25
|
+
await runPromise("yarn version --minor --no-git-tag-version", { cwd: querysubFolder });
|
|
26
|
+
version = readVersion(querysubFolder);
|
|
27
|
+
}
|
|
28
|
+
console.log(magenta(`Publishing querysub ${version}`));
|
|
29
|
+
|
|
30
|
+
let output = await runPromise("yarn publish --non-interactive", { cwd: querysubFolder, nothrow: true, includeStderrInOutput: true });
|
|
31
|
+
if (output.includes(PUBLISH_SUCCESS_TEXT)) return version;
|
|
32
|
+
let loginUrl = LOGIN_URL_PATTERN.exec(output)?.[1];
|
|
33
|
+
if (!loginUrl) {
|
|
34
|
+
throw new Error(`yarn publish of querysub ${version} failed without giving a login link:\n${output}`);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
let otp = await config.getOtp(loginUrl);
|
|
38
|
+
output = await runPromise(`yarn publish --non-interactive --otp ${otp}`, { cwd: querysubFolder, nothrow: true, includeStderrInOutput: true });
|
|
39
|
+
if (!output.includes(PUBLISH_SUCCESS_TEXT)) {
|
|
40
|
+
throw new Error(`yarn publish of querysub ${version} with the one-time code failed:\n${output}`);
|
|
41
|
+
}
|
|
42
|
+
return version;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function readVersion(querysubFolder: string): string {
|
|
46
|
+
return String(JSON.parse(fs.readFileSync(path.join(querysubFolder, "package.json"), "utf8")).version);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Opens the login link, explains the LastPass passkey dance, and waits for the code: typed into this terminal, or written to getPublishOtpFile() (for when something other than a person is running the publish). */
|
|
50
|
+
export async function waitForPublishOtpAsync(loginUrl: string): Promise<string> {
|
|
51
|
+
let otpFile = getPublishOtpFile();
|
|
52
|
+
await fs.promises.rm(otpFile, { force: true });
|
|
53
|
+
openInBrowser(loginUrl);
|
|
54
|
+
console.log(yellow(`npm needs a one-time code to publish. Opened ${loginUrl}`));
|
|
55
|
+
console.log(yellow(`LastPass is NOT enabled on npmjs.com by default: click the LastPass icon, enable it for this site, then sign in with the passkey from LastPass. Copy the code the page gives you.`));
|
|
56
|
+
console.log(yellow(`Type the code here and press enter, or write it to ${otpFile}`));
|
|
57
|
+
|
|
58
|
+
let done = false;
|
|
59
|
+
let fromTerminal = new Promise<string>(resolve => {
|
|
60
|
+
if (!process.stdin.isTTY) return;
|
|
61
|
+
let reader = readline.createInterface({ input: process.stdin, output: process.stdout });
|
|
62
|
+
reader.question("One-time code: ", answer => {
|
|
63
|
+
reader.close();
|
|
64
|
+
resolve(answer);
|
|
65
|
+
});
|
|
66
|
+
});
|
|
67
|
+
let fromFile = (async () => {
|
|
68
|
+
while (!done) {
|
|
69
|
+
let text = await fs.promises.readFile(otpFile, "utf8").catch(() => "");
|
|
70
|
+
if (text.trim()) return text;
|
|
71
|
+
await delay(OTP_FILE_POLL_INTERVAL);
|
|
72
|
+
}
|
|
73
|
+
return "";
|
|
74
|
+
})();
|
|
75
|
+
let code = (await Promise.race([fromTerminal, fromFile])).trim();
|
|
76
|
+
done = true;
|
|
77
|
+
await fs.promises.rm(otpFile, { force: true });
|
|
78
|
+
if (!/^\w+$/.test(code)) {
|
|
79
|
+
throw new Error(`Expected a one-time code of letters and digits, was ${JSON.stringify(code)}`);
|
|
80
|
+
}
|
|
81
|
+
return code;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function openInBrowser(url: string) {
|
|
85
|
+
let child = process.platform === "win32"
|
|
86
|
+
? child_process.spawn("cmd", ["/c", "start", "", url], { detached: true, stdio: "ignore" })
|
|
87
|
+
: child_process.spawn("xdg-open", [url], { detached: true, stdio: "ignore" });
|
|
88
|
+
child.unref();
|
|
89
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
module.hotreload = false;
|
|
2
|
+
|
|
3
|
+
{
|
|
4
|
+
let argv = process.argv;
|
|
5
|
+
let cwdIdx = argv.indexOf("--cwd");
|
|
6
|
+
if (cwdIdx >= 0 && argv[cwdIdx + 1]) {
|
|
7
|
+
process.chdir(argv[cwdIdx + 1]);
|
|
8
|
+
argv.splice(cwdIdx, 2);
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
import "../inject";
|
|
13
|
+
|
|
14
|
+
import yargs from "yargs";
|
|
15
|
+
import { green } from "socket-function/src/formatting/logColors";
|
|
16
|
+
import { disablePathAuditer } from "../diagnostics/pathAuditerCallback";
|
|
17
|
+
import { MachineServiceControllerBase } from "./machineSchema";
|
|
18
|
+
|
|
19
|
+
let args = yargs(process.argv.slice(2))
|
|
20
|
+
.option("message", { type: "string", demandOption: true, describe: "commit message for both querysub and the service repo" })
|
|
21
|
+
.strict()
|
|
22
|
+
.argv;
|
|
23
|
+
|
|
24
|
+
async function main() {
|
|
25
|
+
disablePathAuditer();
|
|
26
|
+
let controller = new MachineServiceControllerBase();
|
|
27
|
+
await controller.commitPushAndPublishQuerysub(args.message);
|
|
28
|
+
await controller.commitPushService(args.message);
|
|
29
|
+
console.log(green(`Published querysub, pointed ${process.cwd()} at it, and pushed both repos`));
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
main().then(
|
|
33
|
+
() => process.exit(),
|
|
34
|
+
e => {
|
|
35
|
+
console.error((e as Error).stack || e);
|
|
36
|
+
process.exit(1);
|
|
37
|
+
},
|
|
38
|
+
);
|
|
@@ -6,6 +6,9 @@ import { LogStreamer } from "./LogStreamer";
|
|
|
6
6
|
import { LogArchives, TimeFileTree } from "./TimeFileTree";
|
|
7
7
|
import { blue, green, magenta } from "socket-function/src/formatting/logColors";
|
|
8
8
|
import { formatDateTime, formatNumber, formatTime } from "socket-function/src/formatting/format";
|
|
9
|
+
import { timeInSecond } from "socket-function/src/misc";
|
|
10
|
+
|
|
11
|
+
const PROGRESS_LOG_INTERVAL = timeInSecond * 5;
|
|
9
12
|
|
|
10
13
|
|
|
11
14
|
|
|
@@ -144,11 +147,20 @@ export async function moveLogsToPublic(config: {
|
|
|
144
147
|
// Get compressed file sizes for all paths in parallel to avoid loading too much data at once
|
|
145
148
|
let sizeThreshold = maxSingleFileData / 10;
|
|
146
149
|
let allPathsFlat = initialGroups.flat();
|
|
150
|
+
let moveStart = Date.now();
|
|
151
|
+
let sizesRead = 0;
|
|
152
|
+
let lastProgressLog = Date.now();
|
|
147
153
|
let pathSizes = await Promise.all(allPathsFlat.map(async path => {
|
|
148
154
|
let info = await localLogs.getInfo(path.fullPath);
|
|
155
|
+
sizesRead++;
|
|
156
|
+
if (Date.now() - lastProgressLog >= PROGRESS_LOG_INTERVAL) {
|
|
157
|
+
lastProgressLog = Date.now();
|
|
158
|
+
console.log(magenta(`Reading log file sizes ${sizesRead}/${allPathsFlat.length} (${blue(config.loggerName)})`));
|
|
159
|
+
}
|
|
149
160
|
return { path, size: info?.size || 0 };
|
|
150
161
|
}));
|
|
151
162
|
let sizeMap = new Map(pathSizes.map(x => [x.path, x.size]));
|
|
163
|
+
let totalBytes = pathSizes.reduce((acc, x) => acc + x.size, 0);
|
|
152
164
|
|
|
153
165
|
// Split individual groups if their compressed file sizes exceed threshold
|
|
154
166
|
// but keep groups separate (don't mix startTimes)
|
|
@@ -169,6 +181,11 @@ export async function moveLogsToPublic(config: {
|
|
|
169
181
|
}
|
|
170
182
|
}
|
|
171
183
|
|
|
184
|
+
console.log(magenta(`Read ${allPathsFlat.length} log file sizes (${formatNumber(totalBytes)}B) in ${formatTime(Date.now() - moveStart)}, moving them in ${groups.length} groups (${blue(config.loggerName)})`));
|
|
185
|
+
|
|
186
|
+
let filesDone = 0;
|
|
187
|
+
let bytesDone = 0;
|
|
188
|
+
let groupsStart = Date.now();
|
|
172
189
|
for (let i = 0; i < groups.length; i++) {
|
|
173
190
|
let group = groups[i];
|
|
174
191
|
let time = Date.now();
|
|
@@ -261,6 +278,11 @@ export async function moveLogsToPublic(config: {
|
|
|
261
278
|
await localLogs.del(path.fullPath + indexExtension);
|
|
262
279
|
}));
|
|
263
280
|
|
|
281
|
+
filesDone += group.length;
|
|
282
|
+
bytesDone += group.reduce((acc, x) => acc + (sizeMap.get(x) || 0), 0);
|
|
283
|
+
let elapsed = Date.now() - groupsStart;
|
|
284
|
+
let remaining = bytesDone && elapsed * (totalBytes - bytesDone) / bytesDone || 0;
|
|
285
|
+
console.log(magenta(`Moved ${filesDone}/${allPathsFlat.length} files, ${formatNumber(bytesDone)}B/${formatNumber(totalBytes)}B, ${formatTime(elapsed)} so far, about ${formatTime(remaining)} left (${blue(config.loggerName)})`));
|
|
264
286
|
console.log(green(`(${i + 1}/${groups.length}) Wrote ${encoded.length} log files to public from ${group.length} input files, ${formatDateTime(startTime)} to ${formatDateTime(endTime)} (${formatNumber(encoded.reduce((acc, x) => acc + x.uncompressedSize, 0))}B compressed to ${formatNumber(encoded.reduce((acc, x) => acc + x.compressedSize, 0))}B + ${formatNumber(encoded.reduce((acc, x) => acc + x.index.length, 0))}B index) in ${formatTime(Date.now() - time)}`) + ` (${blue(config.loggerName)})`);
|
|
265
287
|
}
|
|
266
288
|
|
|
@@ -32,6 +32,7 @@ let args = yargs(process.argv.slice(2))
|
|
|
32
32
|
.option("columns", { type: "array", string: true, describe: `fields to print (default ${DEFAULT_COLUMNS.join(", ")})` })
|
|
33
33
|
.option("limit", { type: "number", default: DEFAULT_LIMIT })
|
|
34
34
|
.option("refresh", { type: "boolean", default: false, describe: "re-walk the log folders instead of using the cached file listing" })
|
|
35
|
+
.option("every", { type: "number", default: 1, describe: "print only every Nth row (oldest first), to see a trend over many rows" })
|
|
35
36
|
.strict()
|
|
36
37
|
.argv;
|
|
37
38
|
|
|
@@ -68,7 +69,9 @@ async function main() {
|
|
|
68
69
|
});
|
|
69
70
|
let rows = result.results.slice();
|
|
70
71
|
rows.sort((a, b) => Number(a.time) - Number(b.time));
|
|
71
|
-
for (let
|
|
72
|
+
for (let index = 0; index < rows.length; index++) {
|
|
73
|
+
if (index % args.every && index !== rows.length - 1) continue;
|
|
74
|
+
let row = rows[index];
|
|
72
75
|
let fields = columns.filter(x => row[x] !== undefined).map(x => x.startsWith("param") ? row[x] : `${x}=${row[x]}`);
|
|
73
76
|
console.log(blue(formatDateTime(Number(row.time))), ...fields);
|
|
74
77
|
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import yargs from "yargs";
|
|
2
|
+
import { runPromise } from "socket-function/src/runPromise";
|
|
3
|
+
import { green, magenta } from "socket-function/src/formatting/logColors";
|
|
4
|
+
|
|
5
|
+
const REMOTE_TOOLS_FOLDER = "~/querysub-tools";
|
|
6
|
+
const SAFE_VALUE = /^[\w.@:/~-]+$/;
|
|
7
|
+
|
|
8
|
+
let args = yargs(process.argv.slice(2))
|
|
9
|
+
.option("host", { type: "string", demandOption: true, describe: "machine to flush, ex 10.0.0.188 (or root@10.0.0.188)" })
|
|
10
|
+
.option("repo", { type: "string", describe: "git url of the service repo to run flushlogs from (default: the origin of --cwd / the current folder)" })
|
|
11
|
+
.option("cwd", { type: "string", describe: "local service repo whose origin is used as --repo" })
|
|
12
|
+
.strict()
|
|
13
|
+
.argv;
|
|
14
|
+
|
|
15
|
+
function assertSafe(value: string, label: string): string {
|
|
16
|
+
if (!SAFE_VALUE.test(value)) {
|
|
17
|
+
throw new Error(`Expected ${label} to match ${SAFE_VALUE}, was ${JSON.stringify(value)}`);
|
|
18
|
+
}
|
|
19
|
+
return value;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
async function main() {
|
|
23
|
+
let host = assertSafe(args.host.includes("@") ? args.host : `root@${args.host}`, "host");
|
|
24
|
+
let repoUrl = assertSafe(args.repo || (await runPromise("git remote get-url origin", { cwd: args.cwd, quiet: true })).trim(), "repo");
|
|
25
|
+
let repoName = assertSafe(repoUrl.split("/").pop()?.replace(/\.git$/, "") || "", "repo name");
|
|
26
|
+
let folder = `${REMOTE_TOOLS_FOLDER}/${repoName}`;
|
|
27
|
+
let ssh = async (command: string) => {
|
|
28
|
+
console.log(magenta(`${host}: ${command}`));
|
|
29
|
+
await runPromise(`ssh -o BatchMode=yes -o ConnectTimeout=20 ${host} "${command}"`);
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
await ssh(`mkdir -p ${REMOTE_TOOLS_FOLDER} && (test -d ${folder}/.git || git clone ${repoUrl} ${folder})`);
|
|
33
|
+
await ssh(`cd ${folder} && git pull --ff-only`);
|
|
34
|
+
await ssh(`cd ${folder} && yarn install`);
|
|
35
|
+
await ssh(`cd ${folder} && yarn flushlogs`);
|
|
36
|
+
console.log(green(`Flushed the logs on ${host}`));
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
main().then(
|
|
40
|
+
() => process.exit(),
|
|
41
|
+
e => {
|
|
42
|
+
console.error((e as Error).stack || e);
|
|
43
|
+
process.exit(1);
|
|
44
|
+
},
|
|
45
|
+
);
|