querysub 0.728.0 → 0.730.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/debugging.md +2 -0
- package/package.json +4 -3
- package/src/0-path-value-core/PathValueCommitter.ts +1 -0
- package/src/deployManager/components/ServicesListPage.tsx +1 -0
- package/src/deployManager/components/deployButtons.tsx +1 -0
- package/src/deployManager/publishQuerysub.ts +5 -5
- package/src/deployManager/serviceCategories.ts +18 -1
- package/src/deployManager/webOtpMain.ts +25 -0
- package/src/diagnostics/logs/TimeRangeSelector.tsx +4 -4
- package/src/diagnostics/logs/errorNotifications2/suppressionPatternMatcher.ts +3 -2
- package/src/diagnostics/logs/logCategories/LogBreakdownPage.tsx +3 -3
- package/src/diagnostics/managementPages.tsx +12 -1
package/debugging.md
CHANGED
|
@@ -17,6 +17,8 @@ Every service runs as its own process in its own tmux session, `<service key>-<i
|
|
|
17
17
|
|
|
18
18
|
The rest are background tasks: `movelogs` (moves pending logs to public storage and categorizes them), `errorwatch`, `gc`, `thin`, `join`, `ssh-trust`, `ping`, and the app's `queue` and `gpuworker` workers. They can go down without taking the site down.
|
|
19
19
|
|
|
20
|
+
The **auditing** services, `site-audit` (a headless browser driving the live site) and `storage-audit` (a write and read back on every storage route), run every 30 seconds on the Hetzner machine and record every run for the Uptime Verification management page (before Gross Stats in the management header). Results are written to the local disk and flushed to storage once an hour. Deploy them on their own, never together with anything else, or deploy-induced failures go unaudited.
|
|
21
|
+
|
|
20
22
|
## 1. What nodes exist
|
|
21
23
|
|
|
22
24
|
```
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "querysub",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.730.0",
|
|
4
4
|
"main": "index.js",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
|
|
@@ -40,6 +40,7 @@
|
|
|
40
40
|
"watch-service": "yarn typenode ./src/diagnostics/network/watchServiceMain.ts",
|
|
41
41
|
"remote-flushlogs": "yarn typenode ./src/diagnostics/network/remoteFlushLogsMain.ts",
|
|
42
42
|
"publish-querysub": "yarn typenode ./src/deployManager/publishQuerysubMain.ts",
|
|
43
|
+
"web-otp-claude": "yarn typenode ./src/deployManager/webOtpMain.ts",
|
|
43
44
|
"parameters-timeline": "yarn typenode ./src/deployManager/parametersTimelineMain.ts",
|
|
44
45
|
"setup-machine": "yarn typenode ./src/deployManager/setupMachineMain.ts",
|
|
45
46
|
"stop-machine": "yarn typenode ./src/deployManager/stopMachineMain.ts",
|
|
@@ -95,8 +96,8 @@
|
|
|
95
96
|
"node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
|
|
96
97
|
"pako": "^2.1.0",
|
|
97
98
|
"peggy": "^5.0.6",
|
|
98
|
-
"sliftutils": "
|
|
99
|
-
"socket-function": "
|
|
99
|
+
"sliftutils": "1.9.22",
|
|
100
|
+
"socket-function": "1.3.13",
|
|
100
101
|
"terser": "^5.31.0",
|
|
101
102
|
"typenode": "^6.6.3",
|
|
102
103
|
"typesafecss": "^0.32.0",
|
|
@@ -462,6 +462,7 @@ class PathValueCommitter {
|
|
|
462
462
|
}
|
|
463
463
|
|
|
464
464
|
export const pathValueCommitter = new PathValueCommitter();
|
|
465
|
+
(globalThis as any).pathValueCommitter = pathValueCommitter;
|
|
465
466
|
|
|
466
467
|
registerShutdownHandler(async () => {
|
|
467
468
|
await pathValueCommitter.waitForValuesToCommit();
|
|
@@ -177,6 +177,7 @@ export class ServicesListPage extends qreact.Component {
|
|
|
177
177
|
<div className={css.vbox(2).pad2(8).fillWidth.hsl(groupInfo.hue, 40, 90).bord2(groupInfo.hue, 40, 60)}>
|
|
178
178
|
<div className={css.fontSize(18).boldStyle}>{groupInfo.emoji} {groupInfo.label} ({groupServices.length})</div>
|
|
179
179
|
<div className={css.fontSize(12).colorhsl(0, 0, 35)}>{groupInfo.description}</div>
|
|
180
|
+
{groupInfo.note && <div className={css.fontSize(12).colorhsl(groupInfo.hue, 60, 35).boldStyle}>{groupInfo.note}</div>}
|
|
180
181
|
</div>
|
|
181
182
|
{this.renderServices({ services: groupServices, keyCounts, serviceNodeIds, getMachineInfo, getMachineConfig, now })}
|
|
182
183
|
</div>;
|
|
@@ -202,6 +202,7 @@ export class DeployGroupButtons extends qreact.Component<{
|
|
|
202
202
|
<div className={css.fontSize(12).colorhsl(0, 0, 40)}>
|
|
203
203
|
{outdatedServices.length} of {allServices.length} service{allServices.length === 1 ? "" : "s"} outdated
|
|
204
204
|
</div>
|
|
205
|
+
{groupInfo.note && <div className={css.fontSize(12).maxWidth(260).colorhsl(groupInfo.hue, 60, 35).boldStyle}>{groupInfo.note}</div>}
|
|
205
206
|
</div>;
|
|
206
207
|
if (!gitInfo || outdatedServices.length === 0) {
|
|
207
208
|
return <div className={css.hbox(10).pad2(8).hsl(groupInfo.hue, 40, 95).bord2(groupInfo.hue, 40, 70)}>
|
|
@@ -44,7 +44,7 @@ export async function publishQuerysubAsync(config: { querysubFolder: string }):
|
|
|
44
44
|
throw new Error(`yarn publish of querysub ${version} as ${user} failed, and didn't ask for a one-time code:\n${output}`);
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
-
let otp = await getWebOtpAsync();
|
|
47
|
+
let otp = await getWebOtpAsync("querysub");
|
|
48
48
|
output = await runPromise(`${publishCommand} --otp ${otp}`, { cwd: querysubFolder, nothrow: true, includeStderrInOutput: true });
|
|
49
49
|
if (!output.includes(PUBLISH_SUCCESS_TEXT)) {
|
|
50
50
|
throw new Error(`yarn publish of querysub ${version} as ${user} with the one-time code from the browser failed:\n${output}`);
|
|
@@ -56,21 +56,21 @@ function readVersion(querysubFolder: string): string {
|
|
|
56
56
|
return String(JSON.parse(fs.readFileSync(path.join(querysubFolder, "package.json"), "utf8")).version);
|
|
57
57
|
}
|
|
58
58
|
|
|
59
|
-
async function getWebOtpAsync(): Promise<string> {
|
|
59
|
+
export async function getWebOtpAsync(packageName: string): Promise<string> {
|
|
60
60
|
let npmrcPath = path.join(os.homedir(), ".npmrc");
|
|
61
61
|
let token = NPMRC_TOKEN_PATTERN.exec(await fs.promises.readFile(npmrcPath, "utf8"))?.[1]?.trim();
|
|
62
62
|
if (!token) {
|
|
63
63
|
throw new Error(`Expected an auth token for ${PUBLISH_REGISTRY} in ${npmrcPath} after logging in, but there wasn't one`);
|
|
64
64
|
}
|
|
65
|
-
let latest = (await runPromise(`npm view
|
|
65
|
+
let latest = (await runPromise(`npm view ${packageName} dist-tags.latest`, { quiet: true })).trim();
|
|
66
66
|
if (!latest) {
|
|
67
|
-
throw new Error(`Expected npm view
|
|
67
|
+
throw new Error(`Expected npm view ${packageName} dist-tags.latest to give a version, was empty`);
|
|
68
68
|
}
|
|
69
69
|
|
|
70
70
|
let npmVersion = (await runPromise(`npm --version`, { quiet: true })).trim();
|
|
71
71
|
let userAgent = `npm/${npmVersion} node/${process.version} ${process.platform} ${process.arch} workspaces/false`;
|
|
72
72
|
|
|
73
|
-
let probeUrl = `${PUBLISH_REGISTRY}/-/package/
|
|
73
|
+
let probeUrl = `${PUBLISH_REGISTRY}/-/package/${packageName}/dist-tags/latest`;
|
|
74
74
|
let probe = await registryRequestAsync({ url: probeUrl, method: "PUT", token, userAgent, payload: JSON.stringify(latest) });
|
|
75
75
|
if (probe.statusCode !== 401) {
|
|
76
76
|
throw new Error(`Expected setting dist-tag latest to its current ${latest} to require two-factor (401), was ${probe.statusCode}: ${probe.body.slice(0, ERROR_BODY_LIMIT)}`);
|
|
@@ -27,7 +27,7 @@ export function getServiceCategory(parameters: ServiceParameters): ServiceCatego
|
|
|
27
27
|
return CATEGORIES.find(x => command.startsWith(x.prefix))?.category;
|
|
28
28
|
}
|
|
29
29
|
|
|
30
|
-
export type ServiceDeployGroup = "storageCore" | "background" | "frontendProxy" | "mainSite";
|
|
30
|
+
export type ServiceDeployGroup = "storageCore" | "background" | "frontendProxy" | "mainSite" | "auditing";
|
|
31
31
|
|
|
32
32
|
export type DeployGroupInfo = {
|
|
33
33
|
group: ServiceDeployGroup;
|
|
@@ -35,6 +35,7 @@ export type DeployGroupInfo = {
|
|
|
35
35
|
emoji: string;
|
|
36
36
|
hue: number;
|
|
37
37
|
description: string;
|
|
38
|
+
note?: string;
|
|
38
39
|
deployWarning?: { title: string; paragraphs: string[] };
|
|
39
40
|
noOverlap?: boolean;
|
|
40
41
|
};
|
|
@@ -61,6 +62,7 @@ export const DEPLOY_GROUPS: DeployGroupInfo[] = [
|
|
|
61
62
|
emoji: "🚪",
|
|
62
63
|
hue: 35,
|
|
63
64
|
description: "The HTTP bootstrapper. If it goes down users can't load the site, but it does very little, so it rarely needs updating. It binds port 443, so it is always deployed without overlap: an overlapped deploy leaves the new instance unable to bind the port, and the site goes down when the old one exits.",
|
|
65
|
+
note: "Only host the bootstrapper. You almost never need to update this. Until we have multiple, any updates will temporarily take down the site.",
|
|
64
66
|
deployWarning: {
|
|
65
67
|
title: "You probably don't need to update the front-end proxy",
|
|
66
68
|
paragraphs: [
|
|
@@ -71,12 +73,22 @@ export const DEPLOY_GROUPS: DeployGroupInfo[] = [
|
|
|
71
73
|
},
|
|
72
74
|
noOverlap: true,
|
|
73
75
|
},
|
|
76
|
+
{
|
|
77
|
+
group: "auditing",
|
|
78
|
+
label: "Auditing",
|
|
79
|
+
emoji: "🔍",
|
|
80
|
+
hue: 280,
|
|
81
|
+
description: "The uptime verification tests (the site test and the storage test), which record their results for the Uptime Verification page. Nothing depends on them and users never notice when they break, so deploying them is risk free. Always deployed without overlap.",
|
|
82
|
+
note: "Don't deploy this with anything else. Otherwise, deploy-induced failures won't be noticed by our auditing.",
|
|
83
|
+
noOverlap: true,
|
|
84
|
+
},
|
|
74
85
|
{
|
|
75
86
|
group: "storageCore",
|
|
76
87
|
label: "Storage Core",
|
|
77
88
|
emoji: "🗄️",
|
|
78
89
|
hue: 0,
|
|
79
90
|
description: "The storage server. Everything else depends on it.",
|
|
91
|
+
note: "Don't break this. If this goes down, you'll need to deploy via the CLI.",
|
|
80
92
|
deployWarning: {
|
|
81
93
|
title: "The storage core is very delicate",
|
|
82
94
|
paragraphs: [
|
|
@@ -88,6 +100,10 @@ export const DEPLOY_GROUPS: DeployGroupInfo[] = [
|
|
|
88
100
|
];
|
|
89
101
|
|
|
90
102
|
const BOOTSTRAP_ONLY_FLAG = "--bootstraponly";
|
|
103
|
+
const AUDITING_COMMAND_PREFIXES = [
|
|
104
|
+
"yarn site-audit",
|
|
105
|
+
"yarn storage-audit",
|
|
106
|
+
];
|
|
91
107
|
const MAIN_SITE_COMMAND_PREFIXES = [
|
|
92
108
|
PATHVALUE_COMMAND_PREFIX,
|
|
93
109
|
FUNCTION_RUNNER_COMMAND_PREFIX,
|
|
@@ -114,5 +130,6 @@ export function getInferredDeployGroup(parameters: ServiceParameters): ServiceDe
|
|
|
114
130
|
if (command.startsWith(STORAGE_COMMAND_PREFIX)) return "storageCore";
|
|
115
131
|
if (command.split(/\s+/).includes(BOOTSTRAP_ONLY_FLAG)) return "frontendProxy";
|
|
116
132
|
if (MAIN_SITE_COMMAND_PREFIXES.some(prefix => command === prefix || command.startsWith(prefix + " "))) return "mainSite";
|
|
133
|
+
if (AUDITING_COMMAND_PREFIXES.some(prefix => command === prefix || command.startsWith(prefix + " "))) return "auditing";
|
|
117
134
|
return "background";
|
|
118
135
|
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
module.hotreload = false;
|
|
2
|
+
|
|
3
|
+
import fs from "fs";
|
|
4
|
+
import yargs from "yargs";
|
|
5
|
+
import { getWebOtpAsync } from "./publishQuerysub";
|
|
6
|
+
|
|
7
|
+
let args = yargs(process.argv.slice(2))
|
|
8
|
+
.option("package", { type: "string", demandOption: true, describe: "npm package to get a publish one-time code for" })
|
|
9
|
+
.option("out", { type: "string", demandOption: true, describe: "file to write the one-time code to" })
|
|
10
|
+
.strict()
|
|
11
|
+
.argv;
|
|
12
|
+
|
|
13
|
+
async function main() {
|
|
14
|
+
let otp = await getWebOtpAsync(args.package);
|
|
15
|
+
await fs.promises.writeFile(args.out, otp);
|
|
16
|
+
console.log(`Wrote the one-time code for ${args.package} to ${args.out}`);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
main().then(
|
|
20
|
+
() => process.exit(),
|
|
21
|
+
e => {
|
|
22
|
+
console.error((e as Error).stack || e);
|
|
23
|
+
process.exit(1);
|
|
24
|
+
},
|
|
25
|
+
);
|
|
@@ -16,8 +16,8 @@ export const searchFromStartParam = new URLParam("searchFromStart", false);
|
|
|
16
16
|
|
|
17
17
|
let now = Date.now();
|
|
18
18
|
/** Get the time range with default values if not set */
|
|
19
|
-
export function getTimeRange(): { startTime: number; endTime: number; searchFromStart: boolean } {
|
|
20
|
-
const defaultStart = now - timeInHour * 24;
|
|
19
|
+
export function getTimeRange(config?: { defaultRange?: number }): { startTime: number; endTime: number; searchFromStart: boolean } {
|
|
20
|
+
const defaultStart = now - (config?.defaultRange || timeInHour * 24);
|
|
21
21
|
const defaultEnd = now + timeInHour;
|
|
22
22
|
|
|
23
23
|
return {
|
|
@@ -27,9 +27,9 @@ export function getTimeRange(): { startTime: number; endTime: number; searchFrom
|
|
|
27
27
|
};
|
|
28
28
|
}
|
|
29
29
|
|
|
30
|
-
export class TimeRangeSelector extends qreact.Component<{ dark?: boolean }> {
|
|
30
|
+
export class TimeRangeSelector extends qreact.Component<{ dark?: boolean; defaultRange?: number }> {
|
|
31
31
|
render() {
|
|
32
|
-
const timeRange = getTimeRange();
|
|
32
|
+
const timeRange = getTimeRange({ defaultRange: this.props.defaultRange });
|
|
33
33
|
|
|
34
34
|
const resetToLastDay = () => {
|
|
35
35
|
now = Date.now();
|
|
@@ -78,8 +78,9 @@ export function createPatternSetMatcher(patterns: string[]): PatternSetMatcher {
|
|
|
78
78
|
|
|
79
79
|
function matchesWithIds(structure: IdStructure): boolean {
|
|
80
80
|
if (structure.type === "exact") return foundStamp[structure.id] === stamp;
|
|
81
|
-
if (structure.type
|
|
82
|
-
|
|
81
|
+
if (structure.type !== "wildcard") {
|
|
82
|
+
return structure.type === "or" ? structure.parts.some(matchesWithIds) : structure.parts.every(matchesWithIds);
|
|
83
|
+
}
|
|
83
84
|
if (!structure.ids.every(id => foundStamp[id] === stamp)) return false;
|
|
84
85
|
let position = 0;
|
|
85
86
|
for (let segment of structure.parts) {
|
|
@@ -168,7 +168,7 @@ export class LogBreakdownPage extends qreact.Component {
|
|
|
168
168
|
</div>}
|
|
169
169
|
</div>
|
|
170
170
|
<div className={css.hbox(16)}>
|
|
171
|
-
<ButtonSelector
|
|
171
|
+
<ButtonSelector<GroupBy>
|
|
172
172
|
title="Group by"
|
|
173
173
|
value={groupBy}
|
|
174
174
|
options={[
|
|
@@ -244,7 +244,7 @@ export class LogBreakdownPage extends qreact.Component {
|
|
|
244
244
|
{shownRows.map(row => {
|
|
245
245
|
let share = getShare(row.total);
|
|
246
246
|
return <div className={css.relative.fillWidth.hsl(0, 0, 14)}>
|
|
247
|
-
<div className={css.absolute.pos(0, 0).fillHeight.width(`${
|
|
247
|
+
<div className={css.absolute.pos(0, 0).fillHeight.width(`${Math.round(share * 10) / 10}%`).hsla(200, 60, 45, 0.35)} />
|
|
248
248
|
<div className={css.relative.hbox(12).pad2(4, 8).fillWidth.colorhsl(0, 0, 100)}>
|
|
249
249
|
<span className={css.width(LABEL_WIDTH).ellipsis} title={row.label}>{row.label}</span>
|
|
250
250
|
<span className={css.width(SHARE_WIDTH)}>{share.toFixed(1)}%</span>
|
|
@@ -268,7 +268,7 @@ export class LogBreakdownPage extends qreact.Component {
|
|
|
268
268
|
</div>;
|
|
269
269
|
})}
|
|
270
270
|
{otherRows.length > 0 && <div className={css.relative.fillWidth.hsl(0, 0, 14)}>
|
|
271
|
-
<div className={css.absolute.pos(0, 0).fillHeight.width(`${
|
|
271
|
+
<div className={css.absolute.pos(0, 0).fillHeight.width(`${Math.round(otherShare * 10) / 10}%`).hsla(0, 0, 60, 0.25)} />
|
|
272
272
|
<div className={css.relative.hbox(12).pad2(4, 8).fillWidth.colorhsl(0, 0, 100)}>
|
|
273
273
|
<span className={css.width(LABEL_WIDTH)}>{`+ ${otherRows.length} other categories under ${MIN_SHARE_PERCENT}% each`}</span>
|
|
274
274
|
<span className={css.width(SHARE_WIDTH)}>{otherShare.toFixed(1)}%</span>
|
|
@@ -67,6 +67,7 @@ export async function registerManagementPages2(config: {
|
|
|
67
67
|
notificationComponentName?: string;
|
|
68
68
|
|
|
69
69
|
title?: string;
|
|
70
|
+
before?: string;
|
|
70
71
|
}[];
|
|
71
72
|
}) {
|
|
72
73
|
// Wait, so this happens asynchronously
|
|
@@ -184,7 +185,17 @@ export async function registerManagementPages2(config: {
|
|
|
184
185
|
// controllerName: "SyncTestController",
|
|
185
186
|
// getModule: () => import("./SyncTestPage"),
|
|
186
187
|
// });
|
|
187
|
-
|
|
188
|
+
for (let page of config.pages) {
|
|
189
|
+
let beforeIndex = page.before ? inputPages.findIndex(x => x.componentName === page.before) : -1;
|
|
190
|
+
if (page.before && beforeIndex < 0) {
|
|
191
|
+
throw new Error(`Management page ${page.componentName} wants to go before ${page.before}, but no page has that componentName. The pages are ${inputPages.map(x => x.componentName).join(", ")}`);
|
|
192
|
+
}
|
|
193
|
+
if (beforeIndex < 0) {
|
|
194
|
+
inputPages.push(page);
|
|
195
|
+
} else {
|
|
196
|
+
inputPages.splice(beforeIndex, 0, page);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
188
199
|
|
|
189
200
|
// NOTE: We don't store the UI in the database (here, or anywhere else, at least
|
|
190
201
|
// not yet), but we do want to store the permission function call, as we may
|