querysub 0.729.0 → 0.730.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/debugging.md CHANGED
@@ -17,6 +17,8 @@ Every service runs as its own process in its own tmux session, `<service key>-<i
17
17
 
18
18
  The rest are background tasks: `movelogs` (moves pending logs to public storage and categorizes them), `errorwatch`, `gc`, `thin`, `join`, `ssh-trust`, `ping`, and the app's `queue` and `gpuworker` workers. They can go down without taking the site down.
19
19
 
20
+ The **auditing** services, `site-audit` (a headless browser driving the live site) and `storage-audit` (a write and read back on every storage route), run every 30 seconds on the Hetzner machine and record every run for the Uptime Verification management page (before Gross Stats in the management header). Results are written to the local disk and flushed to storage once an hour. Deploy them on their own, never together with anything else, or deploy-induced failures go unaudited.
21
+
20
22
  ## 1. What nodes exist
21
23
 
22
24
  ```
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "querysub",
3
- "version": "0.729.0",
3
+ "version": "0.730.0",
4
4
  "main": "index.js",
5
5
  "license": "MIT",
6
6
  "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -177,6 +177,7 @@ export class ServicesListPage extends qreact.Component {
177
177
  <div className={css.vbox(2).pad2(8).fillWidth.hsl(groupInfo.hue, 40, 90).bord2(groupInfo.hue, 40, 60)}>
178
178
  <div className={css.fontSize(18).boldStyle}>{groupInfo.emoji} {groupInfo.label} ({groupServices.length})</div>
179
179
  <div className={css.fontSize(12).colorhsl(0, 0, 35)}>{groupInfo.description}</div>
180
+ {groupInfo.note && <div className={css.fontSize(12).colorhsl(groupInfo.hue, 60, 35).boldStyle}>{groupInfo.note}</div>}
180
181
  </div>
181
182
  {this.renderServices({ services: groupServices, keyCounts, serviceNodeIds, getMachineInfo, getMachineConfig, now })}
182
183
  </div>;
@@ -202,6 +202,7 @@ export class DeployGroupButtons extends qreact.Component<{
202
202
  <div className={css.fontSize(12).colorhsl(0, 0, 40)}>
203
203
  {outdatedServices.length} of {allServices.length} service{allServices.length === 1 ? "" : "s"} outdated
204
204
  </div>
205
+ {groupInfo.note && <div className={css.fontSize(12).maxWidth(260).colorhsl(groupInfo.hue, 60, 35).boldStyle}>{groupInfo.note}</div>}
205
206
  </div>;
206
207
  if (!gitInfo || outdatedServices.length === 0) {
207
208
  return <div className={css.hbox(10).pad2(8).hsl(groupInfo.hue, 40, 95).bord2(groupInfo.hue, 40, 70)}>
@@ -27,7 +27,7 @@ export function getServiceCategory(parameters: ServiceParameters): ServiceCatego
27
27
  return CATEGORIES.find(x => command.startsWith(x.prefix))?.category;
28
28
  }
29
29
 
30
- export type ServiceDeployGroup = "storageCore" | "background" | "frontendProxy" | "mainSite";
30
+ export type ServiceDeployGroup = "storageCore" | "background" | "frontendProxy" | "mainSite" | "auditing";
31
31
 
32
32
  export type DeployGroupInfo = {
33
33
  group: ServiceDeployGroup;
@@ -35,6 +35,7 @@ export type DeployGroupInfo = {
35
35
  emoji: string;
36
36
  hue: number;
37
37
  description: string;
38
+ note?: string;
38
39
  deployWarning?: { title: string; paragraphs: string[] };
39
40
  noOverlap?: boolean;
40
41
  };
@@ -61,6 +62,7 @@ export const DEPLOY_GROUPS: DeployGroupInfo[] = [
61
62
  emoji: "🚪",
62
63
  hue: 35,
63
64
  description: "The HTTP bootstrapper. If it goes down users can't load the site, but it does very little, so it rarely needs updating. It binds port 443, so it is always deployed without overlap: an overlapped deploy leaves the new instance unable to bind the port, and the site goes down when the old one exits.",
65
+ note: "Only host the bootstrapper. You almost never need to update this. Until we have multiple, any updates will temporarily take down the site.",
64
66
  deployWarning: {
65
67
  title: "You probably don't need to update the front-end proxy",
66
68
  paragraphs: [
@@ -71,12 +73,22 @@ export const DEPLOY_GROUPS: DeployGroupInfo[] = [
71
73
  },
72
74
  noOverlap: true,
73
75
  },
76
+ {
77
+ group: "auditing",
78
+ label: "Auditing",
79
+ emoji: "🔍",
80
+ hue: 280,
81
+ description: "The uptime verification tests (the site test and the storage test), which record their results for the Uptime Verification page. Nothing depends on them and users never notice when they break, so deploying them is risk free. Always deployed without overlap.",
82
+ note: "Don't deploy this with anything else. Otherwise, deploy-induced failures won't be noticed by our auditing.",
83
+ noOverlap: true,
84
+ },
74
85
  {
75
86
  group: "storageCore",
76
87
  label: "Storage Core",
77
88
  emoji: "🗄️",
78
89
  hue: 0,
79
90
  description: "The storage server. Everything else depends on it.",
91
+ note: "Don't break this. If this goes down, you'll need to deploy via the CLI.",
80
92
  deployWarning: {
81
93
  title: "The storage core is very delicate",
82
94
  paragraphs: [
@@ -88,6 +100,10 @@ export const DEPLOY_GROUPS: DeployGroupInfo[] = [
88
100
  ];
89
101
 
90
102
  const BOOTSTRAP_ONLY_FLAG = "--bootstraponly";
103
+ const AUDITING_COMMAND_PREFIXES = [
104
+ "yarn site-audit",
105
+ "yarn storage-audit",
106
+ ];
91
107
  const MAIN_SITE_COMMAND_PREFIXES = [
92
108
  PATHVALUE_COMMAND_PREFIX,
93
109
  FUNCTION_RUNNER_COMMAND_PREFIX,
@@ -114,5 +130,6 @@ export function getInferredDeployGroup(parameters: ServiceParameters): ServiceDe
114
130
  if (command.startsWith(STORAGE_COMMAND_PREFIX)) return "storageCore";
115
131
  if (command.split(/\s+/).includes(BOOTSTRAP_ONLY_FLAG)) return "frontendProxy";
116
132
  if (MAIN_SITE_COMMAND_PREFIXES.some(prefix => command === prefix || command.startsWith(prefix + " "))) return "mainSite";
133
+ if (AUDITING_COMMAND_PREFIXES.some(prefix => command === prefix || command.startsWith(prefix + " "))) return "auditing";
117
134
  return "background";
118
135
  }
@@ -16,8 +16,8 @@ export const searchFromStartParam = new URLParam("searchFromStart", false);
16
16
 
17
17
  let now = Date.now();
18
18
  /** Get the time range with default values if not set */
19
- export function getTimeRange(): { startTime: number; endTime: number; searchFromStart: boolean } {
20
- const defaultStart = now - timeInHour * 24;
19
+ export function getTimeRange(config?: { defaultRange?: number }): { startTime: number; endTime: number; searchFromStart: boolean } {
20
+ const defaultStart = now - (config?.defaultRange || timeInHour * 24);
21
21
  const defaultEnd = now + timeInHour;
22
22
 
23
23
  return {
@@ -27,9 +27,9 @@ export function getTimeRange(): { startTime: number; endTime: number; searchFrom
27
27
  };
28
28
  }
29
29
 
30
- export class TimeRangeSelector extends qreact.Component<{ dark?: boolean }> {
30
+ export class TimeRangeSelector extends qreact.Component<{ dark?: boolean; defaultRange?: number }> {
31
31
  render() {
32
- const timeRange = getTimeRange();
32
+ const timeRange = getTimeRange({ defaultRange: this.props.defaultRange });
33
33
 
34
34
  const resetToLastDay = () => {
35
35
  now = Date.now();
@@ -78,8 +78,9 @@ export function createPatternSetMatcher(patterns: string[]): PatternSetMatcher {
78
78
 
79
79
  function matchesWithIds(structure: IdStructure): boolean {
80
80
  if (structure.type === "exact") return foundStamp[structure.id] === stamp;
81
- if (structure.type === "or") return structure.parts.some(matchesWithIds);
82
- if (structure.type === "and") return structure.parts.every(matchesWithIds);
81
+ if (structure.type !== "wildcard") {
82
+ return structure.type === "or" ? structure.parts.some(matchesWithIds) : structure.parts.every(matchesWithIds);
83
+ }
83
84
  if (!structure.ids.every(id => foundStamp[id] === stamp)) return false;
84
85
  let position = 0;
85
86
  for (let segment of structure.parts) {
@@ -168,7 +168,7 @@ export class LogBreakdownPage extends qreact.Component {
168
168
  </div>}
169
169
  </div>
170
170
  <div className={css.hbox(16)}>
171
- <ButtonSelector
171
+ <ButtonSelector<GroupBy>
172
172
  title="Group by"
173
173
  value={groupBy}
174
174
  options={[
@@ -244,7 +244,7 @@ export class LogBreakdownPage extends qreact.Component {
244
244
  {shownRows.map(row => {
245
245
  let share = getShare(row.total);
246
246
  return <div className={css.relative.fillWidth.hsl(0, 0, 14)}>
247
- <div className={css.absolute.pos(0, 0).fillHeight.width(`${share.toFixed(1)}%`).hsla(200, 60, 45, 0.35)} />
247
+ <div className={css.absolute.pos(0, 0).fillHeight.width(`${Math.round(share * 10) / 10}%`).hsla(200, 60, 45, 0.35)} />
248
248
  <div className={css.relative.hbox(12).pad2(4, 8).fillWidth.colorhsl(0, 0, 100)}>
249
249
  <span className={css.width(LABEL_WIDTH).ellipsis} title={row.label}>{row.label}</span>
250
250
  <span className={css.width(SHARE_WIDTH)}>{share.toFixed(1)}%</span>
@@ -268,7 +268,7 @@ export class LogBreakdownPage extends qreact.Component {
268
268
  </div>;
269
269
  })}
270
270
  {otherRows.length > 0 && <div className={css.relative.fillWidth.hsl(0, 0, 14)}>
271
- <div className={css.absolute.pos(0, 0).fillHeight.width(`${otherShare.toFixed(1)}%`).hsla(0, 0, 60, 0.25)} />
271
+ <div className={css.absolute.pos(0, 0).fillHeight.width(`${Math.round(otherShare * 10) / 10}%`).hsla(0, 0, 60, 0.25)} />
272
272
  <div className={css.relative.hbox(12).pad2(4, 8).fillWidth.colorhsl(0, 0, 100)}>
273
273
  <span className={css.width(LABEL_WIDTH)}>{`+ ${otherRows.length} other categories under ${MIN_SHARE_PERCENT}% each`}</span>
274
274
  <span className={css.width(SHARE_WIDTH)}>{otherShare.toFixed(1)}%</span>
@@ -67,6 +67,7 @@ export async function registerManagementPages2(config: {
67
67
  notificationComponentName?: string;
68
68
 
69
69
  title?: string;
70
+ before?: string;
70
71
  }[];
71
72
  }) {
72
73
  // Wait, so this happens asynchronously
@@ -184,7 +185,17 @@ export async function registerManagementPages2(config: {
184
185
  // controllerName: "SyncTestController",
185
186
  // getModule: () => import("./SyncTestPage"),
186
187
  // });
187
- inputPages.push(...config.pages);
188
+ for (let page of config.pages) {
189
+ let beforeIndex = page.before ? inputPages.findIndex(x => x.componentName === page.before) : -1;
190
+ if (page.before && beforeIndex < 0) {
191
+ throw new Error(`Management page ${page.componentName} wants to go before ${page.before}, but no page has that componentName. The pages are ${inputPages.map(x => x.componentName).join(", ")}`);
192
+ }
193
+ if (beforeIndex < 0) {
194
+ inputPages.push(page);
195
+ } else {
196
+ inputPages.splice(beforeIndex, 0, page);
197
+ }
198
+ }
188
199
 
189
200
  // NOTE: We don't store the UI in the database (here, or anywhere else, at least
190
201
  // not yet), but we do want to store the permission function call, as we may
@@ -1172,38 +1172,5 @@ export async function nodeJSLoginToUser(userId: string) {
1172
1172
  });
1173
1173
  }
1174
1174
 
1175
- export const TEST_USER_ID_PREFIX = "test-";
1176
- export async function scriptAddTrust(config: { userId: string; machineId: string; }) {
1177
- const { userId, machineId } = config;
1178
- if (!userId.startsWith(TEST_USER_ID_PREFIX)) {
1179
- throw new Error(`Trust can only be added to test accounts, whose user ids start with "${TEST_USER_ID_PREFIX}", was ${userId}`);
1180
- }
1181
- const ips = [await getExternalIP(), "127.0.0.1"];
1182
- let now = Date.now();
1183
- await Querysub.commitSynced(() => {
1184
- if (!(userId in data().users)) {
1185
- data().users[userId] = {
1186
- userId,
1187
- email: `${userId} is a test account without email login`,
1188
- settings: { displayName: userId },
1189
- userType: "user",
1190
- createTime: now,
1191
- machineIds: {},
1192
- allowedIPs: {},
1193
- loginTokens: {},
1194
- lastPageLoadsIPs: {},
1195
- invitedUsers2: {},
1196
- invitesRemaining: 0,
1197
- };
1198
- }
1199
- let userObj = data().users[userId];
1200
- for (let ip of ips) {
1201
- userObj.allowedIPs[ip] = atomicObjectWrite({ time: now, machineId });
1202
- }
1203
- userObj.machineIds[machineId] = atomicObjectWrite({ time: now, ip: ips[0] });
1204
- data().machineSecure[machineId].userId = userId;
1205
- });
1206
- }
1207
-
1208
1175
  export const user_data = data;
1209
1176
  export const user_functions = functions;