querysub 0.725.0 → 0.727.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/bin/backfill-log-categories.js +7 -0
  2. package/package.json +8 -2
  3. package/src/-a-archives/archiveCache.ts +7 -7
  4. package/src/-a-archives/archivesDisk.ts +20 -7
  5. package/src/0-path-value-core/AuthorityLookup.ts +113 -19
  6. package/src/deployManager/components/DeployPage.tsx +2 -2
  7. package/src/deployManager/components/ServiceDetailPage.tsx +17 -4
  8. package/src/deployManager/components/ServicesListPage.tsx +61 -8
  9. package/src/deployManager/components/deployButtons.tsx +166 -101
  10. package/src/deployManager/machineSchema.ts +13 -2
  11. package/src/deployManager/publishQuerysub.ts +121 -49
  12. package/src/deployManager/serviceCategories.ts +90 -1
  13. package/src/diagnostics/logs/IndexedLogs/BufferIndexLogsOptimizationConstants.ts +2 -0
  14. package/src/diagnostics/logs/IndexedLogs/FindProgressTracker.ts +2 -1
  15. package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts +40 -1
  16. package/src/diagnostics/logs/IndexedLogs/LogStreamer.ts +6 -2
  17. package/src/diagnostics/logs/IndexedLogs/MCPIndexedLogs.ts +2 -6
  18. package/src/diagnostics/logs/IndexedLogs/TimeFileTree.ts +34 -0
  19. package/src/diagnostics/logs/IndexedLogs/limitPublicLogsSize.ts +119 -0
  20. package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts +13 -5
  21. package/src/diagnostics/logs/IndexedLogs/moveLogsEntry.ts +2 -0
  22. package/src/diagnostics/logs/errorNotifications2/ErrorNotificationPage.tsx +93 -21
  23. package/src/diagnostics/logs/errorNotifications2/openRouterHelper.ts +4 -2
  24. package/src/diagnostics/logs/errorNotifications2/suppressionPatternMatcher.ts +1 -1
  25. package/src/diagnostics/logs/logCategories/LogBreakdownPage.tsx +421 -0
  26. package/src/diagnostics/logs/logCategories/logCategories.ts +273 -0
  27. package/src/diagnostics/managementPages.tsx +6 -0
  28. package/src/diagnostics/network/backfillLogCategoriesMain.ts +128 -0
  29. package/src/diagnostics/network/deleteOldLogsMain.ts +57 -0
  30. package/src/diagnostics/network/listLogFilesMain.ts +60 -0
  31. package/src/diagnostics/network/logFilesMain.ts +13 -15
  32. package/src/diagnostics/network/probeLogDeleteMain.ts +71 -0
  33. package/src/diagnostics/network/watchServiceMain.ts +106 -0
  34. package/src/fs.ts +29 -0
  35. package/src/misc/moveFolderContents.ts +67 -0
@@ -1,4 +1,4 @@
1
- import { ServiceParameters } from "./machineSchema";
1
+ import { ServiceConfig, ServiceParameters } from "./machineSchema";
2
2
  import { ViewType } from "./urlParams";
3
3
  import { DISK_PATHVALUE_COLOR, PATHVALUE_COLOR, FUNCTION_RUNNER_COLOR } from "../misc/nodeCategoryColors";
4
4
 
@@ -26,3 +26,92 @@ export function getServiceCategory(parameters: ServiceParameters): ServiceCatego
26
26
  let command = parameters.command.trimStart();
27
27
  return CATEGORIES.find(x => command.startsWith(x.prefix))?.category;
28
28
  }
29
+
30
+ export type ServiceDeployGroup = "storageCore" | "background" | "frontendProxy" | "mainSite";
31
+
32
+ export type DeployGroupInfo = {
33
+ group: ServiceDeployGroup;
34
+ label: string;
35
+ emoji: string;
36
+ hue: number;
37
+ description: string;
38
+ deployWarning?: { title: string; paragraphs: string[] };
39
+ noOverlap?: boolean;
40
+ };
41
+
42
+ export const DEPLOY_GROUPS: DeployGroupInfo[] = [
43
+ {
44
+ group: "background",
45
+ label: "Background Tasks",
46
+ emoji: "🔧",
47
+ hue: 200,
48
+ description: "Non-essential services. They can go down without immediately affecting the site (eventually they will). Also the catch-all, for tests, pings, etc. Deployed without overlap, two copies of a background task must never run at once.",
49
+ noOverlap: true,
50
+ },
51
+ {
52
+ group: "mainSite",
53
+ label: "Main Site",
54
+ emoji: "🌐",
55
+ hue: 120,
56
+ description: "Everything the site needs to work right now.",
57
+ },
58
+ {
59
+ group: "frontendProxy",
60
+ label: "Front-end Proxy",
61
+ emoji: "🚪",
62
+ hue: 35,
63
+ description: "The HTTP bootstrapper. If it goes down users can't load the site, but it does very little, so it rarely needs updating.",
64
+ deployWarning: {
65
+ title: "You probably don't need to update the front-end proxy",
66
+ paragraphs: [
67
+ "It does very little. It only serves the bootstrapper, so it doesn't need to be updated when you change most of the framework code.",
68
+ "While it restarts, the site is temporarily unavailable.",
69
+ "It's usually best not to touch it.",
70
+ ],
71
+ },
72
+ },
73
+ {
74
+ group: "storageCore",
75
+ label: "Storage Core",
76
+ emoji: "🗄️",
77
+ hue: 0,
78
+ description: "The storage server. Everything else depends on it.",
79
+ deployWarning: {
80
+ title: "The storage core is very delicate",
81
+ paragraphs: [
82
+ "If it breaks, everything breaks, and it is very hard to fix.",
83
+ "Be careful when changing it.",
84
+ ],
85
+ },
86
+ },
87
+ ];
88
+
89
+ const BOOTSTRAP_ONLY_FLAG = "--bootstraponly";
90
+ const MAIN_SITE_COMMAND_PREFIXES = [
91
+ PATHVALUE_COMMAND_PREFIX,
92
+ FUNCTION_RUNNER_COMMAND_PREFIX,
93
+ "yarn cyoa-public",
94
+ "yarn queue-public",
95
+ "yarn queue-gpu-public",
96
+ "yarn worker-public",
97
+ ];
98
+
99
+ export function getDeployGroupInfo(group: ServiceDeployGroup): DeployGroupInfo {
100
+ let info = DEPLOY_GROUPS.find(x => x.group === group);
101
+ if (!info) throw new Error(`Unknown deploy group ${JSON.stringify(group)}, expected one of ${DEPLOY_GROUPS.map(x => x.group).join(", ")}`);
102
+ return info;
103
+ }
104
+
105
+ /** The group set on the service, or for services that never had one set, the group its command implies. */
106
+ export function getServiceDeployGroup(config: ServiceConfig): ServiceDeployGroup {
107
+ if (config.info.deployGroup) return config.info.deployGroup;
108
+ return getInferredDeployGroup(config.parameters);
109
+ }
110
+
111
+ export function getInferredDeployGroup(parameters: ServiceParameters): ServiceDeployGroup {
112
+ let command = parameters.command.trimStart();
113
+ if (command.startsWith(STORAGE_COMMAND_PREFIX)) return "storageCore";
114
+ if (command.split(/\s+/).includes(BOOTSTRAP_ONLY_FLAG)) return "frontendProxy";
115
+ if (MAIN_SITE_COMMAND_PREFIXES.some(prefix => command === prefix || command.startsWith(prefix + " "))) return "mainSite";
116
+ return "background";
117
+ }
@@ -18,6 +18,8 @@ export const MAX_SINGLE_FILE_DATA = 1024 * 1024 * 64;
18
18
 
19
19
  export const MOVING_TIMEOUT = timeInMinute * 5;
20
20
 
21
+ export const MAX_PUBLIC_LOG_BYTES = 1024 * 1024 * 1024 * 10;
22
+
21
23
  export const DISK_FLUSH_INTERVAL = timeInSecond * 15;
22
24
  export const MAX_COUNT_PER_FILE = 1000 * 1000 * 10;
23
25
 
@@ -4,7 +4,7 @@ import { SearchParams } from "./BufferIndexHelpers";
4
4
  export class FindProgressTracker<T> {
5
5
  public constructor(private config: {
6
6
  params: SearchParams;
7
- deserialize: (result: Buffer) => T;
7
+ deserialize: (result: Buffer) => T | undefined;
8
8
  getTime: (result: T) => number | undefined;
9
9
  onResult: (result: T) => void;
10
10
  }) { }
@@ -14,6 +14,7 @@ export class FindProgressTracker<T> {
14
14
 
15
15
  public addResult(buffer: Buffer, source: { startTime: number; endTime: number; }): boolean {
16
16
  let result = this.config.deserialize(buffer);
17
+ if (!result) return false;
17
18
  let time = this.config.getTime(result) ?? source.endTime;
18
19
 
19
20
  if (time < this.config.params.startTime) return false;
@@ -18,7 +18,8 @@ import { getOwnMachineId, isNodeIdOnOwnMachineId, isOwnNodeId } from "../../../-
18
18
  import { LogArchives, TimeFilePath, TimeFileTree } from "./TimeFileTree";
19
19
  import { LogStreamer } from "./LogStreamer";
20
20
  import { moveLogsToPublic } from "./moveIndexLogsToPublic";
21
- import { MAX_SINGLE_FILE_DATA, MAX_COUNT_PER_FILE, DISK_FLUSH_INTERVAL, PUBLIC_MOVE_THRESHOLD, MOVING_TIMEOUT } from "./BufferIndexLogsOptimizationConstants";
21
+ import { MAX_SINGLE_FILE_DATA, MAX_COUNT_PER_FILE, DISK_FLUSH_INTERVAL, PUBLIC_MOVE_THRESHOLD, MOVING_TIMEOUT, MAX_PUBLIC_LOG_BYTES } from "./BufferIndexLogsOptimizationConstants";
22
+ import { LimitPublicLogsResult, limitPublicLogsSize } from "./limitPublicLogsSize";
22
23
  import { FindProgressTracker } from "./FindProgressTracker";
23
24
  import { SocketFunction } from "socket-function/SocketFunction";
24
25
  import { assertIsManagementUser } from "../../managementPages";
@@ -30,6 +31,7 @@ import { NodeCapabilitiesController } from "../../../-g-core-values/NodeCapabili
30
31
  import { getLoggers2Async } from "../diskLogger";
31
32
  import { watchAllValues } from "../errorNotifications2/logWatcher";
32
33
  import { wrapArchivesWithCache } from "../../../-a-archives/archiveCache";
34
+ import { LOG_CATEGORIES_EXTENSION, LogCategoryCounts, createLogCategoriesFile, moveLogsAICallLimiter, parseLogCategoriesFile } from "../logCategories/logCategories";
33
35
 
34
36
  // Ensure it's available so that the controller is listening on all servers
35
37
  watchAllValues;
@@ -96,6 +98,9 @@ export class IndexedLogs<T> {
96
98
  });
97
99
  return archives;
98
100
  });
101
+ private getPublicLogArchivesUncached = lazy((): LogArchives => {
102
+ return getArchives2("final-indexed-logs/" + this.config.name);
103
+ });
99
104
  private getPublicLogsBase(loadPublic: boolean): LogArchives {
100
105
  if (loadPublic) {
101
106
  return createArchivesMemoryCache2(getArchives2("final-indexed-logs/" + this.config.name), {
@@ -152,6 +157,26 @@ export class IndexedLogs<T> {
152
157
  private getRealPublicLogs = lazy((): LogArchives => {
153
158
  return this.getPublicLogsBase(true);
154
159
  });
160
+ public getPublicLogArchives(): LogArchives {
161
+ return this.getPublicLogArchivesUncached();
162
+ }
163
+ public async deletePublicLogFile(path: TimeFilePath): Promise<void> {
164
+ let archives = this.getPublicLogArchives();
165
+ await Promise.all([
166
+ archives.del(path.fullPath + INDEX_EXTENSION),
167
+ archives.del(path.fullPath + LOG_CATEGORIES_EXTENSION),
168
+ ]);
169
+ await archives.del(path.fullPath);
170
+ }
171
+ public async listPublicLogPaths(config: { startTime: number; endTime: number }): Promise<TimeFilePath[]> {
172
+ return await new TimeFileTree(this.getPublicLogArchives()).findAllPaths(config);
173
+ }
174
+ // Category counts live beside the public logs, so dev and production both read this same storage
175
+ public async readLogCategoryCounts(fullPath: string): Promise<LogCategoryCounts | undefined> {
176
+ let buffer = await this.getPublicLogArchives().get(fullPath + LOG_CATEGORIES_EXTENSION);
177
+ if (!buffer) return undefined;
178
+ return parseLogCategoriesFile(buffer);
179
+ }
155
180
 
156
181
  private getTimeBlock(time: number): { startTime: number, endTime: number } {
157
182
  let startTime = Math.floor(time / timeInHour) * timeInHour;
@@ -696,9 +721,23 @@ export class IndexedLogs<T> {
696
721
  maxSingleFileData: this.config.maxSingleFileData || MAX_SINGLE_FILE_DATA,
697
722
  movingTimeout: MOVING_TIMEOUT,
698
723
  indexExtension: INDEX_EXTENSION,
724
+ companion: {
725
+ extension: LOG_CATEGORIES_EXTENSION,
726
+ create: entries => createLogCategoriesFile({ loggerName: this.config.name, entries, aiCallLimiter: moveLogsAICallLimiter }),
727
+ },
699
728
  });
729
+ await this.limitPublicLogsSize();
700
730
  }
701
731
 
732
+ public async limitPublicLogsSize(config?: { dryRun?: boolean }): Promise<LimitPublicLogsResult> {
733
+ return await limitPublicLogsSize({
734
+ loggerName: this.config.name,
735
+ publicLogs: this.getPublicLogs(),
736
+ maxBytes: MAX_PUBLIC_LOG_BYTES,
737
+ siblingExtensions: [INDEX_EXTENSION, LOG_CATEGORIES_EXTENSION],
738
+ dryRun: config?.dryRun,
739
+ });
740
+ }
702
741
 
703
742
  // #region Client Calls
704
743
  private findCallbacks = new Map<string, (match: T) => void>();
@@ -40,8 +40,12 @@ export class LogStreamer<T> {
40
40
  this.insideFlush = false;
41
41
  }
42
42
  });
43
- public static deserialize<T>(data: Buffer): T {
44
- return JSON.parse(data.toString());
43
+ public static deserialize<T>(data: Buffer): T | undefined {
44
+ try {
45
+ return JSON.parse(data.toString());
46
+ } catch {
47
+ return undefined;
48
+ }
45
49
  }
46
50
  }
47
51
 
@@ -458,12 +458,8 @@ export class MCPIndexedLogs {
458
458
  let { buf, matchesPattern, columns, startTime, endTime, sink } = append;
459
459
  if (!matchesPattern(buf)) return;
460
460
 
461
- let datum: LogDatum;
462
- try {
463
- datum = LogStreamer.deserialize<LogDatum>(buf);
464
- } catch {
465
- return;
466
- }
461
+ let datum = LogStreamer.deserialize<LogDatum>(buf);
462
+ if (!datum) return;
467
463
 
468
464
  if (typeof datum.time !== "number") return;
469
465
  if (datum.time < startTime || datum.time > endTime) return;
@@ -127,6 +127,7 @@ export class TimeFileTree {
127
127
  public async findAllPaths(config: {
128
128
  startTime: number;
129
129
  endTime: number;
130
+ onDayFolderListed?: (info: { dayFolder: string; fileCount: number }) => void;
130
131
  }): Promise<TimeFilePath[]> {
131
132
  // For short ranges, guessing the day folder paths and reading them all in parallel is
132
133
  // much faster than recursively listing year/month/day folders.
@@ -141,6 +142,7 @@ export class TimeFileTree {
141
142
  private async findAllPathsByDayGuess(config: {
142
143
  startTime: number;
143
144
  endTime: number;
145
+ onDayFolderListed?: (info: { dayFolder: string; fileCount: number }) => void;
144
146
  }): Promise<TimeFilePath[]> {
145
147
  // Build the list of UTC day folder prefixes spanning the range.
146
148
  const dayFolders: string[] = [];
@@ -159,6 +161,7 @@ export class TimeFileTree {
159
161
 
160
162
  await Promise.all(dayFolders.map(async (dayFolder) => {
161
163
  const files = await this.archives.find(dayFolder, { shallow: true, type: "files" });
164
+ config.onDayFolderListed?.({ dayFolder, fileCount: files.length });
162
165
 
163
166
  for (const file of files) {
164
167
  const decoded = decodeLogFilePath(file);
@@ -180,6 +183,7 @@ export class TimeFileTree {
180
183
  private async findAllPathsByScan(config: {
181
184
  startTime: number;
182
185
  endTime: number;
186
+ onDayFolderListed?: (info: { dayFolder: string; fileCount: number }) => void;
183
187
  }): Promise<TimeFilePath[]> {
184
188
  const results: TimeFilePath[] = [];
185
189
 
@@ -241,6 +245,7 @@ export class TimeFileTree {
241
245
 
242
246
  // Find all log files in this day
243
247
  const files = await this.archives.find(dayFolder + "/", { shallow: true, type: "files" });
248
+ config.onDayFolderListed?.({ dayFolder, fileCount: files.length });
244
249
 
245
250
  for (const file of files) {
246
251
  const decoded = decodeLogFilePath(file);
@@ -261,6 +266,35 @@ export class TimeFileTree {
261
266
  return results;
262
267
  }
263
268
 
269
+ public async findDayFolders(): Promise<string[]> {
270
+ let dayFolders: string[] = [];
271
+ let years = await this.archives.find("", { shallow: true, type: "folders" });
272
+ for (let yearFolder of years) {
273
+ if (isNaN(parseInt(yearFolder.split("/").at(-1) || "", 10))) continue;
274
+ let months = await this.archives.find(yearFolder + "/", { shallow: true, type: "folders" });
275
+ await Promise.all(months.map(async monthFolder => {
276
+ if (isNaN(parseInt(monthFolder.split("/").at(-1) || "", 10))) return;
277
+ let days = await this.archives.find(monthFolder + "/", { shallow: true, type: "folders" });
278
+ for (let dayFolder of days) {
279
+ if (isNaN(parseInt(dayFolder.split("/").at(-1) || "", 10))) continue;
280
+ dayFolders.push(dayFolder + "/");
281
+ }
282
+ }));
283
+ }
284
+ dayFolders.sort();
285
+ return dayFolders;
286
+ }
287
+
288
+ public async findPathsInDayFolder(dayFolder: string): Promise<TimeFilePath[]> {
289
+ let files = await this.archives.find(dayFolder, { shallow: true, type: "files" });
290
+ let paths: TimeFilePath[] = [];
291
+ for (let file of files) {
292
+ let decoded = decodeLogFilePath(file);
293
+ if (decoded) paths.push(decoded);
294
+ }
295
+ return paths;
296
+ }
297
+
264
298
  public async deleteAll() {
265
299
  let rootFolders = await this.archives.find("", { shallow: true, type: "folders" });
266
300
  for (let rootFolder of rootFolders) {
@@ -0,0 +1,119 @@
1
+ import { formatDateTime, formatNumber, formatTime } from "socket-function/src/formatting/format";
2
+ import { blue, magenta, yellow } from "socket-function/src/formatting/logColors";
3
+ import { sort, timeInSecond } from "socket-function/src/misc";
4
+ import { LogArchives, TimeFilePath, TimeFileTree } from "./TimeFileTree";
5
+
6
+ const TARGET_FILES_PER_BATCH = 1000;
7
+ const PARALLEL_CALLS = 32;
8
+ const PROGRESS_LOG_INTERVAL = timeInSecond * 5;
9
+ const DELETE_ATTEMPTS = 3;
10
+
11
+ export type LimitPublicLogsResult = {
12
+ keptFiles: number;
13
+ keptBytes: number;
14
+ deletedFiles: number;
15
+ deletedBytes: number;
16
+ failedFiles: number;
17
+ };
18
+
19
+ export async function limitPublicLogsSize(config: {
20
+ loggerName: string;
21
+ publicLogs: LogArchives;
22
+ maxBytes: number;
23
+ siblingExtensions: string[];
24
+ dryRun?: boolean;
25
+ }): Promise<LimitPublicLogsResult> {
26
+ let { loggerName, publicLogs, maxBytes, siblingExtensions, dryRun } = config;
27
+ let start = Date.now();
28
+ let lastProgressLog = Date.now();
29
+ let tree = new TimeFileTree(publicLogs);
30
+ let dayFolders = (await tree.findDayFolders()).reverse();
31
+
32
+ let files: { path: TimeFilePath; bytes: number }[] = [];
33
+ let daysRead = 0;
34
+ let daysPerBatch = 1;
35
+ while (daysRead < dayFolders.length) {
36
+ let batch = dayFolders.slice(daysRead, daysRead + daysPerBatch);
37
+ let paths = (await Promise.all(batch.map(dayFolder => tree.findPathsInDayFolder(dayFolder)))).flat();
38
+ daysRead += batch.length;
39
+ let pathIndex = 0;
40
+ async function sizeWorker() {
41
+ while (pathIndex < paths.length) {
42
+ let path = paths[pathIndex++];
43
+ let infos = await Promise.all([path.fullPath, ...siblingExtensions.map(extension => path.fullPath + extension)].map(file => publicLogs.getInfo(file)));
44
+ files.push({ path, bytes: infos.reduce((acc, x) => acc + (x?.size || 0), 0) });
45
+ }
46
+ }
47
+ await Promise.all(Array.from({ length: PARALLEL_CALLS }, () => sizeWorker()));
48
+ daysPerBatch = Math.max(1, Math.round(TARGET_FILES_PER_BATCH / Math.max(1, files.length / daysRead)));
49
+
50
+ if (Date.now() - lastProgressLog >= PROGRESS_LOG_INTERVAL) {
51
+ lastProgressLog = Date.now();
52
+ console.log(magenta(`Sizing public ${blue(loggerName)} logs: ${daysRead}/${dayFolders.length} day folders, ${formatNumber(files.length)} files, ${formatTime(Date.now() - start)} so far`));
53
+ }
54
+ }
55
+
56
+ sort(files, x => -x.path.startTime);
57
+
58
+ let keptBytes = 0;
59
+ let cutoffIndex = files.length;
60
+ for (let i = 0; i < files.length; i++) {
61
+ if (keptBytes + files[i].bytes > maxBytes) {
62
+ cutoffIndex = i;
63
+ break;
64
+ }
65
+ keptBytes += files[i].bytes;
66
+ }
67
+ let keptFiles = cutoffIndex;
68
+ let toDelete = files.slice(cutoffIndex);
69
+
70
+ let deletedBytes = toDelete.reduce((acc, x) => acc + x.bytes, 0);
71
+ let result: LimitPublicLogsResult = { keptFiles, keptBytes, deletedFiles: toDelete.length, deletedBytes, failedFiles: 0 };
72
+ if (!toDelete.length) {
73
+ if (dryRun) {
74
+ console.log(magenta(`Public ${blue(loggerName)} logs are within the limit: ${keptFiles} files, ${formatNumber(keptBytes)}B of ${formatNumber(maxBytes)}B (dry run, sized in ${formatTime(Date.now() - start)})`));
75
+ }
76
+ return result;
77
+ }
78
+
79
+ let oldest = Math.min(...toDelete.map(x => x.path.startTime));
80
+ let newest = Math.max(...toDelete.map(x => x.path.startTime));
81
+ console.log(yellow(`Public ${blue(loggerName)} logs are over the ${formatNumber(maxBytes)}B limit: keeping the newest ${keptFiles} files (${formatNumber(keptBytes)}B), ${dryRun && "would delete" || "deleting"} ${toDelete.length} older files (${formatNumber(deletedBytes)}B) starting ${formatDateTime(oldest)} to ${formatDateTime(newest)}`));
82
+ if (dryRun) return result;
83
+
84
+ let deleted = 0;
85
+ let failed: { path: TimeFilePath; error: Error }[] = [];
86
+ let deleteStart = Date.now();
87
+ let nextIndex = 0;
88
+ async function deleteWorker() {
89
+ while (nextIndex < toDelete.length) {
90
+ let path = toDelete[nextIndex++].path;
91
+ for (let attempt = 1; attempt <= DELETE_ATTEMPTS; attempt++) {
92
+ try {
93
+ for (let extension of siblingExtensions) {
94
+ await publicLogs.del(path.fullPath + extension);
95
+ }
96
+ await publicLogs.del(path.fullPath);
97
+ deleted++;
98
+ break;
99
+ } catch (e) {
100
+ if (attempt < DELETE_ATTEMPTS) continue;
101
+ failed.push({ path, error: e as Error });
102
+ }
103
+ }
104
+ if (Date.now() - lastProgressLog < PROGRESS_LOG_INTERVAL) continue;
105
+ lastProgressLog = Date.now();
106
+ let elapsed = Date.now() - deleteStart;
107
+ let done = deleted + failed.length;
108
+ console.log(magenta(`Deleting public ${blue(loggerName)} logs: ${deleted}/${toDelete.length} files, ${failed.length} failed, ${formatTime(elapsed)} so far, about ${formatTime(elapsed * (toDelete.length - done) / Math.max(1, done))} left`));
109
+ }
110
+ }
111
+ await Promise.all(Array.from({ length: PARALLEL_CALLS }, () => deleteWorker()));
112
+ console.log(magenta(`Deleted ${deleted} of ${toDelete.length} public ${blue(loggerName)} log files (${formatNumber(deletedBytes)}B targeted) in ${formatTime(Date.now() - deleteStart)}`));
113
+ if (failed.length) {
114
+ console.warn(`Failed to delete ${failed.length} of ${toDelete.length} public ${loggerName} log files after ${DELETE_ATTEMPTS} attempts each; the next limit pass will try them again. First failure, ${failed[0].path.fullPath}:`, failed[0].error.stack || failed[0].error);
115
+ }
116
+ result.deletedFiles = deleted;
117
+ result.failedFiles = failed.length;
118
+ return result;
119
+ }
@@ -22,8 +22,12 @@ export async function moveLogsToPublic(config: {
22
22
  localLogs: LogArchives;
23
23
  publicLogs: LogArchives;
24
24
  indexExtension: string;
25
+ companion: {
26
+ extension: string;
27
+ create: (entries: Buffer[]) => Promise<Buffer>;
28
+ };
25
29
  }) {
26
- let { forceAll, localLogs, publicLogs, publicMoveThreshold, maxSingleFileData, indexExtension, movingTimeout } = config;
30
+ let { forceAll, localLogs, publicLogs, publicMoveThreshold, maxSingleFileData, indexExtension, movingTimeout, companion } = config;
27
31
  let now = Date.now();
28
32
  let threadId = getOwnThreadId();
29
33
  let ourMovingFileName = `${now}-${threadId}.moving`;
@@ -237,6 +241,7 @@ export async function moveLogsToPublic(config: {
237
241
  let encoded: {
238
242
  data: Buffer;
239
243
  index: Buffer;
244
+ companion: Buffer;
240
245
  compressedSize: number;
241
246
  uncompressedSize: number;
242
247
  logCount: number;
@@ -252,11 +257,12 @@ export async function moveLogsToPublic(config: {
252
257
  encoded.push({
253
258
  data: obj.data,
254
259
  index: obj.index,
260
+ companion: await companion.create(group),
255
261
  compressedSize: obj.data.length,
256
262
  uncompressedSize,
257
263
  logCount: group.length,
258
264
  });
259
- // Wait, so if we have to encode a lot of data, we don't block for too long?
265
+ // Wait, so if we have to encode a lot of data, we don't block for too long?
260
266
  await delay(0);
261
267
  }
262
268
 
@@ -269,8 +275,10 @@ export async function moveLogsToPublic(config: {
269
275
  startTime,
270
276
  endTime,
271
277
  });
272
- let indexPath = path + indexExtension;
273
- await publicLogs.set(indexPath, obj.index);
278
+ await Promise.all([
279
+ publicLogs.set(path + indexExtension, obj.index),
280
+ publicLogs.set(path + companion.extension, obj.companion),
281
+ ]);
274
282
  await publicLogs.set(path, obj.data);
275
283
  }
276
284
 
@@ -284,7 +292,7 @@ export async function moveLogsToPublic(config: {
284
292
  let elapsed = Date.now() - groupsStart;
285
293
  let remaining = bytesDone && elapsed * (totalBytes - bytesDone) / bytesDone || 0;
286
294
  console.log(magenta(`Moved ${filesDone}/${allPathsFlat.length} files, ${formatNumber(bytesDone)}B/${formatNumber(totalBytes)}B, ${formatTime(elapsed)} so far, about ${formatTime(remaining)} left (${blue(config.loggerName)})`));
287
- console.log(green(`(${i + 1}/${groups.length}) Wrote ${encoded.length} log files to public from ${group.length} input files, ${formatDateTime(startTime)} to ${formatDateTime(endTime)} (${formatNumber(encoded.reduce((acc, x) => acc + x.uncompressedSize, 0))}B compressed to ${formatNumber(encoded.reduce((acc, x) => acc + x.compressedSize, 0))}B + ${formatNumber(encoded.reduce((acc, x) => acc + x.index.length, 0))}B index) in ${formatTime(Date.now() - time)}`) + ` (${blue(config.loggerName)})`);
295
+ console.log(green(`(${i + 1}/${groups.length}) Wrote ${encoded.length} log files to public from ${group.length} input files, ${formatDateTime(startTime)} to ${formatDateTime(endTime)} (${formatNumber(encoded.reduce((acc, x) => acc + x.uncompressedSize, 0))}B compressed to ${formatNumber(encoded.reduce((acc, x) => acc + x.compressedSize, 0))}B + ${formatNumber(encoded.reduce((acc, x) => acc + x.index.length, 0))}B index + ${formatNumber(encoded.reduce((acc, x) => acc + x.companion.length, 0))}B ${companion.extension}) in ${formatTime(Date.now() - time)}`) + ` (${blue(config.loggerName)})`);
288
296
  }
289
297
 
290
298
  // Clean up orphaned index files (index files without corresponding data files)
@@ -1,9 +1,11 @@
1
1
  import "../../../inject";
2
2
 
3
3
  import { logErrors } from "../../../errors";
4
+ import { Querysub } from "../../../4-querysub/Querysub";
4
5
  import { IndexedLogs } from "./IndexedLogs";
5
6
 
6
7
  async function main() {
8
+ await Querysub.hostService("MoveLogs");
7
9
  void IndexedLogs.runLogMoveLoop();
8
10
  }
9
11
 
@@ -19,6 +19,18 @@ import { ATag, URLOverride } from "../../../library-components/ATag";
19
19
  import { Button } from "../../../library-components/Button";
20
20
  import { createTicketForError, findTicketForPattern, goToTicket } from "../errorTickets/TicketPage";
21
21
 
22
+ const INITIAL_ERRORS_SHOWN = 5;
23
+ const INITIAL_SUPPRESSIONS_SHOWN = 100;
24
+ const SHOW_MORE_STEP = 100;
25
+
26
+ function getSuppressionMatchCount(matchData: { history: Map<number, { count: number }> } | undefined): number {
27
+ let total = 0;
28
+ for (let data of matchData?.history.values() || []) {
29
+ total += data.count;
30
+ }
31
+ return total;
32
+ }
33
+
22
34
  function openTicket(datum: LogDatum, pattern: string | undefined) {
23
35
  // datum may be synchronized state (props/manager state), which cannot be read inside the async callback — snapshot it while we're still in the synchronized event context.
24
36
  let datumSnapshot: LogDatum = JSON.parse(JSON.stringify(datum));
@@ -211,7 +223,10 @@ export class ErrorNotificationPage extends qreact.Component {
211
223
  newFixedPattern: t.atomic<string>(""),
212
224
  newNotABugPattern: t.atomic<string>(""),
213
225
  testErrorMessage: t.atomic<string>("example error"),
214
- showingAllErrors: t.atomic<boolean>(false),
226
+ errorsShown: t.atomic<number>(INITIAL_ERRORS_SHOWN),
227
+ suppressionsShown: t.atomic<number>(INITIAL_SUPPRESSIONS_SHOWN),
228
+ hideZeroMatchSuppressions: t.atomic<boolean>(false),
229
+ sortSuppressionsByMatches: t.atomic<boolean>(false),
215
230
  });
216
231
 
217
232
  manager = getErrorNotificationsManager();
@@ -241,10 +256,19 @@ export class ErrorNotificationPage extends qreact.Component {
241
256
 
242
257
  let sortedErrors = [...this.manager.state.unmatchedErrors];
243
258
  sort(sortedErrors, x => x.time);
244
- let displayedErrors = this.state.showingAllErrors && sortedErrors || sortedErrors.slice(0, 5);
259
+ let displayedErrors = sortedErrors.slice(0, this.state.errorsShown);
245
260
 
261
+ let suppressionMatches = this.manager.state.suppressionMatches;
246
262
  let sortedSuppressions = [...this.manager.state.suppressionEntries];
247
- sort(sortedSuppressions, x => -x.lastUpdatedTime);
263
+ if (this.state.hideZeroMatchSuppressions) {
264
+ sortedSuppressions = sortedSuppressions.filter(x => getSuppressionMatchCount(suppressionMatches.get(x.id)) > 0);
265
+ }
266
+ if (this.state.sortSuppressionsByMatches) {
267
+ sort(sortedSuppressions, x => -getSuppressionMatchCount(suppressionMatches.get(x.id)));
268
+ } else {
269
+ sort(sortedSuppressions, x => -x.lastUpdatedTime);
270
+ }
271
+ let displayedSuppressions = sortedSuppressions.slice(0, this.state.suppressionsShown);
248
272
 
249
273
  return <div className={css.vbox(16).pad2(16).fillWidth}>
250
274
  <h2>Error Notifications</h2>
@@ -279,16 +303,28 @@ export class ErrorNotificationPage extends qreact.Component {
279
303
  <LogDatumRenderer key={idx} datum={error} />
280
304
  ))}
281
305
  </div>
282
- {sortedErrors.length > 5 && (
283
- <Button
284
- hue={200}
285
- onClick={() => {
286
- this.state.showingAllErrors = !this.state.showingAllErrors;
287
- }}
288
- >
289
- {this.state.showingAllErrors && "Show Less" || `Show More (${sortedErrors.length - 5} more)`}
290
- </Button>
291
- )}
306
+ <div className={css.hbox(8)}>
307
+ {sortedErrors.length > displayedErrors.length && (
308
+ <Button
309
+ hue={200}
310
+ onClick={() => {
311
+ this.state.errorsShown = this.state.errorsShown + SHOW_MORE_STEP;
312
+ }}
313
+ >
314
+ {`Show ${Math.min(SHOW_MORE_STEP, sortedErrors.length - displayedErrors.length)} More Errors (${sortedErrors.length - displayedErrors.length} hidden)`}
315
+ </Button>
316
+ )}
317
+ {this.state.errorsShown > INITIAL_ERRORS_SHOWN && (
318
+ <Button
319
+ hue={200}
320
+ onClick={() => {
321
+ this.state.errorsShown = INITIAL_ERRORS_SHOWN;
322
+ }}
323
+ >
324
+ Collapse Errors
325
+ </Button>
326
+ )}
327
+ </div>
292
328
  </div>
293
329
 
294
330
  <div className={css.vbox(12)}>
@@ -342,9 +378,28 @@ export class ErrorNotificationPage extends qreact.Component {
342
378
  />
343
379
  </div>
344
380
 
381
+ <div className={css.hbox(16)}>
382
+ <InputLabel
383
+ label="Hide zero-match suppressions"
384
+ checkbox
385
+ checked={this.state.hideZeroMatchSuppressions}
386
+ onChange={e => {
387
+ this.state.hideZeroMatchSuppressions = e.currentTarget.checked;
388
+ }}
389
+ />
390
+ <InputLabel
391
+ label="Sort by matches"
392
+ checkbox
393
+ checked={this.state.sortSuppressionsByMatches}
394
+ onChange={e => {
395
+ this.state.sortSuppressionsByMatches = e.currentTarget.checked;
396
+ }}
397
+ />
398
+ </div>
399
+
345
400
  <div className={css.vbox(12)}>
346
- {sortedSuppressions.map((suppression) => {
347
- let matchData = this.manager.state.suppressionMatches.get(suppression.id);
401
+ {displayedSuppressions.map((suppression) => {
402
+ let matchData = suppressionMatches.get(suppression.id);
348
403
 
349
404
  return <SuppressionItem
350
405
  key={suppression.id}
@@ -353,6 +408,28 @@ export class ErrorNotificationPage extends qreact.Component {
353
408
  />;
354
409
  })}
355
410
  </div>
411
+ <div className={css.hbox(8)}>
412
+ {sortedSuppressions.length > displayedSuppressions.length && (
413
+ <Button
414
+ hue={200}
415
+ onClick={() => {
416
+ this.state.suppressionsShown = this.state.suppressionsShown + SHOW_MORE_STEP;
417
+ }}
418
+ >
419
+ {`Show ${Math.min(SHOW_MORE_STEP, sortedSuppressions.length - displayedSuppressions.length)} More Suppressions (${sortedSuppressions.length - displayedSuppressions.length} hidden)`}
420
+ </Button>
421
+ )}
422
+ {this.state.suppressionsShown > INITIAL_SUPPRESSIONS_SHOWN && (
423
+ <Button
424
+ hue={200}
425
+ onClick={() => {
426
+ this.state.suppressionsShown = INITIAL_SUPPRESSIONS_SHOWN;
427
+ }}
428
+ >
429
+ Collapse Suppressions
430
+ </Button>
431
+ )}
432
+ </div>
356
433
  </div>
357
434
  </div>;
358
435
  }
@@ -398,12 +475,7 @@ class SuppressionItem extends qreact.Component<{
398
475
  render() {
399
476
  let suppression = this.props.suppression;
400
477
  let matchData = this.props.matchData;
401
- let totalCount = 0;
402
- if (matchData) {
403
- for (let [chunk, data] of matchData.history) {
404
- totalCount += data.count;
405
- }
406
- }
478
+ let totalCount = getSuppressionMatchCount(matchData);
407
479
 
408
480
  let timeoutText: preact.ComponentChild;
409
481
  if (suppression.timeout >= Number.MAX_SAFE_INTEGER) {