querysub 0.725.0 → 0.727.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/backfill-log-categories.js +7 -0
- package/package.json +8 -2
- package/src/-a-archives/archiveCache.ts +7 -7
- package/src/-a-archives/archivesDisk.ts +20 -7
- package/src/0-path-value-core/AuthorityLookup.ts +113 -19
- package/src/deployManager/components/DeployPage.tsx +2 -2
- package/src/deployManager/components/ServiceDetailPage.tsx +17 -4
- package/src/deployManager/components/ServicesListPage.tsx +61 -8
- package/src/deployManager/components/deployButtons.tsx +166 -101
- package/src/deployManager/machineSchema.ts +13 -2
- package/src/deployManager/publishQuerysub.ts +121 -49
- package/src/deployManager/serviceCategories.ts +90 -1
- package/src/diagnostics/logs/IndexedLogs/BufferIndexLogsOptimizationConstants.ts +2 -0
- package/src/diagnostics/logs/IndexedLogs/FindProgressTracker.ts +2 -1
- package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts +40 -1
- package/src/diagnostics/logs/IndexedLogs/LogStreamer.ts +6 -2
- package/src/diagnostics/logs/IndexedLogs/MCPIndexedLogs.ts +2 -6
- package/src/diagnostics/logs/IndexedLogs/TimeFileTree.ts +34 -0
- package/src/diagnostics/logs/IndexedLogs/limitPublicLogsSize.ts +119 -0
- package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts +13 -5
- package/src/diagnostics/logs/IndexedLogs/moveLogsEntry.ts +2 -0
- package/src/diagnostics/logs/errorNotifications2/ErrorNotificationPage.tsx +93 -21
- package/src/diagnostics/logs/errorNotifications2/openRouterHelper.ts +4 -2
- package/src/diagnostics/logs/errorNotifications2/suppressionPatternMatcher.ts +1 -1
- package/src/diagnostics/logs/logCategories/LogBreakdownPage.tsx +421 -0
- package/src/diagnostics/logs/logCategories/logCategories.ts +273 -0
- package/src/diagnostics/managementPages.tsx +6 -0
- package/src/diagnostics/network/backfillLogCategoriesMain.ts +128 -0
- package/src/diagnostics/network/deleteOldLogsMain.ts +57 -0
- package/src/diagnostics/network/listLogFilesMain.ts +60 -0
- package/src/diagnostics/network/logFilesMain.ts +13 -15
- package/src/diagnostics/network/probeLogDeleteMain.ts +71 -0
- package/src/diagnostics/network/watchServiceMain.ts +106 -0
- package/src/fs.ts +29 -0
- package/src/misc/moveFolderContents.ts +67 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { ServiceParameters } from "./machineSchema";
|
|
1
|
+
import { ServiceConfig, ServiceParameters } from "./machineSchema";
|
|
2
2
|
import { ViewType } from "./urlParams";
|
|
3
3
|
import { DISK_PATHVALUE_COLOR, PATHVALUE_COLOR, FUNCTION_RUNNER_COLOR } from "../misc/nodeCategoryColors";
|
|
4
4
|
|
|
@@ -26,3 +26,92 @@ export function getServiceCategory(parameters: ServiceParameters): ServiceCatego
|
|
|
26
26
|
let command = parameters.command.trimStart();
|
|
27
27
|
return CATEGORIES.find(x => command.startsWith(x.prefix))?.category;
|
|
28
28
|
}
|
|
29
|
+
|
|
30
|
+
export type ServiceDeployGroup = "storageCore" | "background" | "frontendProxy" | "mainSite";
|
|
31
|
+
|
|
32
|
+
export type DeployGroupInfo = {
|
|
33
|
+
group: ServiceDeployGroup;
|
|
34
|
+
label: string;
|
|
35
|
+
emoji: string;
|
|
36
|
+
hue: number;
|
|
37
|
+
description: string;
|
|
38
|
+
deployWarning?: { title: string; paragraphs: string[] };
|
|
39
|
+
noOverlap?: boolean;
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
export const DEPLOY_GROUPS: DeployGroupInfo[] = [
|
|
43
|
+
{
|
|
44
|
+
group: "background",
|
|
45
|
+
label: "Background Tasks",
|
|
46
|
+
emoji: "🔧",
|
|
47
|
+
hue: 200,
|
|
48
|
+
description: "Non-essential services. They can go down without immediately affecting the site (eventually they will). Also the catch-all, for tests, pings, etc. Deployed without overlap, two copies of a background task must never run at once.",
|
|
49
|
+
noOverlap: true,
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
group: "mainSite",
|
|
53
|
+
label: "Main Site",
|
|
54
|
+
emoji: "🌐",
|
|
55
|
+
hue: 120,
|
|
56
|
+
description: "Everything the site needs to work right now.",
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
group: "frontendProxy",
|
|
60
|
+
label: "Front-end Proxy",
|
|
61
|
+
emoji: "🚪",
|
|
62
|
+
hue: 35,
|
|
63
|
+
description: "The HTTP bootstrapper. If it goes down users can't load the site, but it does very little, so it rarely needs updating.",
|
|
64
|
+
deployWarning: {
|
|
65
|
+
title: "You probably don't need to update the front-end proxy",
|
|
66
|
+
paragraphs: [
|
|
67
|
+
"It does very little. It only serves the bootstrapper, so it doesn't need to be updated when you change most of the framework code.",
|
|
68
|
+
"While it restarts, the site is temporarily unavailable.",
|
|
69
|
+
"It's usually best not to touch it.",
|
|
70
|
+
],
|
|
71
|
+
},
|
|
72
|
+
},
|
|
73
|
+
{
|
|
74
|
+
group: "storageCore",
|
|
75
|
+
label: "Storage Core",
|
|
76
|
+
emoji: "🗄️",
|
|
77
|
+
hue: 0,
|
|
78
|
+
description: "The storage server. Everything else depends on it.",
|
|
79
|
+
deployWarning: {
|
|
80
|
+
title: "The storage core is very delicate",
|
|
81
|
+
paragraphs: [
|
|
82
|
+
"If it breaks, everything breaks, and it is very hard to fix.",
|
|
83
|
+
"Be careful when changing it.",
|
|
84
|
+
],
|
|
85
|
+
},
|
|
86
|
+
},
|
|
87
|
+
];
|
|
88
|
+
|
|
89
|
+
const BOOTSTRAP_ONLY_FLAG = "--bootstraponly";
|
|
90
|
+
const MAIN_SITE_COMMAND_PREFIXES = [
|
|
91
|
+
PATHVALUE_COMMAND_PREFIX,
|
|
92
|
+
FUNCTION_RUNNER_COMMAND_PREFIX,
|
|
93
|
+
"yarn cyoa-public",
|
|
94
|
+
"yarn queue-public",
|
|
95
|
+
"yarn queue-gpu-public",
|
|
96
|
+
"yarn worker-public",
|
|
97
|
+
];
|
|
98
|
+
|
|
99
|
+
export function getDeployGroupInfo(group: ServiceDeployGroup): DeployGroupInfo {
|
|
100
|
+
let info = DEPLOY_GROUPS.find(x => x.group === group);
|
|
101
|
+
if (!info) throw new Error(`Unknown deploy group ${JSON.stringify(group)}, expected one of ${DEPLOY_GROUPS.map(x => x.group).join(", ")}`);
|
|
102
|
+
return info;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** The group set on the service, or for services that never had one set, the group its command implies. */
|
|
106
|
+
export function getServiceDeployGroup(config: ServiceConfig): ServiceDeployGroup {
|
|
107
|
+
if (config.info.deployGroup) return config.info.deployGroup;
|
|
108
|
+
return getInferredDeployGroup(config.parameters);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export function getInferredDeployGroup(parameters: ServiceParameters): ServiceDeployGroup {
|
|
112
|
+
let command = parameters.command.trimStart();
|
|
113
|
+
if (command.startsWith(STORAGE_COMMAND_PREFIX)) return "storageCore";
|
|
114
|
+
if (command.split(/\s+/).includes(BOOTSTRAP_ONLY_FLAG)) return "frontendProxy";
|
|
115
|
+
if (MAIN_SITE_COMMAND_PREFIXES.some(prefix => command === prefix || command.startsWith(prefix + " "))) return "mainSite";
|
|
116
|
+
return "background";
|
|
117
|
+
}
|
|
@@ -18,6 +18,8 @@ export const MAX_SINGLE_FILE_DATA = 1024 * 1024 * 64;
|
|
|
18
18
|
|
|
19
19
|
export const MOVING_TIMEOUT = timeInMinute * 5;
|
|
20
20
|
|
|
21
|
+
export const MAX_PUBLIC_LOG_BYTES = 1024 * 1024 * 1024 * 10;
|
|
22
|
+
|
|
21
23
|
export const DISK_FLUSH_INTERVAL = timeInSecond * 15;
|
|
22
24
|
export const MAX_COUNT_PER_FILE = 1000 * 1000 * 10;
|
|
23
25
|
|
|
@@ -4,7 +4,7 @@ import { SearchParams } from "./BufferIndexHelpers";
|
|
|
4
4
|
export class FindProgressTracker<T> {
|
|
5
5
|
public constructor(private config: {
|
|
6
6
|
params: SearchParams;
|
|
7
|
-
deserialize: (result: Buffer) => T;
|
|
7
|
+
deserialize: (result: Buffer) => T | undefined;
|
|
8
8
|
getTime: (result: T) => number | undefined;
|
|
9
9
|
onResult: (result: T) => void;
|
|
10
10
|
}) { }
|
|
@@ -14,6 +14,7 @@ export class FindProgressTracker<T> {
|
|
|
14
14
|
|
|
15
15
|
public addResult(buffer: Buffer, source: { startTime: number; endTime: number; }): boolean {
|
|
16
16
|
let result = this.config.deserialize(buffer);
|
|
17
|
+
if (!result) return false;
|
|
17
18
|
let time = this.config.getTime(result) ?? source.endTime;
|
|
18
19
|
|
|
19
20
|
if (time < this.config.params.startTime) return false;
|
|
@@ -18,7 +18,8 @@ import { getOwnMachineId, isNodeIdOnOwnMachineId, isOwnNodeId } from "../../../-
|
|
|
18
18
|
import { LogArchives, TimeFilePath, TimeFileTree } from "./TimeFileTree";
|
|
19
19
|
import { LogStreamer } from "./LogStreamer";
|
|
20
20
|
import { moveLogsToPublic } from "./moveIndexLogsToPublic";
|
|
21
|
-
import { MAX_SINGLE_FILE_DATA, MAX_COUNT_PER_FILE, DISK_FLUSH_INTERVAL, PUBLIC_MOVE_THRESHOLD, MOVING_TIMEOUT } from "./BufferIndexLogsOptimizationConstants";
|
|
21
|
+
import { MAX_SINGLE_FILE_DATA, MAX_COUNT_PER_FILE, DISK_FLUSH_INTERVAL, PUBLIC_MOVE_THRESHOLD, MOVING_TIMEOUT, MAX_PUBLIC_LOG_BYTES } from "./BufferIndexLogsOptimizationConstants";
|
|
22
|
+
import { LimitPublicLogsResult, limitPublicLogsSize } from "./limitPublicLogsSize";
|
|
22
23
|
import { FindProgressTracker } from "./FindProgressTracker";
|
|
23
24
|
import { SocketFunction } from "socket-function/SocketFunction";
|
|
24
25
|
import { assertIsManagementUser } from "../../managementPages";
|
|
@@ -30,6 +31,7 @@ import { NodeCapabilitiesController } from "../../../-g-core-values/NodeCapabili
|
|
|
30
31
|
import { getLoggers2Async } from "../diskLogger";
|
|
31
32
|
import { watchAllValues } from "../errorNotifications2/logWatcher";
|
|
32
33
|
import { wrapArchivesWithCache } from "../../../-a-archives/archiveCache";
|
|
34
|
+
import { LOG_CATEGORIES_EXTENSION, LogCategoryCounts, createLogCategoriesFile, moveLogsAICallLimiter, parseLogCategoriesFile } from "../logCategories/logCategories";
|
|
33
35
|
|
|
34
36
|
// Ensure it's available so that the controller is listening on all servers
|
|
35
37
|
watchAllValues;
|
|
@@ -96,6 +98,9 @@ export class IndexedLogs<T> {
|
|
|
96
98
|
});
|
|
97
99
|
return archives;
|
|
98
100
|
});
|
|
101
|
+
private getPublicLogArchivesUncached = lazy((): LogArchives => {
|
|
102
|
+
return getArchives2("final-indexed-logs/" + this.config.name);
|
|
103
|
+
});
|
|
99
104
|
private getPublicLogsBase(loadPublic: boolean): LogArchives {
|
|
100
105
|
if (loadPublic) {
|
|
101
106
|
return createArchivesMemoryCache2(getArchives2("final-indexed-logs/" + this.config.name), {
|
|
@@ -152,6 +157,26 @@ export class IndexedLogs<T> {
|
|
|
152
157
|
private getRealPublicLogs = lazy((): LogArchives => {
|
|
153
158
|
return this.getPublicLogsBase(true);
|
|
154
159
|
});
|
|
160
|
+
public getPublicLogArchives(): LogArchives {
|
|
161
|
+
return this.getPublicLogArchivesUncached();
|
|
162
|
+
}
|
|
163
|
+
public async deletePublicLogFile(path: TimeFilePath): Promise<void> {
|
|
164
|
+
let archives = this.getPublicLogArchives();
|
|
165
|
+
await Promise.all([
|
|
166
|
+
archives.del(path.fullPath + INDEX_EXTENSION),
|
|
167
|
+
archives.del(path.fullPath + LOG_CATEGORIES_EXTENSION),
|
|
168
|
+
]);
|
|
169
|
+
await archives.del(path.fullPath);
|
|
170
|
+
}
|
|
171
|
+
public async listPublicLogPaths(config: { startTime: number; endTime: number }): Promise<TimeFilePath[]> {
|
|
172
|
+
return await new TimeFileTree(this.getPublicLogArchives()).findAllPaths(config);
|
|
173
|
+
}
|
|
174
|
+
// Category counts live beside the public logs, so dev and production both read this same storage
|
|
175
|
+
public async readLogCategoryCounts(fullPath: string): Promise<LogCategoryCounts | undefined> {
|
|
176
|
+
let buffer = await this.getPublicLogArchives().get(fullPath + LOG_CATEGORIES_EXTENSION);
|
|
177
|
+
if (!buffer) return undefined;
|
|
178
|
+
return parseLogCategoriesFile(buffer);
|
|
179
|
+
}
|
|
155
180
|
|
|
156
181
|
private getTimeBlock(time: number): { startTime: number, endTime: number } {
|
|
157
182
|
let startTime = Math.floor(time / timeInHour) * timeInHour;
|
|
@@ -696,9 +721,23 @@ export class IndexedLogs<T> {
|
|
|
696
721
|
maxSingleFileData: this.config.maxSingleFileData || MAX_SINGLE_FILE_DATA,
|
|
697
722
|
movingTimeout: MOVING_TIMEOUT,
|
|
698
723
|
indexExtension: INDEX_EXTENSION,
|
|
724
|
+
companion: {
|
|
725
|
+
extension: LOG_CATEGORIES_EXTENSION,
|
|
726
|
+
create: entries => createLogCategoriesFile({ loggerName: this.config.name, entries, aiCallLimiter: moveLogsAICallLimiter }),
|
|
727
|
+
},
|
|
699
728
|
});
|
|
729
|
+
await this.limitPublicLogsSize();
|
|
700
730
|
}
|
|
701
731
|
|
|
732
|
+
public async limitPublicLogsSize(config?: { dryRun?: boolean }): Promise<LimitPublicLogsResult> {
|
|
733
|
+
return await limitPublicLogsSize({
|
|
734
|
+
loggerName: this.config.name,
|
|
735
|
+
publicLogs: this.getPublicLogs(),
|
|
736
|
+
maxBytes: MAX_PUBLIC_LOG_BYTES,
|
|
737
|
+
siblingExtensions: [INDEX_EXTENSION, LOG_CATEGORIES_EXTENSION],
|
|
738
|
+
dryRun: config?.dryRun,
|
|
739
|
+
});
|
|
740
|
+
}
|
|
702
741
|
|
|
703
742
|
// #region Client Calls
|
|
704
743
|
private findCallbacks = new Map<string, (match: T) => void>();
|
|
@@ -40,8 +40,12 @@ export class LogStreamer<T> {
|
|
|
40
40
|
this.insideFlush = false;
|
|
41
41
|
}
|
|
42
42
|
});
|
|
43
|
-
public static deserialize<T>(data: Buffer): T {
|
|
44
|
-
|
|
43
|
+
public static deserialize<T>(data: Buffer): T | undefined {
|
|
44
|
+
try {
|
|
45
|
+
return JSON.parse(data.toString());
|
|
46
|
+
} catch {
|
|
47
|
+
return undefined;
|
|
48
|
+
}
|
|
45
49
|
}
|
|
46
50
|
}
|
|
47
51
|
|
|
@@ -458,12 +458,8 @@ export class MCPIndexedLogs {
|
|
|
458
458
|
let { buf, matchesPattern, columns, startTime, endTime, sink } = append;
|
|
459
459
|
if (!matchesPattern(buf)) return;
|
|
460
460
|
|
|
461
|
-
let datum
|
|
462
|
-
|
|
463
|
-
datum = LogStreamer.deserialize<LogDatum>(buf);
|
|
464
|
-
} catch {
|
|
465
|
-
return;
|
|
466
|
-
}
|
|
461
|
+
let datum = LogStreamer.deserialize<LogDatum>(buf);
|
|
462
|
+
if (!datum) return;
|
|
467
463
|
|
|
468
464
|
if (typeof datum.time !== "number") return;
|
|
469
465
|
if (datum.time < startTime || datum.time > endTime) return;
|
|
@@ -127,6 +127,7 @@ export class TimeFileTree {
|
|
|
127
127
|
public async findAllPaths(config: {
|
|
128
128
|
startTime: number;
|
|
129
129
|
endTime: number;
|
|
130
|
+
onDayFolderListed?: (info: { dayFolder: string; fileCount: number }) => void;
|
|
130
131
|
}): Promise<TimeFilePath[]> {
|
|
131
132
|
// For short ranges, guessing the day folder paths and reading them all in parallel is
|
|
132
133
|
// much faster than recursively listing year/month/day folders.
|
|
@@ -141,6 +142,7 @@ export class TimeFileTree {
|
|
|
141
142
|
private async findAllPathsByDayGuess(config: {
|
|
142
143
|
startTime: number;
|
|
143
144
|
endTime: number;
|
|
145
|
+
onDayFolderListed?: (info: { dayFolder: string; fileCount: number }) => void;
|
|
144
146
|
}): Promise<TimeFilePath[]> {
|
|
145
147
|
// Build the list of UTC day folder prefixes spanning the range.
|
|
146
148
|
const dayFolders: string[] = [];
|
|
@@ -159,6 +161,7 @@ export class TimeFileTree {
|
|
|
159
161
|
|
|
160
162
|
await Promise.all(dayFolders.map(async (dayFolder) => {
|
|
161
163
|
const files = await this.archives.find(dayFolder, { shallow: true, type: "files" });
|
|
164
|
+
config.onDayFolderListed?.({ dayFolder, fileCount: files.length });
|
|
162
165
|
|
|
163
166
|
for (const file of files) {
|
|
164
167
|
const decoded = decodeLogFilePath(file);
|
|
@@ -180,6 +183,7 @@ export class TimeFileTree {
|
|
|
180
183
|
private async findAllPathsByScan(config: {
|
|
181
184
|
startTime: number;
|
|
182
185
|
endTime: number;
|
|
186
|
+
onDayFolderListed?: (info: { dayFolder: string; fileCount: number }) => void;
|
|
183
187
|
}): Promise<TimeFilePath[]> {
|
|
184
188
|
const results: TimeFilePath[] = [];
|
|
185
189
|
|
|
@@ -241,6 +245,7 @@ export class TimeFileTree {
|
|
|
241
245
|
|
|
242
246
|
// Find all log files in this day
|
|
243
247
|
const files = await this.archives.find(dayFolder + "/", { shallow: true, type: "files" });
|
|
248
|
+
config.onDayFolderListed?.({ dayFolder, fileCount: files.length });
|
|
244
249
|
|
|
245
250
|
for (const file of files) {
|
|
246
251
|
const decoded = decodeLogFilePath(file);
|
|
@@ -261,6 +266,35 @@ export class TimeFileTree {
|
|
|
261
266
|
return results;
|
|
262
267
|
}
|
|
263
268
|
|
|
269
|
+
public async findDayFolders(): Promise<string[]> {
|
|
270
|
+
let dayFolders: string[] = [];
|
|
271
|
+
let years = await this.archives.find("", { shallow: true, type: "folders" });
|
|
272
|
+
for (let yearFolder of years) {
|
|
273
|
+
if (isNaN(parseInt(yearFolder.split("/").at(-1) || "", 10))) continue;
|
|
274
|
+
let months = await this.archives.find(yearFolder + "/", { shallow: true, type: "folders" });
|
|
275
|
+
await Promise.all(months.map(async monthFolder => {
|
|
276
|
+
if (isNaN(parseInt(monthFolder.split("/").at(-1) || "", 10))) return;
|
|
277
|
+
let days = await this.archives.find(monthFolder + "/", { shallow: true, type: "folders" });
|
|
278
|
+
for (let dayFolder of days) {
|
|
279
|
+
if (isNaN(parseInt(dayFolder.split("/").at(-1) || "", 10))) continue;
|
|
280
|
+
dayFolders.push(dayFolder + "/");
|
|
281
|
+
}
|
|
282
|
+
}));
|
|
283
|
+
}
|
|
284
|
+
dayFolders.sort();
|
|
285
|
+
return dayFolders;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
public async findPathsInDayFolder(dayFolder: string): Promise<TimeFilePath[]> {
|
|
289
|
+
let files = await this.archives.find(dayFolder, { shallow: true, type: "files" });
|
|
290
|
+
let paths: TimeFilePath[] = [];
|
|
291
|
+
for (let file of files) {
|
|
292
|
+
let decoded = decodeLogFilePath(file);
|
|
293
|
+
if (decoded) paths.push(decoded);
|
|
294
|
+
}
|
|
295
|
+
return paths;
|
|
296
|
+
}
|
|
297
|
+
|
|
264
298
|
public async deleteAll() {
|
|
265
299
|
let rootFolders = await this.archives.find("", { shallow: true, type: "folders" });
|
|
266
300
|
for (let rootFolder of rootFolders) {
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import { formatDateTime, formatNumber, formatTime } from "socket-function/src/formatting/format";
|
|
2
|
+
import { blue, magenta, yellow } from "socket-function/src/formatting/logColors";
|
|
3
|
+
import { sort, timeInSecond } from "socket-function/src/misc";
|
|
4
|
+
import { LogArchives, TimeFilePath, TimeFileTree } from "./TimeFileTree";
|
|
5
|
+
|
|
6
|
+
const TARGET_FILES_PER_BATCH = 1000;
|
|
7
|
+
const PARALLEL_CALLS = 32;
|
|
8
|
+
const PROGRESS_LOG_INTERVAL = timeInSecond * 5;
|
|
9
|
+
const DELETE_ATTEMPTS = 3;
|
|
10
|
+
|
|
11
|
+
export type LimitPublicLogsResult = {
|
|
12
|
+
keptFiles: number;
|
|
13
|
+
keptBytes: number;
|
|
14
|
+
deletedFiles: number;
|
|
15
|
+
deletedBytes: number;
|
|
16
|
+
failedFiles: number;
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
export async function limitPublicLogsSize(config: {
|
|
20
|
+
loggerName: string;
|
|
21
|
+
publicLogs: LogArchives;
|
|
22
|
+
maxBytes: number;
|
|
23
|
+
siblingExtensions: string[];
|
|
24
|
+
dryRun?: boolean;
|
|
25
|
+
}): Promise<LimitPublicLogsResult> {
|
|
26
|
+
let { loggerName, publicLogs, maxBytes, siblingExtensions, dryRun } = config;
|
|
27
|
+
let start = Date.now();
|
|
28
|
+
let lastProgressLog = Date.now();
|
|
29
|
+
let tree = new TimeFileTree(publicLogs);
|
|
30
|
+
let dayFolders = (await tree.findDayFolders()).reverse();
|
|
31
|
+
|
|
32
|
+
let files: { path: TimeFilePath; bytes: number }[] = [];
|
|
33
|
+
let daysRead = 0;
|
|
34
|
+
let daysPerBatch = 1;
|
|
35
|
+
while (daysRead < dayFolders.length) {
|
|
36
|
+
let batch = dayFolders.slice(daysRead, daysRead + daysPerBatch);
|
|
37
|
+
let paths = (await Promise.all(batch.map(dayFolder => tree.findPathsInDayFolder(dayFolder)))).flat();
|
|
38
|
+
daysRead += batch.length;
|
|
39
|
+
let pathIndex = 0;
|
|
40
|
+
async function sizeWorker() {
|
|
41
|
+
while (pathIndex < paths.length) {
|
|
42
|
+
let path = paths[pathIndex++];
|
|
43
|
+
let infos = await Promise.all([path.fullPath, ...siblingExtensions.map(extension => path.fullPath + extension)].map(file => publicLogs.getInfo(file)));
|
|
44
|
+
files.push({ path, bytes: infos.reduce((acc, x) => acc + (x?.size || 0), 0) });
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
await Promise.all(Array.from({ length: PARALLEL_CALLS }, () => sizeWorker()));
|
|
48
|
+
daysPerBatch = Math.max(1, Math.round(TARGET_FILES_PER_BATCH / Math.max(1, files.length / daysRead)));
|
|
49
|
+
|
|
50
|
+
if (Date.now() - lastProgressLog >= PROGRESS_LOG_INTERVAL) {
|
|
51
|
+
lastProgressLog = Date.now();
|
|
52
|
+
console.log(magenta(`Sizing public ${blue(loggerName)} logs: ${daysRead}/${dayFolders.length} day folders, ${formatNumber(files.length)} files, ${formatTime(Date.now() - start)} so far`));
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
sort(files, x => -x.path.startTime);
|
|
57
|
+
|
|
58
|
+
let keptBytes = 0;
|
|
59
|
+
let cutoffIndex = files.length;
|
|
60
|
+
for (let i = 0; i < files.length; i++) {
|
|
61
|
+
if (keptBytes + files[i].bytes > maxBytes) {
|
|
62
|
+
cutoffIndex = i;
|
|
63
|
+
break;
|
|
64
|
+
}
|
|
65
|
+
keptBytes += files[i].bytes;
|
|
66
|
+
}
|
|
67
|
+
let keptFiles = cutoffIndex;
|
|
68
|
+
let toDelete = files.slice(cutoffIndex);
|
|
69
|
+
|
|
70
|
+
let deletedBytes = toDelete.reduce((acc, x) => acc + x.bytes, 0);
|
|
71
|
+
let result: LimitPublicLogsResult = { keptFiles, keptBytes, deletedFiles: toDelete.length, deletedBytes, failedFiles: 0 };
|
|
72
|
+
if (!toDelete.length) {
|
|
73
|
+
if (dryRun) {
|
|
74
|
+
console.log(magenta(`Public ${blue(loggerName)} logs are within the limit: ${keptFiles} files, ${formatNumber(keptBytes)}B of ${formatNumber(maxBytes)}B (dry run, sized in ${formatTime(Date.now() - start)})`));
|
|
75
|
+
}
|
|
76
|
+
return result;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
let oldest = Math.min(...toDelete.map(x => x.path.startTime));
|
|
80
|
+
let newest = Math.max(...toDelete.map(x => x.path.startTime));
|
|
81
|
+
console.log(yellow(`Public ${blue(loggerName)} logs are over the ${formatNumber(maxBytes)}B limit: keeping the newest ${keptFiles} files (${formatNumber(keptBytes)}B), ${dryRun && "would delete" || "deleting"} ${toDelete.length} older files (${formatNumber(deletedBytes)}B) starting ${formatDateTime(oldest)} to ${formatDateTime(newest)}`));
|
|
82
|
+
if (dryRun) return result;
|
|
83
|
+
|
|
84
|
+
let deleted = 0;
|
|
85
|
+
let failed: { path: TimeFilePath; error: Error }[] = [];
|
|
86
|
+
let deleteStart = Date.now();
|
|
87
|
+
let nextIndex = 0;
|
|
88
|
+
async function deleteWorker() {
|
|
89
|
+
while (nextIndex < toDelete.length) {
|
|
90
|
+
let path = toDelete[nextIndex++].path;
|
|
91
|
+
for (let attempt = 1; attempt <= DELETE_ATTEMPTS; attempt++) {
|
|
92
|
+
try {
|
|
93
|
+
for (let extension of siblingExtensions) {
|
|
94
|
+
await publicLogs.del(path.fullPath + extension);
|
|
95
|
+
}
|
|
96
|
+
await publicLogs.del(path.fullPath);
|
|
97
|
+
deleted++;
|
|
98
|
+
break;
|
|
99
|
+
} catch (e) {
|
|
100
|
+
if (attempt < DELETE_ATTEMPTS) continue;
|
|
101
|
+
failed.push({ path, error: e as Error });
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
if (Date.now() - lastProgressLog < PROGRESS_LOG_INTERVAL) continue;
|
|
105
|
+
lastProgressLog = Date.now();
|
|
106
|
+
let elapsed = Date.now() - deleteStart;
|
|
107
|
+
let done = deleted + failed.length;
|
|
108
|
+
console.log(magenta(`Deleting public ${blue(loggerName)} logs: ${deleted}/${toDelete.length} files, ${failed.length} failed, ${formatTime(elapsed)} so far, about ${formatTime(elapsed * (toDelete.length - done) / Math.max(1, done))} left`));
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
await Promise.all(Array.from({ length: PARALLEL_CALLS }, () => deleteWorker()));
|
|
112
|
+
console.log(magenta(`Deleted ${deleted} of ${toDelete.length} public ${blue(loggerName)} log files (${formatNumber(deletedBytes)}B targeted) in ${formatTime(Date.now() - deleteStart)}`));
|
|
113
|
+
if (failed.length) {
|
|
114
|
+
console.warn(`Failed to delete ${failed.length} of ${toDelete.length} public ${loggerName} log files after ${DELETE_ATTEMPTS} attempts each; the next limit pass will try them again. First failure, ${failed[0].path.fullPath}:`, failed[0].error.stack || failed[0].error);
|
|
115
|
+
}
|
|
116
|
+
result.deletedFiles = deleted;
|
|
117
|
+
result.failedFiles = failed.length;
|
|
118
|
+
return result;
|
|
119
|
+
}
|
|
@@ -22,8 +22,12 @@ export async function moveLogsToPublic(config: {
|
|
|
22
22
|
localLogs: LogArchives;
|
|
23
23
|
publicLogs: LogArchives;
|
|
24
24
|
indexExtension: string;
|
|
25
|
+
companion: {
|
|
26
|
+
extension: string;
|
|
27
|
+
create: (entries: Buffer[]) => Promise<Buffer>;
|
|
28
|
+
};
|
|
25
29
|
}) {
|
|
26
|
-
let { forceAll, localLogs, publicLogs, publicMoveThreshold, maxSingleFileData, indexExtension, movingTimeout } = config;
|
|
30
|
+
let { forceAll, localLogs, publicLogs, publicMoveThreshold, maxSingleFileData, indexExtension, movingTimeout, companion } = config;
|
|
27
31
|
let now = Date.now();
|
|
28
32
|
let threadId = getOwnThreadId();
|
|
29
33
|
let ourMovingFileName = `${now}-${threadId}.moving`;
|
|
@@ -237,6 +241,7 @@ export async function moveLogsToPublic(config: {
|
|
|
237
241
|
let encoded: {
|
|
238
242
|
data: Buffer;
|
|
239
243
|
index: Buffer;
|
|
244
|
+
companion: Buffer;
|
|
240
245
|
compressedSize: number;
|
|
241
246
|
uncompressedSize: number;
|
|
242
247
|
logCount: number;
|
|
@@ -252,11 +257,12 @@ export async function moveLogsToPublic(config: {
|
|
|
252
257
|
encoded.push({
|
|
253
258
|
data: obj.data,
|
|
254
259
|
index: obj.index,
|
|
260
|
+
companion: await companion.create(group),
|
|
255
261
|
compressedSize: obj.data.length,
|
|
256
262
|
uncompressedSize,
|
|
257
263
|
logCount: group.length,
|
|
258
264
|
});
|
|
259
|
-
// Wait, so if we have to encode a lot of data, we don't block for too long?
|
|
265
|
+
// Wait, so if we have to encode a lot of data, we don't block for too long?
|
|
260
266
|
await delay(0);
|
|
261
267
|
}
|
|
262
268
|
|
|
@@ -269,8 +275,10 @@ export async function moveLogsToPublic(config: {
|
|
|
269
275
|
startTime,
|
|
270
276
|
endTime,
|
|
271
277
|
});
|
|
272
|
-
|
|
273
|
-
|
|
278
|
+
await Promise.all([
|
|
279
|
+
publicLogs.set(path + indexExtension, obj.index),
|
|
280
|
+
publicLogs.set(path + companion.extension, obj.companion),
|
|
281
|
+
]);
|
|
274
282
|
await publicLogs.set(path, obj.data);
|
|
275
283
|
}
|
|
276
284
|
|
|
@@ -284,7 +292,7 @@ export async function moveLogsToPublic(config: {
|
|
|
284
292
|
let elapsed = Date.now() - groupsStart;
|
|
285
293
|
let remaining = bytesDone && elapsed * (totalBytes - bytesDone) / bytesDone || 0;
|
|
286
294
|
console.log(magenta(`Moved ${filesDone}/${allPathsFlat.length} files, ${formatNumber(bytesDone)}B/${formatNumber(totalBytes)}B, ${formatTime(elapsed)} so far, about ${formatTime(remaining)} left (${blue(config.loggerName)})`));
|
|
287
|
-
console.log(green(`(${i + 1}/${groups.length}) Wrote ${encoded.length} log files to public from ${group.length} input files, ${formatDateTime(startTime)} to ${formatDateTime(endTime)} (${formatNumber(encoded.reduce((acc, x) => acc + x.uncompressedSize, 0))}B compressed to ${formatNumber(encoded.reduce((acc, x) => acc + x.compressedSize, 0))}B + ${formatNumber(encoded.reduce((acc, x) => acc + x.index.length, 0))}B index) in ${formatTime(Date.now() - time)}`) + ` (${blue(config.loggerName)})`);
|
|
295
|
+
console.log(green(`(${i + 1}/${groups.length}) Wrote ${encoded.length} log files to public from ${group.length} input files, ${formatDateTime(startTime)} to ${formatDateTime(endTime)} (${formatNumber(encoded.reduce((acc, x) => acc + x.uncompressedSize, 0))}B compressed to ${formatNumber(encoded.reduce((acc, x) => acc + x.compressedSize, 0))}B + ${formatNumber(encoded.reduce((acc, x) => acc + x.index.length, 0))}B index + ${formatNumber(encoded.reduce((acc, x) => acc + x.companion.length, 0))}B ${companion.extension}) in ${formatTime(Date.now() - time)}`) + ` (${blue(config.loggerName)})`);
|
|
288
296
|
}
|
|
289
297
|
|
|
290
298
|
// Clean up orphaned index files (index files without corresponding data files)
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import "../../../inject";
|
|
2
2
|
|
|
3
3
|
import { logErrors } from "../../../errors";
|
|
4
|
+
import { Querysub } from "../../../4-querysub/Querysub";
|
|
4
5
|
import { IndexedLogs } from "./IndexedLogs";
|
|
5
6
|
|
|
6
7
|
async function main() {
|
|
8
|
+
await Querysub.hostService("MoveLogs");
|
|
7
9
|
void IndexedLogs.runLogMoveLoop();
|
|
8
10
|
}
|
|
9
11
|
|
|
@@ -19,6 +19,18 @@ import { ATag, URLOverride } from "../../../library-components/ATag";
|
|
|
19
19
|
import { Button } from "../../../library-components/Button";
|
|
20
20
|
import { createTicketForError, findTicketForPattern, goToTicket } from "../errorTickets/TicketPage";
|
|
21
21
|
|
|
22
|
+
const INITIAL_ERRORS_SHOWN = 5;
|
|
23
|
+
const INITIAL_SUPPRESSIONS_SHOWN = 100;
|
|
24
|
+
const SHOW_MORE_STEP = 100;
|
|
25
|
+
|
|
26
|
+
function getSuppressionMatchCount(matchData: { history: Map<number, { count: number }> } | undefined): number {
|
|
27
|
+
let total = 0;
|
|
28
|
+
for (let data of matchData?.history.values() || []) {
|
|
29
|
+
total += data.count;
|
|
30
|
+
}
|
|
31
|
+
return total;
|
|
32
|
+
}
|
|
33
|
+
|
|
22
34
|
function openTicket(datum: LogDatum, pattern: string | undefined) {
|
|
23
35
|
// datum may be synchronized state (props/manager state), which cannot be read inside the async callback — snapshot it while we're still in the synchronized event context.
|
|
24
36
|
let datumSnapshot: LogDatum = JSON.parse(JSON.stringify(datum));
|
|
@@ -211,7 +223,10 @@ export class ErrorNotificationPage extends qreact.Component {
|
|
|
211
223
|
newFixedPattern: t.atomic<string>(""),
|
|
212
224
|
newNotABugPattern: t.atomic<string>(""),
|
|
213
225
|
testErrorMessage: t.atomic<string>("example error"),
|
|
214
|
-
|
|
226
|
+
errorsShown: t.atomic<number>(INITIAL_ERRORS_SHOWN),
|
|
227
|
+
suppressionsShown: t.atomic<number>(INITIAL_SUPPRESSIONS_SHOWN),
|
|
228
|
+
hideZeroMatchSuppressions: t.atomic<boolean>(false),
|
|
229
|
+
sortSuppressionsByMatches: t.atomic<boolean>(false),
|
|
215
230
|
});
|
|
216
231
|
|
|
217
232
|
manager = getErrorNotificationsManager();
|
|
@@ -241,10 +256,19 @@ export class ErrorNotificationPage extends qreact.Component {
|
|
|
241
256
|
|
|
242
257
|
let sortedErrors = [...this.manager.state.unmatchedErrors];
|
|
243
258
|
sort(sortedErrors, x => x.time);
|
|
244
|
-
let displayedErrors =
|
|
259
|
+
let displayedErrors = sortedErrors.slice(0, this.state.errorsShown);
|
|
245
260
|
|
|
261
|
+
let suppressionMatches = this.manager.state.suppressionMatches;
|
|
246
262
|
let sortedSuppressions = [...this.manager.state.suppressionEntries];
|
|
247
|
-
|
|
263
|
+
if (this.state.hideZeroMatchSuppressions) {
|
|
264
|
+
sortedSuppressions = sortedSuppressions.filter(x => getSuppressionMatchCount(suppressionMatches.get(x.id)) > 0);
|
|
265
|
+
}
|
|
266
|
+
if (this.state.sortSuppressionsByMatches) {
|
|
267
|
+
sort(sortedSuppressions, x => -getSuppressionMatchCount(suppressionMatches.get(x.id)));
|
|
268
|
+
} else {
|
|
269
|
+
sort(sortedSuppressions, x => -x.lastUpdatedTime);
|
|
270
|
+
}
|
|
271
|
+
let displayedSuppressions = sortedSuppressions.slice(0, this.state.suppressionsShown);
|
|
248
272
|
|
|
249
273
|
return <div className={css.vbox(16).pad2(16).fillWidth}>
|
|
250
274
|
<h2>Error Notifications</h2>
|
|
@@ -279,16 +303,28 @@ export class ErrorNotificationPage extends qreact.Component {
|
|
|
279
303
|
<LogDatumRenderer key={idx} datum={error} />
|
|
280
304
|
))}
|
|
281
305
|
</div>
|
|
282
|
-
{
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
306
|
+
<div className={css.hbox(8)}>
|
|
307
|
+
{sortedErrors.length > displayedErrors.length && (
|
|
308
|
+
<Button
|
|
309
|
+
hue={200}
|
|
310
|
+
onClick={() => {
|
|
311
|
+
this.state.errorsShown = this.state.errorsShown + SHOW_MORE_STEP;
|
|
312
|
+
}}
|
|
313
|
+
>
|
|
314
|
+
{`Show ${Math.min(SHOW_MORE_STEP, sortedErrors.length - displayedErrors.length)} More Errors (${sortedErrors.length - displayedErrors.length} hidden)`}
|
|
315
|
+
</Button>
|
|
316
|
+
)}
|
|
317
|
+
{this.state.errorsShown > INITIAL_ERRORS_SHOWN && (
|
|
318
|
+
<Button
|
|
319
|
+
hue={200}
|
|
320
|
+
onClick={() => {
|
|
321
|
+
this.state.errorsShown = INITIAL_ERRORS_SHOWN;
|
|
322
|
+
}}
|
|
323
|
+
>
|
|
324
|
+
Collapse Errors
|
|
325
|
+
</Button>
|
|
326
|
+
)}
|
|
327
|
+
</div>
|
|
292
328
|
</div>
|
|
293
329
|
|
|
294
330
|
<div className={css.vbox(12)}>
|
|
@@ -342,9 +378,28 @@ export class ErrorNotificationPage extends qreact.Component {
|
|
|
342
378
|
/>
|
|
343
379
|
</div>
|
|
344
380
|
|
|
381
|
+
<div className={css.hbox(16)}>
|
|
382
|
+
<InputLabel
|
|
383
|
+
label="Hide zero-match suppressions"
|
|
384
|
+
checkbox
|
|
385
|
+
checked={this.state.hideZeroMatchSuppressions}
|
|
386
|
+
onChange={e => {
|
|
387
|
+
this.state.hideZeroMatchSuppressions = e.currentTarget.checked;
|
|
388
|
+
}}
|
|
389
|
+
/>
|
|
390
|
+
<InputLabel
|
|
391
|
+
label="Sort by matches"
|
|
392
|
+
checkbox
|
|
393
|
+
checked={this.state.sortSuppressionsByMatches}
|
|
394
|
+
onChange={e => {
|
|
395
|
+
this.state.sortSuppressionsByMatches = e.currentTarget.checked;
|
|
396
|
+
}}
|
|
397
|
+
/>
|
|
398
|
+
</div>
|
|
399
|
+
|
|
345
400
|
<div className={css.vbox(12)}>
|
|
346
|
-
{
|
|
347
|
-
let matchData =
|
|
401
|
+
{displayedSuppressions.map((suppression) => {
|
|
402
|
+
let matchData = suppressionMatches.get(suppression.id);
|
|
348
403
|
|
|
349
404
|
return <SuppressionItem
|
|
350
405
|
key={suppression.id}
|
|
@@ -353,6 +408,28 @@ export class ErrorNotificationPage extends qreact.Component {
|
|
|
353
408
|
/>;
|
|
354
409
|
})}
|
|
355
410
|
</div>
|
|
411
|
+
<div className={css.hbox(8)}>
|
|
412
|
+
{sortedSuppressions.length > displayedSuppressions.length && (
|
|
413
|
+
<Button
|
|
414
|
+
hue={200}
|
|
415
|
+
onClick={() => {
|
|
416
|
+
this.state.suppressionsShown = this.state.suppressionsShown + SHOW_MORE_STEP;
|
|
417
|
+
}}
|
|
418
|
+
>
|
|
419
|
+
{`Show ${Math.min(SHOW_MORE_STEP, sortedSuppressions.length - displayedSuppressions.length)} More Suppressions (${sortedSuppressions.length - displayedSuppressions.length} hidden)`}
|
|
420
|
+
</Button>
|
|
421
|
+
)}
|
|
422
|
+
{this.state.suppressionsShown > INITIAL_SUPPRESSIONS_SHOWN && (
|
|
423
|
+
<Button
|
|
424
|
+
hue={200}
|
|
425
|
+
onClick={() => {
|
|
426
|
+
this.state.suppressionsShown = INITIAL_SUPPRESSIONS_SHOWN;
|
|
427
|
+
}}
|
|
428
|
+
>
|
|
429
|
+
Collapse Suppressions
|
|
430
|
+
</Button>
|
|
431
|
+
)}
|
|
432
|
+
</div>
|
|
356
433
|
</div>
|
|
357
434
|
</div>;
|
|
358
435
|
}
|
|
@@ -398,12 +475,7 @@ class SuppressionItem extends qreact.Component<{
|
|
|
398
475
|
render() {
|
|
399
476
|
let suppression = this.props.suppression;
|
|
400
477
|
let matchData = this.props.matchData;
|
|
401
|
-
let totalCount =
|
|
402
|
-
if (matchData) {
|
|
403
|
-
for (let [chunk, data] of matchData.history) {
|
|
404
|
-
totalCount += data.count;
|
|
405
|
-
}
|
|
406
|
-
}
|
|
478
|
+
let totalCount = getSuppressionMatchCount(matchData);
|
|
407
479
|
|
|
408
480
|
let timeoutText: preact.ComponentChild;
|
|
409
481
|
if (suppression.timeout >= Number.MAX_SAFE_INTEGER) {
|