querysub 0.377.0 → 0.379.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.cursorrules +2 -0
- package/bin/error-watch-public.js +7 -0
- package/bin/error-watch.js +6 -0
- package/package.json +7 -4
- package/src/-f-node-discovery/NodeDiscovery.ts +7 -0
- package/src/-g-core-values/NodeCapabilities.ts +28 -14
- package/src/3-path-functions/PathFunctionRunnerMain.ts +0 -4
- package/src/diagnostics/MachineThreadInfo.tsx +33 -10
- package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts +24 -1
- package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx +6 -2
- package/src/diagnostics/logs/diskLogger.ts +26 -27
- package/src/diagnostics/logs/diskShimConsoleLogs.ts +4 -0
- package/src/diagnostics/logs/errorNotifications2/ErrorNotificationPage.tsx +505 -0
- package/src/diagnostics/logs/errorNotifications2/ErrorWarning.tsx +32 -0
- package/src/diagnostics/logs/errorNotifications2/errorNotifications.ts +629 -0
- package/src/diagnostics/logs/errorNotifications2/errorWatchEntry.ts +13 -0
- package/src/diagnostics/logs/errorNotifications2/errorWatcher.ts +168 -0
- package/src/diagnostics/logs/errorNotifications2/logWatcher.ts +104 -0
- package/src/diagnostics/logs/errorNotifications2/openRouterHelper.ts +77 -0
- package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx +17 -19
- package/src/diagnostics/managementPages.tsx +12 -2
- package/src/server.ts +0 -8
- package/src/user-implementation/SecurityPage.tsx +6 -0
- package/src/user-implementation/userData.ts +6 -1
- package/test.ts +16 -6
- package/test2.ts +20 -0
- package/src/diagnostics/logs/errorNotifications2/errorNotifications2.ts +0 -9
- package/src/diagnostics/logs/lifeCycleAnalysis/test.ts +0 -0
- /package/src/library-components/{errorNotifications.tsx → uncaughtToast.tsx} +0 -0
|
@@ -0,0 +1,629 @@
|
|
|
1
|
+
import { cacheLimited, lazy } from "socket-function/src/caching";
|
|
2
|
+
import { nestArchives } from "../../../-a-archives/archives";
|
|
3
|
+
import { getArchivesBackblaze } from "../../../-a-archives/archivesBackBlaze";
|
|
4
|
+
import { archiveJSONT } from "../../../-a-archives/archivesJSONT";
|
|
5
|
+
import { Querysub } from "../../../4-querysub/QuerysubController";
|
|
6
|
+
import { getDomain } from "../../../config";
|
|
7
|
+
import { MachineInfo } from "../../../deployManager/machineSchema";
|
|
8
|
+
import { createMatchesPattern } from "../IndexedLogs/bufferSearchFindMatcher";
|
|
9
|
+
import { LogDatum, getErrorLogs } from "../diskLogger";
|
|
10
|
+
import { watchAllValues } from "./logWatcher";
|
|
11
|
+
import { batchFunction, runInSerial, runInfinitePollCallAtStart } from "socket-function/src/batching";
|
|
12
|
+
import { nextId, sort, throttleFunction, timeInMinute } from "socket-function/src/misc";
|
|
13
|
+
import { SocketFunction } from "socket-function/SocketFunction";
|
|
14
|
+
import { assertIsManagementUser } from "../../managementPages";
|
|
15
|
+
import { getSyncedController } from "../../../library-components/SyncedController";
|
|
16
|
+
import { getControllerNodeId } from "../../../-g-core-values/NodeCapabilities";
|
|
17
|
+
import { t } from "../../../2-proxy/schema2";
|
|
18
|
+
import { sendDiscordMessage } from "../../../email_ims_notifications/discord";
|
|
19
|
+
import { user_data } from "../../../user-implementation/userData";
|
|
20
|
+
import { callOpenRouterJSON } from "./openRouterHelper";
|
|
21
|
+
import { createLink } from "../../../library-components/ATag";
|
|
22
|
+
import { showingManagementURL, managementPageURL } from "../../managementPages";
|
|
23
|
+
import { atomic } from "../../../2-proxy/PathValueProxyWatcher";
|
|
24
|
+
import { magenta } from "socket-function/src/formatting/logColors";
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
type SuppressionEntry = {
|
|
28
|
+
id: string;
|
|
29
|
+
notes?: string;
|
|
30
|
+
pattern: string;
|
|
31
|
+
|
|
32
|
+
// After this time, this suppression won't suppress data anymore.
|
|
33
|
+
timeout: number;
|
|
34
|
+
|
|
35
|
+
createdTime: number;
|
|
36
|
+
lastUpdatedTime: number;
|
|
37
|
+
};
|
|
38
|
+
const suppression = archiveJSONT<SuppressionEntry>(() => nestArchives("logs/error-suppression/", getArchivesBackblaze(getDomain())));
|
|
39
|
+
let suppressionCache: SuppressionEntry[] = [];
|
|
40
|
+
|
|
41
|
+
// In-memory Discord notification throttling
|
|
42
|
+
const timeInHour = 60 * 60 * 1000;
|
|
43
|
+
const NEW_SUPPRESSION_THROTTLE = timeInHour * 6;
|
|
44
|
+
const EXISTING_SUPPRESSION_THROTTLE = timeInHour * 24;
|
|
45
|
+
|
|
46
|
+
let isThrottlingNewSuppressions = false;
|
|
47
|
+
let isThrottlingExistingSuppressions = false;
|
|
48
|
+
let pendingDiscordNotifications = new Map<string, {
|
|
49
|
+
errorCount: number;
|
|
50
|
+
isNew: boolean;
|
|
51
|
+
}>();
|
|
52
|
+
let ensureWatching = lazy(async () => {
|
|
53
|
+
await runInfinitePollCallAtStart(timeInMinute * 5, updateNow);
|
|
54
|
+
});
|
|
55
|
+
async function updateNow() {
|
|
56
|
+
suppressionCache = await suppression.values();
|
|
57
|
+
// Can't wait, because that will cause a cyclic loop
|
|
58
|
+
void reapplySuppressions();
|
|
59
|
+
}
|
|
60
|
+
async function getSuppressionEntries(): Promise<SuppressionEntry[]> {
|
|
61
|
+
await ensureWatching();
|
|
62
|
+
return suppressionCache;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
let getMatcher = cacheLimited(10000, (text: string) => createMatchesPattern(Buffer.from(text), false));
|
|
66
|
+
|
|
67
|
+
const MAX_UNMATCHED = 10000;
|
|
68
|
+
const MAX_EXAMPLES = 100;
|
|
69
|
+
let unmatchedIndex = 0;
|
|
70
|
+
let unmatchedErrors: LogDatum[] = [];
|
|
71
|
+
export type SuppressionMatch = {
|
|
72
|
+
id: string;
|
|
73
|
+
// getHistoryChunk =>
|
|
74
|
+
history: Map<number, {
|
|
75
|
+
count: number;
|
|
76
|
+
}>;
|
|
77
|
+
exampleIndex: number;
|
|
78
|
+
examples: LogDatum[];
|
|
79
|
+
};
|
|
80
|
+
let suppressionMatches = new Map<string, SuppressionMatch>();
|
|
81
|
+
const chunkUnit = timeInMinute * 5;
|
|
82
|
+
export function getHistoryChunk(time: number) {
|
|
83
|
+
return Math.floor(time / chunkUnit) * chunkUnit;
|
|
84
|
+
}
|
|
85
|
+
export function getChunkEndTime(chunk: number) {
|
|
86
|
+
chunk = getHistoryChunk(chunk);
|
|
87
|
+
return chunk + chunkUnit;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
type SuppressionStatus = "suppressed" | "matched-timeout-passed" | "no-match";
|
|
91
|
+
|
|
92
|
+
function applySuppression(error: LogDatum, suppressionEntry: SuppressionEntry): SuppressionStatus {
|
|
93
|
+
let errorBuffer = Buffer.from(JSON.stringify(error));
|
|
94
|
+
let isMatch = getMatcher(suppressionEntry.pattern);
|
|
95
|
+
if (!isMatch(errorBuffer)) return "no-match";
|
|
96
|
+
|
|
97
|
+
let history = suppressionMatches.get(suppressionEntry.id);
|
|
98
|
+
if (!history) {
|
|
99
|
+
history = {
|
|
100
|
+
id: suppressionEntry.id,
|
|
101
|
+
history: new Map(),
|
|
102
|
+
exampleIndex: 0,
|
|
103
|
+
examples: [],
|
|
104
|
+
};
|
|
105
|
+
suppressionMatches.set(suppressionEntry.id, history);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
if (history.examples.length > MAX_EXAMPLES) {
|
|
109
|
+
history.examples[history.exampleIndex] = error;
|
|
110
|
+
history.exampleIndex = (history.exampleIndex + 1) % MAX_EXAMPLES;
|
|
111
|
+
} else {
|
|
112
|
+
history.examples.push(error);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
let historyChunk = getHistoryChunk(error.time);
|
|
116
|
+
let historyEntry = history.history.get(historyChunk);
|
|
117
|
+
if (!historyEntry) {
|
|
118
|
+
historyEntry = { count: 0 };
|
|
119
|
+
history.history.set(historyChunk, historyEntry);
|
|
120
|
+
}
|
|
121
|
+
historyEntry.count++;
|
|
122
|
+
|
|
123
|
+
if (error.time > suppressionEntry.timeout) {
|
|
124
|
+
return "matched-timeout-passed";
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
return "suppressed";
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
async function reapplySuppressions() {
|
|
131
|
+
let suppressionEntries = await getSuppressionEntries();
|
|
132
|
+
let newUnmatchedErrors: LogDatum[] = [];
|
|
133
|
+
let changedSuppressionIds = new Set<string>();
|
|
134
|
+
|
|
135
|
+
for (let error of unmatchedErrors) {
|
|
136
|
+
let suppressed = false;
|
|
137
|
+
for (let suppressionEntry of suppressionEntries) {
|
|
138
|
+
let status = applySuppression(error, suppressionEntry);
|
|
139
|
+
if (status === "suppressed" || status === "matched-timeout-passed") {
|
|
140
|
+
changedSuppressionIds.add(suppressionEntry.id);
|
|
141
|
+
}
|
|
142
|
+
if (status === "suppressed") {
|
|
143
|
+
suppressed = true;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
if (!suppressed) {
|
|
147
|
+
newUnmatchedErrors.push(error);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
unmatchedErrors = newUnmatchedErrors;
|
|
152
|
+
unmatchedIndex = unmatchedErrors.length % MAX_UNMATCHED;
|
|
153
|
+
|
|
154
|
+
for (let id of changedSuppressionIds) {
|
|
155
|
+
void ErrorNotificationService.triggerSuppressionUpdate(id);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
async function onSuppressionsChanged() {
|
|
160
|
+
// NOTE: Update Now also reapplies suppression internally. after it finishes the update.
|
|
161
|
+
void updateNow();
|
|
162
|
+
await reapplySuppressions();
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
type PatternResponse = {
|
|
166
|
+
pattern: string;
|
|
167
|
+
};
|
|
168
|
+
|
|
169
|
+
const PATTERN_GENERATION_LIMIT = 100;
|
|
170
|
+
const PATTERN_GENERATION_WINDOW = 15 * 60 * 1000;
|
|
171
|
+
let patternGenerationTimes: number[] = [];
|
|
172
|
+
|
|
173
|
+
async function generatePatternForError(error: LogDatum): Promise<string> {
|
|
174
|
+
const now = Date.now();
|
|
175
|
+
const windowStart = now - PATTERN_GENERATION_WINDOW;
|
|
176
|
+
|
|
177
|
+
patternGenerationTimes = patternGenerationTimes.filter(time => time > windowStart);
|
|
178
|
+
|
|
179
|
+
if (patternGenerationTimes.length >= PATTERN_GENERATION_LIMIT) {
|
|
180
|
+
console.log(magenta(`Rate limit exceeded for pattern generation (${patternGenerationTimes.length}/${PATTERN_GENERATION_LIMIT} in last 15 minutes)`));
|
|
181
|
+
return "";
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
patternGenerationTimes.push(now);
|
|
185
|
+
const openRouterAPIKey = await Querysub.commitAsync(() => atomic(user_data().secure.openRouterAPIKey));
|
|
186
|
+
if (!openRouterAPIKey) {
|
|
187
|
+
throw new Error("OpenRouter API key not set");
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
console.log(magenta(`Createing pattern for error: ${error.param0}`));
|
|
191
|
+
|
|
192
|
+
const param0 = error.param0 && String(error.param0) || "(no message)";
|
|
193
|
+
|
|
194
|
+
const prompt = `You are helping to categorize error messages. We categorize them by finding specific unique text inside of the error message which will be the same even if the variables and the instance changes.
|
|
195
|
+
|
|
196
|
+
Here is an error message:
|
|
197
|
+
${param0}
|
|
198
|
+
|
|
199
|
+
Please provide a SHORT search string that would match this type of error. The string should:
|
|
200
|
+
- Be specific enough to identify this error type
|
|
201
|
+
- Be generic enough to match similar errors (avoid dynamic values like timestamps, IDs, or specific numbers)
|
|
202
|
+
- Be a simple substring (no special operators)
|
|
203
|
+
|
|
204
|
+
Respond with a JSON object with a "pattern" field containing the search string.`;
|
|
205
|
+
|
|
206
|
+
const response = await callOpenRouterJSON<PatternResponse>(openRouterAPIKey, prompt);
|
|
207
|
+
|
|
208
|
+
const pattern = response.pattern && response.pattern.trim();
|
|
209
|
+
|
|
210
|
+
if (!pattern) {
|
|
211
|
+
throw new Error("AI returned an empty pattern");
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
return pattern;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function queueDiscordNotification(suppressionId: string, errorCount: number, isNew: boolean) {
|
|
218
|
+
const existing = pendingDiscordNotifications.get(suppressionId);
|
|
219
|
+
if (existing) {
|
|
220
|
+
existing.errorCount += errorCount;
|
|
221
|
+
} else {
|
|
222
|
+
pendingDiscordNotifications.set(suppressionId, { errorCount, isNew });
|
|
223
|
+
}
|
|
224
|
+
void trySendDiscordNotifications(isNew);
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
function trySendDiscordNotifications(isNew: boolean) {
|
|
228
|
+
if (isNew && isThrottlingNewSuppressions) return;
|
|
229
|
+
if (isThrottlingExistingSuppressions) return;
|
|
230
|
+
if (pendingDiscordNotifications.size === 0) return;
|
|
231
|
+
void trySendDiscordNotificationsBase();
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
const trySendDiscordNotificationsBase = batchFunction({ delay: 1000 * 3 }, async function trySendDiscordNotificationsBase(nothing: void[]) {
|
|
235
|
+
if (pendingDiscordNotifications.size === 0) return;
|
|
236
|
+
|
|
237
|
+
const webhookURL = await Querysub.commitAsync(() => atomic(user_data().secure.notifyDiscordWebhookURL));
|
|
238
|
+
if (!webhookURL) {
|
|
239
|
+
return;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
const newSuppressions: Array<{ id: string; count: number; pattern: string }> = [];
|
|
244
|
+
const existingSuppressions: Array<{ id: string; count: number; pattern: string }> = [];
|
|
245
|
+
|
|
246
|
+
for (const [suppressionId, data] of pendingDiscordNotifications.entries()) {
|
|
247
|
+
const suppressionEntry = suppressionCache.find(e => e.id === suppressionId);
|
|
248
|
+
if (!suppressionEntry) continue;
|
|
249
|
+
|
|
250
|
+
if (data.isNew) {
|
|
251
|
+
newSuppressions.push({ id: suppressionId, count: data.errorCount, pattern: suppressionEntry.pattern });
|
|
252
|
+
} else {
|
|
253
|
+
existingSuppressions.push({ id: suppressionId, count: data.errorCount, pattern: suppressionEntry.pattern });
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
let messageLines: string[] = [];
|
|
258
|
+
|
|
259
|
+
for (const sup of newSuppressions) {
|
|
260
|
+
const line = `(${sup.count}) "${sup.pattern.slice(0, 50)}" [NEW]`;
|
|
261
|
+
if (messageLines.join("\n").length + line.length < 400) {
|
|
262
|
+
messageLines.push(line);
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
for (const sup of existingSuppressions) {
|
|
267
|
+
const line = `(${sup.count}) "${sup.pattern.slice(0, 50)}" [EXISTING]`;
|
|
268
|
+
if (messageLines.join("\n").length + line.length < 400) {
|
|
269
|
+
messageLines.push(line);
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
const link = createLink([
|
|
274
|
+
{ param: showingManagementURL, value: true },
|
|
275
|
+
{ param: managementPageURL, value: "ErrorNotificationPage" },
|
|
276
|
+
]);
|
|
277
|
+
const messageText = messageLines.join("\n");
|
|
278
|
+
const message = `[${messageText}](${link})`;
|
|
279
|
+
|
|
280
|
+
try {
|
|
281
|
+
await sendDiscordMessage({ webhookURL, message });
|
|
282
|
+
|
|
283
|
+
isThrottlingNewSuppressions = true;
|
|
284
|
+
isThrottlingExistingSuppressions = true;
|
|
285
|
+
setTimeout(() => {
|
|
286
|
+
isThrottlingNewSuppressions = false;
|
|
287
|
+
let anyNew = Array.from(pendingDiscordNotifications.values()).some(data => data.isNew);
|
|
288
|
+
void trySendDiscordNotifications(anyNew);
|
|
289
|
+
}, NEW_SUPPRESSION_THROTTLE);
|
|
290
|
+
setTimeout(() => {
|
|
291
|
+
isThrottlingExistingSuppressions = false;
|
|
292
|
+
void trySendDiscordNotifications(false);
|
|
293
|
+
}, EXISTING_SUPPRESSION_THROTTLE);
|
|
294
|
+
|
|
295
|
+
pendingDiscordNotifications.clear();
|
|
296
|
+
} catch (error) {
|
|
297
|
+
console.error("Failed to send Discord notification:", error);
|
|
298
|
+
}
|
|
299
|
+
});
|
|
300
|
+
|
|
301
|
+
export async function exposeErrorWatchService() {
|
|
302
|
+
SocketFunction.expose(ErrorNotificationServiceBase);
|
|
303
|
+
|
|
304
|
+
let errorLogs = await getErrorLogs();
|
|
305
|
+
for await (let error of watchAllValues(errorLogs)) {
|
|
306
|
+
let suppressionEntries = await getSuppressionEntries();
|
|
307
|
+
let suppressed = false;
|
|
308
|
+
let matchedSuppressionId: string | undefined = undefined;
|
|
309
|
+
|
|
310
|
+
for (let suppressionEntry of suppressionEntries) {
|
|
311
|
+
let status = applySuppression(error, suppressionEntry);
|
|
312
|
+
if (status === "suppressed" || status === "matched-timeout-passed") {
|
|
313
|
+
void ErrorNotificationService.triggerSuppressionUpdate(suppressionEntry.id);
|
|
314
|
+
}
|
|
315
|
+
if (status === "suppressed") {
|
|
316
|
+
suppressed = true;
|
|
317
|
+
matchedSuppressionId = suppressionEntry.id;
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
if (suppressed && matchedSuppressionId) {
|
|
322
|
+
queueDiscordNotification(matchedSuppressionId, 1, false);
|
|
323
|
+
} else if (!suppressed) {
|
|
324
|
+
if (unmatchedErrors.length > unmatchedIndex) {
|
|
325
|
+
unmatchedErrors[unmatchedIndex] = error;
|
|
326
|
+
unmatchedIndex = (unmatchedIndex + 1) % MAX_UNMATCHED;
|
|
327
|
+
} else {
|
|
328
|
+
unmatchedErrors.push(error);
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
const pattern = await generatePatternForError(error);
|
|
332
|
+
if (pattern) {
|
|
333
|
+
const newEntry: SuppressionEntry = {
|
|
334
|
+
id: nextId(),
|
|
335
|
+
pattern,
|
|
336
|
+
timeout: 0,
|
|
337
|
+
createdTime: Date.now(),
|
|
338
|
+
lastUpdatedTime: Date.now(),
|
|
339
|
+
};
|
|
340
|
+
|
|
341
|
+
suppressionCache.push(newEntry);
|
|
342
|
+
void suppression.set(newEntry.id, newEntry);
|
|
343
|
+
applySuppression(error, newEntry);
|
|
344
|
+
queueDiscordNotification(newEntry.id, 1, true);
|
|
345
|
+
|
|
346
|
+
void ErrorNotificationService.triggerSuppressionUpdate(newEntry.id);
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
void ErrorNotificationService.triggerUnmatchedError(error);
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
class ErrorNotificationService {
|
|
355
|
+
public async getData() {
|
|
356
|
+
return {
|
|
357
|
+
unmatchedErrors,
|
|
358
|
+
suppressionMatches,
|
|
359
|
+
};
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
public async getUnmatchedErrorsLimited() {
|
|
363
|
+
// Oldest first, otherwise, while you're trying to deal with one match, it might keep changing, which is annoying
|
|
364
|
+
sort(unmatchedErrors, x => x.time);
|
|
365
|
+
// We have to sort from oldest to first, otherwise the way we write it will be weird, as we write it in the order from oldest to first already. However, we don't want to clobber the ones we're returning, so we start right after them. So we're going to keep the oldest ones for a little bit longer after we call this function.
|
|
366
|
+
unmatchedIndex = 5;
|
|
367
|
+
return unmatchedErrors.slice(0, 5);
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
private static watchersSERVICE = new Set<string>();
|
|
371
|
+
public async watchUnmatchedErrorsSERVICE(): Promise<void> {
|
|
372
|
+
let caller = SocketFunction.getCaller();
|
|
373
|
+
ErrorNotificationService.watchersSERVICE.add(caller.nodeId);
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
public static triggerUnmatchedError = batchFunction({ delay: 100 }, (datums: LogDatum[]) => {
|
|
377
|
+
for (let nodeId of ErrorNotificationService.watchersSERVICE) {
|
|
378
|
+
void (async () => {
|
|
379
|
+
try {
|
|
380
|
+
await ErrorNotificationDataBase.nodes[nodeId].receiveErrorHTTP(datums);
|
|
381
|
+
} catch {
|
|
382
|
+
ErrorNotificationService.watchersSERVICE.delete(nodeId);
|
|
383
|
+
}
|
|
384
|
+
})();
|
|
385
|
+
}
|
|
386
|
+
});
|
|
387
|
+
|
|
388
|
+
public static triggerSuppressionUpdate = batchFunction({ delay: 100 }, (ids: string[]) => {
|
|
389
|
+
let uniqueIds = new Set(ids);
|
|
390
|
+
let matches: SuppressionMatch[] = [];
|
|
391
|
+
for (let id of uniqueIds) {
|
|
392
|
+
let match = suppressionMatches.get(id);
|
|
393
|
+
if (match) {
|
|
394
|
+
matches.push(match);
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
for (let nodeId of ErrorNotificationService.watchersSERVICE) {
|
|
399
|
+
void (async () => {
|
|
400
|
+
try {
|
|
401
|
+
await ErrorNotificationDataBase.nodes[nodeId].receiveSuppressionHTTP(matches);
|
|
402
|
+
} catch {
|
|
403
|
+
ErrorNotificationService.watchersSERVICE.delete(nodeId);
|
|
404
|
+
}
|
|
405
|
+
})();
|
|
406
|
+
}
|
|
407
|
+
});
|
|
408
|
+
|
|
409
|
+
public async getSuppressionEntries() {
|
|
410
|
+
return await getSuppressionEntries();
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
public async setSuppressionEntry(entry: SuppressionEntry) {
|
|
414
|
+
let suppressionPromise = suppression.set(entry.id, entry);
|
|
415
|
+
let prevEntry = suppressionCache.findIndex(e => e.id === entry.id);
|
|
416
|
+
if (prevEntry !== -1) {
|
|
417
|
+
suppressionCache[prevEntry] = entry;
|
|
418
|
+
} else {
|
|
419
|
+
suppressionCache.push(entry);
|
|
420
|
+
}
|
|
421
|
+
await onSuppressionsChanged();
|
|
422
|
+
void suppressionPromise.finally(() => {
|
|
423
|
+
void onSuppressionsChanged();
|
|
424
|
+
});
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
public async deleteSuppressionEntry(id: string) {
|
|
428
|
+
let suppressionPromise = suppression.delete(id);
|
|
429
|
+
let prevEntry = suppressionCache.findIndex(e => e.id === id);
|
|
430
|
+
if (prevEntry !== -1) {
|
|
431
|
+
suppressionCache.splice(prevEntry, 1);
|
|
432
|
+
}
|
|
433
|
+
await onSuppressionsChanged();
|
|
434
|
+
void suppressionPromise.finally(() => {
|
|
435
|
+
void onSuppressionsChanged();
|
|
436
|
+
});
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
public async updateSuppressionNotes(id: string, notes: string | undefined) {
|
|
440
|
+
let entry = suppressionCache.find(e => e.id === id);
|
|
441
|
+
if (!entry) {
|
|
442
|
+
throw new Error(`Suppression entry ${id} not found`);
|
|
443
|
+
}
|
|
444
|
+
let updatedEntry = { ...entry, notes, lastUpdatedTime: Date.now() };
|
|
445
|
+
suppressionCache[suppressionCache.findIndex(e => e.id === id)] = updatedEntry;
|
|
446
|
+
void suppression.set(id, updatedEntry);
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
const ErrorNotificationServiceBase = SocketFunction.register(
|
|
451
|
+
"ErrorNotificationService-019c9cae-8333-7708-a4e7-500f5fc23175",
|
|
452
|
+
new ErrorNotificationService(),
|
|
453
|
+
() => ({
|
|
454
|
+
getData: {},
|
|
455
|
+
getUnmatchedErrorsLimited: {},
|
|
456
|
+
watchUnmatchedErrorsSERVICE: {},
|
|
457
|
+
getSuppressionEntries: {},
|
|
458
|
+
setSuppressionEntry: {},
|
|
459
|
+
deleteSuppressionEntry: {},
|
|
460
|
+
updateSuppressionNotes: {},
|
|
461
|
+
}),
|
|
462
|
+
() => ({
|
|
463
|
+
hooks: [assertIsManagementUser],
|
|
464
|
+
}),
|
|
465
|
+
{
|
|
466
|
+
noAutoExpose: true,
|
|
467
|
+
}
|
|
468
|
+
);
|
|
469
|
+
|
|
470
|
+
class ErrorNotificationData {
|
|
471
|
+
public async getData() {
|
|
472
|
+
let controllerNodeId = await getControllerNodeId(ErrorNotificationServiceBase);
|
|
473
|
+
if (!controllerNodeId) {
|
|
474
|
+
throw new Error(`Could not find node exposing controller ErrorNotificationServiceBase`);
|
|
475
|
+
}
|
|
476
|
+
return await ErrorNotificationServiceBase.nodes[controllerNodeId].getData();
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
public async getUnmatchedErrorsLimited() {
|
|
480
|
+
let controllerNodeId = await getControllerNodeId(ErrorNotificationServiceBase);
|
|
481
|
+
if (!controllerNodeId) {
|
|
482
|
+
throw new Error(`Could not find node exposing controller ErrorNotificationServiceBase`);
|
|
483
|
+
}
|
|
484
|
+
return await ErrorNotificationServiceBase.nodes[controllerNodeId].getUnmatchedErrorsLimited();
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
static browserErrorCallbacks = new Set<(datum: LogDatum[]) => void>();
|
|
488
|
+
static browserSuppressionCallbacks = new Set<(matches: SuppressionMatch[]) => void>();
|
|
489
|
+
public static async watchUnmatchedErrors(config: {
|
|
490
|
+
errorCallback: (datum: LogDatum[]) => void;
|
|
491
|
+
suppressionCallback: (matches: SuppressionMatch[]) => void;
|
|
492
|
+
}) {
|
|
493
|
+
ErrorNotificationData.browserErrorCallbacks.add(config.errorCallback);
|
|
494
|
+
ErrorNotificationData.browserSuppressionCallbacks.add(config.suppressionCallback);
|
|
495
|
+
await ErrorNotificationData.ensureWatchingErrorsBrowser();
|
|
496
|
+
return () => {
|
|
497
|
+
ErrorNotificationData.browserErrorCallbacks.delete(config.errorCallback);
|
|
498
|
+
ErrorNotificationData.browserSuppressionCallbacks.delete(config.suppressionCallback);
|
|
499
|
+
};
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
private static ensureWatchingErrorsBrowser = lazy(async () => {
|
|
503
|
+
await ErrorNotificationDataBase.nodes[SocketFunction.getBrowserNodeId()].watchUnmatchedErrorsHTTP();
|
|
504
|
+
});
|
|
505
|
+
|
|
506
|
+
public async receiveErrorBrowser(datums: LogDatum[]) {
|
|
507
|
+
for (let callback of ErrorNotificationData.browserErrorCallbacks) {
|
|
508
|
+
callback(datums);
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
public async receiveSuppressionBrowser(matches: SuppressionMatch[]) {
|
|
513
|
+
for (let callback of ErrorNotificationData.browserSuppressionCallbacks) {
|
|
514
|
+
callback(matches);
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
private watchersHTTP = new Set<string>();
|
|
519
|
+
public async watchUnmatchedErrorsHTTP() {
|
|
520
|
+
let caller = SocketFunction.getCaller();
|
|
521
|
+
this.watchersHTTP.add(caller.nodeId);
|
|
522
|
+
await ErrorNotificationData.ensureWatchingErrorsHTTP();
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
private static ensureWatchingErrorsHTTP = lazy(async () => {
|
|
526
|
+
let controllerNodeId = await getControllerNodeId(ErrorNotificationServiceBase);
|
|
527
|
+
if (!controllerNodeId) {
|
|
528
|
+
ErrorNotificationData.ensureWatchingErrorsHTTP.reset();
|
|
529
|
+
throw new Error(`Could not find node exposing controller ErrorNotificationServiceBase`);
|
|
530
|
+
}
|
|
531
|
+
SocketFunction.onNextDisconnect(controllerNodeId, () => {
|
|
532
|
+
ErrorNotificationData.ensureWatchingErrorsHTTP.reset();
|
|
533
|
+
});
|
|
534
|
+
await ErrorNotificationServiceBase.nodes[controllerNodeId].watchUnmatchedErrorsSERVICE();
|
|
535
|
+
});
|
|
536
|
+
|
|
537
|
+
public async receiveErrorHTTP(datums: LogDatum[]) {
|
|
538
|
+
for (let nodeId of this.watchersHTTP) {
|
|
539
|
+
void (async () => {
|
|
540
|
+
try {
|
|
541
|
+
await ErrorNotificationDataBase.nodes[nodeId].receiveErrorBrowser(datums);
|
|
542
|
+
} catch {
|
|
543
|
+
this.watchersHTTP.delete(nodeId);
|
|
544
|
+
}
|
|
545
|
+
})();
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
public async receiveSuppressionHTTP(matches: SuppressionMatch[]) {
|
|
550
|
+
for (let nodeId of this.watchersHTTP) {
|
|
551
|
+
void (async () => {
|
|
552
|
+
try {
|
|
553
|
+
await ErrorNotificationDataBase.nodes[nodeId].receiveSuppressionBrowser(matches);
|
|
554
|
+
} catch {
|
|
555
|
+
this.watchersHTTP.delete(nodeId);
|
|
556
|
+
}
|
|
557
|
+
})();
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
public async getSuppressionEntries() {
|
|
562
|
+
let controllerNodeId = await getControllerNodeId(ErrorNotificationServiceBase);
|
|
563
|
+
if (!controllerNodeId) {
|
|
564
|
+
throw new Error(`Could not find node exposing controller ErrorNotificationServiceBase`);
|
|
565
|
+
}
|
|
566
|
+
return await ErrorNotificationServiceBase.nodes[controllerNodeId].getSuppressionEntries();
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
public async setSuppressionEntry(entry: SuppressionEntry) {
|
|
570
|
+
let controllerNodeId = await getControllerNodeId(ErrorNotificationServiceBase);
|
|
571
|
+
if (!controllerNodeId) {
|
|
572
|
+
throw new Error(`Could not find node exposing controller ErrorNotificationServiceBase`);
|
|
573
|
+
}
|
|
574
|
+
await ErrorNotificationServiceBase.nodes[controllerNodeId].setSuppressionEntry(entry);
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
public async deleteSuppressionEntry(id: string) {
|
|
578
|
+
let controllerNodeId = await getControllerNodeId(ErrorNotificationServiceBase);
|
|
579
|
+
if (!controllerNodeId) {
|
|
580
|
+
throw new Error(`Could not find node exposing controller ErrorNotificationServiceBase`);
|
|
581
|
+
}
|
|
582
|
+
await ErrorNotificationServiceBase.nodes[controllerNodeId].deleteSuppressionEntry(id);
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
public async logTestError(errorMessage: string) {
|
|
586
|
+
console.error(errorMessage);
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
public async updateSuppressionNotes(id: string, notes: string | undefined) {
|
|
590
|
+
let controllerNodeId = await getControllerNodeId(ErrorNotificationServiceBase);
|
|
591
|
+
if (!controllerNodeId) {
|
|
592
|
+
throw new Error(`Could not find node exposing controller ErrorNotificationServiceBase`);
|
|
593
|
+
}
|
|
594
|
+
await ErrorNotificationServiceBase.nodes[controllerNodeId].updateSuppressionNotes(id, notes);
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
const ErrorNotificationDataBase = SocketFunction.register(
|
|
599
|
+
"ErrorNotificationData-019c9cae-8333-7708-a4e7-500f5fc23174",
|
|
600
|
+
new ErrorNotificationData(),
|
|
601
|
+
() => ({
|
|
602
|
+
getData: {},
|
|
603
|
+
getUnmatchedErrorsLimited: {},
|
|
604
|
+
receiveErrorBrowser: {},
|
|
605
|
+
receiveSuppressionBrowser: {},
|
|
606
|
+
receiveErrorHTTP: {},
|
|
607
|
+
receiveSuppressionHTTP: {},
|
|
608
|
+
watchUnmatchedErrorsHTTP: {},
|
|
609
|
+
getSuppressionEntries: {},
|
|
610
|
+
setSuppressionEntry: {},
|
|
611
|
+
deleteSuppressionEntry: {},
|
|
612
|
+
logTestError: {},
|
|
613
|
+
updateSuppressionNotes: {},
|
|
614
|
+
}),
|
|
615
|
+
() => ({
|
|
616
|
+
hooks: [assertIsManagementUser],
|
|
617
|
+
})
|
|
618
|
+
);
|
|
619
|
+
export const ErrorNotificationsController = getSyncedController(ErrorNotificationDataBase);
|
|
620
|
+
|
|
621
|
+
export function watchUnmatchedErrors(config: {
|
|
622
|
+
errorCallback: (datum: LogDatum[]) => void;
|
|
623
|
+
suppressionCallback: (matches: SuppressionMatch[]) => void;
|
|
624
|
+
}) {
|
|
625
|
+
return ErrorNotificationData.watchUnmatchedErrors({
|
|
626
|
+
errorCallback: config.errorCallback,
|
|
627
|
+
suppressionCallback: config.suppressionCallback,
|
|
628
|
+
});
|
|
629
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import "../../../forceProduction";
|
|
2
|
+
import "../../../inject";
|
|
3
|
+
|
|
4
|
+
import { logErrors } from "../../../errors";
|
|
5
|
+
import { Querysub } from "../../../4-querysub/QuerysubController";
|
|
6
|
+
import { exposeErrorWatchService } from "./errorNotifications";
|
|
7
|
+
|
|
8
|
+
logErrors(main());
|
|
9
|
+
|
|
10
|
+
async function main() {
|
|
11
|
+
await Querysub.hostService("ErrorWatch");
|
|
12
|
+
await exposeErrorWatchService();
|
|
13
|
+
}
|