querysub 0.725.0 → 0.727.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/backfill-log-categories.js +7 -0
- package/package.json +8 -2
- package/src/-a-archives/archiveCache.ts +7 -7
- package/src/-a-archives/archivesDisk.ts +20 -7
- package/src/0-path-value-core/AuthorityLookup.ts +113 -19
- package/src/deployManager/components/DeployPage.tsx +2 -2
- package/src/deployManager/components/ServiceDetailPage.tsx +17 -4
- package/src/deployManager/components/ServicesListPage.tsx +61 -8
- package/src/deployManager/components/deployButtons.tsx +166 -101
- package/src/deployManager/machineSchema.ts +13 -2
- package/src/deployManager/publishQuerysub.ts +121 -49
- package/src/deployManager/serviceCategories.ts +90 -1
- package/src/diagnostics/logs/IndexedLogs/BufferIndexLogsOptimizationConstants.ts +2 -0
- package/src/diagnostics/logs/IndexedLogs/FindProgressTracker.ts +2 -1
- package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts +40 -1
- package/src/diagnostics/logs/IndexedLogs/LogStreamer.ts +6 -2
- package/src/diagnostics/logs/IndexedLogs/MCPIndexedLogs.ts +2 -6
- package/src/diagnostics/logs/IndexedLogs/TimeFileTree.ts +34 -0
- package/src/diagnostics/logs/IndexedLogs/limitPublicLogsSize.ts +119 -0
- package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts +13 -5
- package/src/diagnostics/logs/IndexedLogs/moveLogsEntry.ts +2 -0
- package/src/diagnostics/logs/errorNotifications2/ErrorNotificationPage.tsx +93 -21
- package/src/diagnostics/logs/errorNotifications2/openRouterHelper.ts +4 -2
- package/src/diagnostics/logs/errorNotifications2/suppressionPatternMatcher.ts +1 -1
- package/src/diagnostics/logs/logCategories/LogBreakdownPage.tsx +421 -0
- package/src/diagnostics/logs/logCategories/logCategories.ts +273 -0
- package/src/diagnostics/managementPages.tsx +6 -0
- package/src/diagnostics/network/backfillLogCategoriesMain.ts +128 -0
- package/src/diagnostics/network/deleteOldLogsMain.ts +57 -0
- package/src/diagnostics/network/listLogFilesMain.ts +60 -0
- package/src/diagnostics/network/logFilesMain.ts +13 -15
- package/src/diagnostics/network/probeLogDeleteMain.ts +71 -0
- package/src/diagnostics/network/watchServiceMain.ts +106 -0
- package/src/fs.ts +29 -0
- package/src/misc/moveFolderContents.ts +67 -0
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
const DEFAULT_OPENROUTER_MODEL = "google/gemini-3-flash-preview";
|
|
2
|
+
|
|
1
3
|
export type OpenRouterEffort = "none" | "low" | "medium" | "high";
|
|
2
4
|
|
|
3
5
|
export type OpenRouterResult = {
|
|
@@ -133,7 +135,7 @@ export async function callOpenRouter(apiKey: string, prompt: string): Promise<st
|
|
|
133
135
|
"Content-Type": "application/json",
|
|
134
136
|
},
|
|
135
137
|
body: JSON.stringify({
|
|
136
|
-
model:
|
|
138
|
+
model: DEFAULT_OPENROUTER_MODEL,
|
|
137
139
|
messages: [
|
|
138
140
|
{
|
|
139
141
|
role: "user",
|
|
@@ -170,7 +172,7 @@ export async function callOpenRouterJSON<T>(apiKey: string, prompt: string): Pro
|
|
|
170
172
|
"Content-Type": "application/json",
|
|
171
173
|
},
|
|
172
174
|
body: JSON.stringify({
|
|
173
|
-
model:
|
|
175
|
+
model: DEFAULT_OPENROUTER_MODEL,
|
|
174
176
|
messages: [
|
|
175
177
|
{
|
|
176
178
|
role: "user",
|
|
@@ -115,7 +115,7 @@ function parseStructure(text: string): Structure {
|
|
|
115
115
|
if (andParts.length > 1) return { type: "and", parts: andParts.map(parseStructure) };
|
|
116
116
|
let wildcardParts = text.split(WILD_CARD_CHAR).map(x => x.trim()).filter(x => x.length > 0);
|
|
117
117
|
if (wildcardParts.length > 1) return { type: "wildcard", parts: wildcardParts.map(x => Buffer.from(x)) };
|
|
118
|
-
return { type: "exact", value: Buffer.from(text) };
|
|
118
|
+
return { type: "exact", value: Buffer.from(wildcardParts[0] || text) };
|
|
119
119
|
}
|
|
120
120
|
|
|
121
121
|
function buildAutomaton(literals: Buffer[]) {
|
|
@@ -0,0 +1,421 @@
|
|
|
1
|
+
import { qreact } from "../../../4-dom/qreact";
|
|
2
|
+
import { t } from "../../../2-proxy/schema2";
|
|
3
|
+
import { css } from "typesafecss";
|
|
4
|
+
import { SocketFunction } from "socket-function/SocketFunction";
|
|
5
|
+
import { runInParallel } from "socket-function/src/batching";
|
|
6
|
+
import { formatDateTime, formatNumber, formatTime } from "socket-function/src/formatting/format";
|
|
7
|
+
import { deepCloneJSON, keyBy, sort, timeInDay, timeInSecond } from "socket-function/src/misc";
|
|
8
|
+
import { Querysub } from "../../../4-querysub/Querysub";
|
|
9
|
+
import { Button } from "../../../library-components/Button";
|
|
10
|
+
import { ButtonSelector } from "../../../library-components/ButtonSelector";
|
|
11
|
+
import { InputLabel } from "../../../library-components/InputLabel";
|
|
12
|
+
import { ATag } from "../../../library-components/ATag";
|
|
13
|
+
import { atomic } from "../../../2-proxy/PathValueProxyWatcher";
|
|
14
|
+
import { user_data } from "../../../user-implementation/userData";
|
|
15
|
+
import { assertIsManagementUser, managementPageURL, showingManagementURL } from "../../managementPages";
|
|
16
|
+
import { getLoggers2Async } from "../diskLogger";
|
|
17
|
+
import { TimeRangeSelector, getTimeRange } from "../TimeRangeSelector";
|
|
18
|
+
import { getSuppressionEntries } from "../errorNotifications2/errorNotifications";
|
|
19
|
+
import { TimeFilePath } from "../IndexedLogs/TimeFileTree";
|
|
20
|
+
import { CATEGORY_AI_LIMIT_HIT_ID, CATEGORY_NO_API_KEY_ID, CATEGORY_PATTERN_FAILED_ID, LogCategoryCounts, deleteLogCategory, getLogCategories } from "./logCategories";
|
|
21
|
+
|
|
22
|
+
const READ_BATCH_SIZE = 200;
|
|
23
|
+
const PARALLEL_BATCHES = 8;
|
|
24
|
+
const RENDER_INTERVAL = timeInSecond * 5;
|
|
25
|
+
const MIN_SHARE_PERCENT = 1;
|
|
26
|
+
const LOGGER_PANEL_WIDTH = 940;
|
|
27
|
+
const LOGGER_PANEL_EDIT_WIDTH = 1300;
|
|
28
|
+
const LABEL_WIDTH = 440;
|
|
29
|
+
const SHARE_WIDTH = 60;
|
|
30
|
+
const NUMBER_WIDTH = 90;
|
|
31
|
+
const CREATED_WIDTH = 170;
|
|
32
|
+
|
|
33
|
+
const SPECIAL_CATEGORY_LABELS: { [id: string]: string } = {
|
|
34
|
+
[CATEGORY_AI_LIMIT_HIT_ID]: "Uncategorized: AI categorization limit hit",
|
|
35
|
+
[CATEGORY_NO_API_KEY_ID]: "Uncategorized: no OpenRouter API key was set",
|
|
36
|
+
[CATEGORY_PATTERN_FAILED_ID]: "Uncategorized: AI pattern failed or didn't match the entry it was made from",
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
type GroupBy = "category" | "day";
|
|
40
|
+
|
|
41
|
+
type BreakdownCategory = {
|
|
42
|
+
id: string;
|
|
43
|
+
pattern: string;
|
|
44
|
+
createdTime: number;
|
|
45
|
+
loggerName: string;
|
|
46
|
+
source: "category" | "suppression";
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
type Total = {
|
|
50
|
+
count: number;
|
|
51
|
+
bytes: number;
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
type LoggerBreakdown = {
|
|
55
|
+
filesTotal: number;
|
|
56
|
+
filesStartTime: number;
|
|
57
|
+
filesEndTime: number;
|
|
58
|
+
filesWithCategories: number;
|
|
59
|
+
filesWithoutCategories: number;
|
|
60
|
+
byCategory: LogCategoryCounts;
|
|
61
|
+
byDay: { [dayStart: string]: Total };
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
type BreakdownSnapshot = {
|
|
65
|
+
loggers: { [loggerName: string]: LoggerBreakdown };
|
|
66
|
+
startTime: number;
|
|
67
|
+
doneTime?: number;
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
type Row = {
|
|
71
|
+
id: string;
|
|
72
|
+
label: string;
|
|
73
|
+
total: Total;
|
|
74
|
+
category?: BreakdownCategory;
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
function getController() {
|
|
78
|
+
return LogBreakdownController.nodes[SocketFunction.getBrowserNodeId()];
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function addTotal(target: Total, value: Total) {
|
|
82
|
+
target.count += value.count;
|
|
83
|
+
target.bytes += value.bytes;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function averageBytes(total: Total): number {
|
|
87
|
+
return total.count && total.bytes / total.count || 0;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function perSecond(count: number, breakdown: LoggerBreakdown): number {
|
|
91
|
+
let seconds = (breakdown.filesEndTime - breakdown.filesStartTime) / timeInSecond;
|
|
92
|
+
return seconds > 0 && count / seconds || 0;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function getDayStart(time: number): number {
|
|
96
|
+
return Math.floor(time / timeInDay) * timeInDay;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function formatDay(dayStart: number): string {
|
|
100
|
+
return new Date(dayStart).toISOString().slice(0, 10);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export class LogBreakdownPage extends qreact.Component {
|
|
104
|
+
state = t.state({
|
|
105
|
+
snapshot: t.atomic<BreakdownSnapshot>(),
|
|
106
|
+
categories: t.atomic<BreakdownCategory[]>([]),
|
|
107
|
+
reading: t.atomic<boolean>(false),
|
|
108
|
+
shareByCount: t.atomic<boolean>(false),
|
|
109
|
+
groupBy: t.atomic<GroupBy>("category"),
|
|
110
|
+
editing: t.atomic<boolean>(false),
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
runId = 0;
|
|
114
|
+
|
|
115
|
+
componentDidMount() {
|
|
116
|
+
void this.loadCategories();
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
componentWillUnmount() {
|
|
120
|
+
this.runId++;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
render() {
|
|
124
|
+
let snapshot = this.state.snapshot;
|
|
125
|
+
let hasOpenRouterAPIKey = !!atomic(user_data().secure.openRouterAPIKey);
|
|
126
|
+
let categoriesById = keyBy(this.state.categories, x => x.id);
|
|
127
|
+
let shareByCount = this.state.shareByCount;
|
|
128
|
+
let groupBy = this.state.groupBy;
|
|
129
|
+
let editing = this.state.editing && groupBy === "category";
|
|
130
|
+
let getValue = (total: Total) => shareByCount && total.count || total.bytes;
|
|
131
|
+
let loggerEntries = snapshot && Object.entries(snapshot.loggers) || [];
|
|
132
|
+
sort(loggerEntries, ([, breakdown]) => -Object.values(breakdown.byCategory).reduce((acc, x) => acc + getValue(x), 0));
|
|
133
|
+
return <div className={css.vbox(16).pad2(16).fillWidth}>
|
|
134
|
+
<h2>Log Breakdown</h2>
|
|
135
|
+
{!hasOpenRouterAPIKey && <div className={css.hbox(8).pad2(12).hsl(40, 60, 20).colorhsl(40, 80, 80)}>
|
|
136
|
+
<span>No OpenRouter API key is set, so new log entries can't get categories (they count as "{SPECIAL_CATEGORY_LABELS[CATEGORY_NO_API_KEY_ID]}").</span>
|
|
137
|
+
<ATag values={[showingManagementURL.getOverride(true), managementPageURL.getOverride("SecurityPage")]}>
|
|
138
|
+
Set the OpenRouter API key
|
|
139
|
+
</ATag>
|
|
140
|
+
</div>}
|
|
141
|
+
<TimeRangeSelector />
|
|
142
|
+
<div className={css.hbox(8)}>
|
|
143
|
+
{!this.state.reading && <Button
|
|
144
|
+
hue={200}
|
|
145
|
+
onClick={() => {
|
|
146
|
+
let range = getTimeRange();
|
|
147
|
+
this.state.reading = true;
|
|
148
|
+
Querysub.onCommitFinished(async () => {
|
|
149
|
+
await this.readBreakdown({ runId: ++this.runId, startTime: range.startTime, endTime: range.endTime });
|
|
150
|
+
});
|
|
151
|
+
}}
|
|
152
|
+
>
|
|
153
|
+
Read Log Breakdown
|
|
154
|
+
</Button>}
|
|
155
|
+
{this.state.reading && <Button
|
|
156
|
+
hue={0}
|
|
157
|
+
onClick={() => {
|
|
158
|
+
this.state.reading = false;
|
|
159
|
+
Querysub.onCommitFinished(() => {
|
|
160
|
+
this.runId++;
|
|
161
|
+
});
|
|
162
|
+
}}
|
|
163
|
+
>
|
|
164
|
+
Stop Reading
|
|
165
|
+
</Button>}
|
|
166
|
+
{snapshot && <div>
|
|
167
|
+
{snapshot.doneTime && `Finished in ${formatTime(snapshot.doneTime - snapshot.startTime)}` || `Reading, started ${formatDateTime(snapshot.startTime)}`}
|
|
168
|
+
</div>}
|
|
169
|
+
</div>
|
|
170
|
+
<div className={css.hbox(16)}>
|
|
171
|
+
<ButtonSelector
|
|
172
|
+
title="Group by"
|
|
173
|
+
value={groupBy}
|
|
174
|
+
options={[
|
|
175
|
+
{ value: "category", title: "Category" },
|
|
176
|
+
{ value: "day", title: "Day" },
|
|
177
|
+
]}
|
|
178
|
+
onChange={value => {
|
|
179
|
+
this.state.groupBy = value;
|
|
180
|
+
}}
|
|
181
|
+
/>
|
|
182
|
+
<ButtonSelector
|
|
183
|
+
title="Share by"
|
|
184
|
+
value={shareByCount}
|
|
185
|
+
options={[
|
|
186
|
+
{ value: false, title: "Bytes" },
|
|
187
|
+
{ value: true, title: "Count" },
|
|
188
|
+
]}
|
|
189
|
+
onChange={value => {
|
|
190
|
+
this.state.shareByCount = value;
|
|
191
|
+
}}
|
|
192
|
+
/>
|
|
193
|
+
{groupBy === "category" && <InputLabel
|
|
194
|
+
label="Edit categories"
|
|
195
|
+
checkbox
|
|
196
|
+
checked={this.state.editing}
|
|
197
|
+
onChange={e => {
|
|
198
|
+
this.state.editing = e.currentTarget.checked;
|
|
199
|
+
}}
|
|
200
|
+
/>}
|
|
201
|
+
</div>
|
|
202
|
+
<div className={css.hbox(16, 24).wrap.alignItems("start").fillWidth}>
|
|
203
|
+
{loggerEntries.map(([loggerName, breakdown]) => {
|
|
204
|
+
let rows: Row[];
|
|
205
|
+
if (groupBy === "day") {
|
|
206
|
+
rows = Object.entries(breakdown.byDay).map(([dayStart, total]): Row => ({ id: dayStart, label: formatDay(+dayStart), total }));
|
|
207
|
+
sort(rows, x => -x.id);
|
|
208
|
+
} else {
|
|
209
|
+
rows = Object.entries(breakdown.byCategory).map(([id, total]): Row => {
|
|
210
|
+
let category = categoriesById.get(id);
|
|
211
|
+
return { id, label: SPECIAL_CATEGORY_LABELS[id] || category?.pattern || `Deleted category ${id}`, total, category };
|
|
212
|
+
});
|
|
213
|
+
sort(rows, x => -getValue(x.total));
|
|
214
|
+
}
|
|
215
|
+
let totalBytes = rows.reduce((acc, x) => acc + x.total.bytes, 0);
|
|
216
|
+
let totalCount = rows.reduce((acc, x) => acc + x.total.count, 0);
|
|
217
|
+
let total = shareByCount && totalCount || totalBytes;
|
|
218
|
+
let getShare = (value: Total) => total && getValue(value) / total * 100 || 0;
|
|
219
|
+
let shownRows = groupBy === "day" && rows || rows.filter(x => getShare(x.total) >= MIN_SHARE_PERCENT);
|
|
220
|
+
let otherRows = groupBy === "day" && [] || rows.filter(x => getShare(x.total) < MIN_SHARE_PERCENT);
|
|
221
|
+
let otherShare = otherRows.reduce((acc, x) => acc + getShare(x.total), 0);
|
|
222
|
+
let otherTotal: Total = { count: 0, bytes: 0 };
|
|
223
|
+
for (let row of otherRows) {
|
|
224
|
+
addTotal(otherTotal, row.total);
|
|
225
|
+
}
|
|
226
|
+
let filesChecked = breakdown.filesWithCategories + breakdown.filesWithoutCategories;
|
|
227
|
+
return <div className={css.vbox(8).width(editing && LOGGER_PANEL_EDIT_WIDTH || LOGGER_PANEL_WIDTH)}>
|
|
228
|
+
<h3 className={css.colorhsl(0, 0, 0)}>{loggerName}</h3>
|
|
229
|
+
<div>
|
|
230
|
+
{`${formatNumber(breakdown.filesWithCategories)} of ${formatNumber(breakdown.filesTotal)} log files have categories (${formatNumber(breakdown.filesWithoutCategories)} without), ${formatNumber(totalCount)} entries, ${formatNumber(totalBytes)}B uncompressed (${formatNumber(averageBytes({ count: totalCount, bytes: totalBytes }))}B per entry, ${formatNumber(perSecond(totalCount, breakdown))} entries/s), ${Object.keys(breakdown.byCategory).length} categories over ${Object.keys(breakdown.byDay).length} days`}
|
|
231
|
+
{breakdown.filesTotal > 0 && `, files span ${formatDateTime(breakdown.filesStartTime)} to ${formatDateTime(breakdown.filesEndTime)} (${formatTime(breakdown.filesEndTime - breakdown.filesStartTime)})`}
|
|
232
|
+
{filesChecked < breakdown.filesTotal && ` (still reading, ${formatNumber(filesChecked)}/${formatNumber(breakdown.filesTotal)} files checked)`}
|
|
233
|
+
</div>
|
|
234
|
+
<div className={css.vbox(2).fillWidth}>
|
|
235
|
+
<div className={css.hbox(12).pad2(4, 8).fillWidth.colorhsl(0, 0, 0)}>
|
|
236
|
+
<span className={css.width(LABEL_WIDTH)}>{groupBy === "day" && "Day (UTC)" || "Category"}</span>
|
|
237
|
+
<span className={css.width(SHARE_WIDTH)} title={shareByCount && "Share of entries" || "Share of bytes"}>Share</span>
|
|
238
|
+
<span className={css.width(NUMBER_WIDTH)}>Entries</span>
|
|
239
|
+
<span className={css.width(NUMBER_WIDTH)}>Bytes</span>
|
|
240
|
+
<span className={css.width(NUMBER_WIDTH)} title="Average uncompressed bytes per entry">Per entry</span>
|
|
241
|
+
<span className={css.width(NUMBER_WIDTH)} title="Entries per second over the time span of the listed log files">Per second</span>
|
|
242
|
+
{editing && <span className={css.width(CREATED_WIDTH)}>Created</span>}
|
|
243
|
+
</div>
|
|
244
|
+
{shownRows.map(row => {
|
|
245
|
+
let share = getShare(row.total);
|
|
246
|
+
return <div className={css.relative.fillWidth.hsl(0, 0, 14)}>
|
|
247
|
+
<div className={css.absolute.pos(0, 0).fillHeight.width(`${share.toFixed(1)}%`).hsla(200, 60, 45, 0.35)} />
|
|
248
|
+
<div className={css.relative.hbox(12).pad2(4, 8).fillWidth.colorhsl(0, 0, 100)}>
|
|
249
|
+
<span className={css.width(LABEL_WIDTH).ellipsis} title={row.label}>{row.label}</span>
|
|
250
|
+
<span className={css.width(SHARE_WIDTH)}>{share.toFixed(1)}%</span>
|
|
251
|
+
<span className={css.width(NUMBER_WIDTH)}>{formatNumber(row.total.count)}</span>
|
|
252
|
+
<span className={css.width(NUMBER_WIDTH)}>{formatNumber(row.total.bytes)}B</span>
|
|
253
|
+
<span className={css.width(NUMBER_WIDTH)}>{formatNumber(averageBytes(row.total))}B</span>
|
|
254
|
+
<span className={css.width(NUMBER_WIDTH)}>{formatNumber(perSecond(row.total.count, breakdown))}/s</span>
|
|
255
|
+
{editing && <span className={css.width(CREATED_WIDTH)}>{row.category && formatDateTime(row.category.createdTime) || ""}</span>}
|
|
256
|
+
{editing && row.category?.source === "category" && <Button
|
|
257
|
+
hue={0}
|
|
258
|
+
onClick={() => {
|
|
259
|
+
Querysub.onCommitFinished(async () => {
|
|
260
|
+
await getController().deleteCategory(row.id);
|
|
261
|
+
await this.loadCategories();
|
|
262
|
+
});
|
|
263
|
+
}}
|
|
264
|
+
>
|
|
265
|
+
Delete Category
|
|
266
|
+
</Button>}
|
|
267
|
+
</div>
|
|
268
|
+
</div>;
|
|
269
|
+
})}
|
|
270
|
+
{otherRows.length > 0 && <div className={css.relative.fillWidth.hsl(0, 0, 14)}>
|
|
271
|
+
<div className={css.absolute.pos(0, 0).fillHeight.width(`${otherShare.toFixed(1)}%`).hsla(0, 0, 60, 0.25)} />
|
|
272
|
+
<div className={css.relative.hbox(12).pad2(4, 8).fillWidth.colorhsl(0, 0, 100)}>
|
|
273
|
+
<span className={css.width(LABEL_WIDTH)}>{`+ ${otherRows.length} other categories under ${MIN_SHARE_PERCENT}% each`}</span>
|
|
274
|
+
<span className={css.width(SHARE_WIDTH)}>{otherShare.toFixed(1)}%</span>
|
|
275
|
+
<span className={css.width(NUMBER_WIDTH)}>{formatNumber(otherTotal.count)}</span>
|
|
276
|
+
<span className={css.width(NUMBER_WIDTH)}>{formatNumber(otherTotal.bytes)}B</span>
|
|
277
|
+
<span className={css.width(NUMBER_WIDTH)}>{formatNumber(averageBytes(otherTotal))}B</span>
|
|
278
|
+
<span className={css.width(NUMBER_WIDTH)}>{formatNumber(perSecond(otherTotal.count, breakdown))}/s</span>
|
|
279
|
+
</div>
|
|
280
|
+
</div>}
|
|
281
|
+
</div>
|
|
282
|
+
</div>;
|
|
283
|
+
})}
|
|
284
|
+
</div>
|
|
285
|
+
</div>;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
private async loadCategories() {
|
|
289
|
+
let categories = await getController().getCategories();
|
|
290
|
+
Querysub.commit(() => {
|
|
291
|
+
this.state.categories = categories;
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
private async readBreakdown(config: { runId: number; startTime: number; endTime: number }) {
|
|
296
|
+
let { runId, startTime, endTime } = config;
|
|
297
|
+
let controller = getController();
|
|
298
|
+
let loggerNames = await controller.getLoggerNames();
|
|
299
|
+
let snapshot: BreakdownSnapshot = { loggers: {}, startTime: Date.now() };
|
|
300
|
+
for (let loggerName of loggerNames) {
|
|
301
|
+
snapshot.loggers[loggerName] = { filesTotal: 0, filesStartTime: 0, filesEndTime: 0, filesWithCategories: 0, filesWithoutCategories: 0, byCategory: {}, byDay: {} };
|
|
302
|
+
}
|
|
303
|
+
let lastRender = 0;
|
|
304
|
+
let render = (force: boolean) => {
|
|
305
|
+
if (runId !== this.runId) return;
|
|
306
|
+
if (!force && Date.now() - lastRender < RENDER_INTERVAL) return;
|
|
307
|
+
lastRender = Date.now();
|
|
308
|
+
let copy = deepCloneJSON(snapshot);
|
|
309
|
+
Querysub.commit(() => {
|
|
310
|
+
this.state.snapshot = copy;
|
|
311
|
+
});
|
|
312
|
+
};
|
|
313
|
+
render(true);
|
|
314
|
+
|
|
315
|
+
await Promise.all(loggerNames.map(async loggerName => {
|
|
316
|
+
let breakdown = snapshot.loggers[loggerName];
|
|
317
|
+
let files = await controller.listLogFiles({ loggerName, startTime, endTime });
|
|
318
|
+
breakdown.filesTotal = files.length;
|
|
319
|
+
if (files.length) {
|
|
320
|
+
breakdown.filesStartTime = Math.min(...files.map(x => x.startTime));
|
|
321
|
+
breakdown.filesEndTime = Math.max(...files.map(x => x.endTime));
|
|
322
|
+
}
|
|
323
|
+
render(false);
|
|
324
|
+
let batches: TimeFilePath[][] = [];
|
|
325
|
+
for (let i = 0; i < files.length; i += READ_BATCH_SIZE) {
|
|
326
|
+
batches.push(files.slice(i, i + READ_BATCH_SIZE));
|
|
327
|
+
}
|
|
328
|
+
let readBatch = runInParallel({ parallelCount: PARALLEL_BATCHES }, async (batch: TimeFilePath[]) => {
|
|
329
|
+
if (runId !== this.runId) return;
|
|
330
|
+
let results = await controller.readLogCategoryCounts({ loggerName, fullPaths: batch.map(x => x.fullPath) });
|
|
331
|
+
for (let i = 0; i < results.length; i++) {
|
|
332
|
+
let counts = results[i];
|
|
333
|
+
if (!counts) {
|
|
334
|
+
breakdown.filesWithoutCategories++;
|
|
335
|
+
continue;
|
|
336
|
+
}
|
|
337
|
+
breakdown.filesWithCategories++;
|
|
338
|
+
let dayKey = String(getDayStart(batch[i].startTime));
|
|
339
|
+
let dayTotal = breakdown.byDay[dayKey];
|
|
340
|
+
if (!dayTotal) {
|
|
341
|
+
dayTotal = { count: 0, bytes: 0 };
|
|
342
|
+
breakdown.byDay[dayKey] = dayTotal;
|
|
343
|
+
}
|
|
344
|
+
for (let [id, value] of Object.entries(counts)) {
|
|
345
|
+
let categoryTotal = breakdown.byCategory[id];
|
|
346
|
+
if (!categoryTotal) {
|
|
347
|
+
categoryTotal = { count: 0, bytes: 0 };
|
|
348
|
+
breakdown.byCategory[id] = categoryTotal;
|
|
349
|
+
}
|
|
350
|
+
addTotal(categoryTotal, value);
|
|
351
|
+
addTotal(dayTotal, value);
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
render(false);
|
|
355
|
+
});
|
|
356
|
+
await Promise.all(batches.map(batch => readBatch(batch)));
|
|
357
|
+
}));
|
|
358
|
+
|
|
359
|
+
if (runId !== this.runId) return;
|
|
360
|
+
snapshot.doneTime = Date.now();
|
|
361
|
+
render(true);
|
|
362
|
+
await this.loadCategories();
|
|
363
|
+
Querysub.commit(() => {
|
|
364
|
+
this.state.reading = false;
|
|
365
|
+
});
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
async function getLoggerByName(loggerName: string) {
|
|
370
|
+
let loggers = Object.values(await getLoggers2Async());
|
|
371
|
+
let logger = loggers.find(x => x.config.name === loggerName);
|
|
372
|
+
if (!logger) {
|
|
373
|
+
throw new Error(`Expected logger to be one of ${loggers.map(x => x.config.name).join(", ")}, was ${JSON.stringify(loggerName)}`);
|
|
374
|
+
}
|
|
375
|
+
return logger;
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
class LogBreakdownControllerBase {
|
|
379
|
+
public async getLoggerNames(): Promise<string[]> {
|
|
380
|
+
return Object.values(await getLoggers2Async()).map(x => x.config.name);
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
public async listLogFiles(config: { loggerName: string; startTime: number; endTime: number }): Promise<TimeFilePath[]> {
|
|
384
|
+
let logger = await getLoggerByName(config.loggerName);
|
|
385
|
+
let paths = await logger.listPublicLogPaths({ startTime: config.startTime, endTime: config.endTime });
|
|
386
|
+
sort(paths, x => -x.startTime);
|
|
387
|
+
return paths;
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
public async readLogCategoryCounts(config: { loggerName: string; fullPaths: string[] }): Promise<(LogCategoryCounts | undefined)[]> {
|
|
391
|
+
let logger = await getLoggerByName(config.loggerName);
|
|
392
|
+
return await Promise.all(config.fullPaths.map(fullPath => logger.readLogCategoryCounts(fullPath)));
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
public async getCategories(): Promise<BreakdownCategory[]> {
|
|
396
|
+
let [categories, suppressions] = await Promise.all([getLogCategories(), getSuppressionEntries()]);
|
|
397
|
+
return [
|
|
398
|
+
...categories.map((x): BreakdownCategory => ({ id: x.id, pattern: x.pattern, createdTime: x.createdTime, loggerName: x.loggerName, source: "category" })),
|
|
399
|
+
...suppressions.map((x): BreakdownCategory => ({ id: x.id, pattern: x.pattern, createdTime: x.createdTime, loggerName: "logs/error", source: "suppression" })),
|
|
400
|
+
];
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
public async deleteCategory(id: string): Promise<void> {
|
|
404
|
+
await deleteLogCategory(id);
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
export const LogBreakdownController = SocketFunction.register(
|
|
409
|
+
"LogBreakdownController-019c9a41-5e2b-7c3d-9f1a-6b8e2d4c7a10",
|
|
410
|
+
new LogBreakdownControllerBase(),
|
|
411
|
+
() => ({
|
|
412
|
+
getLoggerNames: {},
|
|
413
|
+
listLogFiles: { compress: true },
|
|
414
|
+
readLogCategoryCounts: { compress: true },
|
|
415
|
+
getCategories: { compress: true },
|
|
416
|
+
deleteCategory: {},
|
|
417
|
+
}),
|
|
418
|
+
() => ({
|
|
419
|
+
hooks: [assertIsManagementUser],
|
|
420
|
+
}),
|
|
421
|
+
);
|