querysub 0.356.0 → 0.357.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.cursorrules +8 -0
- package/bin/movelogs.js +4 -0
- package/package.json +12 -6
- package/scripts/postinstall.js +23 -0
- package/src/-a-archives/archiveCache.ts +10 -12
- package/src/-a-archives/archives.ts +29 -0
- package/src/-a-archives/archivesBackBlaze.ts +60 -12
- package/src/-a-archives/archivesDisk.ts +27 -8
- package/src/-a-archives/archivesLimitedCache.ts +21 -0
- package/src/-a-archives/archivesMemoryCache.ts +350 -0
- package/src/-a-archives/archivesPrivateFileSystem.ts +22 -0
- package/src/-g-core-values/NodeCapabilities.ts +3 -0
- package/src/0-path-value-core/auditLogs.ts +5 -1
- package/src/0-path-value-core/pathValueCore.ts +7 -7
- package/src/4-dom/qreact.tsx +1 -0
- package/src/4-querysub/Querysub.ts +1 -5
- package/src/config.ts +5 -0
- package/src/diagnostics/MachineThreadInfo.tsx +235 -0
- package/src/diagnostics/NodeViewer.tsx +3 -2
- package/src/diagnostics/logs/FastArchiveAppendable.ts +79 -42
- package/src/diagnostics/logs/FastArchiveController.ts +102 -63
- package/src/diagnostics/logs/FastArchiveViewer.tsx +36 -8
- package/src/diagnostics/logs/IndexedLogs/BufferIndex.ts +461 -0
- package/src/diagnostics/logs/IndexedLogs/BufferIndexCPP.cpp +327 -0
- package/src/diagnostics/logs/IndexedLogs/BufferIndexCPP.d.ts +18 -0
- package/src/diagnostics/logs/IndexedLogs/BufferIndexCPP.js +1 -0
- package/src/diagnostics/logs/IndexedLogs/BufferIndexHelpers.ts +140 -0
- package/src/diagnostics/logs/IndexedLogs/BufferIndexLogsOptimizationConstants.ts +22 -0
- package/src/diagnostics/logs/IndexedLogs/BufferIndexWAT.wat +1145 -0
- package/src/diagnostics/logs/IndexedLogs/BufferIndexWAT.wat.d.ts +178 -0
- package/src/diagnostics/logs/IndexedLogs/BufferListStreamer.ts +206 -0
- package/src/diagnostics/logs/IndexedLogs/BufferUnitIndex.ts +719 -0
- package/src/diagnostics/logs/IndexedLogs/BufferUnitSet.ts +146 -0
- package/src/diagnostics/logs/IndexedLogs/FilePathSelector.tsx +408 -0
- package/src/diagnostics/logs/IndexedLogs/FindProgressTracker.ts +45 -0
- package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts +598 -0
- package/src/diagnostics/logs/IndexedLogs/LogStreamer.ts +47 -0
- package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx +702 -0
- package/src/diagnostics/logs/IndexedLogs/TimeFileTree.ts +236 -0
- package/src/diagnostics/logs/IndexedLogs/binding.gyp +23 -0
- package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts +221 -0
- package/src/diagnostics/logs/IndexedLogs/moveLogsEntry.ts +10 -0
- package/src/diagnostics/logs/LogViewer2.tsx +120 -55
- package/src/diagnostics/logs/TimeRangeSelector.tsx +5 -2
- package/src/diagnostics/logs/diskLogger.ts +32 -48
- package/src/diagnostics/logs/errorNotifications/ErrorNotificationController.ts +3 -2
- package/src/diagnostics/logs/errorNotifications/errorDigests.tsx +1 -0
- package/src/diagnostics/logs/lifeCycleAnalysis/LifeCyclePages.tsx +150 -0
- package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx +132 -15
- package/src/diagnostics/logs/lifeCycleAnalysis/test.ts +180 -0
- package/src/diagnostics/logs/lifeCycleAnalysis/test.wat +106 -0
- package/src/diagnostics/logs/lifeCycleAnalysis/test.wat.d.ts +2 -0
- package/src/diagnostics/logs/lifeCycleAnalysis/testHoist.ts +5 -0
- package/src/diagnostics/logs/logViewerExtractField.ts +2 -3
- package/src/diagnostics/managementPages.tsx +10 -0
- package/src/diagnostics/trackResources.ts +1 -1
- package/src/misc/lz4_wasm_nodejs.d.ts +34 -0
- package/src/misc/lz4_wasm_nodejs.js +178 -0
- package/src/misc/lz4_wasm_nodejs_bg.js +94 -0
- package/src/misc/lz4_wasm_nodejs_bg.wasm +0 -0
- package/src/misc/lz4_wasm_nodejs_bg.wasm.d.ts +15 -0
- package/src/storage/CompressedStream.ts +13 -0
- package/src/storage/LZ4.ts +32 -0
- package/src/storage/ZSTD.ts +10 -0
- package/src/wat/watCompiler.ts +1716 -0
- package/src/wat/watGrammar.pegjs +93 -0
- package/src/wat/watHandler.ts +179 -0
- package/src/wat/watInstructions.txt +707 -0
- package/src/zip.ts +3 -89
- package/src/diagnostics/logs/lifeCycleAnalysis/spec.md +0 -125
|
@@ -0,0 +1,702 @@
|
|
|
1
|
+
module.hotreload = true;
|
|
2
|
+
import { qreact } from "../../../4-dom/qreact";
|
|
3
|
+
import { css } from "../../../4-dom/css";
|
|
4
|
+
import { t } from "../../../2-proxy/schema2";
|
|
5
|
+
import { Button } from "../../../library-components/Button";
|
|
6
|
+
import { InputLabel, InputLabelURL } from "../../../library-components/InputLabel";
|
|
7
|
+
import { getLoggers2, getLoggers2Async, LogDatum } from "../diskLogger";
|
|
8
|
+
import { list, timeInDay, keyByArray, sort, throttleFunction } from "socket-function/src/misc";
|
|
9
|
+
import { formatDateTime, formatDateTimeDetailed, formatNumber, formatTime, formatPercent } from "socket-function/src/formatting/format";
|
|
10
|
+
import { IndexedLogResults, IndexedLogs, TimeFilePathWithSize } from "./IndexedLogs";
|
|
11
|
+
import { Querysub } from "../../../4-querysub/QuerysubController";
|
|
12
|
+
import { URLParam } from "../../../library-components/URLParam";
|
|
13
|
+
import { getOwnMachineId, getOwnThreadId } from "../../../-f-node-discovery/NodeDiscovery";
|
|
14
|
+
import { FilePathSelector, FilePathsByMachine } from "./FilePathSelector";
|
|
15
|
+
import { MachineThreadInfo } from "../../MachineThreadInfo";
|
|
16
|
+
import { TimeRangeSelector, getTimeRange } from "../TimeRangeSelector";
|
|
17
|
+
import { LZ4 } from "../../../storage/LZ4";
|
|
18
|
+
import { ColumnType, Table, TableType } from "../../../5-diagnostics/Table";
|
|
19
|
+
import { InputPicker } from "../../../library-components/InputPicker";
|
|
20
|
+
import { ObjectDisplay } from "../ObjectDisplay";
|
|
21
|
+
import { formatDateJSX } from "../../../misc/formatJSX";
|
|
22
|
+
import { PUBLIC_MOVE_THRESHOLD } from "./BufferIndexLogsOptimizationConstants";
|
|
23
|
+
import { atomic } from "../../../2-proxy/PathValueProxyWatcher";
|
|
24
|
+
|
|
25
|
+
let searchText = new URLParam("searchText", "");
|
|
26
|
+
let readLiveData = new URLParam("readLiveData", false);
|
|
27
|
+
let excludePendingResults = new URLParam("excludePendingResults", false);
|
|
28
|
+
let limitURL = new URLParam("limit", 100);
|
|
29
|
+
|
|
30
|
+
let savedPathsURL = new URLParam("savedPaths", "");
|
|
31
|
+
let selectedFieldsURL = new URLParam("selectedFields", {} as Record<string, boolean>);
|
|
32
|
+
let useRelativeTimeURL = new URLParam("useRelativeTime", true);
|
|
33
|
+
|
|
34
|
+
// Only auto-search if the string is long enough. Otherwise, we're going to get way too many results, which could slow down the server and won't be useful to us.
|
|
35
|
+
const MIN_AUTO_SEARCH_LENGTH = 7;
|
|
36
|
+
|
|
37
|
+
const defaultSelectedFields = {
|
|
38
|
+
param0: true,
|
|
39
|
+
time: true,
|
|
40
|
+
__NAME__: true,
|
|
41
|
+
__machineId: true,
|
|
42
|
+
__threadId: true,
|
|
43
|
+
__entry: true,
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
function createTestLog(text: string, type: "log" | "warn" | "info" | "error"): LogDatum {
|
|
47
|
+
return {
|
|
48
|
+
time: 0,
|
|
49
|
+
message: text,
|
|
50
|
+
__LOG_TYPE: type,
|
|
51
|
+
__machineId: getOwnMachineId(),
|
|
52
|
+
__threadId: getOwnThreadId(),
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export class LogViewer3 extends qreact.Component {
|
|
57
|
+
static renderInProgress = true;
|
|
58
|
+
state = t.state({
|
|
59
|
+
enableLogs: t.boolean(true),
|
|
60
|
+
enableInfos: t.boolean(true),
|
|
61
|
+
enableWarnings: t.boolean(true),
|
|
62
|
+
enableErrors: t.boolean(true),
|
|
63
|
+
results: t.atomic<LogDatum[]>([
|
|
64
|
+
{ "time": 1771825252649.44, "__LOG_TYPE": "log", "__machineId": "1ed7a7340a015680", "__mountId": "1d590f0919b1f438.1ed7a7340a015680.querysubtest.com:7007", "__threadId": "1d590f0919b1f438", "__port": 7007, "__nodeId": "1d590f0919b1f438.1ed7a7340a015680.querysubtest.com:7007", "__entry": "D:\\repos\\qs-cyoa\\src\\server.ts", "__hostName": "DESKTOP-05MP449", "__accountName": "desktop-05mp449\\quent", "__externalIP": "99.250.124.91", "__os": "win32", "__pid": 75888, "param0": "new non-local WATCH", "path": ".,querysubtest._com.,PathFunctionRunner.,book.,Data.,edittingBooks.,hkwnmoiwhuyqenbi.,books.,bf0713806bf1c4338._querysubtest._com_1755390476654._5208_1.,nodes.,1770378259589._5535.,aiCalls.,", "watcher": "client:127.0.0.1:1771825250819.3994:0.634106584461968", "diskAudit": true },
|
|
65
|
+
{ "time": 1771825252649.4397, "__LOG_TYPE": "log", "__machineId": "1ed7a7340a015680", "__mountId": "1d590f0919b1f438.1ed7a7340a015680.querysubtest.com:7007", "__threadId": "x1d590f0919b1f438", "__port": 7007, "__nodeId": "1d590f0919b1f438.1ed7a7340a015680.querysubtest.com:7007", "__entry": "D:\\repos\\qs-cyoa\\src\\server.ts", "__hostName": "DESKTOP-05MP449", "__accountName": "desktop-05mp449\\quent", "__externalIP": "99.250.124.91", "__os": "win32", "__pid": 75888, "param0": "new non-local WATCH", "path": ".,querysubtest._com.,PathFunctionRunner.,book.,Data.,edittingBooks.,hkwnmoiwhuyqenbi.,books.,bf0713806bf1c4338._querysubtest._com_1755390476654._5208_1.,nodes.,1770378259589._5535.,totalTime.,", "watcher": "client:127.0.0.1:1771825250819.3994:0.634106584461968", "diskAudit": true },
|
|
66
|
+
]),
|
|
67
|
+
searching: t.boolean,
|
|
68
|
+
searchingLogs: t.boolean,
|
|
69
|
+
searchingInfos: t.boolean,
|
|
70
|
+
searchingWarnings: t.boolean,
|
|
71
|
+
searchingErrors: t.boolean,
|
|
72
|
+
paths: t.atomic<TimeFilePathWithSize[]>([]),
|
|
73
|
+
loadingPaths: t.boolean,
|
|
74
|
+
stats: t.atomic<IndexedLogResults | undefined>(undefined),
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
componentDidMount(): void {
|
|
78
|
+
void this.loadPaths();
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
getPaths(): TimeFilePathWithSize[] {
|
|
82
|
+
let paths: TimeFilePathWithSize[] = [];
|
|
83
|
+
if (savedPathsURL.value) {
|
|
84
|
+
const compressed = Buffer.from(savedPathsURL.value, "base64");
|
|
85
|
+
const decompressed = LZ4.decompress(compressed);
|
|
86
|
+
paths = JSON.parse(decompressed.toString("utf8"));
|
|
87
|
+
} else {
|
|
88
|
+
paths = this.state.paths;
|
|
89
|
+
}
|
|
90
|
+
if (excludePendingResults.value) {
|
|
91
|
+
paths = paths.filter(x => x.logCount !== undefined);
|
|
92
|
+
}
|
|
93
|
+
return paths;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
clearPaths() {
|
|
97
|
+
this.state.paths = [];
|
|
98
|
+
this.state.searchingLogs = false;
|
|
99
|
+
this.state.searchingInfos = false;
|
|
100
|
+
this.state.searchingWarnings = false;
|
|
101
|
+
this.state.searchingErrors = false;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
freezePaths() {
|
|
105
|
+
const json = JSON.stringify(this.state.paths);
|
|
106
|
+
const buffer = Buffer.from(json, "utf8");
|
|
107
|
+
const compressed = LZ4.compress(buffer);
|
|
108
|
+
savedPathsURL.value = compressed.toString("base64");
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
clearFrozenPaths() {
|
|
112
|
+
savedPathsURL.value = "";
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
renderSearchStats() {
|
|
116
|
+
const stats = this.state.stats;
|
|
117
|
+
if (!stats) return undefined;
|
|
118
|
+
|
|
119
|
+
let totalSizeRead = 0;
|
|
120
|
+
let cachedSize = 0;
|
|
121
|
+
let uncachedSize = 0;
|
|
122
|
+
let uncachedCount = 0;
|
|
123
|
+
let uncachedRemoteSize = 0;
|
|
124
|
+
let uncachedRemoteCount = 0;
|
|
125
|
+
let totalSize = 0;
|
|
126
|
+
|
|
127
|
+
for (let read of stats.reads) {
|
|
128
|
+
totalSizeRead += read.size;
|
|
129
|
+
if (read.cached) {
|
|
130
|
+
cachedSize += read.size;
|
|
131
|
+
} else {
|
|
132
|
+
uncachedSize += read.size;
|
|
133
|
+
uncachedCount += read.count;
|
|
134
|
+
totalSize += read.totalSize;
|
|
135
|
+
}
|
|
136
|
+
if (read.remote && !read.cached) {
|
|
137
|
+
uncachedRemoteSize += read.size;
|
|
138
|
+
uncachedRemoteCount += read.count;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
const renderStage = (title: string, items: preact.ComponentChild[], hue: number | undefined, isFirst: boolean, isLast: boolean) => {
|
|
143
|
+
const triangleWidth = 20;
|
|
144
|
+
|
|
145
|
+
let color = hue ? css.hsl(hue, 70, 80) : css.hsl(0, 0, 100);
|
|
146
|
+
let colorhsl = hue ? css.hslcolor(hue, 80, 30) : css;
|
|
147
|
+
|
|
148
|
+
return (
|
|
149
|
+
<div className={css.vbox(2).relative.minWidth(200).paddingTop(8).paddingBottom(8).paddingRight(10).paddingLeft(isFirst ? 10 : 10 + triangleWidth) + color}>
|
|
150
|
+
<div className={css.fontWeight(600) + colorhsl}>{title}</div>
|
|
151
|
+
{items.map((item, i) => (
|
|
152
|
+
<div key={i}>{item}</div>
|
|
153
|
+
))}
|
|
154
|
+
{!isLast && (
|
|
155
|
+
<svg width={triangleWidth} height="100" viewBox="0 0 20 100" preserveAspectRatio="none" className={css.absolute.right(-triangleWidth).top(0).fillHeight.zIndex(1)}>
|
|
156
|
+
<polygon points="0,0 20,50 0,100" fill={`hsl(${hue}, 70%, 80%)`} />
|
|
157
|
+
</svg>
|
|
158
|
+
)}
|
|
159
|
+
</div>
|
|
160
|
+
);
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
const fileItems = [
|
|
164
|
+
`${formatTime(stats.fileFindTime)} | ${formatNumber(stats.localFilesSearched + stats.backblazeFilesSearched)} | ${formatNumber(totalSize)}B`,
|
|
165
|
+
`pending ${formatNumber(stats.localFilesSearched)}`,
|
|
166
|
+
];
|
|
167
|
+
|
|
168
|
+
const indexItems = [
|
|
169
|
+
`${formatTime(stats.indexSearchTime)} | ${formatNumber(stats.indexesSearched)} | ${formatNumber(stats.indexSize)}B`,
|
|
170
|
+
];
|
|
171
|
+
|
|
172
|
+
const blockItems = [
|
|
173
|
+
`${formatTime(stats.blockSearchTime)} | ${formatNumber(stats.blockCheckedCount)} | ${formatNumber(stats.blocksCheckedCompressedSize)}B | ${formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X`,
|
|
174
|
+
`total blocks ${formatNumber(stats.totalBlockCount)}`,
|
|
175
|
+
];
|
|
176
|
+
let cacheItems = [
|
|
177
|
+
`disk read ${formatNumber(uncachedSize)}B`,
|
|
178
|
+
`remote ${formatPercent(uncachedRemoteSize / totalSizeRead)} (${formatNumber(uncachedRemoteCount)})`,
|
|
179
|
+
];
|
|
180
|
+
|
|
181
|
+
const resultItems = [
|
|
182
|
+
`${formatTime(stats.timeToFirstMatch)} until first result`,
|
|
183
|
+
`${formatNumber(stats.matchCount)} results / ${limitURL.value} limit`,
|
|
184
|
+
];
|
|
185
|
+
|
|
186
|
+
const doneItems = [
|
|
187
|
+
`${formatTime(stats.totalSearchTime)} total search time`,
|
|
188
|
+
];
|
|
189
|
+
|
|
190
|
+
let hasErrors = stats.fileErrors.length > 0 || stats.blockErrors.length > 0;
|
|
191
|
+
|
|
192
|
+
return (
|
|
193
|
+
<div className={css.hbox(0).alignItems("stretch")}>
|
|
194
|
+
{renderStage("Files", fileItems, 290, true, false)}
|
|
195
|
+
{renderStage("Indexes", indexItems, 250, false, false)}
|
|
196
|
+
{renderStage("Blocks", blockItems, 210, false, false)}
|
|
197
|
+
{renderStage("Cache", cacheItems, 160, false, false)}
|
|
198
|
+
{renderStage("Results", resultItems, 120, false, false)}
|
|
199
|
+
{renderStage("Done", doneItems, undefined, false, !hasErrors)}
|
|
200
|
+
{(() => {
|
|
201
|
+
if (!hasErrors) return undefined;
|
|
202
|
+
let errorItems: preact.ComponentChild[] = [];
|
|
203
|
+
if (stats.fileErrors.length > 0) {
|
|
204
|
+
errorItems.push(<div title={stats.fileErrors.join("\n")}>{stats.fileErrors.length} files failed</div>);
|
|
205
|
+
}
|
|
206
|
+
if (stats.blockErrors.length > 0) {
|
|
207
|
+
errorItems.push(<div title={stats.blockErrors.join("\n")}>{stats.blockErrors.length} blocks failed</div>);
|
|
208
|
+
}
|
|
209
|
+
return renderStage("Errors", errorItems, 0, false, true);
|
|
210
|
+
})()}
|
|
211
|
+
</div>
|
|
212
|
+
);
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
renderPendingLogWarnings() {
|
|
216
|
+
const paths = this.getPaths();
|
|
217
|
+
const now = Date.now();
|
|
218
|
+
const threshold = now - (2 * PUBLIC_MOVE_THRESHOLD);
|
|
219
|
+
|
|
220
|
+
const machineWarnings: Record<string, { machineId: string; oldestTime: number }> = {};
|
|
221
|
+
|
|
222
|
+
for (const path of paths) {
|
|
223
|
+
if (path.logCount !== undefined) continue;
|
|
224
|
+
|
|
225
|
+
if (path.startTime >= threshold) continue;
|
|
226
|
+
|
|
227
|
+
if (!path.machineId) continue;
|
|
228
|
+
|
|
229
|
+
const existing = machineWarnings[path.machineId];
|
|
230
|
+
if (!existing || path.startTime < existing.oldestTime) {
|
|
231
|
+
machineWarnings[path.machineId] = {
|
|
232
|
+
machineId: path.machineId,
|
|
233
|
+
oldestTime: path.startTime,
|
|
234
|
+
};
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
const warnings = Object.values(machineWarnings);
|
|
239
|
+
if (warnings.length === 0) return undefined;
|
|
240
|
+
|
|
241
|
+
return (
|
|
242
|
+
<div className={css.vbox(10).pad2(10).hsl(40, 50, 50).colorhsl(60, 50, 100)}>
|
|
243
|
+
<div className={css.fontWeight(600)}>Machines not moving logs to remote storage:</div>
|
|
244
|
+
{warnings.map((warning) => (
|
|
245
|
+
<div key={warning.machineId} className={css.hbox(10)}>
|
|
246
|
+
<MachineThreadInfo machineId={warning.machineId} />
|
|
247
|
+
<span>is not moving logs to remote storage. Logs are {formatTime(now - warning.oldestTime)} old</span>
|
|
248
|
+
</div>
|
|
249
|
+
))}
|
|
250
|
+
</div>
|
|
251
|
+
);
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
async loadPaths() {
|
|
255
|
+
if (savedPathsURL.value) {
|
|
256
|
+
return;
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
this.state.loadingPaths = true;
|
|
260
|
+
|
|
261
|
+
let loggers = await getLoggers2Async();
|
|
262
|
+
let selectedLoggers: typeof loggers.logLogs[] = [];
|
|
263
|
+
Querysub.localRead(() => {
|
|
264
|
+
if (this.state.enableLogs) selectedLoggers.push(loggers.logLogs);
|
|
265
|
+
if (this.state.enableInfos) selectedLoggers.push(loggers.infoLogs);
|
|
266
|
+
if (this.state.enableWarnings) selectedLoggers.push(loggers.warnLogs);
|
|
267
|
+
if (this.state.enableErrors) selectedLoggers.push(loggers.errorLogs);
|
|
268
|
+
});
|
|
269
|
+
|
|
270
|
+
if (selectedLoggers.length === 0) {
|
|
271
|
+
console.error("No log sources selected");
|
|
272
|
+
Querysub.commitLocal(() => {
|
|
273
|
+
this.state.loadingPaths = false;
|
|
274
|
+
});
|
|
275
|
+
return;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
let allPaths: TimeFilePathWithSize[] = [];
|
|
279
|
+
let range = Querysub.localRead(() => getTimeRange());
|
|
280
|
+
|
|
281
|
+
for (let logger of selectedLoggers) {
|
|
282
|
+
let paths = await logger.clientGetPaths({
|
|
283
|
+
startTime: range.startTime,
|
|
284
|
+
endTime: range.endTime,
|
|
285
|
+
});
|
|
286
|
+
allPaths.push(...paths);
|
|
287
|
+
}
|
|
288
|
+
sort(allPaths, x => -x.startTime);
|
|
289
|
+
|
|
290
|
+
Querysub.commitLocal(() => {
|
|
291
|
+
this.state.paths = allPaths;
|
|
292
|
+
this.state.loadingPaths = false;
|
|
293
|
+
});
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
search = throttleFunction(200, async () => {
|
|
298
|
+
Querysub.commitLocal(() => {
|
|
299
|
+
this.state.searching = true;
|
|
300
|
+
this.state.results = [];
|
|
301
|
+
this.state.stats = undefined;
|
|
302
|
+
this.state.searchingLogs = false;
|
|
303
|
+
this.state.searchingInfos = false;
|
|
304
|
+
this.state.searchingWarnings = false;
|
|
305
|
+
this.state.searchingErrors = false;
|
|
306
|
+
});
|
|
307
|
+
|
|
308
|
+
let startTime = Date.now();
|
|
309
|
+
|
|
310
|
+
let hasPaths = Querysub.localRead(() => this.getPaths().length > 0);
|
|
311
|
+
let getFilesTime = 0;
|
|
312
|
+
if (readLiveData.value || !hasPaths) {
|
|
313
|
+
let startTime = Date.now();
|
|
314
|
+
await this.loadPaths();
|
|
315
|
+
getFilesTime = Date.now() - startTime;
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
let loggers = await getLoggers2Async();
|
|
319
|
+
|
|
320
|
+
let selectedLoggers: typeof loggers.logLogs[] = [];
|
|
321
|
+
Querysub.localRead(() => {
|
|
322
|
+
if (this.state.enableLogs) {
|
|
323
|
+
selectedLoggers.push(loggers.logLogs);
|
|
324
|
+
this.state.searchingLogs = true;
|
|
325
|
+
}
|
|
326
|
+
if (this.state.enableInfos) {
|
|
327
|
+
selectedLoggers.push(loggers.infoLogs);
|
|
328
|
+
this.state.searchingInfos = true;
|
|
329
|
+
}
|
|
330
|
+
if (this.state.enableWarnings) {
|
|
331
|
+
selectedLoggers.push(loggers.warnLogs);
|
|
332
|
+
this.state.searchingWarnings = true;
|
|
333
|
+
}
|
|
334
|
+
if (this.state.enableErrors) {
|
|
335
|
+
selectedLoggers.push(loggers.errorLogs);
|
|
336
|
+
this.state.searchingErrors = true;
|
|
337
|
+
}
|
|
338
|
+
});
|
|
339
|
+
|
|
340
|
+
if (selectedLoggers.length === 0) {
|
|
341
|
+
console.error("No log sources selected");
|
|
342
|
+
Querysub.commitLocal(() => {
|
|
343
|
+
this.state.searching = false;
|
|
344
|
+
});
|
|
345
|
+
return;
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
let searchBuffer = Querysub.localRead(() => Buffer.from(searchText.value, "utf8"));
|
|
349
|
+
let results: LogDatum[] = [];
|
|
350
|
+
let stats: IndexedLogResults = {
|
|
351
|
+
matchCount: 0,
|
|
352
|
+
reads: [],
|
|
353
|
+
localFilesSearched: 0,
|
|
354
|
+
backblazeFilesSearched: 0,
|
|
355
|
+
totalBlockCount: 0,
|
|
356
|
+
blockCheckedCount: 0,
|
|
357
|
+
blocksCheckedCompressedSize: 0,
|
|
358
|
+
blocksCheckedDecompressedSize: 0,
|
|
359
|
+
blockErrors: [],
|
|
360
|
+
fileErrors: [],
|
|
361
|
+
indexesSearched: 0,
|
|
362
|
+
indexSize: 0,
|
|
363
|
+
timeToFirstMatch: 0,
|
|
364
|
+
fileFindTime: 0,
|
|
365
|
+
indexSearchTime: 0,
|
|
366
|
+
blockSearchTime: 0,
|
|
367
|
+
totalSearchTime: 0,
|
|
368
|
+
};
|
|
369
|
+
stats.fileFindTime += getFilesTime;
|
|
370
|
+
|
|
371
|
+
let paths = Querysub.localRead(() => this.getPaths());
|
|
372
|
+
|
|
373
|
+
let range = Querysub.localRead(() => getTimeRange());
|
|
374
|
+
|
|
375
|
+
let updateResults = throttleFunction(100, () => {
|
|
376
|
+
Querysub.commitLocal(() => {
|
|
377
|
+
this.state.results = results;
|
|
378
|
+
});
|
|
379
|
+
});
|
|
380
|
+
|
|
381
|
+
await Promise.all(selectedLoggers.map(async (logger) => {
|
|
382
|
+
let result = await logger.clientFind({
|
|
383
|
+
params: {
|
|
384
|
+
startTime: range.startTime,
|
|
385
|
+
endTime: range.endTime,
|
|
386
|
+
limit: limitURL.value,
|
|
387
|
+
findBuffer: searchBuffer,
|
|
388
|
+
pathOverrides: paths,
|
|
389
|
+
},
|
|
390
|
+
onResult: (match: LogDatum) => {
|
|
391
|
+
console.log("onResult", match);
|
|
392
|
+
results.push(match);
|
|
393
|
+
void updateResults();
|
|
394
|
+
},
|
|
395
|
+
});
|
|
396
|
+
stats.reads.push(...result.reads);
|
|
397
|
+
stats.matchCount += result.matchCount;
|
|
398
|
+
stats.localFilesSearched += result.localFilesSearched;
|
|
399
|
+
stats.backblazeFilesSearched += result.backblazeFilesSearched;
|
|
400
|
+
stats.totalBlockCount += result.totalBlockCount;
|
|
401
|
+
stats.blockCheckedCount += result.blockCheckedCount;
|
|
402
|
+
stats.blocksCheckedCompressedSize += result.blocksCheckedCompressedSize;
|
|
403
|
+
stats.blocksCheckedDecompressedSize += result.blocksCheckedDecompressedSize;
|
|
404
|
+
stats.indexesSearched += result.indexesSearched;
|
|
405
|
+
stats.indexSize += result.indexSize;
|
|
406
|
+
stats.timeToFirstMatch = Math.min(stats.timeToFirstMatch, result.timeToFirstMatch);
|
|
407
|
+
stats.fileFindTime += result.fileFindTime;
|
|
408
|
+
stats.indexSearchTime += result.indexSearchTime;
|
|
409
|
+
stats.blockSearchTime += result.blockSearchTime;
|
|
410
|
+
stats.totalSearchTime = Date.now() - startTime;
|
|
411
|
+
Querysub.commitLocal(() => {
|
|
412
|
+
this.state.results = results;
|
|
413
|
+
this.state.stats = stats;
|
|
414
|
+
if (logger === loggers.logLogs) this.state.searchingLogs = false;
|
|
415
|
+
if (logger === loggers.infoLogs) this.state.searchingInfos = false;
|
|
416
|
+
if (logger === loggers.warnLogs) this.state.searchingWarnings = false;
|
|
417
|
+
if (logger === loggers.errorLogs) this.state.searchingErrors = false;
|
|
418
|
+
});
|
|
419
|
+
}));
|
|
420
|
+
|
|
421
|
+
stats.totalSearchTime = Date.now() - startTime;
|
|
422
|
+
|
|
423
|
+
Querysub.commitLocal(() => {
|
|
424
|
+
this.state.results = results;
|
|
425
|
+
this.state.searching = false;
|
|
426
|
+
this.state.stats = stats;
|
|
427
|
+
});
|
|
428
|
+
});
|
|
429
|
+
|
|
430
|
+
renderResults() {
|
|
431
|
+
let fieldNames = new Set<string>();
|
|
432
|
+
for (let i = 0; i < Math.min(1000, this.state.results.length); i++) {
|
|
433
|
+
let result = this.state.results[i];
|
|
434
|
+
for (let key of Object.keys(result)) {
|
|
435
|
+
fieldNames.add(key);
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
let selectedFields: string[] = [];
|
|
440
|
+
for (let field of Object.keys(defaultSelectedFields)) {
|
|
441
|
+
if (atomic(selectedFieldsURL.value[field]) === undefined) {
|
|
442
|
+
selectedFields.push(field);
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
for (let [key, value] of Object.entries(selectedFieldsURL.value)) {
|
|
446
|
+
if (value && !selectedFields.includes(key)) {
|
|
447
|
+
selectedFields.splice(1, 0, key);
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
let columns: TableType<LogDatum>["columns"] = {};
|
|
452
|
+
columns["log"] = {
|
|
453
|
+
title: "Log",
|
|
454
|
+
formatter: (x, context) => {
|
|
455
|
+
return <Button onClick={() => console.log(context?.row)}>
|
|
456
|
+
Log
|
|
457
|
+
</Button>;
|
|
458
|
+
}
|
|
459
|
+
};
|
|
460
|
+
|
|
461
|
+
for (let field of selectedFields) {
|
|
462
|
+
let column: ColumnType<unknown, LogDatum> = {};
|
|
463
|
+
if (field === "time") {
|
|
464
|
+
column.formatter = (x: unknown) => useRelativeTimeURL.value ? formatDateJSX(Number(x)) : <span title={formatDateTimeDetailed(Number(x))}>{formatDateTime(Number(x))}</span>;
|
|
465
|
+
} else if (field === "__machineId") {
|
|
466
|
+
column.formatter = (x: unknown, context) => {
|
|
467
|
+
if (!context?.row || !context.row.__machineId) return <ObjectDisplay value={x} />;
|
|
468
|
+
return <MachineThreadInfo machineId={context.row.__machineId} threadId={context.row.__threadId || undefined} />;
|
|
469
|
+
};
|
|
470
|
+
}
|
|
471
|
+
if (!column.formatter) {
|
|
472
|
+
column.formatter = (x: unknown) => <ObjectDisplay value={x} />;
|
|
473
|
+
}
|
|
474
|
+
columns[field] = column;
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
columns["fields"] = {
|
|
478
|
+
title: "Fields",
|
|
479
|
+
formatter: (x, context) => {
|
|
480
|
+
if (!context?.row) return undefined;
|
|
481
|
+
let datum = context.row;
|
|
482
|
+
let fields = Object.keys(datum);
|
|
483
|
+
fields = fields.filter(x => !(x in columns) && !x.startsWith("_"));
|
|
484
|
+
return <div className={css.hbox(4, 4).wrap}>
|
|
485
|
+
{fields.map(x => <div
|
|
486
|
+
key={x}
|
|
487
|
+
className={css.pad2(4, 0).hsl(0, 0, 80).button}
|
|
488
|
+
onClick={() => {
|
|
489
|
+
let newValues = { ...selectedFieldsURL.value };
|
|
490
|
+
newValues[x] = true;
|
|
491
|
+
selectedFieldsURL.value = newValues;
|
|
492
|
+
}}
|
|
493
|
+
>
|
|
494
|
+
{x}
|
|
495
|
+
</div>)}
|
|
496
|
+
</div>;
|
|
497
|
+
}
|
|
498
|
+
};
|
|
499
|
+
|
|
500
|
+
return <>
|
|
501
|
+
<div className={css.hbox(10)}>
|
|
502
|
+
<InputPicker
|
|
503
|
+
label={<div className={css.hbox(10)}>
|
|
504
|
+
<div className={css.flexShrink0}>
|
|
505
|
+
Selected Fields
|
|
506
|
+
</div>
|
|
507
|
+
<Button onClick={() => {
|
|
508
|
+
let newValues = { ...selectedFieldsURL.value };
|
|
509
|
+
for (let key of Object.keys(newValues)) {
|
|
510
|
+
newValues[key] = key in defaultSelectedFields;
|
|
511
|
+
}
|
|
512
|
+
selectedFieldsURL.value = newValues;
|
|
513
|
+
}}>
|
|
514
|
+
Reset
|
|
515
|
+
</Button>
|
|
516
|
+
</div>}
|
|
517
|
+
picked={selectedFields}
|
|
518
|
+
options={Array.from(fieldNames).map(x => ({ value: x, label: x }))}
|
|
519
|
+
addPicked={x => {
|
|
520
|
+
let newValues = { ...selectedFieldsURL.value };
|
|
521
|
+
newValues[x] = true;
|
|
522
|
+
selectedFieldsURL.value = newValues;
|
|
523
|
+
}}
|
|
524
|
+
removePicked={x => {
|
|
525
|
+
let newValues = { ...selectedFieldsURL.value };
|
|
526
|
+
newValues[x] = false;
|
|
527
|
+
selectedFieldsURL.value = newValues;
|
|
528
|
+
}}
|
|
529
|
+
/>
|
|
530
|
+
<InputLabelURL
|
|
531
|
+
label="Use Relative Time"
|
|
532
|
+
checkbox
|
|
533
|
+
url={useRelativeTimeURL}
|
|
534
|
+
/>
|
|
535
|
+
</div>
|
|
536
|
+
|
|
537
|
+
{this.state.results.length === 0 && !this.state.searching && <div className={css.hsl(40, 50, 50).colorhsl(60, 50, 100).boldStyle.pad2(10).ellipsis}>
|
|
538
|
+
No logs matched, try adjusting your search or time range.
|
|
539
|
+
</div>}
|
|
540
|
+
|
|
541
|
+
<Table
|
|
542
|
+
rows={this.state.results}
|
|
543
|
+
columns={columns}
|
|
544
|
+
lineLimit={4}
|
|
545
|
+
initialLimit={10}
|
|
546
|
+
characterLimit={400}
|
|
547
|
+
getRowAttributes={row => {
|
|
548
|
+
let hue = -1;
|
|
549
|
+
if (row.__LOG_TYPE === "warn") hue = 40;
|
|
550
|
+
if (row.__LOG_TYPE === "error") hue = 0;
|
|
551
|
+
if (row.__LOG_TYPE === "info") hue = 200;
|
|
552
|
+
return {
|
|
553
|
+
className: hue !== -1 && css.hsl(hue, 40, 50).colorhsl(hue, 0, 100) || undefined,
|
|
554
|
+
};
|
|
555
|
+
}}
|
|
556
|
+
/>
|
|
557
|
+
</>;
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
render() {
|
|
561
|
+
return (
|
|
562
|
+
<div className={css.vbox(20).pad2(20).fillBoth}>
|
|
563
|
+
<div className={css.hbox(20)}>
|
|
564
|
+
<div>Log Viewer 3</div>
|
|
565
|
+
<InputLabelURL
|
|
566
|
+
checkbox
|
|
567
|
+
label="Always Read Live Data"
|
|
568
|
+
url={readLiveData}
|
|
569
|
+
/>
|
|
570
|
+
<InputLabelURL
|
|
571
|
+
checkbox
|
|
572
|
+
label="Exclude Pending Results"
|
|
573
|
+
url={excludePendingResults}
|
|
574
|
+
/>
|
|
575
|
+
<InputLabelURL
|
|
576
|
+
label="Limit"
|
|
577
|
+
number
|
|
578
|
+
url={limitURL}
|
|
579
|
+
/>
|
|
580
|
+
</div>
|
|
581
|
+
|
|
582
|
+
<div className={css.hbox(10)}>
|
|
583
|
+
<Button
|
|
584
|
+
onClick={() => {
|
|
585
|
+
this.state.enableLogs = !this.state.enableLogs;
|
|
586
|
+
this.clearPaths();
|
|
587
|
+
}}
|
|
588
|
+
hue={this.state.enableLogs && 210 || undefined}
|
|
589
|
+
className={this.state.searchingLogs && css.opacity(0.5) || undefined}
|
|
590
|
+
>
|
|
591
|
+
Logs
|
|
592
|
+
</Button>
|
|
593
|
+
<Button
|
|
594
|
+
onClick={() => {
|
|
595
|
+
this.state.enableInfos = !this.state.enableInfos;
|
|
596
|
+
this.clearPaths();
|
|
597
|
+
}}
|
|
598
|
+
hue={this.state.enableInfos && 200 || undefined}
|
|
599
|
+
className={this.state.searchingInfos && css.opacity(0.5) || undefined}
|
|
600
|
+
>
|
|
601
|
+
Infos
|
|
602
|
+
</Button>
|
|
603
|
+
<Button
|
|
604
|
+
onClick={() => {
|
|
605
|
+
this.state.enableWarnings = !this.state.enableWarnings;
|
|
606
|
+
this.clearPaths();
|
|
607
|
+
}}
|
|
608
|
+
hue={this.state.enableWarnings && 40 || undefined}
|
|
609
|
+
className={this.state.searchingWarnings && css.opacity(0.5) || undefined}
|
|
610
|
+
>
|
|
611
|
+
Warnings
|
|
612
|
+
</Button>
|
|
613
|
+
<Button
|
|
614
|
+
onClick={() => {
|
|
615
|
+
this.state.enableErrors = !this.state.enableErrors;
|
|
616
|
+
this.clearPaths();
|
|
617
|
+
}}
|
|
618
|
+
hue={this.state.enableErrors ? 0 : undefined}
|
|
619
|
+
className={this.state.searchingErrors && css.opacity(0.5) || undefined}
|
|
620
|
+
>
|
|
621
|
+
Errors
|
|
622
|
+
</Button>
|
|
623
|
+
<TimeRangeSelector />
|
|
624
|
+
</div>
|
|
625
|
+
|
|
626
|
+
<div className={css.hbox(10).fillWidth}>
|
|
627
|
+
<InputLabelURL
|
|
628
|
+
flavor="large"
|
|
629
|
+
fillWidth
|
|
630
|
+
focusOnMount
|
|
631
|
+
url={searchText}
|
|
632
|
+
onChangeValue={(value) => {
|
|
633
|
+
if (value.length >= MIN_AUTO_SEARCH_LENGTH) {
|
|
634
|
+
void this.search();
|
|
635
|
+
}
|
|
636
|
+
}}
|
|
637
|
+
onKeyDown={(e) => {
|
|
638
|
+
if (e.key === "Enter") {
|
|
639
|
+
searchText.value = e.currentTarget.value;
|
|
640
|
+
void this.search();
|
|
641
|
+
}
|
|
642
|
+
}}
|
|
643
|
+
/>
|
|
644
|
+
{!savedPathsURL.value && (
|
|
645
|
+
<Button
|
|
646
|
+
flavor="large"
|
|
647
|
+
onClick={() => void this.loadPaths()}
|
|
648
|
+
hue={120}
|
|
649
|
+
>
|
|
650
|
+
Preview Files
|
|
651
|
+
</Button>
|
|
652
|
+
)}
|
|
653
|
+
{savedPathsURL.value && (
|
|
654
|
+
<Button
|
|
655
|
+
flavor="large"
|
|
656
|
+
onClick={() => this.clearFrozenPaths()}
|
|
657
|
+
hue={0}
|
|
658
|
+
>
|
|
659
|
+
Clear Frozen Files ({this.getPaths().length})
|
|
660
|
+
</Button>
|
|
661
|
+
)}
|
|
662
|
+
<Button
|
|
663
|
+
flavor="large"
|
|
664
|
+
hotkeys={["enter"]}
|
|
665
|
+
showHotkeys
|
|
666
|
+
onClick={() => void this.search()}
|
|
667
|
+
hue={210}
|
|
668
|
+
>
|
|
669
|
+
Search
|
|
670
|
+
</Button>
|
|
671
|
+
</div>
|
|
672
|
+
|
|
673
|
+
{this.state.loadingPaths && <div>Loading paths...</div>}
|
|
674
|
+
{this.state.searching && <div>Searching...</div>}
|
|
675
|
+
|
|
676
|
+
{this.getPaths().length > 0 && (
|
|
677
|
+
<div className={css.hbox(10).fillWidth}>
|
|
678
|
+
<FilePathSelector
|
|
679
|
+
paths={this.getPaths()}
|
|
680
|
+
onChange={(paths) => {
|
|
681
|
+
this.state.paths = paths;
|
|
682
|
+
}}
|
|
683
|
+
/>
|
|
684
|
+
<Button
|
|
685
|
+
onClick={() => this.freezePaths()}
|
|
686
|
+
hue={180}
|
|
687
|
+
>
|
|
688
|
+
Freeze Files ({this.state.paths.length})
|
|
689
|
+
</Button>
|
|
690
|
+
</div>
|
|
691
|
+
)}
|
|
692
|
+
|
|
693
|
+
{this.renderPendingLogWarnings()}
|
|
694
|
+
|
|
695
|
+
{this.renderSearchStats()}
|
|
696
|
+
|
|
697
|
+
{this.renderResults()}
|
|
698
|
+
</div>
|
|
699
|
+
);
|
|
700
|
+
}
|
|
701
|
+
}
|
|
702
|
+
|