querysub 0.356.0 → 0.357.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/.cursorrules +8 -0
  2. package/bin/movelogs.js +4 -0
  3. package/package.json +12 -6
  4. package/scripts/postinstall.js +23 -0
  5. package/src/-a-archives/archiveCache.ts +10 -12
  6. package/src/-a-archives/archives.ts +29 -0
  7. package/src/-a-archives/archivesBackBlaze.ts +60 -12
  8. package/src/-a-archives/archivesDisk.ts +27 -8
  9. package/src/-a-archives/archivesLimitedCache.ts +21 -0
  10. package/src/-a-archives/archivesMemoryCache.ts +350 -0
  11. package/src/-a-archives/archivesPrivateFileSystem.ts +22 -0
  12. package/src/-g-core-values/NodeCapabilities.ts +3 -0
  13. package/src/0-path-value-core/auditLogs.ts +5 -1
  14. package/src/0-path-value-core/pathValueCore.ts +7 -7
  15. package/src/4-dom/qreact.tsx +1 -0
  16. package/src/4-querysub/Querysub.ts +1 -5
  17. package/src/config.ts +5 -0
  18. package/src/diagnostics/MachineThreadInfo.tsx +235 -0
  19. package/src/diagnostics/NodeViewer.tsx +3 -2
  20. package/src/diagnostics/logs/FastArchiveAppendable.ts +79 -42
  21. package/src/diagnostics/logs/FastArchiveController.ts +102 -63
  22. package/src/diagnostics/logs/FastArchiveViewer.tsx +36 -8
  23. package/src/diagnostics/logs/IndexedLogs/BufferIndex.ts +461 -0
  24. package/src/diagnostics/logs/IndexedLogs/BufferIndexCPP.cpp +327 -0
  25. package/src/diagnostics/logs/IndexedLogs/BufferIndexCPP.d.ts +18 -0
  26. package/src/diagnostics/logs/IndexedLogs/BufferIndexCPP.js +1 -0
  27. package/src/diagnostics/logs/IndexedLogs/BufferIndexHelpers.ts +140 -0
  28. package/src/diagnostics/logs/IndexedLogs/BufferIndexLogsOptimizationConstants.ts +22 -0
  29. package/src/diagnostics/logs/IndexedLogs/BufferIndexWAT.wat +1145 -0
  30. package/src/diagnostics/logs/IndexedLogs/BufferIndexWAT.wat.d.ts +178 -0
  31. package/src/diagnostics/logs/IndexedLogs/BufferListStreamer.ts +206 -0
  32. package/src/diagnostics/logs/IndexedLogs/BufferUnitIndex.ts +719 -0
  33. package/src/diagnostics/logs/IndexedLogs/BufferUnitSet.ts +146 -0
  34. package/src/diagnostics/logs/IndexedLogs/FilePathSelector.tsx +408 -0
  35. package/src/diagnostics/logs/IndexedLogs/FindProgressTracker.ts +45 -0
  36. package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts +598 -0
  37. package/src/diagnostics/logs/IndexedLogs/LogStreamer.ts +47 -0
  38. package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx +702 -0
  39. package/src/diagnostics/logs/IndexedLogs/TimeFileTree.ts +236 -0
  40. package/src/diagnostics/logs/IndexedLogs/binding.gyp +23 -0
  41. package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts +221 -0
  42. package/src/diagnostics/logs/IndexedLogs/moveLogsEntry.ts +10 -0
  43. package/src/diagnostics/logs/LogViewer2.tsx +120 -55
  44. package/src/diagnostics/logs/TimeRangeSelector.tsx +5 -2
  45. package/src/diagnostics/logs/diskLogger.ts +32 -48
  46. package/src/diagnostics/logs/errorNotifications/ErrorNotificationController.ts +3 -2
  47. package/src/diagnostics/logs/errorNotifications/errorDigests.tsx +1 -0
  48. package/src/diagnostics/logs/lifeCycleAnalysis/LifeCyclePages.tsx +150 -0
  49. package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx +132 -15
  50. package/src/diagnostics/logs/lifeCycleAnalysis/test.ts +180 -0
  51. package/src/diagnostics/logs/lifeCycleAnalysis/test.wat +106 -0
  52. package/src/diagnostics/logs/lifeCycleAnalysis/test.wat.d.ts +2 -0
  53. package/src/diagnostics/logs/lifeCycleAnalysis/testHoist.ts +5 -0
  54. package/src/diagnostics/logs/logViewerExtractField.ts +2 -3
  55. package/src/diagnostics/managementPages.tsx +10 -0
  56. package/src/diagnostics/trackResources.ts +1 -1
  57. package/src/misc/lz4_wasm_nodejs.d.ts +34 -0
  58. package/src/misc/lz4_wasm_nodejs.js +178 -0
  59. package/src/misc/lz4_wasm_nodejs_bg.js +94 -0
  60. package/src/misc/lz4_wasm_nodejs_bg.wasm +0 -0
  61. package/src/misc/lz4_wasm_nodejs_bg.wasm.d.ts +15 -0
  62. package/src/storage/CompressedStream.ts +13 -0
  63. package/src/storage/LZ4.ts +32 -0
  64. package/src/storage/ZSTD.ts +10 -0
  65. package/src/wat/watCompiler.ts +1716 -0
  66. package/src/wat/watGrammar.pegjs +93 -0
  67. package/src/wat/watHandler.ts +179 -0
  68. package/src/wat/watInstructions.txt +707 -0
  69. package/src/zip.ts +3 -89
  70. package/src/diagnostics/logs/lifeCycleAnalysis/spec.md +0 -125
@@ -0,0 +1,702 @@
1
+ module.hotreload = true;
2
+ import { qreact } from "../../../4-dom/qreact";
3
+ import { css } from "../../../4-dom/css";
4
+ import { t } from "../../../2-proxy/schema2";
5
+ import { Button } from "../../../library-components/Button";
6
+ import { InputLabel, InputLabelURL } from "../../../library-components/InputLabel";
7
+ import { getLoggers2, getLoggers2Async, LogDatum } from "../diskLogger";
8
+ import { list, timeInDay, keyByArray, sort, throttleFunction } from "socket-function/src/misc";
9
+ import { formatDateTime, formatDateTimeDetailed, formatNumber, formatTime, formatPercent } from "socket-function/src/formatting/format";
10
+ import { IndexedLogResults, IndexedLogs, TimeFilePathWithSize } from "./IndexedLogs";
11
+ import { Querysub } from "../../../4-querysub/QuerysubController";
12
+ import { URLParam } from "../../../library-components/URLParam";
13
+ import { getOwnMachineId, getOwnThreadId } from "../../../-f-node-discovery/NodeDiscovery";
14
+ import { FilePathSelector, FilePathsByMachine } from "./FilePathSelector";
15
+ import { MachineThreadInfo } from "../../MachineThreadInfo";
16
+ import { TimeRangeSelector, getTimeRange } from "../TimeRangeSelector";
17
+ import { LZ4 } from "../../../storage/LZ4";
18
+ import { ColumnType, Table, TableType } from "../../../5-diagnostics/Table";
19
+ import { InputPicker } from "../../../library-components/InputPicker";
20
+ import { ObjectDisplay } from "../ObjectDisplay";
21
+ import { formatDateJSX } from "../../../misc/formatJSX";
22
+ import { PUBLIC_MOVE_THRESHOLD } from "./BufferIndexLogsOptimizationConstants";
23
+ import { atomic } from "../../../2-proxy/PathValueProxyWatcher";
24
+
25
+ let searchText = new URLParam("searchText", "");
26
+ let readLiveData = new URLParam("readLiveData", false);
27
+ let excludePendingResults = new URLParam("excludePendingResults", false);
28
+ let limitURL = new URLParam("limit", 100);
29
+
30
+ let savedPathsURL = new URLParam("savedPaths", "");
31
+ let selectedFieldsURL = new URLParam("selectedFields", {} as Record<string, boolean>);
32
+ let useRelativeTimeURL = new URLParam("useRelativeTime", true);
33
+
34
+ // Only auto-search if the string is long enough. Otherwise, we're going to get way too many results, which could slow down the server and won't be useful to us.
35
+ const MIN_AUTO_SEARCH_LENGTH = 7;
36
+
37
+ const defaultSelectedFields = {
38
+ param0: true,
39
+ time: true,
40
+ __NAME__: true,
41
+ __machineId: true,
42
+ __threadId: true,
43
+ __entry: true,
44
+ };
45
+
46
+ function createTestLog(text: string, type: "log" | "warn" | "info" | "error"): LogDatum {
47
+ return {
48
+ time: 0,
49
+ message: text,
50
+ __LOG_TYPE: type,
51
+ __machineId: getOwnMachineId(),
52
+ __threadId: getOwnThreadId(),
53
+ };
54
+ }
55
+
56
+ export class LogViewer3 extends qreact.Component {
57
+ static renderInProgress = true;
58
+ state = t.state({
59
+ enableLogs: t.boolean(true),
60
+ enableInfos: t.boolean(true),
61
+ enableWarnings: t.boolean(true),
62
+ enableErrors: t.boolean(true),
63
+ results: t.atomic<LogDatum[]>([
64
+ { "time": 1771825252649.44, "__LOG_TYPE": "log", "__machineId": "1ed7a7340a015680", "__mountId": "1d590f0919b1f438.1ed7a7340a015680.querysubtest.com:7007", "__threadId": "1d590f0919b1f438", "__port": 7007, "__nodeId": "1d590f0919b1f438.1ed7a7340a015680.querysubtest.com:7007", "__entry": "D:\\repos\\qs-cyoa\\src\\server.ts", "__hostName": "DESKTOP-05MP449", "__accountName": "desktop-05mp449\\quent", "__externalIP": "99.250.124.91", "__os": "win32", "__pid": 75888, "param0": "new non-local WATCH", "path": ".,querysubtest._com.,PathFunctionRunner.,book.,Data.,edittingBooks.,hkwnmoiwhuyqenbi.,books.,bf0713806bf1c4338._querysubtest._com_1755390476654._5208_1.,nodes.,1770378259589._5535.,aiCalls.,", "watcher": "client:127.0.0.1:1771825250819.3994:0.634106584461968", "diskAudit": true },
65
+ { "time": 1771825252649.4397, "__LOG_TYPE": "log", "__machineId": "1ed7a7340a015680", "__mountId": "1d590f0919b1f438.1ed7a7340a015680.querysubtest.com:7007", "__threadId": "x1d590f0919b1f438", "__port": 7007, "__nodeId": "1d590f0919b1f438.1ed7a7340a015680.querysubtest.com:7007", "__entry": "D:\\repos\\qs-cyoa\\src\\server.ts", "__hostName": "DESKTOP-05MP449", "__accountName": "desktop-05mp449\\quent", "__externalIP": "99.250.124.91", "__os": "win32", "__pid": 75888, "param0": "new non-local WATCH", "path": ".,querysubtest._com.,PathFunctionRunner.,book.,Data.,edittingBooks.,hkwnmoiwhuyqenbi.,books.,bf0713806bf1c4338._querysubtest._com_1755390476654._5208_1.,nodes.,1770378259589._5535.,totalTime.,", "watcher": "client:127.0.0.1:1771825250819.3994:0.634106584461968", "diskAudit": true },
66
+ ]),
67
+ searching: t.boolean,
68
+ searchingLogs: t.boolean,
69
+ searchingInfos: t.boolean,
70
+ searchingWarnings: t.boolean,
71
+ searchingErrors: t.boolean,
72
+ paths: t.atomic<TimeFilePathWithSize[]>([]),
73
+ loadingPaths: t.boolean,
74
+ stats: t.atomic<IndexedLogResults | undefined>(undefined),
75
+ });
76
+
77
+ componentDidMount(): void {
78
+ void this.loadPaths();
79
+ }
80
+
81
+ getPaths(): TimeFilePathWithSize[] {
82
+ let paths: TimeFilePathWithSize[] = [];
83
+ if (savedPathsURL.value) {
84
+ const compressed = Buffer.from(savedPathsURL.value, "base64");
85
+ const decompressed = LZ4.decompress(compressed);
86
+ paths = JSON.parse(decompressed.toString("utf8"));
87
+ } else {
88
+ paths = this.state.paths;
89
+ }
90
+ if (excludePendingResults.value) {
91
+ paths = paths.filter(x => x.logCount !== undefined);
92
+ }
93
+ return paths;
94
+ }
95
+
96
+ clearPaths() {
97
+ this.state.paths = [];
98
+ this.state.searchingLogs = false;
99
+ this.state.searchingInfos = false;
100
+ this.state.searchingWarnings = false;
101
+ this.state.searchingErrors = false;
102
+ }
103
+
104
+ freezePaths() {
105
+ const json = JSON.stringify(this.state.paths);
106
+ const buffer = Buffer.from(json, "utf8");
107
+ const compressed = LZ4.compress(buffer);
108
+ savedPathsURL.value = compressed.toString("base64");
109
+ }
110
+
111
+ clearFrozenPaths() {
112
+ savedPathsURL.value = "";
113
+ }
114
+
115
+ renderSearchStats() {
116
+ const stats = this.state.stats;
117
+ if (!stats) return undefined;
118
+
119
+ let totalSizeRead = 0;
120
+ let cachedSize = 0;
121
+ let uncachedSize = 0;
122
+ let uncachedCount = 0;
123
+ let uncachedRemoteSize = 0;
124
+ let uncachedRemoteCount = 0;
125
+ let totalSize = 0;
126
+
127
+ for (let read of stats.reads) {
128
+ totalSizeRead += read.size;
129
+ if (read.cached) {
130
+ cachedSize += read.size;
131
+ } else {
132
+ uncachedSize += read.size;
133
+ uncachedCount += read.count;
134
+ totalSize += read.totalSize;
135
+ }
136
+ if (read.remote && !read.cached) {
137
+ uncachedRemoteSize += read.size;
138
+ uncachedRemoteCount += read.count;
139
+ }
140
+ }
141
+
142
+ const renderStage = (title: string, items: preact.ComponentChild[], hue: number | undefined, isFirst: boolean, isLast: boolean) => {
143
+ const triangleWidth = 20;
144
+
145
+ let color = hue ? css.hsl(hue, 70, 80) : css.hsl(0, 0, 100);
146
+ let colorhsl = hue ? css.hslcolor(hue, 80, 30) : css;
147
+
148
+ return (
149
+ <div className={css.vbox(2).relative.minWidth(200).paddingTop(8).paddingBottom(8).paddingRight(10).paddingLeft(isFirst ? 10 : 10 + triangleWidth) + color}>
150
+ <div className={css.fontWeight(600) + colorhsl}>{title}</div>
151
+ {items.map((item, i) => (
152
+ <div key={i}>{item}</div>
153
+ ))}
154
+ {!isLast && (
155
+ <svg width={triangleWidth} height="100" viewBox="0 0 20 100" preserveAspectRatio="none" className={css.absolute.right(-triangleWidth).top(0).fillHeight.zIndex(1)}>
156
+ <polygon points="0,0 20,50 0,100" fill={`hsl(${hue}, 70%, 80%)`} />
157
+ </svg>
158
+ )}
159
+ </div>
160
+ );
161
+ };
162
+
163
+ const fileItems = [
164
+ `${formatTime(stats.fileFindTime)} | ${formatNumber(stats.localFilesSearched + stats.backblazeFilesSearched)} | ${formatNumber(totalSize)}B`,
165
+ `pending ${formatNumber(stats.localFilesSearched)}`,
166
+ ];
167
+
168
+ const indexItems = [
169
+ `${formatTime(stats.indexSearchTime)} | ${formatNumber(stats.indexesSearched)} | ${formatNumber(stats.indexSize)}B`,
170
+ ];
171
+
172
+ const blockItems = [
173
+ `${formatTime(stats.blockSearchTime)} | ${formatNumber(stats.blockCheckedCount)} | ${formatNumber(stats.blocksCheckedCompressedSize)}B | ${formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X`,
174
+ `total blocks ${formatNumber(stats.totalBlockCount)}`,
175
+ ];
176
+ let cacheItems = [
177
+ `disk read ${formatNumber(uncachedSize)}B`,
178
+ `remote ${formatPercent(uncachedRemoteSize / totalSizeRead)} (${formatNumber(uncachedRemoteCount)})`,
179
+ ];
180
+
181
+ const resultItems = [
182
+ `${formatTime(stats.timeToFirstMatch)} until first result`,
183
+ `${formatNumber(stats.matchCount)} results / ${limitURL.value} limit`,
184
+ ];
185
+
186
+ const doneItems = [
187
+ `${formatTime(stats.totalSearchTime)} total search time`,
188
+ ];
189
+
190
+ let hasErrors = stats.fileErrors.length > 0 || stats.blockErrors.length > 0;
191
+
192
+ return (
193
+ <div className={css.hbox(0).alignItems("stretch")}>
194
+ {renderStage("Files", fileItems, 290, true, false)}
195
+ {renderStage("Indexes", indexItems, 250, false, false)}
196
+ {renderStage("Blocks", blockItems, 210, false, false)}
197
+ {renderStage("Cache", cacheItems, 160, false, false)}
198
+ {renderStage("Results", resultItems, 120, false, false)}
199
+ {renderStage("Done", doneItems, undefined, false, !hasErrors)}
200
+ {(() => {
201
+ if (!hasErrors) return undefined;
202
+ let errorItems: preact.ComponentChild[] = [];
203
+ if (stats.fileErrors.length > 0) {
204
+ errorItems.push(<div title={stats.fileErrors.join("\n")}>{stats.fileErrors.length} files failed</div>);
205
+ }
206
+ if (stats.blockErrors.length > 0) {
207
+ errorItems.push(<div title={stats.blockErrors.join("\n")}>{stats.blockErrors.length} blocks failed</div>);
208
+ }
209
+ return renderStage("Errors", errorItems, 0, false, true);
210
+ })()}
211
+ </div>
212
+ );
213
+ }
214
+
215
+ renderPendingLogWarnings() {
216
+ const paths = this.getPaths();
217
+ const now = Date.now();
218
+ const threshold = now - (2 * PUBLIC_MOVE_THRESHOLD);
219
+
220
+ const machineWarnings: Record<string, { machineId: string; oldestTime: number }> = {};
221
+
222
+ for (const path of paths) {
223
+ if (path.logCount !== undefined) continue;
224
+
225
+ if (path.startTime >= threshold) continue;
226
+
227
+ if (!path.machineId) continue;
228
+
229
+ const existing = machineWarnings[path.machineId];
230
+ if (!existing || path.startTime < existing.oldestTime) {
231
+ machineWarnings[path.machineId] = {
232
+ machineId: path.machineId,
233
+ oldestTime: path.startTime,
234
+ };
235
+ }
236
+ }
237
+
238
+ const warnings = Object.values(machineWarnings);
239
+ if (warnings.length === 0) return undefined;
240
+
241
+ return (
242
+ <div className={css.vbox(10).pad2(10).hsl(40, 50, 50).colorhsl(60, 50, 100)}>
243
+ <div className={css.fontWeight(600)}>Machines not moving logs to remote storage:</div>
244
+ {warnings.map((warning) => (
245
+ <div key={warning.machineId} className={css.hbox(10)}>
246
+ <MachineThreadInfo machineId={warning.machineId} />
247
+ <span>is not moving logs to remote storage. Logs are {formatTime(now - warning.oldestTime)} old</span>
248
+ </div>
249
+ ))}
250
+ </div>
251
+ );
252
+ }
253
+
254
+ async loadPaths() {
255
+ if (savedPathsURL.value) {
256
+ return;
257
+ }
258
+
259
+ this.state.loadingPaths = true;
260
+
261
+ let loggers = await getLoggers2Async();
262
+ let selectedLoggers: typeof loggers.logLogs[] = [];
263
+ Querysub.localRead(() => {
264
+ if (this.state.enableLogs) selectedLoggers.push(loggers.logLogs);
265
+ if (this.state.enableInfos) selectedLoggers.push(loggers.infoLogs);
266
+ if (this.state.enableWarnings) selectedLoggers.push(loggers.warnLogs);
267
+ if (this.state.enableErrors) selectedLoggers.push(loggers.errorLogs);
268
+ });
269
+
270
+ if (selectedLoggers.length === 0) {
271
+ console.error("No log sources selected");
272
+ Querysub.commitLocal(() => {
273
+ this.state.loadingPaths = false;
274
+ });
275
+ return;
276
+ }
277
+
278
+ let allPaths: TimeFilePathWithSize[] = [];
279
+ let range = Querysub.localRead(() => getTimeRange());
280
+
281
+ for (let logger of selectedLoggers) {
282
+ let paths = await logger.clientGetPaths({
283
+ startTime: range.startTime,
284
+ endTime: range.endTime,
285
+ });
286
+ allPaths.push(...paths);
287
+ }
288
+ sort(allPaths, x => -x.startTime);
289
+
290
+ Querysub.commitLocal(() => {
291
+ this.state.paths = allPaths;
292
+ this.state.loadingPaths = false;
293
+ });
294
+ }
295
+
296
+
297
+ search = throttleFunction(200, async () => {
298
+ Querysub.commitLocal(() => {
299
+ this.state.searching = true;
300
+ this.state.results = [];
301
+ this.state.stats = undefined;
302
+ this.state.searchingLogs = false;
303
+ this.state.searchingInfos = false;
304
+ this.state.searchingWarnings = false;
305
+ this.state.searchingErrors = false;
306
+ });
307
+
308
+ let startTime = Date.now();
309
+
310
+ let hasPaths = Querysub.localRead(() => this.getPaths().length > 0);
311
+ let getFilesTime = 0;
312
+ if (readLiveData.value || !hasPaths) {
313
+ let startTime = Date.now();
314
+ await this.loadPaths();
315
+ getFilesTime = Date.now() - startTime;
316
+ }
317
+
318
+ let loggers = await getLoggers2Async();
319
+
320
+ let selectedLoggers: typeof loggers.logLogs[] = [];
321
+ Querysub.localRead(() => {
322
+ if (this.state.enableLogs) {
323
+ selectedLoggers.push(loggers.logLogs);
324
+ this.state.searchingLogs = true;
325
+ }
326
+ if (this.state.enableInfos) {
327
+ selectedLoggers.push(loggers.infoLogs);
328
+ this.state.searchingInfos = true;
329
+ }
330
+ if (this.state.enableWarnings) {
331
+ selectedLoggers.push(loggers.warnLogs);
332
+ this.state.searchingWarnings = true;
333
+ }
334
+ if (this.state.enableErrors) {
335
+ selectedLoggers.push(loggers.errorLogs);
336
+ this.state.searchingErrors = true;
337
+ }
338
+ });
339
+
340
+ if (selectedLoggers.length === 0) {
341
+ console.error("No log sources selected");
342
+ Querysub.commitLocal(() => {
343
+ this.state.searching = false;
344
+ });
345
+ return;
346
+ }
347
+
348
+ let searchBuffer = Querysub.localRead(() => Buffer.from(searchText.value, "utf8"));
349
+ let results: LogDatum[] = [];
350
+ let stats: IndexedLogResults = {
351
+ matchCount: 0,
352
+ reads: [],
353
+ localFilesSearched: 0,
354
+ backblazeFilesSearched: 0,
355
+ totalBlockCount: 0,
356
+ blockCheckedCount: 0,
357
+ blocksCheckedCompressedSize: 0,
358
+ blocksCheckedDecompressedSize: 0,
359
+ blockErrors: [],
360
+ fileErrors: [],
361
+ indexesSearched: 0,
362
+ indexSize: 0,
363
+ timeToFirstMatch: 0,
364
+ fileFindTime: 0,
365
+ indexSearchTime: 0,
366
+ blockSearchTime: 0,
367
+ totalSearchTime: 0,
368
+ };
369
+ stats.fileFindTime += getFilesTime;
370
+
371
+ let paths = Querysub.localRead(() => this.getPaths());
372
+
373
+ let range = Querysub.localRead(() => getTimeRange());
374
+
375
+ let updateResults = throttleFunction(100, () => {
376
+ Querysub.commitLocal(() => {
377
+ this.state.results = results;
378
+ });
379
+ });
380
+
381
+ await Promise.all(selectedLoggers.map(async (logger) => {
382
+ let result = await logger.clientFind({
383
+ params: {
384
+ startTime: range.startTime,
385
+ endTime: range.endTime,
386
+ limit: limitURL.value,
387
+ findBuffer: searchBuffer,
388
+ pathOverrides: paths,
389
+ },
390
+ onResult: (match: LogDatum) => {
391
+ console.log("onResult", match);
392
+ results.push(match);
393
+ void updateResults();
394
+ },
395
+ });
396
+ stats.reads.push(...result.reads);
397
+ stats.matchCount += result.matchCount;
398
+ stats.localFilesSearched += result.localFilesSearched;
399
+ stats.backblazeFilesSearched += result.backblazeFilesSearched;
400
+ stats.totalBlockCount += result.totalBlockCount;
401
+ stats.blockCheckedCount += result.blockCheckedCount;
402
+ stats.blocksCheckedCompressedSize += result.blocksCheckedCompressedSize;
403
+ stats.blocksCheckedDecompressedSize += result.blocksCheckedDecompressedSize;
404
+ stats.indexesSearched += result.indexesSearched;
405
+ stats.indexSize += result.indexSize;
406
+ stats.timeToFirstMatch = Math.min(stats.timeToFirstMatch, result.timeToFirstMatch);
407
+ stats.fileFindTime += result.fileFindTime;
408
+ stats.indexSearchTime += result.indexSearchTime;
409
+ stats.blockSearchTime += result.blockSearchTime;
410
+ stats.totalSearchTime = Date.now() - startTime;
411
+ Querysub.commitLocal(() => {
412
+ this.state.results = results;
413
+ this.state.stats = stats;
414
+ if (logger === loggers.logLogs) this.state.searchingLogs = false;
415
+ if (logger === loggers.infoLogs) this.state.searchingInfos = false;
416
+ if (logger === loggers.warnLogs) this.state.searchingWarnings = false;
417
+ if (logger === loggers.errorLogs) this.state.searchingErrors = false;
418
+ });
419
+ }));
420
+
421
+ stats.totalSearchTime = Date.now() - startTime;
422
+
423
+ Querysub.commitLocal(() => {
424
+ this.state.results = results;
425
+ this.state.searching = false;
426
+ this.state.stats = stats;
427
+ });
428
+ });
429
+
430
+ renderResults() {
431
+ let fieldNames = new Set<string>();
432
+ for (let i = 0; i < Math.min(1000, this.state.results.length); i++) {
433
+ let result = this.state.results[i];
434
+ for (let key of Object.keys(result)) {
435
+ fieldNames.add(key);
436
+ }
437
+ }
438
+
439
+ let selectedFields: string[] = [];
440
+ for (let field of Object.keys(defaultSelectedFields)) {
441
+ if (atomic(selectedFieldsURL.value[field]) === undefined) {
442
+ selectedFields.push(field);
443
+ }
444
+ }
445
+ for (let [key, value] of Object.entries(selectedFieldsURL.value)) {
446
+ if (value && !selectedFields.includes(key)) {
447
+ selectedFields.splice(1, 0, key);
448
+ }
449
+ }
450
+
451
+ let columns: TableType<LogDatum>["columns"] = {};
452
+ columns["log"] = {
453
+ title: "Log",
454
+ formatter: (x, context) => {
455
+ return <Button onClick={() => console.log(context?.row)}>
456
+ Log
457
+ </Button>;
458
+ }
459
+ };
460
+
461
+ for (let field of selectedFields) {
462
+ let column: ColumnType<unknown, LogDatum> = {};
463
+ if (field === "time") {
464
+ column.formatter = (x: unknown) => useRelativeTimeURL.value ? formatDateJSX(Number(x)) : <span title={formatDateTimeDetailed(Number(x))}>{formatDateTime(Number(x))}</span>;
465
+ } else if (field === "__machineId") {
466
+ column.formatter = (x: unknown, context) => {
467
+ if (!context?.row || !context.row.__machineId) return <ObjectDisplay value={x} />;
468
+ return <MachineThreadInfo machineId={context.row.__machineId} threadId={context.row.__threadId || undefined} />;
469
+ };
470
+ }
471
+ if (!column.formatter) {
472
+ column.formatter = (x: unknown) => <ObjectDisplay value={x} />;
473
+ }
474
+ columns[field] = column;
475
+ }
476
+
477
+ columns["fields"] = {
478
+ title: "Fields",
479
+ formatter: (x, context) => {
480
+ if (!context?.row) return undefined;
481
+ let datum = context.row;
482
+ let fields = Object.keys(datum);
483
+ fields = fields.filter(x => !(x in columns) && !x.startsWith("_"));
484
+ return <div className={css.hbox(4, 4).wrap}>
485
+ {fields.map(x => <div
486
+ key={x}
487
+ className={css.pad2(4, 0).hsl(0, 0, 80).button}
488
+ onClick={() => {
489
+ let newValues = { ...selectedFieldsURL.value };
490
+ newValues[x] = true;
491
+ selectedFieldsURL.value = newValues;
492
+ }}
493
+ >
494
+ {x}
495
+ </div>)}
496
+ </div>;
497
+ }
498
+ };
499
+
500
+ return <>
501
+ <div className={css.hbox(10)}>
502
+ <InputPicker
503
+ label={<div className={css.hbox(10)}>
504
+ <div className={css.flexShrink0}>
505
+ Selected Fields
506
+ </div>
507
+ <Button onClick={() => {
508
+ let newValues = { ...selectedFieldsURL.value };
509
+ for (let key of Object.keys(newValues)) {
510
+ newValues[key] = key in defaultSelectedFields;
511
+ }
512
+ selectedFieldsURL.value = newValues;
513
+ }}>
514
+ Reset
515
+ </Button>
516
+ </div>}
517
+ picked={selectedFields}
518
+ options={Array.from(fieldNames).map(x => ({ value: x, label: x }))}
519
+ addPicked={x => {
520
+ let newValues = { ...selectedFieldsURL.value };
521
+ newValues[x] = true;
522
+ selectedFieldsURL.value = newValues;
523
+ }}
524
+ removePicked={x => {
525
+ let newValues = { ...selectedFieldsURL.value };
526
+ newValues[x] = false;
527
+ selectedFieldsURL.value = newValues;
528
+ }}
529
+ />
530
+ <InputLabelURL
531
+ label="Use Relative Time"
532
+ checkbox
533
+ url={useRelativeTimeURL}
534
+ />
535
+ </div>
536
+
537
+ {this.state.results.length === 0 && !this.state.searching && <div className={css.hsl(40, 50, 50).colorhsl(60, 50, 100).boldStyle.pad2(10).ellipsis}>
538
+ No logs matched, try adjusting your search or time range.
539
+ </div>}
540
+
541
+ <Table
542
+ rows={this.state.results}
543
+ columns={columns}
544
+ lineLimit={4}
545
+ initialLimit={10}
546
+ characterLimit={400}
547
+ getRowAttributes={row => {
548
+ let hue = -1;
549
+ if (row.__LOG_TYPE === "warn") hue = 40;
550
+ if (row.__LOG_TYPE === "error") hue = 0;
551
+ if (row.__LOG_TYPE === "info") hue = 200;
552
+ return {
553
+ className: hue !== -1 && css.hsl(hue, 40, 50).colorhsl(hue, 0, 100) || undefined,
554
+ };
555
+ }}
556
+ />
557
+ </>;
558
+ }
559
+
560
+ render() {
561
+ return (
562
+ <div className={css.vbox(20).pad2(20).fillBoth}>
563
+ <div className={css.hbox(20)}>
564
+ <div>Log Viewer 3</div>
565
+ <InputLabelURL
566
+ checkbox
567
+ label="Always Read Live Data"
568
+ url={readLiveData}
569
+ />
570
+ <InputLabelURL
571
+ checkbox
572
+ label="Exclude Pending Results"
573
+ url={excludePendingResults}
574
+ />
575
+ <InputLabelURL
576
+ label="Limit"
577
+ number
578
+ url={limitURL}
579
+ />
580
+ </div>
581
+
582
+ <div className={css.hbox(10)}>
583
+ <Button
584
+ onClick={() => {
585
+ this.state.enableLogs = !this.state.enableLogs;
586
+ this.clearPaths();
587
+ }}
588
+ hue={this.state.enableLogs && 210 || undefined}
589
+ className={this.state.searchingLogs && css.opacity(0.5) || undefined}
590
+ >
591
+ Logs
592
+ </Button>
593
+ <Button
594
+ onClick={() => {
595
+ this.state.enableInfos = !this.state.enableInfos;
596
+ this.clearPaths();
597
+ }}
598
+ hue={this.state.enableInfos && 200 || undefined}
599
+ className={this.state.searchingInfos && css.opacity(0.5) || undefined}
600
+ >
601
+ Infos
602
+ </Button>
603
+ <Button
604
+ onClick={() => {
605
+ this.state.enableWarnings = !this.state.enableWarnings;
606
+ this.clearPaths();
607
+ }}
608
+ hue={this.state.enableWarnings && 40 || undefined}
609
+ className={this.state.searchingWarnings && css.opacity(0.5) || undefined}
610
+ >
611
+ Warnings
612
+ </Button>
613
+ <Button
614
+ onClick={() => {
615
+ this.state.enableErrors = !this.state.enableErrors;
616
+ this.clearPaths();
617
+ }}
618
+ hue={this.state.enableErrors ? 0 : undefined}
619
+ className={this.state.searchingErrors && css.opacity(0.5) || undefined}
620
+ >
621
+ Errors
622
+ </Button>
623
+ <TimeRangeSelector />
624
+ </div>
625
+
626
+ <div className={css.hbox(10).fillWidth}>
627
+ <InputLabelURL
628
+ flavor="large"
629
+ fillWidth
630
+ focusOnMount
631
+ url={searchText}
632
+ onChangeValue={(value) => {
633
+ if (value.length >= MIN_AUTO_SEARCH_LENGTH) {
634
+ void this.search();
635
+ }
636
+ }}
637
+ onKeyDown={(e) => {
638
+ if (e.key === "Enter") {
639
+ searchText.value = e.currentTarget.value;
640
+ void this.search();
641
+ }
642
+ }}
643
+ />
644
+ {!savedPathsURL.value && (
645
+ <Button
646
+ flavor="large"
647
+ onClick={() => void this.loadPaths()}
648
+ hue={120}
649
+ >
650
+ Preview Files
651
+ </Button>
652
+ )}
653
+ {savedPathsURL.value && (
654
+ <Button
655
+ flavor="large"
656
+ onClick={() => this.clearFrozenPaths()}
657
+ hue={0}
658
+ >
659
+ Clear Frozen Files ({this.getPaths().length})
660
+ </Button>
661
+ )}
662
+ <Button
663
+ flavor="large"
664
+ hotkeys={["enter"]}
665
+ showHotkeys
666
+ onClick={() => void this.search()}
667
+ hue={210}
668
+ >
669
+ Search
670
+ </Button>
671
+ </div>
672
+
673
+ {this.state.loadingPaths && <div>Loading paths...</div>}
674
+ {this.state.searching && <div>Searching...</div>}
675
+
676
+ {this.getPaths().length > 0 && (
677
+ <div className={css.hbox(10).fillWidth}>
678
+ <FilePathSelector
679
+ paths={this.getPaths()}
680
+ onChange={(paths) => {
681
+ this.state.paths = paths;
682
+ }}
683
+ />
684
+ <Button
685
+ onClick={() => this.freezePaths()}
686
+ hue={180}
687
+ >
688
+ Freeze Files ({this.state.paths.length})
689
+ </Button>
690
+ </div>
691
+ )}
692
+
693
+ {this.renderPendingLogWarnings()}
694
+
695
+ {this.renderSearchStats()}
696
+
697
+ {this.renderResults()}
698
+ </div>
699
+ );
700
+ }
701
+ }
702
+