querysub 0.394.0 → 0.396.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.cursorrules +8 -0
- package/package.json +1 -1
- package/src/-a-archives/archivesJSONT.ts +71 -8
- package/src/-c-identity/IdentityController.ts +19 -1
- package/src/0-path-value-core/pathValueCore.ts +26 -7
- package/src/5-diagnostics/GenericFormat.tsx +1 -1
- package/src/deployManager/components/MachinesListPage.tsx +1 -2
- package/src/diagnostics/logs/IndexedLogs/BufferIndex.ts +12 -3
- package/src/diagnostics/logs/IndexedLogs/BufferIndexHelpers.ts +8 -3
- package/src/diagnostics/logs/IndexedLogs/BufferUnitIndex.ts +24 -9
- package/src/diagnostics/logs/IndexedLogs/BufferUnitSet.ts +0 -1
- package/src/diagnostics/logs/IndexedLogs/FindProgressTracker.ts +21 -5
- package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts +10 -4
- package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx +94 -124
- package/src/diagnostics/logs/IndexedLogs/RenderSearchStats.tsx +127 -0
- package/src/diagnostics/logs/IndexedLogs/bufferSearchFindMatcher.ts +4 -0
- package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts +1 -1
- package/src/diagnostics/logs/TimeRangeSelector.tsx +11 -2
- package/src/diagnostics/logs/errorNotifications2/ErrorNotificationPage.tsx +1 -4
- package/src/diagnostics/logs/errorNotifications2/errorNotifications.ts +1 -1
- package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleEntryEditor.tsx +358 -0
- package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleEntryReadMode.tsx +149 -0
- package/src/diagnostics/logs/lifeCycleAnalysis/LifeCyclePage.tsx +274 -0
- package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleRenderer.tsx +291 -0
- package/src/diagnostics/logs/lifeCycleAnalysis/NestedLifeCycleInfo.tsx +151 -0
- package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycleMatching.ts +49 -0
- package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycleSearch.tsx +580 -0
- package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx +128 -90
- package/src/diagnostics/managementPages.tsx +17 -1
- package/src/diagnostics/misc-pages/ArchiveViewer.tsx +4 -3
- package/src/diagnostics/misc-pages/ArchiveViewerTree.tsx +6 -3
- package/src/functional/{limitProcessing.ts → throttleProcessing.ts} +1 -1
- package/src/library-components/StartEllipsis.tsx +13 -0
- package/src/misc.ts +4 -0
- package/tempnotes.txt +0 -0
- package/src/diagnostics/logs/lifeCycleAnalysis/test.wat +0 -106
- package/src/diagnostics/logs/lifeCycleAnalysis/test.wat.d.ts +0 -2
- package/src/diagnostics/logs/lifeCycleAnalysis/testHoist.ts +0 -5
|
@@ -5,7 +5,7 @@ import { t } from "../../../2-proxy/schema2";
|
|
|
5
5
|
import { Button } from "../../../library-components/Button";
|
|
6
6
|
import { InputLabel, InputLabelURL } from "../../../library-components/InputLabel";
|
|
7
7
|
import { getLoggers2Async, LogDatum } from "../diskLogger";
|
|
8
|
-
import { list, timeInDay, keyByArray, sort, throttleFunction } from "socket-function/src/misc";
|
|
8
|
+
import { list, timeInDay, timeInHour, keyByArray, sort, throttleFunction } from "socket-function/src/misc";
|
|
9
9
|
import { formatDateTime, formatDateTimeDetailed, formatNumber, formatTime, formatPercent } from "socket-function/src/formatting/format";
|
|
10
10
|
import { IndexedLogs, TimeFilePathWithSize } from "./IndexedLogs";
|
|
11
11
|
import { Querysub } from "../../../4-querysub/QuerysubController";
|
|
@@ -27,6 +27,15 @@ import { TimeFilePath } from "./TimeFileTree";
|
|
|
27
27
|
import { isPublic } from "../../../config";
|
|
28
28
|
import { Zip } from "../../../zip";
|
|
29
29
|
import { readProductionLogsURL, searchTextURL } from "./LogViewerParams";
|
|
30
|
+
import { RenderSearchStats } from "./RenderSearchStats";
|
|
31
|
+
import { LifeCyclesController, LifeCycle, LifeCycleEntry } from "../lifeCycleAnalysis/lifeCycles";
|
|
32
|
+
import { getLifecycleMatchesForDatum } from "../lifeCycleAnalysis/lifeCycleMatching";
|
|
33
|
+
import { lifecycleIdURL, additionalSearchURL } from "../lifeCycleAnalysis/LifeCyclePage";
|
|
34
|
+
import { ATag } from "../../../library-components/ATag";
|
|
35
|
+
import { SocketFunction } from "socket-function/SocketFunction";
|
|
36
|
+
import { managementPageURL } from "../../managementPages";
|
|
37
|
+
import { startTimeParam, endTimeParam } from "../TimeRangeSelector";
|
|
38
|
+
import { formatSearchString } from "./LogViewerParams";
|
|
30
39
|
|
|
31
40
|
let excludePendingResults = new URLParam("excludePendingResults", false);
|
|
32
41
|
let limitURL = new URLParam("limit", 100);
|
|
@@ -34,6 +43,7 @@ let limitURL = new URLParam("limit", 100);
|
|
|
34
43
|
let savedPathsURL = new URLParam("savedPaths", "");
|
|
35
44
|
let selectedFieldsURL = new URLParam("selectedFields", {} as Record<string, boolean>);
|
|
36
45
|
let useRelativeTimeURL = new URLParam("useRelativeTime", true);
|
|
46
|
+
let showLifecycleColumnURL = new URLParam("showlifecycle", true);
|
|
37
47
|
|
|
38
48
|
const defaultSelectedFields = {
|
|
39
49
|
param0: true,
|
|
@@ -64,11 +74,16 @@ export class LogViewer3 extends qreact.Component {
|
|
|
64
74
|
hasSearched: t.boolean(false),
|
|
65
75
|
forceMoveStartTime: t.atomic<number | undefined>(undefined),
|
|
66
76
|
forceMoveEndTime: t.atomic<number | undefined>(undefined),
|
|
77
|
+
showLifecycleColumn: t.boolean(true),
|
|
67
78
|
});
|
|
68
79
|
|
|
80
|
+
lifecycleController = LifeCyclesController(SocketFunction.browserNodeId());
|
|
81
|
+
|
|
69
82
|
private searchSequenceNumber = 0;
|
|
70
83
|
|
|
71
84
|
componentDidMount(): void {
|
|
85
|
+
this.state.showLifecycleColumn = showLifecycleColumnURL.value;
|
|
86
|
+
|
|
72
87
|
if (searchTextURL.value) {
|
|
73
88
|
void this.search();
|
|
74
89
|
} else {
|
|
@@ -285,6 +300,7 @@ export class LogViewer3 extends qreact.Component {
|
|
|
285
300
|
pathOverrides: paths,
|
|
286
301
|
only: excludePendingResults.value ? "local" : undefined,
|
|
287
302
|
forceReadProduction: readProductionLogsURL.value,
|
|
303
|
+
searchFromStart: range.searchFromStart,
|
|
288
304
|
},
|
|
289
305
|
onResult: (match: LogDatum) => {
|
|
290
306
|
results.push(match);
|
|
@@ -320,123 +336,6 @@ export class LogViewer3 extends qreact.Component {
|
|
|
320
336
|
});
|
|
321
337
|
};
|
|
322
338
|
|
|
323
|
-
renderSearchStats() {
|
|
324
|
-
const stats = this.state.stats;
|
|
325
|
-
if (!stats) return undefined;
|
|
326
|
-
|
|
327
|
-
let totalSizeRead = 0;
|
|
328
|
-
let cachedSize = 0;
|
|
329
|
-
let cachedCount = 0;
|
|
330
|
-
let uncachedSize = 0;
|
|
331
|
-
let uncachedCount = 0;
|
|
332
|
-
let uncachedRemoteSize = 0;
|
|
333
|
-
let uncachedRemoteCount = 0;
|
|
334
|
-
let totalSize = 0;
|
|
335
|
-
let remoteTotalSize = 0;
|
|
336
|
-
let localTotalSize = 0;
|
|
337
|
-
|
|
338
|
-
for (let read of stats.reads) {
|
|
339
|
-
totalSizeRead += read.size;
|
|
340
|
-
if (read.cached) {
|
|
341
|
-
cachedSize += read.size;
|
|
342
|
-
cachedCount += read.count;
|
|
343
|
-
} else {
|
|
344
|
-
uncachedSize += read.size;
|
|
345
|
-
uncachedCount += read.count;
|
|
346
|
-
totalSize += read.totalSize;
|
|
347
|
-
if (read.remote) {
|
|
348
|
-
remoteTotalSize += read.totalSize;
|
|
349
|
-
} else {
|
|
350
|
-
localTotalSize += read.totalSize;
|
|
351
|
-
}
|
|
352
|
-
}
|
|
353
|
-
if (read.remote && !read.cached) {
|
|
354
|
-
uncachedRemoteSize += read.size;
|
|
355
|
-
uncachedRemoteCount += read.count;
|
|
356
|
-
}
|
|
357
|
-
}
|
|
358
|
-
|
|
359
|
-
const renderStage = (title: string, items: preact.ComponentChild[], hue: number | undefined, isFirst: boolean, isLast: boolean) => {
|
|
360
|
-
const triangleWidth = 20;
|
|
361
|
-
|
|
362
|
-
let color = hue !== undefined ? css.hsl(hue, 70, 80) : css.hsl(0, 0, 100);
|
|
363
|
-
let colorhsl = hue !== undefined ? css.hslcolor(hue, 80, 30) : css;
|
|
364
|
-
|
|
365
|
-
return (
|
|
366
|
-
<div className={css.vbox(2).relative.minWidth(200).paddingTop(8).paddingBottom(8).paddingRight(10).paddingLeft(isFirst ? 10 : 10 + triangleWidth) + color}>
|
|
367
|
-
<div className={css.fontWeight(600) + colorhsl}>{title}</div>
|
|
368
|
-
{items.map((item, i) => (
|
|
369
|
-
<div key={i}>{item}</div>
|
|
370
|
-
))}
|
|
371
|
-
{!isLast && hue !== undefined && (
|
|
372
|
-
<svg width={triangleWidth} height="100" viewBox="0 0 20 100" preserveAspectRatio="none" className={css.absolute.right(-triangleWidth).top(0).fillHeight.zIndex(1)}>
|
|
373
|
-
<polygon points="0,0 20,50 0,100" fill={`hsl(${hue}, 70%, 80%)`} />
|
|
374
|
-
</svg>
|
|
375
|
-
)}
|
|
376
|
-
</div>
|
|
377
|
-
);
|
|
378
|
-
};
|
|
379
|
-
|
|
380
|
-
const fileItems = [
|
|
381
|
-
`${formatTime(stats.fileFindTime)} | ${formatNumber(stats.localFilesSearched + stats.backblazeFilesSearched)} / ${formatNumber(stats.totalLocalFiles + stats.totalBackblazeFiles)} | ${formatNumber(totalSize)}B`,
|
|
382
|
-
`remote ${formatNumber(stats.backblazeFilesSearched)} / ${formatNumber(stats.totalBackblazeFiles)} | ${formatNumber(remoteTotalSize)}B`,
|
|
383
|
-
`pending ${formatNumber(stats.localFilesSearched)} / ${formatNumber(stats.totalLocalFiles)} | ${formatNumber(localTotalSize)}B`,
|
|
384
|
-
];
|
|
385
|
-
|
|
386
|
-
const totalIndexesSearched = stats.remoteIndexesSearched + stats.localIndexesSearched;
|
|
387
|
-
const totalIndexSize = stats.remoteIndexSize + stats.localIndexSize;
|
|
388
|
-
const indexItems = [
|
|
389
|
-
`${formatTime(stats.indexSearchTime)} | ${formatNumber(totalIndexesSearched)} | ${formatNumber(totalIndexSize)}B | ${formatNumber(totalSize / totalIndexSize)}X`,
|
|
390
|
-
`remote ${formatNumber(stats.remoteIndexesSearched)} | ${formatNumber(stats.remoteIndexSize)}B | ${formatNumber(remoteTotalSize / stats.remoteIndexSize)}X`,
|
|
391
|
-
`local ${formatNumber(stats.localIndexesSearched)} | ${formatNumber(stats.localIndexSize)}B | ${formatNumber(localTotalSize / stats.localIndexSize)}X`,
|
|
392
|
-
];
|
|
393
|
-
|
|
394
|
-
const blockItems = [
|
|
395
|
-
<div title={`Scanned size = ${formatNumber(stats.blocksCheckedCompressedSize)}B, Decompressed size = ${formatNumber(stats.blocksCheckedDecompressedSize)}B, Compression ratio = ${formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X`}>{formatTime(stats.blockSearchTime)} | {formatNumber(stats.blockCheckedCount)} | {formatNumber(stats.blocksCheckedCompressedSize)}B | {formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X</div>,
|
|
396
|
-
`total scanned ${formatNumber(stats.totalBlockCount)} (${formatPercent(stats.blockCheckedCount / stats.totalBlockCount)})`,
|
|
397
|
-
`remote ${formatNumber(stats.remoteBlockCheckedCount)} / ${formatNumber(stats.remoteBlockCount)} (${formatPercent(stats.remoteBlockCheckedCount / stats.remoteBlockCount)}) | local ${formatNumber(stats.localBlockCheckedCount)} / ${formatNumber(stats.localBlockCount)} (${formatPercent(stats.localBlockCheckedCount / stats.localBlockCount)})`,
|
|
398
|
-
];
|
|
399
|
-
let cacheItems = [
|
|
400
|
-
`disk read ${formatNumber(uncachedSize)}B (${formatNumber(uncachedCount)})`,
|
|
401
|
-
`remote ${formatPercent(uncachedRemoteSize / totalSizeRead)} (${formatNumber(uncachedRemoteCount)})`,
|
|
402
|
-
`cached ${formatNumber(cachedSize)} (${formatNumber(cachedCount)})`,
|
|
403
|
-
];
|
|
404
|
-
|
|
405
|
-
const resultItems = [
|
|
406
|
-
`${formatTime(stats.timeToFirstMatch)} until first result`,
|
|
407
|
-
`${formatNumber(stats.matchCount)} results / ${limitURL.value} limit`,
|
|
408
|
-
];
|
|
409
|
-
|
|
410
|
-
const doneItems = [
|
|
411
|
-
`${formatTime(stats.totalSearchTime)} total search time`,
|
|
412
|
-
];
|
|
413
|
-
|
|
414
|
-
let hasErrors = stats.fileErrors.length > 0 || stats.blockErrors.length > 0;
|
|
415
|
-
|
|
416
|
-
return (
|
|
417
|
-
<div className={css.hbox(0).alignItems("stretch")}>
|
|
418
|
-
{renderStage("Files", fileItems, 290, true, false)}
|
|
419
|
-
{renderStage("Indexes", indexItems, 250, false, false)}
|
|
420
|
-
{renderStage("Blocks", blockItems, 210, false, false)}
|
|
421
|
-
{renderStage("Reads", cacheItems, 160, false, false)}
|
|
422
|
-
{renderStage("Results", resultItems, 120, false, this.state.searching && !hasErrors)}
|
|
423
|
-
{!this.state.searching && renderStage("Done", doneItems, undefined, false, !hasErrors)}
|
|
424
|
-
{(() => {
|
|
425
|
-
if (!hasErrors) return undefined;
|
|
426
|
-
let errorItems: preact.ComponentChild[] = [];
|
|
427
|
-
if (stats.fileErrors.length > 0) {
|
|
428
|
-
const errorTitle = stats.fileErrors.map(e => `${e.path}:\n${e.error}`).join("\n\n");
|
|
429
|
-
errorItems.push(<div title={errorTitle}>{stats.fileErrors.length} files failed</div>);
|
|
430
|
-
}
|
|
431
|
-
if (stats.blockErrors.length > 0) {
|
|
432
|
-
errorItems.push(<div title={stats.blockErrors.join("\n")}>{stats.blockErrors.length} blocks failed</div>);
|
|
433
|
-
}
|
|
434
|
-
return renderStage("Errors", errorItems, 0, false, true);
|
|
435
|
-
})()}
|
|
436
|
-
</div>
|
|
437
|
-
);
|
|
438
|
-
}
|
|
439
|
-
|
|
440
339
|
renderForceMoveLogs() {
|
|
441
340
|
return <><Button
|
|
442
341
|
onClick={async () => {
|
|
@@ -627,6 +526,64 @@ export class LogViewer3 extends qreact.Component {
|
|
|
627
526
|
}
|
|
628
527
|
};
|
|
629
528
|
|
|
529
|
+
if (this.state.showLifecycleColumn) {
|
|
530
|
+
const lifecycles = this.lifecycleController.getLifeCycles();
|
|
531
|
+
if (lifecycles && lifecycles.length > 0) {
|
|
532
|
+
columns["lifecycles"] = {
|
|
533
|
+
title: "Lifecycles",
|
|
534
|
+
formatter: (x, context) => {
|
|
535
|
+
if (!context?.row) return undefined;
|
|
536
|
+
let datum = context.row;
|
|
537
|
+
let matches = getLifecycleMatchesForDatum(lifecycles, datum);
|
|
538
|
+
if (matches.length === 0) return undefined;
|
|
539
|
+
|
|
540
|
+
let lifecycleId = matches[0].lifecycle.id;
|
|
541
|
+
let lifecycleTitle = matches[0].lifecycle.title;
|
|
542
|
+
|
|
543
|
+
let firstMatch = matches[0];
|
|
544
|
+
let searchFilter = formatSearchString(firstMatch.datum);
|
|
545
|
+
|
|
546
|
+
let minTime = Infinity;
|
|
547
|
+
let maxTime = -Infinity;
|
|
548
|
+
for (let resultDatum of this.state.results) {
|
|
549
|
+
let resultMatches = getLifecycleMatchesForDatum(lifecycles, resultDatum);
|
|
550
|
+
if (resultMatches.some(m => m.lifecycle.id === lifecycleId)) {
|
|
551
|
+
if (resultDatum.time < minTime) minTime = resultDatum.time;
|
|
552
|
+
if (resultDatum.time > maxTime) maxTime = resultDatum.time;
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
return <div className={css.hbox(4).wrap}>
|
|
557
|
+
{matches.map((match, idx) => (
|
|
558
|
+
<div key={idx} className={css.hbox(8)}>
|
|
559
|
+
<ATag values={[
|
|
560
|
+
managementPageURL.getOverride("LifeCyclePage"),
|
|
561
|
+
lifecycleIdURL.getOverride(match.lifecycle.id),
|
|
562
|
+
additionalSearchURL.getOverride(formatSearchString(match.datum)),
|
|
563
|
+
startTimeParam.getOverride(datum.time - timeInHour),
|
|
564
|
+
endTimeParam.getOverride(datum.time + timeInHour),
|
|
565
|
+
]}>
|
|
566
|
+
View {JSON.stringify(match.lifecycle.title)}
|
|
567
|
+
</ATag>
|
|
568
|
+
</div>
|
|
569
|
+
))}
|
|
570
|
+
<ATag values={[
|
|
571
|
+
managementPageURL.getOverride("LifeCyclePage"),
|
|
572
|
+
lifecycleIdURL.getOverride(firstMatch.lifecycle.id),
|
|
573
|
+
additionalSearchURL.getOverride(searchFilter),
|
|
574
|
+
startTimeParam.getOverride(minTime - timeInHour),
|
|
575
|
+
endTimeParam.getOverride(maxTime + timeInHour),
|
|
576
|
+
]}>
|
|
577
|
+
<Button hue={280} className={css.boldStyle}>
|
|
578
|
+
View All
|
|
579
|
+
</Button>
|
|
580
|
+
</ATag>
|
|
581
|
+
</div>;
|
|
582
|
+
}
|
|
583
|
+
};
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
|
|
630
587
|
return <>
|
|
631
588
|
<div className={css.hbox(10)}>
|
|
632
589
|
<InputPicker
|
|
@@ -662,6 +619,14 @@ export class LogViewer3 extends qreact.Component {
|
|
|
662
619
|
checkbox
|
|
663
620
|
url={useRelativeTimeURL}
|
|
664
621
|
/>
|
|
622
|
+
<InputLabelURL
|
|
623
|
+
label="Show Lifecycles"
|
|
624
|
+
checkbox
|
|
625
|
+
url={showLifecycleColumnURL}
|
|
626
|
+
onChange={(e) => {
|
|
627
|
+
this.state.showLifecycleColumn = e.currentTarget.checked;
|
|
628
|
+
}}
|
|
629
|
+
/>
|
|
665
630
|
</div>
|
|
666
631
|
|
|
667
632
|
{this.state.results.length === 0 && !this.state.searching && this.state.hasSearched && <div className={css.hsl(40, 50, 50).colorhsl(60, 50, 100).boldStyle.pad2(10).ellipsis}>
|
|
@@ -674,13 +639,14 @@ export class LogViewer3 extends qreact.Component {
|
|
|
674
639
|
lineLimit={4}
|
|
675
640
|
initialLimit={10}
|
|
676
641
|
characterLimit={400}
|
|
642
|
+
cellClass={css.maxWidth("20vw")}
|
|
677
643
|
getRowAttributes={row => {
|
|
678
644
|
let hue = -1;
|
|
679
645
|
if (row.__LOG_TYPE === "warn") hue = 40;
|
|
680
646
|
if (row.__LOG_TYPE === "error") hue = 0;
|
|
681
647
|
if (row.__LOG_TYPE === "info") hue = 200;
|
|
682
648
|
return {
|
|
683
|
-
className: hue !== -1 && css.hsl(hue, 40,
|
|
649
|
+
className: hue !== -1 && css.hsl(hue, 40, 60).colorhsl(hue, 0, 100) || undefined,
|
|
684
650
|
};
|
|
685
651
|
}}
|
|
686
652
|
/>
|
|
@@ -695,8 +661,8 @@ export class LogViewer3 extends qreact.Component {
|
|
|
695
661
|
checkbox
|
|
696
662
|
label="Exclude Pending Results"
|
|
697
663
|
url={excludePendingResults}
|
|
698
|
-
|
|
699
|
-
if (
|
|
664
|
+
onChange={(e) => {
|
|
665
|
+
if (e.currentTarget.checked) {
|
|
700
666
|
savedPathsURL.value = "";
|
|
701
667
|
void this.loadPaths();
|
|
702
668
|
}
|
|
@@ -706,8 +672,8 @@ export class LogViewer3 extends qreact.Component {
|
|
|
706
672
|
checkbox
|
|
707
673
|
label="Read Production Logs"
|
|
708
674
|
url={readProductionLogsURL}
|
|
709
|
-
|
|
710
|
-
if (
|
|
675
|
+
onChange={(e) => {
|
|
676
|
+
if (e.currentTarget.checked) {
|
|
711
677
|
savedPathsURL.value = "";
|
|
712
678
|
void this.loadPaths();
|
|
713
679
|
}
|
|
@@ -888,7 +854,11 @@ export class LogViewer3 extends qreact.Component {
|
|
|
888
854
|
|
|
889
855
|
{this.renderPendingLogWarnings()}
|
|
890
856
|
|
|
891
|
-
|
|
857
|
+
<RenderSearchStats
|
|
858
|
+
searching={this.state.searching}
|
|
859
|
+
limit={limitURL.value}
|
|
860
|
+
stats={this.state.stats}
|
|
861
|
+
/>
|
|
892
862
|
|
|
893
863
|
{this.renderResults()}
|
|
894
864
|
</div>
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import { formatTime, formatNumber, formatPercent } from "socket-function/src/formatting/format";
|
|
2
|
+
import { css } from "typesafecss";
|
|
3
|
+
import { qreact } from "../../../4-dom/qreact";
|
|
4
|
+
import { IndexedLogResults } from "./BufferIndexHelpers";
|
|
5
|
+
|
|
6
|
+
export class RenderSearchStats extends qreact.Component<{
|
|
7
|
+
stats: IndexedLogResults | undefined;
|
|
8
|
+
searching: boolean;
|
|
9
|
+
limit: number;
|
|
10
|
+
}> {
|
|
11
|
+
render() {
|
|
12
|
+
const stats = this.props.stats;
|
|
13
|
+
if (!stats) return undefined;
|
|
14
|
+
|
|
15
|
+
let totalSizeRead = 0;
|
|
16
|
+
let cachedSize = 0;
|
|
17
|
+
let cachedCount = 0;
|
|
18
|
+
let uncachedSize = 0;
|
|
19
|
+
let uncachedCount = 0;
|
|
20
|
+
let uncachedRemoteSize = 0;
|
|
21
|
+
let uncachedRemoteCount = 0;
|
|
22
|
+
let totalSize = 0;
|
|
23
|
+
let remoteTotalSize = 0;
|
|
24
|
+
let localTotalSize = 0;
|
|
25
|
+
|
|
26
|
+
for (let read of stats.reads) {
|
|
27
|
+
totalSizeRead += read.size;
|
|
28
|
+
if (read.cached) {
|
|
29
|
+
cachedSize += read.size;
|
|
30
|
+
cachedCount += read.count;
|
|
31
|
+
} else {
|
|
32
|
+
uncachedSize += read.size;
|
|
33
|
+
uncachedCount += read.count;
|
|
34
|
+
totalSize += read.totalSize;
|
|
35
|
+
if (read.remote) {
|
|
36
|
+
remoteTotalSize += read.totalSize;
|
|
37
|
+
} else {
|
|
38
|
+
localTotalSize += read.totalSize;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
if (read.remote && !read.cached) {
|
|
42
|
+
uncachedRemoteSize += read.size;
|
|
43
|
+
uncachedRemoteCount += read.count;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const renderStage = (title: string, items: preact.ComponentChild[], hue: number | undefined, isFirst: boolean, isLast: boolean) => {
|
|
48
|
+
const triangleWidth = 20;
|
|
49
|
+
|
|
50
|
+
let color = hue !== undefined ? css.hsl(hue, 70, 80) : css.hsl(0, 0, 100);
|
|
51
|
+
let colorhsl = hue !== undefined ? css.hslcolor(hue, 80, 30) : css;
|
|
52
|
+
|
|
53
|
+
return (
|
|
54
|
+
<div className={css.vbox(2).relative.minWidth(200).paddingTop(8).paddingBottom(8).paddingRight(10).paddingLeft(isFirst ? 10 : 10 + triangleWidth) + color}>
|
|
55
|
+
<div className={css.fontWeight(600) + colorhsl}>{title}</div>
|
|
56
|
+
{items.map((item, i) => (
|
|
57
|
+
<div key={i}>{item}</div>
|
|
58
|
+
))}
|
|
59
|
+
{!isLast && hue !== undefined && (
|
|
60
|
+
<svg width={triangleWidth} height="100" viewBox="0 0 20 100" preserveAspectRatio="none" className={css.absolute.right(-triangleWidth).top(0).fillHeight.zIndex(1)}>
|
|
61
|
+
<polygon points="0,0 20,50 0,100" fill={`hsl(${hue}, 70%, 80%)`} />
|
|
62
|
+
</svg>
|
|
63
|
+
)}
|
|
64
|
+
</div>
|
|
65
|
+
);
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
const fileItems = [
|
|
69
|
+
`${formatTime(stats.fileFindTime)} | ${formatNumber(stats.localFilesSearched + stats.backblazeFilesSearched)} / ${formatNumber(stats.totalLocalFiles + stats.totalBackblazeFiles)} | ${formatNumber(totalSize)}B`,
|
|
70
|
+
`remote ${formatNumber(stats.backblazeFilesSearched)} / ${formatNumber(stats.totalBackblazeFiles)} | ${formatNumber(remoteTotalSize)}B | ${formatNumber(stats.backblazeLogsSearched)} logs`,
|
|
71
|
+
`pending ${formatNumber(stats.localFilesSearched)} / ${formatNumber(stats.totalLocalFiles)} | ${formatNumber(localTotalSize)}B`,
|
|
72
|
+
];
|
|
73
|
+
|
|
74
|
+
const totalIndexesSearched = stats.remoteIndexesSearched + stats.localIndexesSearched;
|
|
75
|
+
const totalIndexSize = stats.remoteIndexSize + stats.localIndexSize;
|
|
76
|
+
const indexItems = [
|
|
77
|
+
`${formatTime(stats.indexSearchTime)} | ${formatNumber(totalIndexesSearched)} | ${formatNumber(totalIndexSize)}B | ${formatNumber(totalSize / totalIndexSize)}X`,
|
|
78
|
+
`remote ${formatNumber(stats.remoteIndexesSearched)} | ${formatNumber(stats.remoteIndexSize)}B | ${formatNumber(remoteTotalSize / stats.remoteIndexSize)}X`,
|
|
79
|
+
`local ${formatNumber(stats.localIndexesSearched)} | ${formatNumber(stats.localIndexSize)}B | ${formatNumber(localTotalSize / stats.localIndexSize)}X`,
|
|
80
|
+
];
|
|
81
|
+
|
|
82
|
+
const blockItems = [
|
|
83
|
+
<div title={`Scanned size = ${formatNumber(stats.blocksCheckedCompressedSize)}B, Decompressed size = ${formatNumber(stats.blocksCheckedDecompressedSize)}B, Compression ratio = ${formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X`}>{formatTime(stats.blockSearchTime)} | {formatNumber(stats.blockCheckedCount)} | {formatNumber(stats.blocksCheckedCompressedSize)}B | {formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X</div>,
|
|
84
|
+
`total scanned ${formatNumber(stats.totalBlockCount)} (${formatPercent(stats.blockCheckedCount / stats.totalBlockCount)})`,
|
|
85
|
+
`remote ${formatNumber(stats.remoteBlockCheckedCount)} / ${formatNumber(stats.remoteBlockCount)} (${formatPercent(stats.remoteBlockCheckedCount / stats.remoteBlockCount)}) | local ${formatNumber(stats.localBlockCheckedCount)} / ${formatNumber(stats.localBlockCount)} (${formatPercent(stats.localBlockCheckedCount / stats.localBlockCount)})`,
|
|
86
|
+
];
|
|
87
|
+
let cacheItems = [
|
|
88
|
+
`disk read ${formatNumber(uncachedSize)}B (${formatNumber(uncachedCount)})`,
|
|
89
|
+
`remote ${formatPercent(uncachedRemoteSize / totalSizeRead)} (${formatNumber(uncachedRemoteCount)})`,
|
|
90
|
+
`cached ${formatNumber(cachedSize)} (${formatNumber(cachedCount)})`,
|
|
91
|
+
];
|
|
92
|
+
|
|
93
|
+
const resultItems = [
|
|
94
|
+
`${formatTime(stats.timeToFirstMatch)} until first result`,
|
|
95
|
+
`${formatNumber(stats.matchCount)} results / ${this.props.limit} limit`,
|
|
96
|
+
];
|
|
97
|
+
|
|
98
|
+
const doneItems = [
|
|
99
|
+
`${formatTime(stats.totalSearchTime)} total search time`,
|
|
100
|
+
];
|
|
101
|
+
|
|
102
|
+
let hasErrors = stats.fileErrors.length > 0 || stats.blockErrors.length > 0;
|
|
103
|
+
|
|
104
|
+
return (
|
|
105
|
+
<div className={css.hbox(0).alignItems("stretch")}>
|
|
106
|
+
{renderStage("Files", fileItems, 290, true, false)}
|
|
107
|
+
{renderStage("Indexes", indexItems, 250, false, false)}
|
|
108
|
+
{renderStage("Blocks", blockItems, 210, false, false)}
|
|
109
|
+
{renderStage("Reads", cacheItems, 160, false, false)}
|
|
110
|
+
{renderStage("Results", resultItems, 120, false, this.props.searching && !hasErrors)}
|
|
111
|
+
{!this.props.searching && renderStage("Done", doneItems, undefined, false, !hasErrors)}
|
|
112
|
+
{(() => {
|
|
113
|
+
if (!hasErrors) return undefined;
|
|
114
|
+
let errorItems: preact.ComponentChild[] = [];
|
|
115
|
+
if (stats.fileErrors.length > 0) {
|
|
116
|
+
const errorTitle = stats.fileErrors.map(e => `${e.path}:\n${e.error}`).join("\n\n");
|
|
117
|
+
errorItems.push(<div title={errorTitle}>{stats.fileErrors.length} files failed</div>);
|
|
118
|
+
}
|
|
119
|
+
if (stats.blockErrors.length > 0) {
|
|
120
|
+
errorItems.push(<div title={stats.blockErrors.join("\n")}>{stats.blockErrors.length} blocks failed</div>);
|
|
121
|
+
}
|
|
122
|
+
return renderStage("Errors", errorItems, 0, false, true);
|
|
123
|
+
})()}
|
|
124
|
+
</div>
|
|
125
|
+
);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { measureWrap } from "socket-function/src/profiling/measure";
|
|
2
2
|
import { Unit, getAllUnits, remapSearchBuffer } from "./BufferIndexHelpers";
|
|
3
|
+
import { cacheLimited } from "socket-function/src/caching";
|
|
3
4
|
|
|
4
5
|
const WILD_CARD_BYTE = 42;
|
|
5
6
|
const WILD_CARD_CHAR = "*";
|
|
@@ -8,6 +9,7 @@ const SEARCH_OR_CHAR = "|";
|
|
|
8
9
|
const SEARCH_AND_BYTE = 38;
|
|
9
10
|
const SEARCH_AND_CHAR = "&";
|
|
10
11
|
|
|
12
|
+
// IMPORTANT! While technically THIS structure can support negation, the index structure above us cannot, as it only stores information at the block level (multiple buffers per block). So... if we supported negation, it couldn't, so it would give us a lot of false positives (cases where negation removes the results). So... never use negation, just turn everything into a positive signal.
|
|
11
13
|
export type MatchStructure = {
|
|
12
14
|
type: "or";
|
|
13
15
|
parts: MatchStructure[];
|
|
@@ -105,6 +107,8 @@ function getSimplerStructure(structure: MatchStructure): Buffer[][] {
|
|
|
105
107
|
|
|
106
108
|
throw new Error(`Unknown structure type: ${type}`);
|
|
107
109
|
}
|
|
110
|
+
export const createMatchesPatternCached = cacheLimited(1000, (pattern: string) => createMatchesPattern(Buffer.from(pattern), false));
|
|
111
|
+
|
|
108
112
|
// Each WILD_CARD_BYTE in pattern acts as a multi-byte wildcard: the segments on either
|
|
109
113
|
// side must appear in order somewhere within buffer.
|
|
110
114
|
// Returns a function that matches buffers against the pre-processed pattern.
|
|
@@ -155,7 +155,7 @@ export async function moveLogsToPublic(config: {
|
|
|
155
155
|
try {
|
|
156
156
|
let decoded = await BufferIndex.decodeAll(buffer.buffer);
|
|
157
157
|
allBuffers.push(decoded);
|
|
158
|
-
} catch (e) {
|
|
158
|
+
} catch (e: any) {
|
|
159
159
|
console.error(`Error decoding buffer, ignoring it`, { path: buffer.path, bufferLength: buffer.buffer.length, bufferFirst100: buffer.buffer.slice(0, 100).toString("hex"), error: String(e.stack) });
|
|
160
160
|
continue;
|
|
161
161
|
}
|
|
@@ -12,16 +12,18 @@ import { timeInDay, timeInHour, timeInMinute } from "socket-function/src/misc";
|
|
|
12
12
|
// URL parameters for time range
|
|
13
13
|
export const startTimeParam = new URLParam("startTime", undefined as number | undefined);
|
|
14
14
|
export const endTimeParam = new URLParam("endTime", undefined as number | undefined);
|
|
15
|
+
export const searchFromStartParam = new URLParam("searchFromStart", false);
|
|
15
16
|
|
|
16
17
|
let now = Date.now();
|
|
17
18
|
/** Get the time range with default values if not set */
|
|
18
|
-
export function getTimeRange(): { startTime: number; endTime: number } {
|
|
19
|
+
export function getTimeRange(): { startTime: number; endTime: number; searchFromStart: boolean } {
|
|
19
20
|
const defaultStart = now - timeInHour * 24;
|
|
20
21
|
const defaultEnd = now + timeInHour;
|
|
21
22
|
|
|
22
23
|
return {
|
|
23
24
|
startTime: startTimeParam.value || defaultStart,
|
|
24
25
|
endTime: endTimeParam.value || defaultEnd,
|
|
26
|
+
searchFromStart: searchFromStartParam.value,
|
|
25
27
|
};
|
|
26
28
|
}
|
|
27
29
|
|
|
@@ -75,7 +77,14 @@ export class TimeRangeSelector extends qreact.Component {
|
|
|
75
77
|
/>
|
|
76
78
|
</div>
|
|
77
79
|
<div className={css.hbox(12).wrap}>
|
|
78
|
-
|
|
80
|
+
<InputLabel
|
|
81
|
+
label="Search from Start"
|
|
82
|
+
checkbox
|
|
83
|
+
value={searchFromStartParam.value ? "true" : ""}
|
|
84
|
+
onChangeValue={value => {
|
|
85
|
+
searchFromStartParam.value = !!value;
|
|
86
|
+
}}
|
|
87
|
+
/>
|
|
79
88
|
<Button
|
|
80
89
|
hue={110} onClick={() => {
|
|
81
90
|
let now = Date.now();
|
|
@@ -450,10 +450,7 @@ class SuppressionItem extends qreact.Component<{
|
|
|
450
450
|
onKeyDown={e => {
|
|
451
451
|
if (e.key === "Enter") {
|
|
452
452
|
let pattern = e.currentTarget.value.trim();
|
|
453
|
-
if (!pattern)
|
|
454
|
-
alert("Pattern cannot be empty");
|
|
455
|
-
return;
|
|
456
|
-
}
|
|
453
|
+
if (!pattern) return;
|
|
457
454
|
this.state.patternValue = pattern;
|
|
458
455
|
Querysub.onCommitFinished(async () => {
|
|
459
456
|
await this.manager.updateSuppression(suppression.id, pattern);
|
|
@@ -170,7 +170,7 @@ const PATTERN_GENERATION_LIMIT = 100;
|
|
|
170
170
|
const PATTERN_GENERATION_WINDOW = 15 * 60 * 1000;
|
|
171
171
|
let patternGenerationTimes: number[] = [];
|
|
172
172
|
|
|
173
|
-
async function generatePatternForError(error: LogDatum): Promise<string> {
|
|
173
|
+
export async function generatePatternForError(error: LogDatum): Promise<string> {
|
|
174
174
|
const now = Date.now();
|
|
175
175
|
const windowStart = now - PATTERN_GENERATION_WINDOW;
|
|
176
176
|
|