querysub 0.394.0 → 0.395.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.cursorrules +8 -0
- package/package.json +1 -1
- package/src/-a-archives/archivesJSONT.ts +71 -8
- package/src/0-path-value-core/pathValueCore.ts +20 -1
- package/src/5-diagnostics/GenericFormat.tsx +1 -1
- package/src/deployManager/components/MachinesListPage.tsx +1 -2
- package/src/diagnostics/logs/IndexedLogs/BufferIndex.ts +12 -3
- package/src/diagnostics/logs/IndexedLogs/BufferIndexHelpers.ts +8 -3
- package/src/diagnostics/logs/IndexedLogs/BufferUnitIndex.ts +24 -9
- package/src/diagnostics/logs/IndexedLogs/BufferUnitSet.ts +0 -1
- package/src/diagnostics/logs/IndexedLogs/FindProgressTracker.ts +21 -5
- package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts +10 -4
- package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx +95 -124
- package/src/diagnostics/logs/IndexedLogs/RenderSearchStats.tsx +127 -0
- package/src/diagnostics/logs/IndexedLogs/bufferSearchFindMatcher.ts +3 -0
- package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts +1 -1
- package/src/diagnostics/logs/TimeRangeSelector.tsx +11 -2
- package/src/diagnostics/logs/errorNotifications2/ErrorNotificationPage.tsx +1 -4
- package/src/diagnostics/logs/errorNotifications2/errorNotifications.ts +1 -1
- package/src/diagnostics/logs/lifeCycleAnalysis/LifeCyclePage.tsx +946 -0
- package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycleMatching.ts +49 -0
- package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycleSearch.tsx +553 -0
- package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx +125 -90
- package/src/diagnostics/managementPages.tsx +17 -1
- package/src/functional/{limitProcessing.ts → throttleProcessing.ts} +1 -1
- package/src/library-components/StartEllipsis.tsx +13 -0
- package/src/misc.ts +4 -0
- package/src/diagnostics/logs/lifeCycleAnalysis/test.wat +0 -106
- package/src/diagnostics/logs/lifeCycleAnalysis/test.wat.d.ts +0 -2
- package/src/diagnostics/logs/lifeCycleAnalysis/testHoist.ts +0 -5
|
@@ -5,7 +5,7 @@ import { t } from "../../../2-proxy/schema2";
|
|
|
5
5
|
import { Button } from "../../../library-components/Button";
|
|
6
6
|
import { InputLabel, InputLabelURL } from "../../../library-components/InputLabel";
|
|
7
7
|
import { getLoggers2Async, LogDatum } from "../diskLogger";
|
|
8
|
-
import { list, timeInDay, keyByArray, sort, throttleFunction } from "socket-function/src/misc";
|
|
8
|
+
import { list, timeInDay, timeInHour, keyByArray, sort, throttleFunction } from "socket-function/src/misc";
|
|
9
9
|
import { formatDateTime, formatDateTimeDetailed, formatNumber, formatTime, formatPercent } from "socket-function/src/formatting/format";
|
|
10
10
|
import { IndexedLogs, TimeFilePathWithSize } from "./IndexedLogs";
|
|
11
11
|
import { Querysub } from "../../../4-querysub/QuerysubController";
|
|
@@ -27,6 +27,16 @@ import { TimeFilePath } from "./TimeFileTree";
|
|
|
27
27
|
import { isPublic } from "../../../config";
|
|
28
28
|
import { Zip } from "../../../zip";
|
|
29
29
|
import { readProductionLogsURL, searchTextURL } from "./LogViewerParams";
|
|
30
|
+
import { RenderSearchStats } from "./RenderSearchStats";
|
|
31
|
+
import { LifeCyclesController, LifeCycle, LifeCycleEntry } from "../lifeCycleAnalysis/lifeCycles";
|
|
32
|
+
import { getLifecycleMatchesForDatum } from "../lifeCycleAnalysis/lifeCycleMatching";
|
|
33
|
+
import { lifecycleIdURL } from "../lifeCycleAnalysis/LifeCyclePage";
|
|
34
|
+
import { ATag } from "../../../library-components/ATag";
|
|
35
|
+
import { SocketFunction } from "socket-function/SocketFunction";
|
|
36
|
+
import { managementPageURL } from "../../managementPages";
|
|
37
|
+
import { startTimeParam, endTimeParam } from "../TimeRangeSelector";
|
|
38
|
+
import { formatSearchString } from "./LogViewerParams";
|
|
39
|
+
import { additionalSearchURL } from "../lifeCycleAnalysis/lifeCycleSearch";
|
|
30
40
|
|
|
31
41
|
let excludePendingResults = new URLParam("excludePendingResults", false);
|
|
32
42
|
let limitURL = new URLParam("limit", 100);
|
|
@@ -34,6 +44,7 @@ let limitURL = new URLParam("limit", 100);
|
|
|
34
44
|
let savedPathsURL = new URLParam("savedPaths", "");
|
|
35
45
|
let selectedFieldsURL = new URLParam("selectedFields", {} as Record<string, boolean>);
|
|
36
46
|
let useRelativeTimeURL = new URLParam("useRelativeTime", true);
|
|
47
|
+
let showLifecycleColumnURL = new URLParam("showlifecycle", true);
|
|
37
48
|
|
|
38
49
|
const defaultSelectedFields = {
|
|
39
50
|
param0: true,
|
|
@@ -64,11 +75,16 @@ export class LogViewer3 extends qreact.Component {
|
|
|
64
75
|
hasSearched: t.boolean(false),
|
|
65
76
|
forceMoveStartTime: t.atomic<number | undefined>(undefined),
|
|
66
77
|
forceMoveEndTime: t.atomic<number | undefined>(undefined),
|
|
78
|
+
showLifecycleColumn: t.boolean(true),
|
|
67
79
|
});
|
|
68
80
|
|
|
81
|
+
lifecycleController = LifeCyclesController(SocketFunction.browserNodeId());
|
|
82
|
+
|
|
69
83
|
private searchSequenceNumber = 0;
|
|
70
84
|
|
|
71
85
|
componentDidMount(): void {
|
|
86
|
+
this.state.showLifecycleColumn = showLifecycleColumnURL.value;
|
|
87
|
+
|
|
72
88
|
if (searchTextURL.value) {
|
|
73
89
|
void this.search();
|
|
74
90
|
} else {
|
|
@@ -285,6 +301,7 @@ export class LogViewer3 extends qreact.Component {
|
|
|
285
301
|
pathOverrides: paths,
|
|
286
302
|
only: excludePendingResults.value ? "local" : undefined,
|
|
287
303
|
forceReadProduction: readProductionLogsURL.value,
|
|
304
|
+
searchFromStart: range.searchFromStart,
|
|
288
305
|
},
|
|
289
306
|
onResult: (match: LogDatum) => {
|
|
290
307
|
results.push(match);
|
|
@@ -320,123 +337,6 @@ export class LogViewer3 extends qreact.Component {
|
|
|
320
337
|
});
|
|
321
338
|
};
|
|
322
339
|
|
|
323
|
-
renderSearchStats() {
|
|
324
|
-
const stats = this.state.stats;
|
|
325
|
-
if (!stats) return undefined;
|
|
326
|
-
|
|
327
|
-
let totalSizeRead = 0;
|
|
328
|
-
let cachedSize = 0;
|
|
329
|
-
let cachedCount = 0;
|
|
330
|
-
let uncachedSize = 0;
|
|
331
|
-
let uncachedCount = 0;
|
|
332
|
-
let uncachedRemoteSize = 0;
|
|
333
|
-
let uncachedRemoteCount = 0;
|
|
334
|
-
let totalSize = 0;
|
|
335
|
-
let remoteTotalSize = 0;
|
|
336
|
-
let localTotalSize = 0;
|
|
337
|
-
|
|
338
|
-
for (let read of stats.reads) {
|
|
339
|
-
totalSizeRead += read.size;
|
|
340
|
-
if (read.cached) {
|
|
341
|
-
cachedSize += read.size;
|
|
342
|
-
cachedCount += read.count;
|
|
343
|
-
} else {
|
|
344
|
-
uncachedSize += read.size;
|
|
345
|
-
uncachedCount += read.count;
|
|
346
|
-
totalSize += read.totalSize;
|
|
347
|
-
if (read.remote) {
|
|
348
|
-
remoteTotalSize += read.totalSize;
|
|
349
|
-
} else {
|
|
350
|
-
localTotalSize += read.totalSize;
|
|
351
|
-
}
|
|
352
|
-
}
|
|
353
|
-
if (read.remote && !read.cached) {
|
|
354
|
-
uncachedRemoteSize += read.size;
|
|
355
|
-
uncachedRemoteCount += read.count;
|
|
356
|
-
}
|
|
357
|
-
}
|
|
358
|
-
|
|
359
|
-
const renderStage = (title: string, items: preact.ComponentChild[], hue: number | undefined, isFirst: boolean, isLast: boolean) => {
|
|
360
|
-
const triangleWidth = 20;
|
|
361
|
-
|
|
362
|
-
let color = hue !== undefined ? css.hsl(hue, 70, 80) : css.hsl(0, 0, 100);
|
|
363
|
-
let colorhsl = hue !== undefined ? css.hslcolor(hue, 80, 30) : css;
|
|
364
|
-
|
|
365
|
-
return (
|
|
366
|
-
<div className={css.vbox(2).relative.minWidth(200).paddingTop(8).paddingBottom(8).paddingRight(10).paddingLeft(isFirst ? 10 : 10 + triangleWidth) + color}>
|
|
367
|
-
<div className={css.fontWeight(600) + colorhsl}>{title}</div>
|
|
368
|
-
{items.map((item, i) => (
|
|
369
|
-
<div key={i}>{item}</div>
|
|
370
|
-
))}
|
|
371
|
-
{!isLast && hue !== undefined && (
|
|
372
|
-
<svg width={triangleWidth} height="100" viewBox="0 0 20 100" preserveAspectRatio="none" className={css.absolute.right(-triangleWidth).top(0).fillHeight.zIndex(1)}>
|
|
373
|
-
<polygon points="0,0 20,50 0,100" fill={`hsl(${hue}, 70%, 80%)`} />
|
|
374
|
-
</svg>
|
|
375
|
-
)}
|
|
376
|
-
</div>
|
|
377
|
-
);
|
|
378
|
-
};
|
|
379
|
-
|
|
380
|
-
const fileItems = [
|
|
381
|
-
`${formatTime(stats.fileFindTime)} | ${formatNumber(stats.localFilesSearched + stats.backblazeFilesSearched)} / ${formatNumber(stats.totalLocalFiles + stats.totalBackblazeFiles)} | ${formatNumber(totalSize)}B`,
|
|
382
|
-
`remote ${formatNumber(stats.backblazeFilesSearched)} / ${formatNumber(stats.totalBackblazeFiles)} | ${formatNumber(remoteTotalSize)}B`,
|
|
383
|
-
`pending ${formatNumber(stats.localFilesSearched)} / ${formatNumber(stats.totalLocalFiles)} | ${formatNumber(localTotalSize)}B`,
|
|
384
|
-
];
|
|
385
|
-
|
|
386
|
-
const totalIndexesSearched = stats.remoteIndexesSearched + stats.localIndexesSearched;
|
|
387
|
-
const totalIndexSize = stats.remoteIndexSize + stats.localIndexSize;
|
|
388
|
-
const indexItems = [
|
|
389
|
-
`${formatTime(stats.indexSearchTime)} | ${formatNumber(totalIndexesSearched)} | ${formatNumber(totalIndexSize)}B | ${formatNumber(totalSize / totalIndexSize)}X`,
|
|
390
|
-
`remote ${formatNumber(stats.remoteIndexesSearched)} | ${formatNumber(stats.remoteIndexSize)}B | ${formatNumber(remoteTotalSize / stats.remoteIndexSize)}X`,
|
|
391
|
-
`local ${formatNumber(stats.localIndexesSearched)} | ${formatNumber(stats.localIndexSize)}B | ${formatNumber(localTotalSize / stats.localIndexSize)}X`,
|
|
392
|
-
];
|
|
393
|
-
|
|
394
|
-
const blockItems = [
|
|
395
|
-
<div title={`Scanned size = ${formatNumber(stats.blocksCheckedCompressedSize)}B, Decompressed size = ${formatNumber(stats.blocksCheckedDecompressedSize)}B, Compression ratio = ${formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X`}>{formatTime(stats.blockSearchTime)} | {formatNumber(stats.blockCheckedCount)} | {formatNumber(stats.blocksCheckedCompressedSize)}B | {formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X</div>,
|
|
396
|
-
`total scanned ${formatNumber(stats.totalBlockCount)} (${formatPercent(stats.blockCheckedCount / stats.totalBlockCount)})`,
|
|
397
|
-
`remote ${formatNumber(stats.remoteBlockCheckedCount)} / ${formatNumber(stats.remoteBlockCount)} (${formatPercent(stats.remoteBlockCheckedCount / stats.remoteBlockCount)}) | local ${formatNumber(stats.localBlockCheckedCount)} / ${formatNumber(stats.localBlockCount)} (${formatPercent(stats.localBlockCheckedCount / stats.localBlockCount)})`,
|
|
398
|
-
];
|
|
399
|
-
let cacheItems = [
|
|
400
|
-
`disk read ${formatNumber(uncachedSize)}B (${formatNumber(uncachedCount)})`,
|
|
401
|
-
`remote ${formatPercent(uncachedRemoteSize / totalSizeRead)} (${formatNumber(uncachedRemoteCount)})`,
|
|
402
|
-
`cached ${formatNumber(cachedSize)} (${formatNumber(cachedCount)})`,
|
|
403
|
-
];
|
|
404
|
-
|
|
405
|
-
const resultItems = [
|
|
406
|
-
`${formatTime(stats.timeToFirstMatch)} until first result`,
|
|
407
|
-
`${formatNumber(stats.matchCount)} results / ${limitURL.value} limit`,
|
|
408
|
-
];
|
|
409
|
-
|
|
410
|
-
const doneItems = [
|
|
411
|
-
`${formatTime(stats.totalSearchTime)} total search time`,
|
|
412
|
-
];
|
|
413
|
-
|
|
414
|
-
let hasErrors = stats.fileErrors.length > 0 || stats.blockErrors.length > 0;
|
|
415
|
-
|
|
416
|
-
return (
|
|
417
|
-
<div className={css.hbox(0).alignItems("stretch")}>
|
|
418
|
-
{renderStage("Files", fileItems, 290, true, false)}
|
|
419
|
-
{renderStage("Indexes", indexItems, 250, false, false)}
|
|
420
|
-
{renderStage("Blocks", blockItems, 210, false, false)}
|
|
421
|
-
{renderStage("Reads", cacheItems, 160, false, false)}
|
|
422
|
-
{renderStage("Results", resultItems, 120, false, this.state.searching && !hasErrors)}
|
|
423
|
-
{!this.state.searching && renderStage("Done", doneItems, undefined, false, !hasErrors)}
|
|
424
|
-
{(() => {
|
|
425
|
-
if (!hasErrors) return undefined;
|
|
426
|
-
let errorItems: preact.ComponentChild[] = [];
|
|
427
|
-
if (stats.fileErrors.length > 0) {
|
|
428
|
-
const errorTitle = stats.fileErrors.map(e => `${e.path}:\n${e.error}`).join("\n\n");
|
|
429
|
-
errorItems.push(<div title={errorTitle}>{stats.fileErrors.length} files failed</div>);
|
|
430
|
-
}
|
|
431
|
-
if (stats.blockErrors.length > 0) {
|
|
432
|
-
errorItems.push(<div title={stats.blockErrors.join("\n")}>{stats.blockErrors.length} blocks failed</div>);
|
|
433
|
-
}
|
|
434
|
-
return renderStage("Errors", errorItems, 0, false, true);
|
|
435
|
-
})()}
|
|
436
|
-
</div>
|
|
437
|
-
);
|
|
438
|
-
}
|
|
439
|
-
|
|
440
340
|
renderForceMoveLogs() {
|
|
441
341
|
return <><Button
|
|
442
342
|
onClick={async () => {
|
|
@@ -627,6 +527,64 @@ export class LogViewer3 extends qreact.Component {
|
|
|
627
527
|
}
|
|
628
528
|
};
|
|
629
529
|
|
|
530
|
+
if (this.state.showLifecycleColumn) {
|
|
531
|
+
const lifecycles = this.lifecycleController.getLifeCycles();
|
|
532
|
+
if (lifecycles && lifecycles.length > 0) {
|
|
533
|
+
columns["lifecycles"] = {
|
|
534
|
+
title: "Lifecycles",
|
|
535
|
+
formatter: (x, context) => {
|
|
536
|
+
if (!context?.row) return undefined;
|
|
537
|
+
let datum = context.row;
|
|
538
|
+
let matches = getLifecycleMatchesForDatum(lifecycles, datum);
|
|
539
|
+
if (matches.length === 0) return undefined;
|
|
540
|
+
|
|
541
|
+
let lifecycleId = matches[0].lifecycle.id;
|
|
542
|
+
let lifecycleTitle = matches[0].lifecycle.title;
|
|
543
|
+
|
|
544
|
+
let firstMatch = matches[0];
|
|
545
|
+
let searchFilter = formatSearchString(firstMatch.datum);
|
|
546
|
+
|
|
547
|
+
let minTime = Infinity;
|
|
548
|
+
let maxTime = -Infinity;
|
|
549
|
+
for (let resultDatum of this.state.results) {
|
|
550
|
+
let resultMatches = getLifecycleMatchesForDatum(lifecycles, resultDatum);
|
|
551
|
+
if (resultMatches.some(m => m.lifecycle.id === lifecycleId)) {
|
|
552
|
+
if (resultDatum.time < minTime) minTime = resultDatum.time;
|
|
553
|
+
if (resultDatum.time > maxTime) maxTime = resultDatum.time;
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
return <div className={css.hbox(4).wrap}>
|
|
558
|
+
{matches.map((match, idx) => (
|
|
559
|
+
<div key={idx} className={css.hbox(8)}>
|
|
560
|
+
<ATag values={[
|
|
561
|
+
managementPageURL.getOverride("LifeCyclePage"),
|
|
562
|
+
lifecycleIdURL.getOverride(match.lifecycle.id),
|
|
563
|
+
additionalSearchURL.getOverride(formatSearchString(match.datum)),
|
|
564
|
+
startTimeParam.getOverride(datum.time - timeInHour),
|
|
565
|
+
endTimeParam.getOverride(datum.time + timeInHour),
|
|
566
|
+
]}>
|
|
567
|
+
{match.lifecycle.title}
|
|
568
|
+
</ATag>
|
|
569
|
+
</div>
|
|
570
|
+
))}
|
|
571
|
+
<ATag values={[
|
|
572
|
+
managementPageURL.getOverride("LifeCyclePage"),
|
|
573
|
+
lifecycleIdURL.getOverride(firstMatch.lifecycle.id),
|
|
574
|
+
additionalSearchURL.getOverride(searchFilter),
|
|
575
|
+
startTimeParam.getOverride(minTime - timeInHour),
|
|
576
|
+
endTimeParam.getOverride(maxTime + timeInHour),
|
|
577
|
+
]}>
|
|
578
|
+
<Button hue={280} className={css.boldStyle}>
|
|
579
|
+
View All {lifecycleTitle}
|
|
580
|
+
</Button>
|
|
581
|
+
</ATag>
|
|
582
|
+
</div>;
|
|
583
|
+
}
|
|
584
|
+
};
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
|
|
630
588
|
return <>
|
|
631
589
|
<div className={css.hbox(10)}>
|
|
632
590
|
<InputPicker
|
|
@@ -662,6 +620,14 @@ export class LogViewer3 extends qreact.Component {
|
|
|
662
620
|
checkbox
|
|
663
621
|
url={useRelativeTimeURL}
|
|
664
622
|
/>
|
|
623
|
+
<InputLabelURL
|
|
624
|
+
label="Show Lifecycles"
|
|
625
|
+
checkbox
|
|
626
|
+
url={showLifecycleColumnURL}
|
|
627
|
+
onChange={(e) => {
|
|
628
|
+
this.state.showLifecycleColumn = e.currentTarget.checked;
|
|
629
|
+
}}
|
|
630
|
+
/>
|
|
665
631
|
</div>
|
|
666
632
|
|
|
667
633
|
{this.state.results.length === 0 && !this.state.searching && this.state.hasSearched && <div className={css.hsl(40, 50, 50).colorhsl(60, 50, 100).boldStyle.pad2(10).ellipsis}>
|
|
@@ -674,13 +640,14 @@ export class LogViewer3 extends qreact.Component {
|
|
|
674
640
|
lineLimit={4}
|
|
675
641
|
initialLimit={10}
|
|
676
642
|
characterLimit={400}
|
|
643
|
+
cellClass={css.maxWidth("20vw")}
|
|
677
644
|
getRowAttributes={row => {
|
|
678
645
|
let hue = -1;
|
|
679
646
|
if (row.__LOG_TYPE === "warn") hue = 40;
|
|
680
647
|
if (row.__LOG_TYPE === "error") hue = 0;
|
|
681
648
|
if (row.__LOG_TYPE === "info") hue = 200;
|
|
682
649
|
return {
|
|
683
|
-
className: hue !== -1 && css.hsl(hue, 40,
|
|
650
|
+
className: hue !== -1 && css.hsl(hue, 40, 60).colorhsl(hue, 0, 100) || undefined,
|
|
684
651
|
};
|
|
685
652
|
}}
|
|
686
653
|
/>
|
|
@@ -695,8 +662,8 @@ export class LogViewer3 extends qreact.Component {
|
|
|
695
662
|
checkbox
|
|
696
663
|
label="Exclude Pending Results"
|
|
697
664
|
url={excludePendingResults}
|
|
698
|
-
|
|
699
|
-
if (
|
|
665
|
+
onChange={(e) => {
|
|
666
|
+
if (e.currentTarget.checked) {
|
|
700
667
|
savedPathsURL.value = "";
|
|
701
668
|
void this.loadPaths();
|
|
702
669
|
}
|
|
@@ -706,8 +673,8 @@ export class LogViewer3 extends qreact.Component {
|
|
|
706
673
|
checkbox
|
|
707
674
|
label="Read Production Logs"
|
|
708
675
|
url={readProductionLogsURL}
|
|
709
|
-
|
|
710
|
-
if (
|
|
676
|
+
onChange={(e) => {
|
|
677
|
+
if (e.currentTarget.checked) {
|
|
711
678
|
savedPathsURL.value = "";
|
|
712
679
|
void this.loadPaths();
|
|
713
680
|
}
|
|
@@ -888,7 +855,11 @@ export class LogViewer3 extends qreact.Component {
|
|
|
888
855
|
|
|
889
856
|
{this.renderPendingLogWarnings()}
|
|
890
857
|
|
|
891
|
-
|
|
858
|
+
<RenderSearchStats
|
|
859
|
+
searching={this.state.searching}
|
|
860
|
+
limit={limitURL.value}
|
|
861
|
+
stats={this.state.stats}
|
|
862
|
+
/>
|
|
892
863
|
|
|
893
864
|
{this.renderResults()}
|
|
894
865
|
</div>
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import { formatTime, formatNumber, formatPercent } from "socket-function/src/formatting/format";
|
|
2
|
+
import { css } from "typesafecss";
|
|
3
|
+
import { qreact } from "../../../4-dom/qreact";
|
|
4
|
+
import { IndexedLogResults } from "./BufferIndexHelpers";
|
|
5
|
+
|
|
6
|
+
export class RenderSearchStats extends qreact.Component<{
|
|
7
|
+
stats: IndexedLogResults | undefined;
|
|
8
|
+
searching: boolean;
|
|
9
|
+
limit: number;
|
|
10
|
+
}> {
|
|
11
|
+
render() {
|
|
12
|
+
const stats = this.props.stats;
|
|
13
|
+
if (!stats) return undefined;
|
|
14
|
+
|
|
15
|
+
let totalSizeRead = 0;
|
|
16
|
+
let cachedSize = 0;
|
|
17
|
+
let cachedCount = 0;
|
|
18
|
+
let uncachedSize = 0;
|
|
19
|
+
let uncachedCount = 0;
|
|
20
|
+
let uncachedRemoteSize = 0;
|
|
21
|
+
let uncachedRemoteCount = 0;
|
|
22
|
+
let totalSize = 0;
|
|
23
|
+
let remoteTotalSize = 0;
|
|
24
|
+
let localTotalSize = 0;
|
|
25
|
+
|
|
26
|
+
for (let read of stats.reads) {
|
|
27
|
+
totalSizeRead += read.size;
|
|
28
|
+
if (read.cached) {
|
|
29
|
+
cachedSize += read.size;
|
|
30
|
+
cachedCount += read.count;
|
|
31
|
+
} else {
|
|
32
|
+
uncachedSize += read.size;
|
|
33
|
+
uncachedCount += read.count;
|
|
34
|
+
totalSize += read.totalSize;
|
|
35
|
+
if (read.remote) {
|
|
36
|
+
remoteTotalSize += read.totalSize;
|
|
37
|
+
} else {
|
|
38
|
+
localTotalSize += read.totalSize;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
if (read.remote && !read.cached) {
|
|
42
|
+
uncachedRemoteSize += read.size;
|
|
43
|
+
uncachedRemoteCount += read.count;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const renderStage = (title: string, items: preact.ComponentChild[], hue: number | undefined, isFirst: boolean, isLast: boolean) => {
|
|
48
|
+
const triangleWidth = 20;
|
|
49
|
+
|
|
50
|
+
let color = hue !== undefined ? css.hsl(hue, 70, 80) : css.hsl(0, 0, 100);
|
|
51
|
+
let colorhsl = hue !== undefined ? css.hslcolor(hue, 80, 30) : css;
|
|
52
|
+
|
|
53
|
+
return (
|
|
54
|
+
<div className={css.vbox(2).relative.minWidth(200).paddingTop(8).paddingBottom(8).paddingRight(10).paddingLeft(isFirst ? 10 : 10 + triangleWidth) + color}>
|
|
55
|
+
<div className={css.fontWeight(600) + colorhsl}>{title}</div>
|
|
56
|
+
{items.map((item, i) => (
|
|
57
|
+
<div key={i}>{item}</div>
|
|
58
|
+
))}
|
|
59
|
+
{!isLast && hue !== undefined && (
|
|
60
|
+
<svg width={triangleWidth} height="100" viewBox="0 0 20 100" preserveAspectRatio="none" className={css.absolute.right(-triangleWidth).top(0).fillHeight.zIndex(1)}>
|
|
61
|
+
<polygon points="0,0 20,50 0,100" fill={`hsl(${hue}, 70%, 80%)`} />
|
|
62
|
+
</svg>
|
|
63
|
+
)}
|
|
64
|
+
</div>
|
|
65
|
+
);
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
const fileItems = [
|
|
69
|
+
`${formatTime(stats.fileFindTime)} | ${formatNumber(stats.localFilesSearched + stats.backblazeFilesSearched)} / ${formatNumber(stats.totalLocalFiles + stats.totalBackblazeFiles)} | ${formatNumber(totalSize)}B`,
|
|
70
|
+
`remote ${formatNumber(stats.backblazeFilesSearched)} / ${formatNumber(stats.totalBackblazeFiles)} | ${formatNumber(remoteTotalSize)}B | ${formatNumber(stats.backblazeLogsSearched)} logs`,
|
|
71
|
+
`pending ${formatNumber(stats.localFilesSearched)} / ${formatNumber(stats.totalLocalFiles)} | ${formatNumber(localTotalSize)}B`,
|
|
72
|
+
];
|
|
73
|
+
|
|
74
|
+
const totalIndexesSearched = stats.remoteIndexesSearched + stats.localIndexesSearched;
|
|
75
|
+
const totalIndexSize = stats.remoteIndexSize + stats.localIndexSize;
|
|
76
|
+
const indexItems = [
|
|
77
|
+
`${formatTime(stats.indexSearchTime)} | ${formatNumber(totalIndexesSearched)} | ${formatNumber(totalIndexSize)}B | ${formatNumber(totalSize / totalIndexSize)}X`,
|
|
78
|
+
`remote ${formatNumber(stats.remoteIndexesSearched)} | ${formatNumber(stats.remoteIndexSize)}B | ${formatNumber(remoteTotalSize / stats.remoteIndexSize)}X`,
|
|
79
|
+
`local ${formatNumber(stats.localIndexesSearched)} | ${formatNumber(stats.localIndexSize)}B | ${formatNumber(localTotalSize / stats.localIndexSize)}X`,
|
|
80
|
+
];
|
|
81
|
+
|
|
82
|
+
const blockItems = [
|
|
83
|
+
<div title={`Scanned size = ${formatNumber(stats.blocksCheckedCompressedSize)}B, Decompressed size = ${formatNumber(stats.blocksCheckedDecompressedSize)}B, Compression ratio = ${formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X`}>{formatTime(stats.blockSearchTime)} | {formatNumber(stats.blockCheckedCount)} | {formatNumber(stats.blocksCheckedCompressedSize)}B | {formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X</div>,
|
|
84
|
+
`total scanned ${formatNumber(stats.totalBlockCount)} (${formatPercent(stats.blockCheckedCount / stats.totalBlockCount)})`,
|
|
85
|
+
`remote ${formatNumber(stats.remoteBlockCheckedCount)} / ${formatNumber(stats.remoteBlockCount)} (${formatPercent(stats.remoteBlockCheckedCount / stats.remoteBlockCount)}) | local ${formatNumber(stats.localBlockCheckedCount)} / ${formatNumber(stats.localBlockCount)} (${formatPercent(stats.localBlockCheckedCount / stats.localBlockCount)})`,
|
|
86
|
+
];
|
|
87
|
+
let cacheItems = [
|
|
88
|
+
`disk read ${formatNumber(uncachedSize)}B (${formatNumber(uncachedCount)})`,
|
|
89
|
+
`remote ${formatPercent(uncachedRemoteSize / totalSizeRead)} (${formatNumber(uncachedRemoteCount)})`,
|
|
90
|
+
`cached ${formatNumber(cachedSize)} (${formatNumber(cachedCount)})`,
|
|
91
|
+
];
|
|
92
|
+
|
|
93
|
+
const resultItems = [
|
|
94
|
+
`${formatTime(stats.timeToFirstMatch)} until first result`,
|
|
95
|
+
`${formatNumber(stats.matchCount)} results / ${this.props.limit} limit`,
|
|
96
|
+
];
|
|
97
|
+
|
|
98
|
+
const doneItems = [
|
|
99
|
+
`${formatTime(stats.totalSearchTime)} total search time`,
|
|
100
|
+
];
|
|
101
|
+
|
|
102
|
+
let hasErrors = stats.fileErrors.length > 0 || stats.blockErrors.length > 0;
|
|
103
|
+
|
|
104
|
+
return (
|
|
105
|
+
<div className={css.hbox(0).alignItems("stretch")}>
|
|
106
|
+
{renderStage("Files", fileItems, 290, true, false)}
|
|
107
|
+
{renderStage("Indexes", indexItems, 250, false, false)}
|
|
108
|
+
{renderStage("Blocks", blockItems, 210, false, false)}
|
|
109
|
+
{renderStage("Reads", cacheItems, 160, false, false)}
|
|
110
|
+
{renderStage("Results", resultItems, 120, false, this.props.searching && !hasErrors)}
|
|
111
|
+
{!this.props.searching && renderStage("Done", doneItems, undefined, false, !hasErrors)}
|
|
112
|
+
{(() => {
|
|
113
|
+
if (!hasErrors) return undefined;
|
|
114
|
+
let errorItems: preact.ComponentChild[] = [];
|
|
115
|
+
if (stats.fileErrors.length > 0) {
|
|
116
|
+
const errorTitle = stats.fileErrors.map(e => `${e.path}:\n${e.error}`).join("\n\n");
|
|
117
|
+
errorItems.push(<div title={errorTitle}>{stats.fileErrors.length} files failed</div>);
|
|
118
|
+
}
|
|
119
|
+
if (stats.blockErrors.length > 0) {
|
|
120
|
+
errorItems.push(<div title={stats.blockErrors.join("\n")}>{stats.blockErrors.length} blocks failed</div>);
|
|
121
|
+
}
|
|
122
|
+
return renderStage("Errors", errorItems, 0, false, true);
|
|
123
|
+
})()}
|
|
124
|
+
</div>
|
|
125
|
+
);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { measureWrap } from "socket-function/src/profiling/measure";
|
|
2
2
|
import { Unit, getAllUnits, remapSearchBuffer } from "./BufferIndexHelpers";
|
|
3
|
+
import { cacheLimited } from "socket-function/src/caching";
|
|
3
4
|
|
|
4
5
|
const WILD_CARD_BYTE = 42;
|
|
5
6
|
const WILD_CARD_CHAR = "*";
|
|
@@ -105,6 +106,8 @@ function getSimplerStructure(structure: MatchStructure): Buffer[][] {
|
|
|
105
106
|
|
|
106
107
|
throw new Error(`Unknown structure type: ${type}`);
|
|
107
108
|
}
|
|
109
|
+
export const createMatchesPatternCached = cacheLimited(1000, (pattern: string) => createMatchesPattern(Buffer.from(pattern), false));
|
|
110
|
+
|
|
108
111
|
// Each WILD_CARD_BYTE in pattern acts as a multi-byte wildcard: the segments on either
|
|
109
112
|
// side must appear in order somewhere within buffer.
|
|
110
113
|
// Returns a function that matches buffers against the pre-processed pattern.
|
|
@@ -155,7 +155,7 @@ export async function moveLogsToPublic(config: {
|
|
|
155
155
|
try {
|
|
156
156
|
let decoded = await BufferIndex.decodeAll(buffer.buffer);
|
|
157
157
|
allBuffers.push(decoded);
|
|
158
|
-
} catch (e) {
|
|
158
|
+
} catch (e: any) {
|
|
159
159
|
console.error(`Error decoding buffer, ignoring it`, { path: buffer.path, bufferLength: buffer.buffer.length, bufferFirst100: buffer.buffer.slice(0, 100).toString("hex"), error: String(e.stack) });
|
|
160
160
|
continue;
|
|
161
161
|
}
|
|
@@ -12,16 +12,18 @@ import { timeInDay, timeInHour, timeInMinute } from "socket-function/src/misc";
|
|
|
12
12
|
// URL parameters for time range
|
|
13
13
|
export const startTimeParam = new URLParam("startTime", undefined as number | undefined);
|
|
14
14
|
export const endTimeParam = new URLParam("endTime", undefined as number | undefined);
|
|
15
|
+
export const searchFromStartParam = new URLParam("searchFromStart", false);
|
|
15
16
|
|
|
16
17
|
let now = Date.now();
|
|
17
18
|
/** Get the time range with default values if not set */
|
|
18
|
-
export function getTimeRange(): { startTime: number; endTime: number } {
|
|
19
|
+
export function getTimeRange(): { startTime: number; endTime: number; searchFromStart: boolean } {
|
|
19
20
|
const defaultStart = now - timeInHour * 24;
|
|
20
21
|
const defaultEnd = now + timeInHour;
|
|
21
22
|
|
|
22
23
|
return {
|
|
23
24
|
startTime: startTimeParam.value || defaultStart,
|
|
24
25
|
endTime: endTimeParam.value || defaultEnd,
|
|
26
|
+
searchFromStart: searchFromStartParam.value,
|
|
25
27
|
};
|
|
26
28
|
}
|
|
27
29
|
|
|
@@ -75,7 +77,14 @@ export class TimeRangeSelector extends qreact.Component {
|
|
|
75
77
|
/>
|
|
76
78
|
</div>
|
|
77
79
|
<div className={css.hbox(12).wrap}>
|
|
78
|
-
|
|
80
|
+
<InputLabel
|
|
81
|
+
label="Search from Start"
|
|
82
|
+
checkbox
|
|
83
|
+
value={searchFromStartParam.value ? "true" : ""}
|
|
84
|
+
onChangeValue={value => {
|
|
85
|
+
searchFromStartParam.value = !!value;
|
|
86
|
+
}}
|
|
87
|
+
/>
|
|
79
88
|
<Button
|
|
80
89
|
hue={110} onClick={() => {
|
|
81
90
|
let now = Date.now();
|
|
@@ -450,10 +450,7 @@ class SuppressionItem extends qreact.Component<{
|
|
|
450
450
|
onKeyDown={e => {
|
|
451
451
|
if (e.key === "Enter") {
|
|
452
452
|
let pattern = e.currentTarget.value.trim();
|
|
453
|
-
if (!pattern)
|
|
454
|
-
alert("Pattern cannot be empty");
|
|
455
|
-
return;
|
|
456
|
-
}
|
|
453
|
+
if (!pattern) return;
|
|
457
454
|
this.state.patternValue = pattern;
|
|
458
455
|
Querysub.onCommitFinished(async () => {
|
|
459
456
|
await this.manager.updateSuppression(suppression.id, pattern);
|
|
@@ -170,7 +170,7 @@ const PATTERN_GENERATION_LIMIT = 100;
|
|
|
170
170
|
const PATTERN_GENERATION_WINDOW = 15 * 60 * 1000;
|
|
171
171
|
let patternGenerationTimes: number[] = [];
|
|
172
172
|
|
|
173
|
-
async function generatePatternForError(error: LogDatum): Promise<string> {
|
|
173
|
+
export async function generatePatternForError(error: LogDatum): Promise<string> {
|
|
174
174
|
const now = Date.now();
|
|
175
175
|
const windowStart = now - PATTERN_GENERATION_WINDOW;
|
|
176
176
|
|