querysub 0.394.0 → 0.395.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. package/.cursorrules +8 -0
  2. package/package.json +1 -1
  3. package/src/-a-archives/archivesJSONT.ts +71 -8
  4. package/src/0-path-value-core/pathValueCore.ts +20 -1
  5. package/src/5-diagnostics/GenericFormat.tsx +1 -1
  6. package/src/deployManager/components/MachinesListPage.tsx +1 -2
  7. package/src/diagnostics/logs/IndexedLogs/BufferIndex.ts +12 -3
  8. package/src/diagnostics/logs/IndexedLogs/BufferIndexHelpers.ts +8 -3
  9. package/src/diagnostics/logs/IndexedLogs/BufferUnitIndex.ts +24 -9
  10. package/src/diagnostics/logs/IndexedLogs/BufferUnitSet.ts +0 -1
  11. package/src/diagnostics/logs/IndexedLogs/FindProgressTracker.ts +21 -5
  12. package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts +10 -4
  13. package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx +95 -124
  14. package/src/diagnostics/logs/IndexedLogs/RenderSearchStats.tsx +127 -0
  15. package/src/diagnostics/logs/IndexedLogs/bufferSearchFindMatcher.ts +3 -0
  16. package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts +1 -1
  17. package/src/diagnostics/logs/TimeRangeSelector.tsx +11 -2
  18. package/src/diagnostics/logs/errorNotifications2/ErrorNotificationPage.tsx +1 -4
  19. package/src/diagnostics/logs/errorNotifications2/errorNotifications.ts +1 -1
  20. package/src/diagnostics/logs/lifeCycleAnalysis/LifeCyclePage.tsx +946 -0
  21. package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycleMatching.ts +49 -0
  22. package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycleSearch.tsx +553 -0
  23. package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx +125 -90
  24. package/src/diagnostics/managementPages.tsx +17 -1
  25. package/src/functional/{limitProcessing.ts → throttleProcessing.ts} +1 -1
  26. package/src/library-components/StartEllipsis.tsx +13 -0
  27. package/src/misc.ts +4 -0
  28. package/src/diagnostics/logs/lifeCycleAnalysis/test.wat +0 -106
  29. package/src/diagnostics/logs/lifeCycleAnalysis/test.wat.d.ts +0 -2
  30. package/src/diagnostics/logs/lifeCycleAnalysis/testHoist.ts +0 -5
@@ -5,7 +5,7 @@ import { t } from "../../../2-proxy/schema2";
5
5
  import { Button } from "../../../library-components/Button";
6
6
  import { InputLabel, InputLabelURL } from "../../../library-components/InputLabel";
7
7
  import { getLoggers2Async, LogDatum } from "../diskLogger";
8
- import { list, timeInDay, keyByArray, sort, throttleFunction } from "socket-function/src/misc";
8
+ import { list, timeInDay, timeInHour, keyByArray, sort, throttleFunction } from "socket-function/src/misc";
9
9
  import { formatDateTime, formatDateTimeDetailed, formatNumber, formatTime, formatPercent } from "socket-function/src/formatting/format";
10
10
  import { IndexedLogs, TimeFilePathWithSize } from "./IndexedLogs";
11
11
  import { Querysub } from "../../../4-querysub/QuerysubController";
@@ -27,6 +27,16 @@ import { TimeFilePath } from "./TimeFileTree";
27
27
  import { isPublic } from "../../../config";
28
28
  import { Zip } from "../../../zip";
29
29
  import { readProductionLogsURL, searchTextURL } from "./LogViewerParams";
30
+ import { RenderSearchStats } from "./RenderSearchStats";
31
+ import { LifeCyclesController, LifeCycle, LifeCycleEntry } from "../lifeCycleAnalysis/lifeCycles";
32
+ import { getLifecycleMatchesForDatum } from "../lifeCycleAnalysis/lifeCycleMatching";
33
+ import { lifecycleIdURL } from "../lifeCycleAnalysis/LifeCyclePage";
34
+ import { ATag } from "../../../library-components/ATag";
35
+ import { SocketFunction } from "socket-function/SocketFunction";
36
+ import { managementPageURL } from "../../managementPages";
37
+ import { startTimeParam, endTimeParam } from "../TimeRangeSelector";
38
+ import { formatSearchString } from "./LogViewerParams";
39
+ import { additionalSearchURL } from "../lifeCycleAnalysis/lifeCycleSearch";
30
40
 
31
41
  let excludePendingResults = new URLParam("excludePendingResults", false);
32
42
  let limitURL = new URLParam("limit", 100);
@@ -34,6 +44,7 @@ let limitURL = new URLParam("limit", 100);
34
44
  let savedPathsURL = new URLParam("savedPaths", "");
35
45
  let selectedFieldsURL = new URLParam("selectedFields", {} as Record<string, boolean>);
36
46
  let useRelativeTimeURL = new URLParam("useRelativeTime", true);
47
+ let showLifecycleColumnURL = new URLParam("showlifecycle", true);
37
48
 
38
49
  const defaultSelectedFields = {
39
50
  param0: true,
@@ -64,11 +75,16 @@ export class LogViewer3 extends qreact.Component {
64
75
  hasSearched: t.boolean(false),
65
76
  forceMoveStartTime: t.atomic<number | undefined>(undefined),
66
77
  forceMoveEndTime: t.atomic<number | undefined>(undefined),
78
+ showLifecycleColumn: t.boolean(true),
67
79
  });
68
80
 
81
+ lifecycleController = LifeCyclesController(SocketFunction.browserNodeId());
82
+
69
83
  private searchSequenceNumber = 0;
70
84
 
71
85
  componentDidMount(): void {
86
+ this.state.showLifecycleColumn = showLifecycleColumnURL.value;
87
+
72
88
  if (searchTextURL.value) {
73
89
  void this.search();
74
90
  } else {
@@ -285,6 +301,7 @@ export class LogViewer3 extends qreact.Component {
285
301
  pathOverrides: paths,
286
302
  only: excludePendingResults.value ? "local" : undefined,
287
303
  forceReadProduction: readProductionLogsURL.value,
304
+ searchFromStart: range.searchFromStart,
288
305
  },
289
306
  onResult: (match: LogDatum) => {
290
307
  results.push(match);
@@ -320,123 +337,6 @@ export class LogViewer3 extends qreact.Component {
320
337
  });
321
338
  };
322
339
 
323
- renderSearchStats() {
324
- const stats = this.state.stats;
325
- if (!stats) return undefined;
326
-
327
- let totalSizeRead = 0;
328
- let cachedSize = 0;
329
- let cachedCount = 0;
330
- let uncachedSize = 0;
331
- let uncachedCount = 0;
332
- let uncachedRemoteSize = 0;
333
- let uncachedRemoteCount = 0;
334
- let totalSize = 0;
335
- let remoteTotalSize = 0;
336
- let localTotalSize = 0;
337
-
338
- for (let read of stats.reads) {
339
- totalSizeRead += read.size;
340
- if (read.cached) {
341
- cachedSize += read.size;
342
- cachedCount += read.count;
343
- } else {
344
- uncachedSize += read.size;
345
- uncachedCount += read.count;
346
- totalSize += read.totalSize;
347
- if (read.remote) {
348
- remoteTotalSize += read.totalSize;
349
- } else {
350
- localTotalSize += read.totalSize;
351
- }
352
- }
353
- if (read.remote && !read.cached) {
354
- uncachedRemoteSize += read.size;
355
- uncachedRemoteCount += read.count;
356
- }
357
- }
358
-
359
- const renderStage = (title: string, items: preact.ComponentChild[], hue: number | undefined, isFirst: boolean, isLast: boolean) => {
360
- const triangleWidth = 20;
361
-
362
- let color = hue !== undefined ? css.hsl(hue, 70, 80) : css.hsl(0, 0, 100);
363
- let colorhsl = hue !== undefined ? css.hslcolor(hue, 80, 30) : css;
364
-
365
- return (
366
- <div className={css.vbox(2).relative.minWidth(200).paddingTop(8).paddingBottom(8).paddingRight(10).paddingLeft(isFirst ? 10 : 10 + triangleWidth) + color}>
367
- <div className={css.fontWeight(600) + colorhsl}>{title}</div>
368
- {items.map((item, i) => (
369
- <div key={i}>{item}</div>
370
- ))}
371
- {!isLast && hue !== undefined && (
372
- <svg width={triangleWidth} height="100" viewBox="0 0 20 100" preserveAspectRatio="none" className={css.absolute.right(-triangleWidth).top(0).fillHeight.zIndex(1)}>
373
- <polygon points="0,0 20,50 0,100" fill={`hsl(${hue}, 70%, 80%)`} />
374
- </svg>
375
- )}
376
- </div>
377
- );
378
- };
379
-
380
- const fileItems = [
381
- `${formatTime(stats.fileFindTime)} | ${formatNumber(stats.localFilesSearched + stats.backblazeFilesSearched)} / ${formatNumber(stats.totalLocalFiles + stats.totalBackblazeFiles)} | ${formatNumber(totalSize)}B`,
382
- `remote ${formatNumber(stats.backblazeFilesSearched)} / ${formatNumber(stats.totalBackblazeFiles)} | ${formatNumber(remoteTotalSize)}B`,
383
- `pending ${formatNumber(stats.localFilesSearched)} / ${formatNumber(stats.totalLocalFiles)} | ${formatNumber(localTotalSize)}B`,
384
- ];
385
-
386
- const totalIndexesSearched = stats.remoteIndexesSearched + stats.localIndexesSearched;
387
- const totalIndexSize = stats.remoteIndexSize + stats.localIndexSize;
388
- const indexItems = [
389
- `${formatTime(stats.indexSearchTime)} | ${formatNumber(totalIndexesSearched)} | ${formatNumber(totalIndexSize)}B | ${formatNumber(totalSize / totalIndexSize)}X`,
390
- `remote ${formatNumber(stats.remoteIndexesSearched)} | ${formatNumber(stats.remoteIndexSize)}B | ${formatNumber(remoteTotalSize / stats.remoteIndexSize)}X`,
391
- `local ${formatNumber(stats.localIndexesSearched)} | ${formatNumber(stats.localIndexSize)}B | ${formatNumber(localTotalSize / stats.localIndexSize)}X`,
392
- ];
393
-
394
- const blockItems = [
395
- <div title={`Scanned size = ${formatNumber(stats.blocksCheckedCompressedSize)}B, Decompressed size = ${formatNumber(stats.blocksCheckedDecompressedSize)}B, Compression ratio = ${formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X`}>{formatTime(stats.blockSearchTime)} | {formatNumber(stats.blockCheckedCount)} | {formatNumber(stats.blocksCheckedCompressedSize)}B | {formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X</div>,
396
- `total scanned ${formatNumber(stats.totalBlockCount)} (${formatPercent(stats.blockCheckedCount / stats.totalBlockCount)})`,
397
- `remote ${formatNumber(stats.remoteBlockCheckedCount)} / ${formatNumber(stats.remoteBlockCount)} (${formatPercent(stats.remoteBlockCheckedCount / stats.remoteBlockCount)}) | local ${formatNumber(stats.localBlockCheckedCount)} / ${formatNumber(stats.localBlockCount)} (${formatPercent(stats.localBlockCheckedCount / stats.localBlockCount)})`,
398
- ];
399
- let cacheItems = [
400
- `disk read ${formatNumber(uncachedSize)}B (${formatNumber(uncachedCount)})`,
401
- `remote ${formatPercent(uncachedRemoteSize / totalSizeRead)} (${formatNumber(uncachedRemoteCount)})`,
402
- `cached ${formatNumber(cachedSize)} (${formatNumber(cachedCount)})`,
403
- ];
404
-
405
- const resultItems = [
406
- `${formatTime(stats.timeToFirstMatch)} until first result`,
407
- `${formatNumber(stats.matchCount)} results / ${limitURL.value} limit`,
408
- ];
409
-
410
- const doneItems = [
411
- `${formatTime(stats.totalSearchTime)} total search time`,
412
- ];
413
-
414
- let hasErrors = stats.fileErrors.length > 0 || stats.blockErrors.length > 0;
415
-
416
- return (
417
- <div className={css.hbox(0).alignItems("stretch")}>
418
- {renderStage("Files", fileItems, 290, true, false)}
419
- {renderStage("Indexes", indexItems, 250, false, false)}
420
- {renderStage("Blocks", blockItems, 210, false, false)}
421
- {renderStage("Reads", cacheItems, 160, false, false)}
422
- {renderStage("Results", resultItems, 120, false, this.state.searching && !hasErrors)}
423
- {!this.state.searching && renderStage("Done", doneItems, undefined, false, !hasErrors)}
424
- {(() => {
425
- if (!hasErrors) return undefined;
426
- let errorItems: preact.ComponentChild[] = [];
427
- if (stats.fileErrors.length > 0) {
428
- const errorTitle = stats.fileErrors.map(e => `${e.path}:\n${e.error}`).join("\n\n");
429
- errorItems.push(<div title={errorTitle}>{stats.fileErrors.length} files failed</div>);
430
- }
431
- if (stats.blockErrors.length > 0) {
432
- errorItems.push(<div title={stats.blockErrors.join("\n")}>{stats.blockErrors.length} blocks failed</div>);
433
- }
434
- return renderStage("Errors", errorItems, 0, false, true);
435
- })()}
436
- </div>
437
- );
438
- }
439
-
440
340
  renderForceMoveLogs() {
441
341
  return <><Button
442
342
  onClick={async () => {
@@ -627,6 +527,64 @@ export class LogViewer3 extends qreact.Component {
627
527
  }
628
528
  };
629
529
 
530
+ if (this.state.showLifecycleColumn) {
531
+ const lifecycles = this.lifecycleController.getLifeCycles();
532
+ if (lifecycles && lifecycles.length > 0) {
533
+ columns["lifecycles"] = {
534
+ title: "Lifecycles",
535
+ formatter: (x, context) => {
536
+ if (!context?.row) return undefined;
537
+ let datum = context.row;
538
+ let matches = getLifecycleMatchesForDatum(lifecycles, datum);
539
+ if (matches.length === 0) return undefined;
540
+
541
+ let lifecycleId = matches[0].lifecycle.id;
542
+ let lifecycleTitle = matches[0].lifecycle.title;
543
+
544
+ let firstMatch = matches[0];
545
+ let searchFilter = formatSearchString(firstMatch.datum);
546
+
547
+ let minTime = Infinity;
548
+ let maxTime = -Infinity;
549
+ for (let resultDatum of this.state.results) {
550
+ let resultMatches = getLifecycleMatchesForDatum(lifecycles, resultDatum);
551
+ if (resultMatches.some(m => m.lifecycle.id === lifecycleId)) {
552
+ if (resultDatum.time < minTime) minTime = resultDatum.time;
553
+ if (resultDatum.time > maxTime) maxTime = resultDatum.time;
554
+ }
555
+ }
556
+
557
+ return <div className={css.hbox(4).wrap}>
558
+ {matches.map((match, idx) => (
559
+ <div key={idx} className={css.hbox(8)}>
560
+ <ATag values={[
561
+ managementPageURL.getOverride("LifeCyclePage"),
562
+ lifecycleIdURL.getOverride(match.lifecycle.id),
563
+ additionalSearchURL.getOverride(formatSearchString(match.datum)),
564
+ startTimeParam.getOverride(datum.time - timeInHour),
565
+ endTimeParam.getOverride(datum.time + timeInHour),
566
+ ]}>
567
+ {match.lifecycle.title}
568
+ </ATag>
569
+ </div>
570
+ ))}
571
+ <ATag values={[
572
+ managementPageURL.getOverride("LifeCyclePage"),
573
+ lifecycleIdURL.getOverride(firstMatch.lifecycle.id),
574
+ additionalSearchURL.getOverride(searchFilter),
575
+ startTimeParam.getOverride(minTime - timeInHour),
576
+ endTimeParam.getOverride(maxTime + timeInHour),
577
+ ]}>
578
+ <Button hue={280} className={css.boldStyle}>
579
+ View All {lifecycleTitle}
580
+ </Button>
581
+ </ATag>
582
+ </div>;
583
+ }
584
+ };
585
+ }
586
+ }
587
+
630
588
  return <>
631
589
  <div className={css.hbox(10)}>
632
590
  <InputPicker
@@ -662,6 +620,14 @@ export class LogViewer3 extends qreact.Component {
662
620
  checkbox
663
621
  url={useRelativeTimeURL}
664
622
  />
623
+ <InputLabelURL
624
+ label="Show Lifecycles"
625
+ checkbox
626
+ url={showLifecycleColumnURL}
627
+ onChange={(e) => {
628
+ this.state.showLifecycleColumn = e.currentTarget.checked;
629
+ }}
630
+ />
665
631
  </div>
666
632
 
667
633
  {this.state.results.length === 0 && !this.state.searching && this.state.hasSearched && <div className={css.hsl(40, 50, 50).colorhsl(60, 50, 100).boldStyle.pad2(10).ellipsis}>
@@ -674,13 +640,14 @@ export class LogViewer3 extends qreact.Component {
674
640
  lineLimit={4}
675
641
  initialLimit={10}
676
642
  characterLimit={400}
643
+ cellClass={css.maxWidth("20vw")}
677
644
  getRowAttributes={row => {
678
645
  let hue = -1;
679
646
  if (row.__LOG_TYPE === "warn") hue = 40;
680
647
  if (row.__LOG_TYPE === "error") hue = 0;
681
648
  if (row.__LOG_TYPE === "info") hue = 200;
682
649
  return {
683
- className: hue !== -1 && css.hsl(hue, 40, 50).colorhsl(hue, 0, 100) || undefined,
650
+ className: hue !== -1 && css.hsl(hue, 40, 60).colorhsl(hue, 0, 100) || undefined,
684
651
  };
685
652
  }}
686
653
  />
@@ -695,8 +662,8 @@ export class LogViewer3 extends qreact.Component {
695
662
  checkbox
696
663
  label="Exclude Pending Results"
697
664
  url={excludePendingResults}
698
- onChangeValue={(newValue) => {
699
- if (newValue) {
665
+ onChange={(e) => {
666
+ if (e.currentTarget.checked) {
700
667
  savedPathsURL.value = "";
701
668
  void this.loadPaths();
702
669
  }
@@ -706,8 +673,8 @@ export class LogViewer3 extends qreact.Component {
706
673
  checkbox
707
674
  label="Read Production Logs"
708
675
  url={readProductionLogsURL}
709
- onChangeValue={(newValue) => {
710
- if (newValue) {
676
+ onChange={(e) => {
677
+ if (e.currentTarget.checked) {
711
678
  savedPathsURL.value = "";
712
679
  void this.loadPaths();
713
680
  }
@@ -888,7 +855,11 @@ export class LogViewer3 extends qreact.Component {
888
855
 
889
856
  {this.renderPendingLogWarnings()}
890
857
 
891
- {this.renderSearchStats()}
858
+ <RenderSearchStats
859
+ searching={this.state.searching}
860
+ limit={limitURL.value}
861
+ stats={this.state.stats}
862
+ />
892
863
 
893
864
  {this.renderResults()}
894
865
  </div>
@@ -0,0 +1,127 @@
1
+ import { formatTime, formatNumber, formatPercent } from "socket-function/src/formatting/format";
2
+ import { css } from "typesafecss";
3
+ import { qreact } from "../../../4-dom/qreact";
4
+ import { IndexedLogResults } from "./BufferIndexHelpers";
5
+
6
+ export class RenderSearchStats extends qreact.Component<{
7
+ stats: IndexedLogResults | undefined;
8
+ searching: boolean;
9
+ limit: number;
10
+ }> {
11
+ render() {
12
+ const stats = this.props.stats;
13
+ if (!stats) return undefined;
14
+
15
+ let totalSizeRead = 0;
16
+ let cachedSize = 0;
17
+ let cachedCount = 0;
18
+ let uncachedSize = 0;
19
+ let uncachedCount = 0;
20
+ let uncachedRemoteSize = 0;
21
+ let uncachedRemoteCount = 0;
22
+ let totalSize = 0;
23
+ let remoteTotalSize = 0;
24
+ let localTotalSize = 0;
25
+
26
+ for (let read of stats.reads) {
27
+ totalSizeRead += read.size;
28
+ if (read.cached) {
29
+ cachedSize += read.size;
30
+ cachedCount += read.count;
31
+ } else {
32
+ uncachedSize += read.size;
33
+ uncachedCount += read.count;
34
+ totalSize += read.totalSize;
35
+ if (read.remote) {
36
+ remoteTotalSize += read.totalSize;
37
+ } else {
38
+ localTotalSize += read.totalSize;
39
+ }
40
+ }
41
+ if (read.remote && !read.cached) {
42
+ uncachedRemoteSize += read.size;
43
+ uncachedRemoteCount += read.count;
44
+ }
45
+ }
46
+
47
+ const renderStage = (title: string, items: preact.ComponentChild[], hue: number | undefined, isFirst: boolean, isLast: boolean) => {
48
+ const triangleWidth = 20;
49
+
50
+ let color = hue !== undefined ? css.hsl(hue, 70, 80) : css.hsl(0, 0, 100);
51
+ let colorhsl = hue !== undefined ? css.hslcolor(hue, 80, 30) : css;
52
+
53
+ return (
54
+ <div className={css.vbox(2).relative.minWidth(200).paddingTop(8).paddingBottom(8).paddingRight(10).paddingLeft(isFirst ? 10 : 10 + triangleWidth) + color}>
55
+ <div className={css.fontWeight(600) + colorhsl}>{title}</div>
56
+ {items.map((item, i) => (
57
+ <div key={i}>{item}</div>
58
+ ))}
59
+ {!isLast && hue !== undefined && (
60
+ <svg width={triangleWidth} height="100" viewBox="0 0 20 100" preserveAspectRatio="none" className={css.absolute.right(-triangleWidth).top(0).fillHeight.zIndex(1)}>
61
+ <polygon points="0,0 20,50 0,100" fill={`hsl(${hue}, 70%, 80%)`} />
62
+ </svg>
63
+ )}
64
+ </div>
65
+ );
66
+ };
67
+
68
+ const fileItems = [
69
+ `${formatTime(stats.fileFindTime)} | ${formatNumber(stats.localFilesSearched + stats.backblazeFilesSearched)} / ${formatNumber(stats.totalLocalFiles + stats.totalBackblazeFiles)} | ${formatNumber(totalSize)}B`,
70
+ `remote ${formatNumber(stats.backblazeFilesSearched)} / ${formatNumber(stats.totalBackblazeFiles)} | ${formatNumber(remoteTotalSize)}B | ${formatNumber(stats.backblazeLogsSearched)} logs`,
71
+ `pending ${formatNumber(stats.localFilesSearched)} / ${formatNumber(stats.totalLocalFiles)} | ${formatNumber(localTotalSize)}B`,
72
+ ];
73
+
74
+ const totalIndexesSearched = stats.remoteIndexesSearched + stats.localIndexesSearched;
75
+ const totalIndexSize = stats.remoteIndexSize + stats.localIndexSize;
76
+ const indexItems = [
77
+ `${formatTime(stats.indexSearchTime)} | ${formatNumber(totalIndexesSearched)} | ${formatNumber(totalIndexSize)}B | ${formatNumber(totalSize / totalIndexSize)}X`,
78
+ `remote ${formatNumber(stats.remoteIndexesSearched)} | ${formatNumber(stats.remoteIndexSize)}B | ${formatNumber(remoteTotalSize / stats.remoteIndexSize)}X`,
79
+ `local ${formatNumber(stats.localIndexesSearched)} | ${formatNumber(stats.localIndexSize)}B | ${formatNumber(localTotalSize / stats.localIndexSize)}X`,
80
+ ];
81
+
82
+ const blockItems = [
83
+ <div title={`Scanned size = ${formatNumber(stats.blocksCheckedCompressedSize)}B, Decompressed size = ${formatNumber(stats.blocksCheckedDecompressedSize)}B, Compression ratio = ${formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X`}>{formatTime(stats.blockSearchTime)} | {formatNumber(stats.blockCheckedCount)} | {formatNumber(stats.blocksCheckedCompressedSize)}B | {formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X</div>,
84
+ `total scanned ${formatNumber(stats.totalBlockCount)} (${formatPercent(stats.blockCheckedCount / stats.totalBlockCount)})`,
85
+ `remote ${formatNumber(stats.remoteBlockCheckedCount)} / ${formatNumber(stats.remoteBlockCount)} (${formatPercent(stats.remoteBlockCheckedCount / stats.remoteBlockCount)}) | local ${formatNumber(stats.localBlockCheckedCount)} / ${formatNumber(stats.localBlockCount)} (${formatPercent(stats.localBlockCheckedCount / stats.localBlockCount)})`,
86
+ ];
87
+ let cacheItems = [
88
+ `disk read ${formatNumber(uncachedSize)}B (${formatNumber(uncachedCount)})`,
89
+ `remote ${formatPercent(uncachedRemoteSize / totalSizeRead)} (${formatNumber(uncachedRemoteCount)})`,
90
+ `cached ${formatNumber(cachedSize)} (${formatNumber(cachedCount)})`,
91
+ ];
92
+
93
+ const resultItems = [
94
+ `${formatTime(stats.timeToFirstMatch)} until first result`,
95
+ `${formatNumber(stats.matchCount)} results / ${this.props.limit} limit`,
96
+ ];
97
+
98
+ const doneItems = [
99
+ `${formatTime(stats.totalSearchTime)} total search time`,
100
+ ];
101
+
102
+ let hasErrors = stats.fileErrors.length > 0 || stats.blockErrors.length > 0;
103
+
104
+ return (
105
+ <div className={css.hbox(0).alignItems("stretch")}>
106
+ {renderStage("Files", fileItems, 290, true, false)}
107
+ {renderStage("Indexes", indexItems, 250, false, false)}
108
+ {renderStage("Blocks", blockItems, 210, false, false)}
109
+ {renderStage("Reads", cacheItems, 160, false, false)}
110
+ {renderStage("Results", resultItems, 120, false, this.props.searching && !hasErrors)}
111
+ {!this.props.searching && renderStage("Done", doneItems, undefined, false, !hasErrors)}
112
+ {(() => {
113
+ if (!hasErrors) return undefined;
114
+ let errorItems: preact.ComponentChild[] = [];
115
+ if (stats.fileErrors.length > 0) {
116
+ const errorTitle = stats.fileErrors.map(e => `${e.path}:\n${e.error}`).join("\n\n");
117
+ errorItems.push(<div title={errorTitle}>{stats.fileErrors.length} files failed</div>);
118
+ }
119
+ if (stats.blockErrors.length > 0) {
120
+ errorItems.push(<div title={stats.blockErrors.join("\n")}>{stats.blockErrors.length} blocks failed</div>);
121
+ }
122
+ return renderStage("Errors", errorItems, 0, false, true);
123
+ })()}
124
+ </div>
125
+ );
126
+ }
127
+ }
@@ -1,5 +1,6 @@
1
1
  import { measureWrap } from "socket-function/src/profiling/measure";
2
2
  import { Unit, getAllUnits, remapSearchBuffer } from "./BufferIndexHelpers";
3
+ import { cacheLimited } from "socket-function/src/caching";
3
4
 
4
5
  const WILD_CARD_BYTE = 42;
5
6
  const WILD_CARD_CHAR = "*";
@@ -105,6 +106,8 @@ function getSimplerStructure(structure: MatchStructure): Buffer[][] {
105
106
 
106
107
  throw new Error(`Unknown structure type: ${type}`);
107
108
  }
109
+ export const createMatchesPatternCached = cacheLimited(1000, (pattern: string) => createMatchesPattern(Buffer.from(pattern), false));
110
+
108
111
  // Each WILD_CARD_BYTE in pattern acts as a multi-byte wildcard: the segments on either
109
112
  // side must appear in order somewhere within buffer.
110
113
  // Returns a function that matches buffers against the pre-processed pattern.
@@ -155,7 +155,7 @@ export async function moveLogsToPublic(config: {
155
155
  try {
156
156
  let decoded = await BufferIndex.decodeAll(buffer.buffer);
157
157
  allBuffers.push(decoded);
158
- } catch (e) {
158
+ } catch (e: any) {
159
159
  console.error(`Error decoding buffer, ignoring it`, { path: buffer.path, bufferLength: buffer.buffer.length, bufferFirst100: buffer.buffer.slice(0, 100).toString("hex"), error: String(e.stack) });
160
160
  continue;
161
161
  }
@@ -12,16 +12,18 @@ import { timeInDay, timeInHour, timeInMinute } from "socket-function/src/misc";
12
12
  // URL parameters for time range
13
13
  export const startTimeParam = new URLParam("startTime", undefined as number | undefined);
14
14
  export const endTimeParam = new URLParam("endTime", undefined as number | undefined);
15
+ export const searchFromStartParam = new URLParam("searchFromStart", false);
15
16
 
16
17
  let now = Date.now();
17
18
  /** Get the time range with default values if not set */
18
- export function getTimeRange(): { startTime: number; endTime: number } {
19
+ export function getTimeRange(): { startTime: number; endTime: number; searchFromStart: boolean } {
19
20
  const defaultStart = now - timeInHour * 24;
20
21
  const defaultEnd = now + timeInHour;
21
22
 
22
23
  return {
23
24
  startTime: startTimeParam.value || defaultStart,
24
25
  endTime: endTimeParam.value || defaultEnd,
26
+ searchFromStart: searchFromStartParam.value,
25
27
  };
26
28
  }
27
29
 
@@ -75,7 +77,14 @@ export class TimeRangeSelector extends qreact.Component {
75
77
  />
76
78
  </div>
77
79
  <div className={css.hbox(12).wrap}>
78
-
80
+ <InputLabel
81
+ label="Search from Start"
82
+ checkbox
83
+ value={searchFromStartParam.value ? "true" : ""}
84
+ onChangeValue={value => {
85
+ searchFromStartParam.value = !!value;
86
+ }}
87
+ />
79
88
  <Button
80
89
  hue={110} onClick={() => {
81
90
  let now = Date.now();
@@ -450,10 +450,7 @@ class SuppressionItem extends qreact.Component<{
450
450
  onKeyDown={e => {
451
451
  if (e.key === "Enter") {
452
452
  let pattern = e.currentTarget.value.trim();
453
- if (!pattern) {
454
- alert("Pattern cannot be empty");
455
- return;
456
- }
453
+ if (!pattern) return;
457
454
  this.state.patternValue = pattern;
458
455
  Querysub.onCommitFinished(async () => {
459
456
  await this.manager.updateSuppression(suppression.id, pattern);
@@ -170,7 +170,7 @@ const PATTERN_GENERATION_LIMIT = 100;
170
170
  const PATTERN_GENERATION_WINDOW = 15 * 60 * 1000;
171
171
  let patternGenerationTimes: number[] = [];
172
172
 
173
- async function generatePatternForError(error: LogDatum): Promise<string> {
173
+ export async function generatePatternForError(error: LogDatum): Promise<string> {
174
174
  const now = Date.now();
175
175
  const windowStart = now - PATTERN_GENERATION_WINDOW;
176
176