querysub 0.394.0 → 0.396.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/.cursorrules +8 -0
  2. package/package.json +1 -1
  3. package/src/-a-archives/archivesJSONT.ts +71 -8
  4. package/src/-c-identity/IdentityController.ts +19 -1
  5. package/src/0-path-value-core/pathValueCore.ts +26 -7
  6. package/src/5-diagnostics/GenericFormat.tsx +1 -1
  7. package/src/deployManager/components/MachinesListPage.tsx +1 -2
  8. package/src/diagnostics/logs/IndexedLogs/BufferIndex.ts +12 -3
  9. package/src/diagnostics/logs/IndexedLogs/BufferIndexHelpers.ts +8 -3
  10. package/src/diagnostics/logs/IndexedLogs/BufferUnitIndex.ts +24 -9
  11. package/src/diagnostics/logs/IndexedLogs/BufferUnitSet.ts +0 -1
  12. package/src/diagnostics/logs/IndexedLogs/FindProgressTracker.ts +21 -5
  13. package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts +10 -4
  14. package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx +94 -124
  15. package/src/diagnostics/logs/IndexedLogs/RenderSearchStats.tsx +127 -0
  16. package/src/diagnostics/logs/IndexedLogs/bufferSearchFindMatcher.ts +4 -0
  17. package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts +1 -1
  18. package/src/diagnostics/logs/TimeRangeSelector.tsx +11 -2
  19. package/src/diagnostics/logs/errorNotifications2/ErrorNotificationPage.tsx +1 -4
  20. package/src/diagnostics/logs/errorNotifications2/errorNotifications.ts +1 -1
  21. package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleEntryEditor.tsx +358 -0
  22. package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleEntryReadMode.tsx +149 -0
  23. package/src/diagnostics/logs/lifeCycleAnalysis/LifeCyclePage.tsx +274 -0
  24. package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleRenderer.tsx +291 -0
  25. package/src/diagnostics/logs/lifeCycleAnalysis/NestedLifeCycleInfo.tsx +151 -0
  26. package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycleMatching.ts +49 -0
  27. package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycleSearch.tsx +580 -0
  28. package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx +128 -90
  29. package/src/diagnostics/managementPages.tsx +17 -1
  30. package/src/diagnostics/misc-pages/ArchiveViewer.tsx +4 -3
  31. package/src/diagnostics/misc-pages/ArchiveViewerTree.tsx +6 -3
  32. package/src/functional/{limitProcessing.ts → throttleProcessing.ts} +1 -1
  33. package/src/library-components/StartEllipsis.tsx +13 -0
  34. package/src/misc.ts +4 -0
  35. package/tempnotes.txt +0 -0
  36. package/src/diagnostics/logs/lifeCycleAnalysis/test.wat +0 -106
  37. package/src/diagnostics/logs/lifeCycleAnalysis/test.wat.d.ts +0 -2
  38. package/src/diagnostics/logs/lifeCycleAnalysis/testHoist.ts +0 -5
@@ -5,7 +5,7 @@ import { t } from "../../../2-proxy/schema2";
5
5
  import { Button } from "../../../library-components/Button";
6
6
  import { InputLabel, InputLabelURL } from "../../../library-components/InputLabel";
7
7
  import { getLoggers2Async, LogDatum } from "../diskLogger";
8
- import { list, timeInDay, keyByArray, sort, throttleFunction } from "socket-function/src/misc";
8
+ import { list, timeInDay, timeInHour, keyByArray, sort, throttleFunction } from "socket-function/src/misc";
9
9
  import { formatDateTime, formatDateTimeDetailed, formatNumber, formatTime, formatPercent } from "socket-function/src/formatting/format";
10
10
  import { IndexedLogs, TimeFilePathWithSize } from "./IndexedLogs";
11
11
  import { Querysub } from "../../../4-querysub/QuerysubController";
@@ -27,6 +27,15 @@ import { TimeFilePath } from "./TimeFileTree";
27
27
  import { isPublic } from "../../../config";
28
28
  import { Zip } from "../../../zip";
29
29
  import { readProductionLogsURL, searchTextURL } from "./LogViewerParams";
30
+ import { RenderSearchStats } from "./RenderSearchStats";
31
+ import { LifeCyclesController, LifeCycle, LifeCycleEntry } from "../lifeCycleAnalysis/lifeCycles";
32
+ import { getLifecycleMatchesForDatum } from "../lifeCycleAnalysis/lifeCycleMatching";
33
+ import { lifecycleIdURL, additionalSearchURL } from "../lifeCycleAnalysis/LifeCyclePage";
34
+ import { ATag } from "../../../library-components/ATag";
35
+ import { SocketFunction } from "socket-function/SocketFunction";
36
+ import { managementPageURL } from "../../managementPages";
37
+ import { startTimeParam, endTimeParam } from "../TimeRangeSelector";
38
+ import { formatSearchString } from "./LogViewerParams";
30
39
 
31
40
  let excludePendingResults = new URLParam("excludePendingResults", false);
32
41
  let limitURL = new URLParam("limit", 100);
@@ -34,6 +43,7 @@ let limitURL = new URLParam("limit", 100);
34
43
  let savedPathsURL = new URLParam("savedPaths", "");
35
44
  let selectedFieldsURL = new URLParam("selectedFields", {} as Record<string, boolean>);
36
45
  let useRelativeTimeURL = new URLParam("useRelativeTime", true);
46
+ let showLifecycleColumnURL = new URLParam("showlifecycle", true);
37
47
 
38
48
  const defaultSelectedFields = {
39
49
  param0: true,
@@ -64,11 +74,16 @@ export class LogViewer3 extends qreact.Component {
64
74
  hasSearched: t.boolean(false),
65
75
  forceMoveStartTime: t.atomic<number | undefined>(undefined),
66
76
  forceMoveEndTime: t.atomic<number | undefined>(undefined),
77
+ showLifecycleColumn: t.boolean(true),
67
78
  });
68
79
 
80
+ lifecycleController = LifeCyclesController(SocketFunction.browserNodeId());
81
+
69
82
  private searchSequenceNumber = 0;
70
83
 
71
84
  componentDidMount(): void {
85
+ this.state.showLifecycleColumn = showLifecycleColumnURL.value;
86
+
72
87
  if (searchTextURL.value) {
73
88
  void this.search();
74
89
  } else {
@@ -285,6 +300,7 @@ export class LogViewer3 extends qreact.Component {
285
300
  pathOverrides: paths,
286
301
  only: excludePendingResults.value ? "local" : undefined,
287
302
  forceReadProduction: readProductionLogsURL.value,
303
+ searchFromStart: range.searchFromStart,
288
304
  },
289
305
  onResult: (match: LogDatum) => {
290
306
  results.push(match);
@@ -320,123 +336,6 @@ export class LogViewer3 extends qreact.Component {
320
336
  });
321
337
  };
322
338
 
323
- renderSearchStats() {
324
- const stats = this.state.stats;
325
- if (!stats) return undefined;
326
-
327
- let totalSizeRead = 0;
328
- let cachedSize = 0;
329
- let cachedCount = 0;
330
- let uncachedSize = 0;
331
- let uncachedCount = 0;
332
- let uncachedRemoteSize = 0;
333
- let uncachedRemoteCount = 0;
334
- let totalSize = 0;
335
- let remoteTotalSize = 0;
336
- let localTotalSize = 0;
337
-
338
- for (let read of stats.reads) {
339
- totalSizeRead += read.size;
340
- if (read.cached) {
341
- cachedSize += read.size;
342
- cachedCount += read.count;
343
- } else {
344
- uncachedSize += read.size;
345
- uncachedCount += read.count;
346
- totalSize += read.totalSize;
347
- if (read.remote) {
348
- remoteTotalSize += read.totalSize;
349
- } else {
350
- localTotalSize += read.totalSize;
351
- }
352
- }
353
- if (read.remote && !read.cached) {
354
- uncachedRemoteSize += read.size;
355
- uncachedRemoteCount += read.count;
356
- }
357
- }
358
-
359
- const renderStage = (title: string, items: preact.ComponentChild[], hue: number | undefined, isFirst: boolean, isLast: boolean) => {
360
- const triangleWidth = 20;
361
-
362
- let color = hue !== undefined ? css.hsl(hue, 70, 80) : css.hsl(0, 0, 100);
363
- let colorhsl = hue !== undefined ? css.hslcolor(hue, 80, 30) : css;
364
-
365
- return (
366
- <div className={css.vbox(2).relative.minWidth(200).paddingTop(8).paddingBottom(8).paddingRight(10).paddingLeft(isFirst ? 10 : 10 + triangleWidth) + color}>
367
- <div className={css.fontWeight(600) + colorhsl}>{title}</div>
368
- {items.map((item, i) => (
369
- <div key={i}>{item}</div>
370
- ))}
371
- {!isLast && hue !== undefined && (
372
- <svg width={triangleWidth} height="100" viewBox="0 0 20 100" preserveAspectRatio="none" className={css.absolute.right(-triangleWidth).top(0).fillHeight.zIndex(1)}>
373
- <polygon points="0,0 20,50 0,100" fill={`hsl(${hue}, 70%, 80%)`} />
374
- </svg>
375
- )}
376
- </div>
377
- );
378
- };
379
-
380
- const fileItems = [
381
- `${formatTime(stats.fileFindTime)} | ${formatNumber(stats.localFilesSearched + stats.backblazeFilesSearched)} / ${formatNumber(stats.totalLocalFiles + stats.totalBackblazeFiles)} | ${formatNumber(totalSize)}B`,
382
- `remote ${formatNumber(stats.backblazeFilesSearched)} / ${formatNumber(stats.totalBackblazeFiles)} | ${formatNumber(remoteTotalSize)}B`,
383
- `pending ${formatNumber(stats.localFilesSearched)} / ${formatNumber(stats.totalLocalFiles)} | ${formatNumber(localTotalSize)}B`,
384
- ];
385
-
386
- const totalIndexesSearched = stats.remoteIndexesSearched + stats.localIndexesSearched;
387
- const totalIndexSize = stats.remoteIndexSize + stats.localIndexSize;
388
- const indexItems = [
389
- `${formatTime(stats.indexSearchTime)} | ${formatNumber(totalIndexesSearched)} | ${formatNumber(totalIndexSize)}B | ${formatNumber(totalSize / totalIndexSize)}X`,
390
- `remote ${formatNumber(stats.remoteIndexesSearched)} | ${formatNumber(stats.remoteIndexSize)}B | ${formatNumber(remoteTotalSize / stats.remoteIndexSize)}X`,
391
- `local ${formatNumber(stats.localIndexesSearched)} | ${formatNumber(stats.localIndexSize)}B | ${formatNumber(localTotalSize / stats.localIndexSize)}X`,
392
- ];
393
-
394
- const blockItems = [
395
- <div title={`Scanned size = ${formatNumber(stats.blocksCheckedCompressedSize)}B, Decompressed size = ${formatNumber(stats.blocksCheckedDecompressedSize)}B, Compression ratio = ${formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X`}>{formatTime(stats.blockSearchTime)} | {formatNumber(stats.blockCheckedCount)} | {formatNumber(stats.blocksCheckedCompressedSize)}B | {formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X</div>,
396
- `total scanned ${formatNumber(stats.totalBlockCount)} (${formatPercent(stats.blockCheckedCount / stats.totalBlockCount)})`,
397
- `remote ${formatNumber(stats.remoteBlockCheckedCount)} / ${formatNumber(stats.remoteBlockCount)} (${formatPercent(stats.remoteBlockCheckedCount / stats.remoteBlockCount)}) | local ${formatNumber(stats.localBlockCheckedCount)} / ${formatNumber(stats.localBlockCount)} (${formatPercent(stats.localBlockCheckedCount / stats.localBlockCount)})`,
398
- ];
399
- let cacheItems = [
400
- `disk read ${formatNumber(uncachedSize)}B (${formatNumber(uncachedCount)})`,
401
- `remote ${formatPercent(uncachedRemoteSize / totalSizeRead)} (${formatNumber(uncachedRemoteCount)})`,
402
- `cached ${formatNumber(cachedSize)} (${formatNumber(cachedCount)})`,
403
- ];
404
-
405
- const resultItems = [
406
- `${formatTime(stats.timeToFirstMatch)} until first result`,
407
- `${formatNumber(stats.matchCount)} results / ${limitURL.value} limit`,
408
- ];
409
-
410
- const doneItems = [
411
- `${formatTime(stats.totalSearchTime)} total search time`,
412
- ];
413
-
414
- let hasErrors = stats.fileErrors.length > 0 || stats.blockErrors.length > 0;
415
-
416
- return (
417
- <div className={css.hbox(0).alignItems("stretch")}>
418
- {renderStage("Files", fileItems, 290, true, false)}
419
- {renderStage("Indexes", indexItems, 250, false, false)}
420
- {renderStage("Blocks", blockItems, 210, false, false)}
421
- {renderStage("Reads", cacheItems, 160, false, false)}
422
- {renderStage("Results", resultItems, 120, false, this.state.searching && !hasErrors)}
423
- {!this.state.searching && renderStage("Done", doneItems, undefined, false, !hasErrors)}
424
- {(() => {
425
- if (!hasErrors) return undefined;
426
- let errorItems: preact.ComponentChild[] = [];
427
- if (stats.fileErrors.length > 0) {
428
- const errorTitle = stats.fileErrors.map(e => `${e.path}:\n${e.error}`).join("\n\n");
429
- errorItems.push(<div title={errorTitle}>{stats.fileErrors.length} files failed</div>);
430
- }
431
- if (stats.blockErrors.length > 0) {
432
- errorItems.push(<div title={stats.blockErrors.join("\n")}>{stats.blockErrors.length} blocks failed</div>);
433
- }
434
- return renderStage("Errors", errorItems, 0, false, true);
435
- })()}
436
- </div>
437
- );
438
- }
439
-
440
339
  renderForceMoveLogs() {
441
340
  return <><Button
442
341
  onClick={async () => {
@@ -627,6 +526,64 @@ export class LogViewer3 extends qreact.Component {
627
526
  }
628
527
  };
629
528
 
529
+ if (this.state.showLifecycleColumn) {
530
+ const lifecycles = this.lifecycleController.getLifeCycles();
531
+ if (lifecycles && lifecycles.length > 0) {
532
+ columns["lifecycles"] = {
533
+ title: "Lifecycles",
534
+ formatter: (x, context) => {
535
+ if (!context?.row) return undefined;
536
+ let datum = context.row;
537
+ let matches = getLifecycleMatchesForDatum(lifecycles, datum);
538
+ if (matches.length === 0) return undefined;
539
+
540
+ let lifecycleId = matches[0].lifecycle.id;
541
+ let lifecycleTitle = matches[0].lifecycle.title;
542
+
543
+ let firstMatch = matches[0];
544
+ let searchFilter = formatSearchString(firstMatch.datum);
545
+
546
+ let minTime = Infinity;
547
+ let maxTime = -Infinity;
548
+ for (let resultDatum of this.state.results) {
549
+ let resultMatches = getLifecycleMatchesForDatum(lifecycles, resultDatum);
550
+ if (resultMatches.some(m => m.lifecycle.id === lifecycleId)) {
551
+ if (resultDatum.time < minTime) minTime = resultDatum.time;
552
+ if (resultDatum.time > maxTime) maxTime = resultDatum.time;
553
+ }
554
+ }
555
+
556
+ return <div className={css.hbox(4).wrap}>
557
+ {matches.map((match, idx) => (
558
+ <div key={idx} className={css.hbox(8)}>
559
+ <ATag values={[
560
+ managementPageURL.getOverride("LifeCyclePage"),
561
+ lifecycleIdURL.getOverride(match.lifecycle.id),
562
+ additionalSearchURL.getOverride(formatSearchString(match.datum)),
563
+ startTimeParam.getOverride(datum.time - timeInHour),
564
+ endTimeParam.getOverride(datum.time + timeInHour),
565
+ ]}>
566
+ View {JSON.stringify(match.lifecycle.title)}
567
+ </ATag>
568
+ </div>
569
+ ))}
570
+ <ATag values={[
571
+ managementPageURL.getOverride("LifeCyclePage"),
572
+ lifecycleIdURL.getOverride(firstMatch.lifecycle.id),
573
+ additionalSearchURL.getOverride(searchFilter),
574
+ startTimeParam.getOverride(minTime - timeInHour),
575
+ endTimeParam.getOverride(maxTime + timeInHour),
576
+ ]}>
577
+ <Button hue={280} className={css.boldStyle}>
578
+ View All
579
+ </Button>
580
+ </ATag>
581
+ </div>;
582
+ }
583
+ };
584
+ }
585
+ }
586
+
630
587
  return <>
631
588
  <div className={css.hbox(10)}>
632
589
  <InputPicker
@@ -662,6 +619,14 @@ export class LogViewer3 extends qreact.Component {
662
619
  checkbox
663
620
  url={useRelativeTimeURL}
664
621
  />
622
+ <InputLabelURL
623
+ label="Show Lifecycles"
624
+ checkbox
625
+ url={showLifecycleColumnURL}
626
+ onChange={(e) => {
627
+ this.state.showLifecycleColumn = e.currentTarget.checked;
628
+ }}
629
+ />
665
630
  </div>
666
631
 
667
632
  {this.state.results.length === 0 && !this.state.searching && this.state.hasSearched && <div className={css.hsl(40, 50, 50).colorhsl(60, 50, 100).boldStyle.pad2(10).ellipsis}>
@@ -674,13 +639,14 @@ export class LogViewer3 extends qreact.Component {
674
639
  lineLimit={4}
675
640
  initialLimit={10}
676
641
  characterLimit={400}
642
+ cellClass={css.maxWidth("20vw")}
677
643
  getRowAttributes={row => {
678
644
  let hue = -1;
679
645
  if (row.__LOG_TYPE === "warn") hue = 40;
680
646
  if (row.__LOG_TYPE === "error") hue = 0;
681
647
  if (row.__LOG_TYPE === "info") hue = 200;
682
648
  return {
683
- className: hue !== -1 && css.hsl(hue, 40, 50).colorhsl(hue, 0, 100) || undefined,
649
+ className: hue !== -1 && css.hsl(hue, 40, 60).colorhsl(hue, 0, 100) || undefined,
684
650
  };
685
651
  }}
686
652
  />
@@ -695,8 +661,8 @@ export class LogViewer3 extends qreact.Component {
695
661
  checkbox
696
662
  label="Exclude Pending Results"
697
663
  url={excludePendingResults}
698
- onChangeValue={(newValue) => {
699
- if (newValue) {
664
+ onChange={(e) => {
665
+ if (e.currentTarget.checked) {
700
666
  savedPathsURL.value = "";
701
667
  void this.loadPaths();
702
668
  }
@@ -706,8 +672,8 @@ export class LogViewer3 extends qreact.Component {
706
672
  checkbox
707
673
  label="Read Production Logs"
708
674
  url={readProductionLogsURL}
709
- onChangeValue={(newValue) => {
710
- if (newValue) {
675
+ onChange={(e) => {
676
+ if (e.currentTarget.checked) {
711
677
  savedPathsURL.value = "";
712
678
  void this.loadPaths();
713
679
  }
@@ -888,7 +854,11 @@ export class LogViewer3 extends qreact.Component {
888
854
 
889
855
  {this.renderPendingLogWarnings()}
890
856
 
891
- {this.renderSearchStats()}
857
+ <RenderSearchStats
858
+ searching={this.state.searching}
859
+ limit={limitURL.value}
860
+ stats={this.state.stats}
861
+ />
892
862
 
893
863
  {this.renderResults()}
894
864
  </div>
@@ -0,0 +1,127 @@
1
+ import { formatTime, formatNumber, formatPercent } from "socket-function/src/formatting/format";
2
+ import { css } from "typesafecss";
3
+ import { qreact } from "../../../4-dom/qreact";
4
+ import { IndexedLogResults } from "./BufferIndexHelpers";
5
+
6
+ export class RenderSearchStats extends qreact.Component<{
7
+ stats: IndexedLogResults | undefined;
8
+ searching: boolean;
9
+ limit: number;
10
+ }> {
11
+ render() {
12
+ const stats = this.props.stats;
13
+ if (!stats) return undefined;
14
+
15
+ let totalSizeRead = 0;
16
+ let cachedSize = 0;
17
+ let cachedCount = 0;
18
+ let uncachedSize = 0;
19
+ let uncachedCount = 0;
20
+ let uncachedRemoteSize = 0;
21
+ let uncachedRemoteCount = 0;
22
+ let totalSize = 0;
23
+ let remoteTotalSize = 0;
24
+ let localTotalSize = 0;
25
+
26
+ for (let read of stats.reads) {
27
+ totalSizeRead += read.size;
28
+ if (read.cached) {
29
+ cachedSize += read.size;
30
+ cachedCount += read.count;
31
+ } else {
32
+ uncachedSize += read.size;
33
+ uncachedCount += read.count;
34
+ totalSize += read.totalSize;
35
+ if (read.remote) {
36
+ remoteTotalSize += read.totalSize;
37
+ } else {
38
+ localTotalSize += read.totalSize;
39
+ }
40
+ }
41
+ if (read.remote && !read.cached) {
42
+ uncachedRemoteSize += read.size;
43
+ uncachedRemoteCount += read.count;
44
+ }
45
+ }
46
+
47
+ const renderStage = (title: string, items: preact.ComponentChild[], hue: number | undefined, isFirst: boolean, isLast: boolean) => {
48
+ const triangleWidth = 20;
49
+
50
+ let color = hue !== undefined ? css.hsl(hue, 70, 80) : css.hsl(0, 0, 100);
51
+ let colorhsl = hue !== undefined ? css.hslcolor(hue, 80, 30) : css;
52
+
53
+ return (
54
+ <div className={css.vbox(2).relative.minWidth(200).paddingTop(8).paddingBottom(8).paddingRight(10).paddingLeft(isFirst ? 10 : 10 + triangleWidth) + color}>
55
+ <div className={css.fontWeight(600) + colorhsl}>{title}</div>
56
+ {items.map((item, i) => (
57
+ <div key={i}>{item}</div>
58
+ ))}
59
+ {!isLast && hue !== undefined && (
60
+ <svg width={triangleWidth} height="100" viewBox="0 0 20 100" preserveAspectRatio="none" className={css.absolute.right(-triangleWidth).top(0).fillHeight.zIndex(1)}>
61
+ <polygon points="0,0 20,50 0,100" fill={`hsl(${hue}, 70%, 80%)`} />
62
+ </svg>
63
+ )}
64
+ </div>
65
+ );
66
+ };
67
+
68
+ const fileItems = [
69
+ `${formatTime(stats.fileFindTime)} | ${formatNumber(stats.localFilesSearched + stats.backblazeFilesSearched)} / ${formatNumber(stats.totalLocalFiles + stats.totalBackblazeFiles)} | ${formatNumber(totalSize)}B`,
70
+ `remote ${formatNumber(stats.backblazeFilesSearched)} / ${formatNumber(stats.totalBackblazeFiles)} | ${formatNumber(remoteTotalSize)}B | ${formatNumber(stats.backblazeLogsSearched)} logs`,
71
+ `pending ${formatNumber(stats.localFilesSearched)} / ${formatNumber(stats.totalLocalFiles)} | ${formatNumber(localTotalSize)}B`,
72
+ ];
73
+
74
+ const totalIndexesSearched = stats.remoteIndexesSearched + stats.localIndexesSearched;
75
+ const totalIndexSize = stats.remoteIndexSize + stats.localIndexSize;
76
+ const indexItems = [
77
+ `${formatTime(stats.indexSearchTime)} | ${formatNumber(totalIndexesSearched)} | ${formatNumber(totalIndexSize)}B | ${formatNumber(totalSize / totalIndexSize)}X`,
78
+ `remote ${formatNumber(stats.remoteIndexesSearched)} | ${formatNumber(stats.remoteIndexSize)}B | ${formatNumber(remoteTotalSize / stats.remoteIndexSize)}X`,
79
+ `local ${formatNumber(stats.localIndexesSearched)} | ${formatNumber(stats.localIndexSize)}B | ${formatNumber(localTotalSize / stats.localIndexSize)}X`,
80
+ ];
81
+
82
+ const blockItems = [
83
+ <div title={`Scanned size = ${formatNumber(stats.blocksCheckedCompressedSize)}B, Decompressed size = ${formatNumber(stats.blocksCheckedDecompressedSize)}B, Compression ratio = ${formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X`}>{formatTime(stats.blockSearchTime)} | {formatNumber(stats.blockCheckedCount)} | {formatNumber(stats.blocksCheckedCompressedSize)}B | {formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X</div>,
84
+ `total scanned ${formatNumber(stats.totalBlockCount)} (${formatPercent(stats.blockCheckedCount / stats.totalBlockCount)})`,
85
+ `remote ${formatNumber(stats.remoteBlockCheckedCount)} / ${formatNumber(stats.remoteBlockCount)} (${formatPercent(stats.remoteBlockCheckedCount / stats.remoteBlockCount)}) | local ${formatNumber(stats.localBlockCheckedCount)} / ${formatNumber(stats.localBlockCount)} (${formatPercent(stats.localBlockCheckedCount / stats.localBlockCount)})`,
86
+ ];
87
+ let cacheItems = [
88
+ `disk read ${formatNumber(uncachedSize)}B (${formatNumber(uncachedCount)})`,
89
+ `remote ${formatPercent(uncachedRemoteSize / totalSizeRead)} (${formatNumber(uncachedRemoteCount)})`,
90
+ `cached ${formatNumber(cachedSize)} (${formatNumber(cachedCount)})`,
91
+ ];
92
+
93
+ const resultItems = [
94
+ `${formatTime(stats.timeToFirstMatch)} until first result`,
95
+ `${formatNumber(stats.matchCount)} results / ${this.props.limit} limit`,
96
+ ];
97
+
98
+ const doneItems = [
99
+ `${formatTime(stats.totalSearchTime)} total search time`,
100
+ ];
101
+
102
+ let hasErrors = stats.fileErrors.length > 0 || stats.blockErrors.length > 0;
103
+
104
+ return (
105
+ <div className={css.hbox(0).alignItems("stretch")}>
106
+ {renderStage("Files", fileItems, 290, true, false)}
107
+ {renderStage("Indexes", indexItems, 250, false, false)}
108
+ {renderStage("Blocks", blockItems, 210, false, false)}
109
+ {renderStage("Reads", cacheItems, 160, false, false)}
110
+ {renderStage("Results", resultItems, 120, false, this.props.searching && !hasErrors)}
111
+ {!this.props.searching && renderStage("Done", doneItems, undefined, false, !hasErrors)}
112
+ {(() => {
113
+ if (!hasErrors) return undefined;
114
+ let errorItems: preact.ComponentChild[] = [];
115
+ if (stats.fileErrors.length > 0) {
116
+ const errorTitle = stats.fileErrors.map(e => `${e.path}:\n${e.error}`).join("\n\n");
117
+ errorItems.push(<div title={errorTitle}>{stats.fileErrors.length} files failed</div>);
118
+ }
119
+ if (stats.blockErrors.length > 0) {
120
+ errorItems.push(<div title={stats.blockErrors.join("\n")}>{stats.blockErrors.length} blocks failed</div>);
121
+ }
122
+ return renderStage("Errors", errorItems, 0, false, true);
123
+ })()}
124
+ </div>
125
+ );
126
+ }
127
+ }
@@ -1,5 +1,6 @@
1
1
  import { measureWrap } from "socket-function/src/profiling/measure";
2
2
  import { Unit, getAllUnits, remapSearchBuffer } from "./BufferIndexHelpers";
3
+ import { cacheLimited } from "socket-function/src/caching";
3
4
 
4
5
  const WILD_CARD_BYTE = 42;
5
6
  const WILD_CARD_CHAR = "*";
@@ -8,6 +9,7 @@ const SEARCH_OR_CHAR = "|";
8
9
  const SEARCH_AND_BYTE = 38;
9
10
  const SEARCH_AND_CHAR = "&";
10
11
 
12
+ // IMPORTANT! While technically THIS structure can support negation, the index structure above us cannot, as it only stores information at the block level (multiple buffers per block). So... if we supported negation, it couldn't, so it would give us a lot of false positives (cases where negation removes the results). So... never use negation, just turn everything into a positive signal.
11
13
  export type MatchStructure = {
12
14
  type: "or";
13
15
  parts: MatchStructure[];
@@ -105,6 +107,8 @@ function getSimplerStructure(structure: MatchStructure): Buffer[][] {
105
107
 
106
108
  throw new Error(`Unknown structure type: ${type}`);
107
109
  }
110
+ export const createMatchesPatternCached = cacheLimited(1000, (pattern: string) => createMatchesPattern(Buffer.from(pattern), false));
111
+
108
112
  // Each WILD_CARD_BYTE in pattern acts as a multi-byte wildcard: the segments on either
109
113
  // side must appear in order somewhere within buffer.
110
114
  // Returns a function that matches buffers against the pre-processed pattern.
@@ -155,7 +155,7 @@ export async function moveLogsToPublic(config: {
155
155
  try {
156
156
  let decoded = await BufferIndex.decodeAll(buffer.buffer);
157
157
  allBuffers.push(decoded);
158
- } catch (e) {
158
+ } catch (e: any) {
159
159
  console.error(`Error decoding buffer, ignoring it`, { path: buffer.path, bufferLength: buffer.buffer.length, bufferFirst100: buffer.buffer.slice(0, 100).toString("hex"), error: String(e.stack) });
160
160
  continue;
161
161
  }
@@ -12,16 +12,18 @@ import { timeInDay, timeInHour, timeInMinute } from "socket-function/src/misc";
12
12
  // URL parameters for time range
13
13
  export const startTimeParam = new URLParam("startTime", undefined as number | undefined);
14
14
  export const endTimeParam = new URLParam("endTime", undefined as number | undefined);
15
+ export const searchFromStartParam = new URLParam("searchFromStart", false);
15
16
 
16
17
  let now = Date.now();
17
18
  /** Get the time range with default values if not set */
18
- export function getTimeRange(): { startTime: number; endTime: number } {
19
+ export function getTimeRange(): { startTime: number; endTime: number; searchFromStart: boolean } {
19
20
  const defaultStart = now - timeInHour * 24;
20
21
  const defaultEnd = now + timeInHour;
21
22
 
22
23
  return {
23
24
  startTime: startTimeParam.value || defaultStart,
24
25
  endTime: endTimeParam.value || defaultEnd,
26
+ searchFromStart: searchFromStartParam.value,
25
27
  };
26
28
  }
27
29
 
@@ -75,7 +77,14 @@ export class TimeRangeSelector extends qreact.Component {
75
77
  />
76
78
  </div>
77
79
  <div className={css.hbox(12).wrap}>
78
-
80
+ <InputLabel
81
+ label="Search from Start"
82
+ checkbox
83
+ value={searchFromStartParam.value ? "true" : ""}
84
+ onChangeValue={value => {
85
+ searchFromStartParam.value = !!value;
86
+ }}
87
+ />
79
88
  <Button
80
89
  hue={110} onClick={() => {
81
90
  let now = Date.now();
@@ -450,10 +450,7 @@ class SuppressionItem extends qreact.Component<{
450
450
  onKeyDown={e => {
451
451
  if (e.key === "Enter") {
452
452
  let pattern = e.currentTarget.value.trim();
453
- if (!pattern) {
454
- alert("Pattern cannot be empty");
455
- return;
456
- }
453
+ if (!pattern) return;
457
454
  this.state.patternValue = pattern;
458
455
  Querysub.onCommitFinished(async () => {
459
456
  await this.manager.updateSuppression(suppression.id, pattern);
@@ -170,7 +170,7 @@ const PATTERN_GENERATION_LIMIT = 100;
170
170
  const PATTERN_GENERATION_WINDOW = 15 * 60 * 1000;
171
171
  let patternGenerationTimes: number[] = [];
172
172
 
173
- async function generatePatternForError(error: LogDatum): Promise<string> {
173
+ export async function generatePatternForError(error: LogDatum): Promise<string> {
174
174
  const now = Date.now();
175
175
  const windowStart = now - PATTERN_GENERATION_WINDOW;
176
176