querysub 0.312.0 → 0.313.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/.cursorrules +1 -1
  2. package/costsBenefits.txt +4 -1
  3. package/package.json +3 -2
  4. package/spec.txt +23 -18
  5. package/src/-0-hooks/hooks.ts +1 -1
  6. package/src/-a-archives/archives.ts +16 -3
  7. package/src/-a-archives/archivesBackBlaze.ts +51 -3
  8. package/src/-a-archives/archivesLimitedCache.ts +175 -0
  9. package/src/-a-archives/archivesPrivateFileSystem.ts +299 -0
  10. package/src/-a-auth/certs.ts +58 -31
  11. package/src/-b-authorities/cdnAuthority.ts +2 -2
  12. package/src/-b-authorities/dnsAuthority.ts +3 -2
  13. package/src/-c-identity/IdentityController.ts +3 -2
  14. package/src/-d-trust/NetworkTrust2.ts +17 -19
  15. package/src/-e-certs/EdgeCertController.ts +3 -4
  16. package/src/-e-certs/certAuthority.ts +1 -2
  17. package/src/-f-node-discovery/NodeDiscovery.ts +9 -7
  18. package/src/-g-core-values/NodeCapabilities.ts +6 -1
  19. package/src/0-path-value-core/NodePathAuthorities.ts +1 -1
  20. package/src/0-path-value-core/PathValueCommitter.ts +3 -3
  21. package/src/0-path-value-core/PathValueController.ts +3 -3
  22. package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts +15 -37
  23. package/src/0-path-value-core/pathValueCore.ts +4 -3
  24. package/src/3-path-functions/PathFunctionRunner.ts +2 -2
  25. package/src/4-dom/qreact.tsx +4 -3
  26. package/src/4-querysub/Querysub.ts +2 -2
  27. package/src/4-querysub/QuerysubController.ts +2 -2
  28. package/src/5-diagnostics/GenericFormat.tsx +1 -0
  29. package/src/5-diagnostics/Table.tsx +3 -0
  30. package/src/5-diagnostics/diskValueAudit.ts +2 -1
  31. package/src/5-diagnostics/nodeMetadata.ts +0 -1
  32. package/src/deployManager/components/MachineDetailPage.tsx +9 -1
  33. package/src/deployManager/components/ServiceDetailPage.tsx +10 -1
  34. package/src/diagnostics/NodeViewer.tsx +3 -4
  35. package/src/diagnostics/logs/FastArchiveAppendable.ts +748 -0
  36. package/src/diagnostics/logs/FastArchiveController.ts +524 -0
  37. package/src/diagnostics/logs/FastArchiveViewer.tsx +863 -0
  38. package/src/diagnostics/logs/LogViewer2.tsx +349 -0
  39. package/src/diagnostics/logs/TimeRangeSelector.tsx +94 -0
  40. package/src/diagnostics/logs/diskLogger.ts +135 -305
  41. package/src/diagnostics/logs/diskShimConsoleLogs.ts +6 -29
  42. package/src/diagnostics/logs/errorNotifications/ErrorNotificationController.ts +577 -0
  43. package/src/diagnostics/logs/errorNotifications/ErrorSuppressionUI.tsx +225 -0
  44. package/src/diagnostics/logs/errorNotifications/ErrorWarning.tsx +207 -0
  45. package/src/diagnostics/logs/importLogsEntry.ts +38 -0
  46. package/src/diagnostics/logs/injectFileLocationToConsole.ts +7 -17
  47. package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx +0 -0
  48. package/src/diagnostics/logs/lifeCycleAnalysis/spec.md +151 -0
  49. package/src/diagnostics/managementPages.tsx +7 -16
  50. package/src/diagnostics/misc-pages/ComponentSyncStats.tsx +0 -1
  51. package/src/diagnostics/periodic.ts +5 -0
  52. package/src/diagnostics/watchdog.ts +2 -2
  53. package/src/functional/SocketChannel.ts +67 -0
  54. package/src/library-components/Input.tsx +1 -1
  55. package/src/library-components/InputLabel.tsx +5 -2
  56. package/src/misc.ts +111 -0
  57. package/src/src.d.ts +34 -1
  58. package/src/user-implementation/userData.ts +4 -3
  59. package/test.ts +13 -0
  60. package/testEntry2.ts +29 -0
  61. package/src/diagnostics/errorLogs/ErrorLogController.ts +0 -535
  62. package/src/diagnostics/errorLogs/ErrorLogCore.ts +0 -274
  63. package/src/diagnostics/errorLogs/LogClassifiers.tsx +0 -308
  64. package/src/diagnostics/errorLogs/LogFilterUI.tsx +0 -84
  65. package/src/diagnostics/errorLogs/LogNotify.tsx +0 -101
  66. package/src/diagnostics/errorLogs/LogTimeSelector.tsx +0 -723
  67. package/src/diagnostics/errorLogs/LogViewer.tsx +0 -757
  68. package/src/diagnostics/errorLogs/logFiltering.tsx +0 -149
  69. package/src/diagnostics/logs/DiskLoggerPage.tsx +0 -613
@@ -0,0 +1,863 @@
1
+ import { measureFnc } from "socket-function/src/profiling/measure";
2
+ import { t } from "../../2-proxy/schema2";
3
+ import { qreact } from "../../4-dom/qreact";
4
+ import { DatumStats, FastArchiveAppendable } from "./FastArchiveAppendable";
5
+ import { Querysub } from "../../4-querysub/Querysub";
6
+ import { URLParam } from "../../library-components/URLParam";
7
+ import { TimeRangeSelector, getTimeRange } from "./TimeRangeSelector";
8
+ import { formatNumber, formatTime, formatVeryNiceDateTime } from "socket-function/src/formatting/format";
9
+ import { list, sort, throttleFunction, timeInHour } from "socket-function/src/misc";
10
+ import { css } from "typesafecss";
11
+ import { logErrors } from "../../errors";
12
+ import { InputLabelURL } from "../../library-components/InputLabel";
13
+ import { clamp } from "../../misc";
14
+ import { ButtonSelector } from "../../library-components/ButtonSelector";
15
+ import { Button } from "../../library-components/Button";
16
+ import { lazy } from "socket-function/src/caching";
17
+ import { LOG_LIMIT_FLAG } from "./diskLogger";
18
+ import { canHaveChildren } from "socket-function/src/types";
19
+ import { niceParse } from "../../niceStringify";
20
+ import { FileMetadata } from "./FastArchiveController";
21
+
22
+ const RENDER_INTERVAL = 1000;
23
+
24
+ const HISTOGRAM_RERENDER_INTERVAL = 10000;
25
+
26
+ export const filterParam = new URLParam("filter", "");
27
+ const cacheBustParam = new URLParam("cacheBust", 0);
28
+ const caseInsensitiveParam = new URLParam("caseInsensitive", false);
29
+
30
+
31
+ export class FastArchiveViewer<T> extends qreact.Component<{
32
+ fastArchives: FastArchiveAppendable<T>[];
33
+ onStart: () => void;
34
+ getWantData?: () => Promise<((posStart: number, posEnd: number, data: Buffer, file: FileMetadata) => boolean) | undefined>;
35
+ onDatums: (source: FastArchiveAppendable<T>, datums: T[], metadata: FileMetadata) => void;
36
+ // Called after onData
37
+ onStats?: (source: FastArchiveAppendable<T>, stats: DatumStats, metadata: FileMetadata) => void;
38
+ onFinish?: () => void;
39
+ }> {
40
+ state = t.state({
41
+ // rootPath =>
42
+ fileMetadata: t.atomic<({
43
+ files: FileMetadata[];
44
+ createTime?: number;
45
+ } | undefined)[]>([]),
46
+ finished: t.atomic(false),
47
+ error: t.atomic<string | undefined>(undefined),
48
+ pendingSyncInitializations: t.atomic<number>(0),
49
+
50
+ renderSeqNum: t.atomic<number>(0),
51
+ });
52
+
53
+ // Histogram data arrays
54
+ private histogramAllDataCounts: Float64Array = new Float64Array(0);
55
+ private histogramSelectedDataCounts: Float64Array = new Float64Array(0);
56
+ private histogramAllDataSizes: Float64Array = new Float64Array(0);
57
+ private histogramSelectedDataSizes: Float64Array = new Float64Array(0);
58
+ private histogramStartTime: number = 0;
59
+ private histogramEndTime: number = 0;
60
+ private histogramBucketTime: number = timeInHour;
61
+
62
+ private allSize = 0;
63
+ private matchedSize = 0;
64
+
65
+ private limitedScanCount = 0;
66
+ private limitedMatchCount = 0;
67
+
68
+ private progressBars: Record<string, {
69
+ section: string,
70
+ value: number,
71
+ max: number,
72
+ initialTime: number,
73
+ lastSetTime: number,
74
+ }> = {};
75
+
76
+ private lastRenderTime = 0;
77
+
78
+ private getHistogramBucketIndex(startTime: number): number {
79
+ const index = Math.floor((startTime - this.histogramStartTime) / this.histogramBucketTime);
80
+ return clamp(index, 0, this.histogramAllDataCounts.length - 1);
81
+ }
82
+
83
+ componentWillMount(): void {
84
+ qreact.onDispose(() => {
85
+ this.cancelAllSynchronizes();
86
+ });
87
+ }
88
+
89
+ cancelAllSynchronizes() {
90
+ for (let fastArchive of this.props.fastArchives) {
91
+ fastArchive.cancelAllSynchronizes();
92
+ }
93
+ }
94
+
95
+ private currentSequenceNumber = 0;
96
+ private latestSequenceNumber = 0;
97
+
98
+ @measureFnc
99
+ private async synchronizeData() {
100
+ let onStart = Querysub.fastRead(() => this.props.onStart);
101
+ let onDatums = Querysub.fastRead(() => this.props.onDatums);
102
+ let onStats = Querysub.fastRead(() => this.props.onStats);
103
+ let fastArchives = Querysub.fastRead(() => this.props.fastArchives);
104
+ let onFinish = Querysub.fastRead(() => this.props.onFinish);
105
+ let getWantData = Querysub.fastRead(() => this.props.getWantData);
106
+ let wantData = await getWantData?.();
107
+ // Increment sequence number for this new sync attempt
108
+ this.currentSequenceNumber++;
109
+ this.latestSequenceNumber = this.currentSequenceNumber;
110
+ const mySequenceNumber = this.currentSequenceNumber;
111
+
112
+ // Helper function to check if this sequence number is still the latest
113
+ const isLatestSync = () => mySequenceNumber === this.latestSequenceNumber;
114
+
115
+ // Helper function to safely update state only if we're the latest sync
116
+ const ifLatest = (updater: () => void) => {
117
+ if (isLatestSync()) {
118
+ Querysub.commit(updater);
119
+ } else {
120
+ console.log(`Ignoring state update from sequence ${mySequenceNumber}, latest is ${this.latestSequenceNumber}`);
121
+ }
122
+ };
123
+
124
+ // Increment sync count before making the call
125
+ Querysub.commit(() => {
126
+ this.state.pendingSyncInitializations++;
127
+ });
128
+
129
+
130
+ const timeRange = getTimeRange();
131
+ let filterString = filterParam.value;
132
+
133
+ let scannedValueCount = 0;
134
+
135
+
136
+ ifLatest(() => {
137
+ this.state.error = undefined;
138
+ this.state.finished = false;
139
+
140
+ // Clear existing progress bars
141
+ for (let key of Object.keys(this.progressBars)) {
142
+ delete this.progressBars[key];
143
+ }
144
+ this.state.fileMetadata = [];
145
+ this.limitedScanCount = 0;
146
+ this.limitedMatchCount = 0;
147
+ this.allSize = 0;
148
+ this.matchedSize = 0;
149
+
150
+ this.histogramBucketTime = Math.max(timeInHour, (timeRange.endTime - timeRange.startTime) / 100);
151
+
152
+ this.histogramStartTime = timeRange.startTime - this.histogramBucketTime * 2;
153
+ this.histogramEndTime = timeRange.endTime + this.histogramBucketTime * 2;
154
+ const bucketCount = Math.ceil((this.histogramEndTime - this.histogramStartTime) / this.histogramBucketTime);
155
+
156
+ this.histogramAllDataCounts = new Float64Array(bucketCount);
157
+ this.histogramSelectedDataCounts = new Float64Array(bucketCount);
158
+ this.histogramAllDataSizes = new Float64Array(bucketCount);
159
+ this.histogramSelectedDataSizes = new Float64Array(bucketCount);
160
+ });
161
+
162
+ const onProgress = (progress: { section: string; value: number; max: number }) => {
163
+ if (!isLatestSync()) return;
164
+ const key = progress.section;
165
+
166
+ // Add or update progress bar
167
+ let existingBar = this.progressBars[key];
168
+ this.progressBars[key] = {
169
+ section: progress.section,
170
+ value: progress.value,
171
+ max: progress.max,
172
+ initialTime: existingBar?.initialTime || Date.now(),
173
+ lastSetTime: Date.now(),
174
+ };
175
+
176
+ let now = Date.now();
177
+ if (now - this.lastRenderTime > RENDER_INTERVAL) {
178
+ this.lastRenderTime = now;
179
+ ifLatest(() => {
180
+ this.state.renderSeqNum++;
181
+ });
182
+ }
183
+ };
184
+ try {
185
+ ifLatest(() => {
186
+ onStart();
187
+ });
188
+
189
+ const caseInsensitive = Querysub.fastRead(() => caseInsensitiveParam.value);
190
+ let caseInsensitiveMapping = new Uint8Array(256);
191
+ for (let i = 0; i < 256; i++) {
192
+ caseInsensitiveMapping[i] = String.fromCharCode(i).toLowerCase().charCodeAt(0);
193
+ }
194
+
195
+ type ScanMatch = (posStart: number, posEnd: number, buffer: Buffer) => boolean;
196
+ type DatumMatch = (datum: unknown) => boolean;
197
+ type MatchObj = {
198
+ scanMatch: ScanMatch,
199
+ datumMatch: DatumMatch,
200
+ };
201
+
202
+ function parsePart(filterString: string): MatchObj {
203
+ if (!filterString.trim()) {
204
+ return {
205
+ scanMatch: () => true,
206
+ datumMatch: () => true,
207
+ };
208
+ }
209
+ let datumMatch = (datum: unknown) => true;
210
+ if (filterString.includes("=")) {
211
+ let equalIndex = filterString.indexOf("=");
212
+ let key = filterString.slice(0, equalIndex).trim();
213
+ let value = niceParse(filterString.slice(equalIndex + 1).trim());
214
+ filterString = `${JSON.stringify(key)}:${JSON.stringify(value)}`;
215
+ datumMatch = (datum: unknown) => {
216
+ if (!canHaveChildren(datum)) return false;
217
+ let curKey = key;
218
+ if (!(curKey in datum)) {
219
+ curKey = curKey.toLowerCase();
220
+ let keys = Object.keys(datum);
221
+ curKey = keys.find(x => x.toLowerCase() === curKey) || "";
222
+ if (!curKey) return false;
223
+ }
224
+ return datum[curKey] === value;
225
+ };
226
+ }
227
+ if (caseInsensitive) {
228
+ filterString = filterString.toLowerCase();
229
+ }
230
+ const filterBuffer = Buffer.from(filterString);
231
+ const char0 = filterBuffer[0];
232
+ return {
233
+ scanMatch: (posStart: number, posEnd: number, buffer: Buffer) => {
234
+ for (let i = posStart; i < posEnd; i++) {
235
+ // NOTE: This is slow for repeated prefixes, but... in practice, even if we have prefixes, they won't repeat, so... this should be quite fast.
236
+ let ch = buffer[i];
237
+ if (caseInsensitive) {
238
+ ch = caseInsensitiveMapping[ch];
239
+ }
240
+ if (ch === char0) {
241
+ if (filterBuffer.length === 1) return true;
242
+ for (let j = 1; j < filterBuffer.length; j++) {
243
+ let ch2 = buffer[i + j];
244
+ if (caseInsensitive) {
245
+ ch2 = caseInsensitiveMapping[ch2];
246
+ }
247
+ if (ch2 !== filterBuffer[j]) {
248
+ break;
249
+ }
250
+ if (j === filterBuffer.length - 1) {
251
+ return true;
252
+ }
253
+ }
254
+ }
255
+ }
256
+ return false;
257
+ },
258
+ datumMatch,
259
+ };
260
+ }
261
+
262
+ function joinMatches(matches: MatchObj[], condition: "||" | "&&"): MatchObj {
263
+ if (condition === "||") {
264
+ return {
265
+ scanMatch: (posStart: number, posEnd: number, buffer: Buffer) => {
266
+ return matches.some(match => match.scanMatch(posStart, posEnd, buffer));
267
+ },
268
+ datumMatch: (datum: unknown) => {
269
+ return matches.some(match => match.datumMatch(datum));
270
+ },
271
+ };
272
+ }
273
+ return {
274
+ scanMatch: (posStart: number, posEnd: number, buffer: Buffer) => {
275
+ return matches.every(match => match.scanMatch(posStart, posEnd, buffer));
276
+ },
277
+ datumMatch: (datum: unknown) => {
278
+ return matches.every(match => match.datumMatch(datum));
279
+ },
280
+ };
281
+ }
282
+
283
+
284
+ let andMatches = filterString.split("|").map(orParts => {
285
+ let andParts = orParts.split("&");
286
+ let andMatches = andParts.map(part => parsePart(part.trim()));
287
+ return joinMatches(andMatches, "&&");
288
+ });
289
+ let { scanMatch, datumMatch } = joinMatches(andMatches, "||");
290
+
291
+
292
+ const limitedBuffer = Buffer.from(LOG_LIMIT_FLAG);
293
+
294
+
295
+ await Promise.all(fastArchives.map(async (fastArchive, index) => {
296
+ const result = await fastArchive.synchronizeData({
297
+ range: timeRange,
298
+ cacheBust: Querysub.fastRead(() => cacheBustParam.value),
299
+ wantData: (posStart: number, posEnd: number, data: Buffer, file) => {
300
+
301
+ let isLimited = false;
302
+ for (let i = posStart; i < posEnd && !isLimited; i++) {
303
+ if (data[i] === limitedBuffer[0]) {
304
+ for (let j = 1; j < limitedBuffer.length; j++) {
305
+ if (data[i + j] !== limitedBuffer[j]) {
306
+ break;
307
+ }
308
+ if (j === limitedBuffer.length - 1) {
309
+ isLimited = true;
310
+ break;
311
+ }
312
+ }
313
+ }
314
+ }
315
+ if (isLimited) {
316
+ this.limitedScanCount++;
317
+ }
318
+
319
+ let matched = true;
320
+ if (wantData) {
321
+ matched = wantData(posStart, posEnd, data, file);
322
+ }
323
+ matched = matched && scanMatch(posStart, posEnd, data);
324
+ if (isLimited && matched) {
325
+ this.limitedMatchCount++;
326
+ }
327
+ return matched;
328
+ },
329
+ onData: (datums, file) => {
330
+ if (!isLatestSync()) return;
331
+ let keptDatums: T[] = [];
332
+ for (let i = 0; i < datums.length; i++) {
333
+ let datum = datums[i];
334
+ if (datumMatch(datum)) {
335
+ keptDatums.push(datum);
336
+ }
337
+ }
338
+ onDatums(fastArchive, keptDatums, file);
339
+ },
340
+ onStats: onStats && ((stats, file) => {
341
+ if (!isLatestSync()) return;
342
+
343
+ onStats!(fastArchive, stats, file);
344
+ this.allSize += stats.matchedSize + stats.notMatchedSize;
345
+ this.matchedSize += stats.matchedSize;
346
+ // Update scanned count and sequence number for rendering
347
+ scannedValueCount += stats.notMatchedCount + stats.matchedCount;
348
+ // Update all data histogram
349
+ const bucketIndex = this.getHistogramBucketIndex(file.startTime);
350
+ this.histogramAllDataCounts[bucketIndex] += stats.notMatchedCount + stats.matchedCount;
351
+ this.histogramSelectedDataCounts[bucketIndex] += stats.matchedCount;
352
+ this.histogramAllDataSizes[bucketIndex] += stats.matchedSize + stats.notMatchedSize;
353
+ this.histogramSelectedDataSizes[bucketIndex] += stats.matchedSize;
354
+ if (scannedValueCount >= HISTOGRAM_RERENDER_INTERVAL) {
355
+ scannedValueCount = 0;
356
+ let now = Date.now();
357
+ if (now - this.lastRenderTime > RENDER_INTERVAL) {
358
+ this.lastRenderTime = now;
359
+ ifLatest(() => {
360
+ this.state.renderSeqNum++;
361
+ });
362
+ }
363
+ }
364
+ }),
365
+ onProgress,
366
+ onFinish: () => {
367
+ ifLatest(() => {
368
+ this.state.renderSeqNum++;
369
+ });
370
+ setTimeout(() => {
371
+ ifLatest(() => {
372
+ this.state.finished = true;
373
+ });
374
+ }, 2500);
375
+ onFinish?.();
376
+ },
377
+ });
378
+ if (result !== "cancelled") {
379
+ ifLatest(() => {
380
+ this.state.fileMetadata[index] = result.metadata;
381
+ });
382
+ }
383
+ return result;
384
+ }));
385
+
386
+ } catch (error: any) {
387
+ ifLatest(() => {
388
+ this.state.error = error.stack || String(error);
389
+ });
390
+ } finally {
391
+ // Decrement sync count when call finishes (regardless of sequence number)
392
+ Querysub.commit(() => {
393
+ this.state.pendingSyncInitializations--;
394
+ });
395
+ }
396
+ }
397
+ private renderProgressBars() {
398
+ let progressEntries = Object.entries(this.progressBars);
399
+
400
+ if (progressEntries.length === 0) {
401
+ return null;
402
+ }
403
+
404
+ const getNowTime = lazy(() => Querysub.timeDelayed(1000));
405
+ if (progressEntries.some(x => x[0].startsWith("Error"))) {
406
+ progressEntries = progressEntries.filter(x => x[0].startsWith("Error"));
407
+ }
408
+ sort(progressEntries, x => x[1].initialTime);
409
+
410
+ return (
411
+ <div className={css.hbox(10).wrap.alignItems("start").fillWidth}>
412
+ {!this.state.finished && (
413
+ <div
414
+ onClick={() => {
415
+ this.cancelAllSynchronizes();
416
+ }}
417
+ className={css.pad2(12).bord2(200, 50, 50).hsl(200, 50, 95).colorhsl(200, 50, 40).button}
418
+ >
419
+ Cancel
420
+ </div>
421
+ )}
422
+ {progressEntries.map(([key, progress]) => {
423
+ const fraction = clamp(progress.max > 0 ? (progress.value / progress.max) * 1 : 1, 0, 1);
424
+ let progressTime = 0;
425
+ if (progress.value >= progress.max) {
426
+ progressTime = progress.lastSetTime;
427
+ } else {
428
+ progressTime = getNowTime();
429
+ }
430
+ const elapsedTime = progressTime - progress.initialTime;
431
+
432
+ return (
433
+ <div key={key} className={
434
+ // NOTE: There is no point making this proportionally to the time, because we stream the data, so the time overlaps, making everything take most of the time.
435
+ css.vbox(4)
436
+ }>
437
+ <div className={css.vbox(2)}>
438
+ <div className={css.fontSize(14).colorhsl(0, 0, 20)}>
439
+ {progress.section}
440
+ </div>
441
+ <div className={css.fontSize(12).colorhsl(0, 0, 40).width(150)}>
442
+ {formatNumber(progress.value)} {progress.value < progress.max && `/ ${formatNumber(progress.max)}`} ({formatTime(elapsedTime)})
443
+ </div>
444
+ </div>
445
+ <div className={css.fillWidth.height(8).bord2(200, 20, 80).hsl(200, 10, 95)}>
446
+ <div
447
+ className={css.height(8).hsl(200, 50, 70).transition("width 200ms")}
448
+ style={{ width: `${fraction * 100}%` }}
449
+ />
450
+ </div>
451
+ </div>
452
+ );
453
+ })}
454
+ </div>
455
+ );
456
+ }
457
+
458
+ private renderHistogram() {
459
+ this.state.renderSeqNum;
460
+
461
+ if (this.histogramAllDataCounts.length === 0) {
462
+ return null;
463
+ }
464
+
465
+ let countsArray = Array.from(this.histogramAllDataCounts);
466
+ let selectedArray = Array.from(this.histogramSelectedDataCounts);
467
+ const maxAllCount = countsArray.reduce((a, b) => Math.max(a, b), 0);
468
+ const selected = css.hsla(120, 40, 50, 1);
469
+ const all = css.hsla(200, 40, 70, 1);
470
+
471
+ return (
472
+ <div className={css.vbox(10).fillWidth}>
473
+ <div
474
+ className={css.relative.fillWidth.height(100).bord2(0, 0, 40).hbox0.overflowHidden}
475
+ >
476
+ {Array.from(this.histogramAllDataCounts).map((count, index) => {
477
+ const selectedCount = this.histogramSelectedDataCounts[index] || 0;
478
+ const allSize = this.histogramAllDataSizes[index] || 0;
479
+ const selectedSize = this.histogramSelectedDataSizes[index] || 0;
480
+ return (
481
+ <div
482
+ key={index}
483
+ className={css.relative.fillHeight.fillWidth.filter("brightness(1.1)", "hover")}
484
+ title={`${formatNumber(selectedCount)} filtered (${formatNumber(selectedSize)}B)\n${formatNumber(count)} all (${formatNumber(allSize)}B)\n${formatVeryNiceDateTime(this.histogramStartTime + index * this.histogramBucketTime)}`}
485
+ >
486
+ {count > 0 && <div
487
+ className={
488
+ all.absolute.bottom(0).left(0).right(1)
489
+ .height(`calc(max(3px, ${(count / maxAllCount) * 100}%))`)
490
+ }
491
+ />}
492
+
493
+ {selectedCount > 0 && <div
494
+ className={
495
+ selected.absolute.bottom(0).left(0).right(1)
496
+ .height(`calc(max(3px, ${(selectedCount / maxAllCount) * 100}%))`)
497
+ }
498
+ />}
499
+ </div>
500
+ );
501
+ })}
502
+ </div>
503
+ <div className={css.hbox(20).fontSize(12)}>
504
+ <div className={css.hbox(5)}>
505
+ <div className={all.size(16, 16)} />
506
+ <span>All data ({formatNumber(countsArray.reduce((a, b) => a + b, 0))} | {formatNumber(this.allSize)}B)</span>
507
+ </div>
508
+ <div className={css.hbox(5)}>
509
+ <div className={selected.size(16, 16)} />
510
+ <span>Filtered data ({formatNumber(selectedArray.reduce((a, b) => a + b, 0))} | {formatNumber(this.matchedSize)}B)</span>
511
+ </div>
512
+ </div>
513
+ </div>
514
+ );
515
+ }
516
+
517
+ public handleDownload = throttleFunction(500, () => {
518
+ console.log("handleDownload");
519
+ Querysub.onCommitFinished(() => {
520
+ logErrors(this.synchronizeData());
521
+ });
522
+ });
523
+ render() {
524
+ let totalFileCount = 0;
525
+ let totalBackblazeByteCount = 0;
526
+ let totalLocalByteCount = 0;
527
+ let totalLocalFileCount = 0;
528
+
529
+ let readLocalTimes: number[] = [];
530
+
531
+
532
+ for (let fileMetadata of this.state.fileMetadata) {
533
+ if (!fileMetadata) continue;
534
+ totalFileCount += fileMetadata.files.length;
535
+ let backblazeFiles = fileMetadata.files.filter(file => !file.nodeId);
536
+ let localFiles = fileMetadata.files.filter(file => file.nodeId);
537
+ if (localFiles.length > 0) {
538
+ readLocalTimes.push(fileMetadata.createTime || 0);
539
+ }
540
+ totalBackblazeByteCount += backblazeFiles.reduce((acc, file) => acc + file.size, 0);
541
+ totalLocalByteCount += localFiles.reduce((acc, file) => acc + file.size, 0);
542
+ totalLocalFileCount += localFiles.length;
543
+ }
544
+
545
+ const infoDisplay = (hue: number) => css.pad2(12).bord2(hue, 50, 50).hsl(hue, 50, 95).colorhsl(hue, 50, 40);
546
+
547
+ return (
548
+ <div className={css.vbox(20).pad2(20).fillBoth}>
549
+ <div className={css.hbox(20).fillWidth}>
550
+ <TimeRangeSelector />
551
+ <div className={infoDisplay(280)}>
552
+ TODO: Filtering for machineId / threadId, if we find the log loading is too slow
553
+ </div>
554
+ </div>
555
+
556
+ <div className={css.vbox(12).fillWidth}>
557
+ <InputLabelURL
558
+ label={
559
+ <div className={css.hbox(10)}>
560
+ <Button onClick={() => {
561
+ void this.handleDownload();
562
+ }}>
563
+ Run
564
+ </Button>
565
+ <span>Filter</span>
566
+ </div>
567
+ }
568
+ url={filterParam}
569
+ hot
570
+ flavor="large"
571
+ fillWidth
572
+ onKeyUp={this.handleDownload}
573
+ ref2={() => {
574
+ void this.handleDownload();
575
+ }}
576
+ noEnterKeyBlur
577
+ placeholder="Filter terms, ex x | y & z"
578
+ />
579
+ <InputLabelURL
580
+ label="Case Insensitive"
581
+ checkbox
582
+ url={caseInsensitiveParam}
583
+ onChange={() => {
584
+ void this.handleDownload();
585
+ }}
586
+ flavor="small"
587
+ />
588
+ <div className={css.vbox(10)}>
589
+ {this.state.fileMetadata
590
+ && (
591
+ <div
592
+ className={infoDisplay(120)}
593
+ title={this.state.fileMetadata.map(x => x?.files || []).flat().map(x =>
594
+ `${x.path} (${formatNumber(x.size)})`
595
+ ).join("\n")}
596
+ >
597
+ File count: {formatNumber(totalFileCount)}, Backblaze size: {formatNumber(totalBackblazeByteCount)}B (compressed), Disk size: {formatNumber(totalLocalByteCount)}B (uncompressed)
598
+ </div>
599
+ )}
600
+ {this.state.finished && <div
601
+ className={infoDisplay(60).button}
602
+ onClick={() => {
603
+ Querysub.commit(() => {
604
+ cacheBustParam.value = Date.now();
605
+ });
606
+ void this.handleDownload();
607
+ }}
608
+ >
609
+ Snapshot from dates: {readLocalTimes.map(x => formatVeryNiceDateTime(x)).join(" | ")}. Clear here to reload data.
610
+ </div>}
611
+ {!this.state.finished && <LoaderAurora />}
612
+ {this.limitedScanCount > 0 && (
613
+ <div className={infoDisplay(60).boldStyle}>
614
+ {formatNumber(this.limitedScanCount)} scanned logs were rate limited. We do not track how many were ignored, there might be infinite missing logs.
615
+ </div>
616
+ )}
617
+ {this.limitedMatchCount > 0 && (
618
+ <div className={infoDisplay(0).boldStyle}>
619
+ {formatNumber(this.limitedMatchCount)} matched logs were rate limited. We do not track how many were ignored, there might be infinite missing logs.
620
+ </div>
621
+ )}
622
+ {this.state.pendingSyncInitializations > 0 && (
623
+ <div className={infoDisplay(200)}>
624
+ <strong>Pending sync initializations:</strong> {formatNumber(this.state.pendingSyncInitializations)}
625
+ </div>
626
+ )}
627
+ </div>
628
+
629
+ {this.state.error && (
630
+ <div className={infoDisplay(0).whiteSpace("pre-wrap")}>
631
+ <strong>Error:</strong> {this.state.error}
632
+ </div>
633
+ )}
634
+
635
+ {this.renderProgressBars()}
636
+
637
+ {this.renderHistogram()}
638
+
639
+ {this.props.children}
640
+ </div>
641
+ </div>
642
+ );
643
+ }
644
+ }
645
+
646
+
647
+ class LoaderAurora extends qreact.Component {
648
+ render() {
649
+ return (
650
+ <>
651
+ <style>{`
652
+ .aurora-container-2 {
653
+ position: relative;
654
+ width: 800px;
655
+ height: 40px;
656
+ overflow: hidden;
657
+ background: linear-gradient(180deg, hsl(280, 30%, 12%), hsl(320, 35%, 15%));
658
+ }
659
+ .aurora-ribbon-2 {
660
+ position: absolute;
661
+ width: 1420px;
662
+ height: 40px;
663
+ filter: blur(12px);
664
+ opacity: 0.9;
665
+ }
666
+
667
+ /* Different gradient patterns for variety */
668
+ .aurora-ribbon-2.gradient1 {
669
+ background: linear-gradient(90deg,
670
+ hsla(160, 80%, 50%, 0) 0%,
671
+ hsla(160, 80%, 50%, 0.7) 30%,
672
+ hsla(280, 80%, 60%, 0.7) 60%,
673
+ hsla(40, 80%, 55%, 0) 100%);
674
+ }
675
+ .aurora-ribbon-2.gradient2 {
676
+ background: linear-gradient(90deg,
677
+ hsla(200, 70%, 60%, 0) 0%,
678
+ hsla(200, 70%, 60%, 0.6) 40%,
679
+ hsla(320, 70%, 55%, 0.6) 70%,
680
+ hsla(60, 70%, 50%, 0) 100%);
681
+ }
682
+ .aurora-ribbon-2.gradient3 {
683
+ background: linear-gradient(90deg,
684
+ hsla(300, 80%, 65%, 0) 0%,
685
+ hsla(300, 80%, 65%, 0.8) 25%,
686
+ hsla(180, 80%, 55%, 0.8) 65%,
687
+ hsla(240, 80%, 60%, 0) 100%);
688
+ }
689
+
690
+ /* Left to right movement */
691
+ .aurora-ribbon-2.r1 {
692
+ top: 0;
693
+ transform: rotate(-10deg);
694
+ animation: auroraSlideLeft-2 3s ease-in-out infinite, auroraRotate1-2 6s linear infinite;
695
+ opacity: 0.7;
696
+ }
697
+ .aurora-ribbon-2.r2 {
698
+ top: 20px;
699
+ transform: rotate(12deg);
700
+ animation: auroraSlideRight-2 2.5s ease-in-out infinite, auroraRotate2-2 8s linear infinite reverse;
701
+ opacity: 0.8;
702
+ }
703
+ .aurora-ribbon-2.r3 {
704
+ top: 40px;
705
+ transform: rotate(18deg);
706
+ animation: auroraSlideDiagonal1-2 4s ease-in-out infinite;
707
+ opacity: 0.6;
708
+ }
709
+ .aurora-ribbon-2.r4 {
710
+ top: -10px;
711
+ transform: rotate(-15deg);
712
+ animation: auroraSlideLeft-2 3.5s ease-in-out infinite, auroraRotate3-2 10s linear infinite;
713
+ opacity: 0.5;
714
+ }
715
+ .aurora-ribbon-2.r5 {
716
+ top: 30px;
717
+ transform: rotate(8deg);
718
+ animation: auroraSlideDiagonal2-2 2.8s ease-in-out infinite;
719
+ opacity: 0.9;
720
+ }
721
+ .aurora-ribbon-2.r6 {
722
+ top: 10px;
723
+ transform: rotate(-5deg);
724
+ animation: auroraSlideRight-2 4.2s ease-in-out infinite, auroraRotate1-2 7s linear infinite;
725
+ opacity: 0.4;
726
+ }
727
+ .aurora-ribbon-2.r7 {
728
+ top: 50px;
729
+ transform: rotate(22deg);
730
+ animation: auroraSlideLeft-2 2.2s ease-in-out infinite;
731
+ opacity: 0.8;
732
+ }
733
+ .aurora-ribbon-2.r8 {
734
+ top: 15px;
735
+ transform: rotate(-8deg);
736
+ animation: auroraSlideDiagonal3-2 3.7s ease-in-out infinite, auroraRotate2-2 9s linear infinite;
737
+ opacity: 0.6;
738
+ }
739
+ .aurora-ribbon-2.r9 {
740
+ top: 35px;
741
+ transform: rotate(15deg);
742
+ animation: auroraSlideRight-2 3.1s ease-in-out infinite, auroraRotate1-2 8.5s linear infinite;
743
+ opacity: 0.7;
744
+ }
745
+ .aurora-ribbon-2.r10 {
746
+ top: -5px;
747
+ transform: rotate(-12deg);
748
+ animation: auroraSlideDiagonal1-2 3.8s ease-in-out infinite;
749
+ opacity: 0.5;
750
+ }
751
+ .aurora-ribbon-2.r11 {
752
+ top: 25px;
753
+ transform: rotate(6deg);
754
+ animation: auroraSlideLeft-2 2.9s ease-in-out infinite, auroraRotate3-2 11s linear infinite;
755
+ opacity: 0.8;
756
+ }
757
+ .aurora-ribbon-2.r12 {
758
+ top: 45px;
759
+ transform: rotate(-20deg);
760
+ animation: auroraSlideDiagonal2-2 4.1s ease-in-out infinite, auroraRotate2-2 7.5s linear infinite reverse;
761
+ opacity: 0.6;
762
+ }
763
+ .aurora-ribbon-2.r13 {
764
+ top: 5px;
765
+ transform: rotate(10deg);
766
+ animation: auroraSlideRight-2 2.7s ease-in-out infinite;
767
+ opacity: 0.9;
768
+ }
769
+ .aurora-ribbon-2.r14 {
770
+ top: 38px;
771
+ transform: rotate(-7deg);
772
+ animation: auroraSlideDiagonal3-2 3.3s ease-in-out infinite, auroraRotate1-2 9.5s linear infinite;
773
+ opacity: 0.4;
774
+ }
775
+ .aurora-ribbon-2.r15 {
776
+ top: 18px;
777
+ transform: rotate(14deg);
778
+ animation: auroraSlideLeft-2 4.3s ease-in-out infinite, auroraRotate3-2 8.2s linear infinite reverse;
779
+ opacity: 0.7;
780
+ }
781
+ .aurora-ribbon-2.r16 {
782
+ top: 12px;
783
+ transform: rotate(-18deg);
784
+ animation: auroraSlideDiagonal1-2 2.6s ease-in-out infinite, auroraRotate2-2 10.5s linear infinite;
785
+ opacity: 0.8;
786
+ }
787
+
788
+ /* Left to right slide */
789
+ @keyframes auroraSlideLeft-2 {
790
+ 0% { left: -920px; }
791
+ 100% { left: 900px; }
792
+ }
793
+
794
+ /* Right to left slide */
795
+ @keyframes auroraSlideRight-2 {
796
+ 0% { left: 900px; }
797
+ 100% { left: -920px; }
798
+ }
799
+
800
+ /* Diagonal movements */
801
+ @keyframes auroraSlideDiagonal1-2 {
802
+ 0% { left: -920px; top: 40px; }
803
+ 50% { left: 400px; top: -5px; }
804
+ 100% { left: 900px; top: 40px; }
805
+ }
806
+
807
+ @keyframes auroraSlideDiagonal2-2 {
808
+ 0% { left: 900px; top: 30px; }
809
+ 50% { left: 400px; top: 45px; }
810
+ 100% { left: -920px; top: 30px; }
811
+ }
812
+
813
+ @keyframes auroraSlideDiagonal3-2 {
814
+ 0% { left: -920px; top: 15px; }
815
+ 30% { left: -200px; top: 5px; }
816
+ 70% { left: 600px; top: 25px; }
817
+ 100% { left: 900px; top: 15px; }
818
+ }
819
+
820
+ /* Rotation animations */
821
+ @keyframes auroraRotate1-2 {
822
+ 0% { transform: rotate(-10deg); }
823
+ 25% { transform: rotate(-5deg); }
824
+ 50% { transform: rotate(-15deg); }
825
+ 75% { transform: rotate(-8deg); }
826
+ 100% { transform: rotate(-10deg); }
827
+ }
828
+
829
+ @keyframes auroraRotate2-2 {
830
+ 0% { transform: rotate(12deg); }
831
+ 33% { transform: rotate(18deg); }
832
+ 66% { transform: rotate(8deg); }
833
+ 100% { transform: rotate(12deg); }
834
+ }
835
+
836
+ @keyframes auroraRotate3-2 {
837
+ 0% { transform: rotate(-15deg); }
838
+ 50% { transform: rotate(-20deg); }
839
+ 100% { transform: rotate(-15deg); }
840
+ }
841
+ `}</style>
842
+ <div className="aurora-container-2">
843
+ <div className="aurora-ribbon-2 r1 gradient1"></div>
844
+ <div className="aurora-ribbon-2 r2 gradient2"></div>
845
+ <div className="aurora-ribbon-2 r3 gradient3"></div>
846
+ <div className="aurora-ribbon-2 r4 gradient1"></div>
847
+ <div className="aurora-ribbon-2 r5 gradient3"></div>
848
+ <div className="aurora-ribbon-2 r6 gradient2"></div>
849
+ <div className="aurora-ribbon-2 r7 gradient1"></div>
850
+ <div className="aurora-ribbon-2 r8 gradient3"></div>
851
+ <div className="aurora-ribbon-2 r9 gradient2"></div>
852
+ <div className="aurora-ribbon-2 r10 gradient1"></div>
853
+ <div className="aurora-ribbon-2 r11 gradient3"></div>
854
+ <div className="aurora-ribbon-2 r12 gradient2"></div>
855
+ <div className="aurora-ribbon-2 r13 gradient1"></div>
856
+ <div className="aurora-ribbon-2 r14 gradient3"></div>
857
+ <div className="aurora-ribbon-2 r15 gradient2"></div>
858
+ <div className="aurora-ribbon-2 r16 gradient1"></div>
859
+ </div>
860
+ </>
861
+ );
862
+ }
863
+ }