querysub 0.374.0 → 0.376.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/package.json +2 -4
  2. package/src/deployManager/components/MachineDetailPage.tsx +2 -5
  3. package/src/deployManager/components/ServiceDetailPage.tsx +2 -5
  4. package/src/deployManager/machineApplyMainCode.ts +7 -0
  5. package/src/diagnostics/NodeViewer.tsx +4 -5
  6. package/src/diagnostics/logs/IndexedLogs/BufferIndex.ts +10 -5
  7. package/src/diagnostics/logs/IndexedLogs/BufferIndexCPP.cpp +20 -0
  8. package/src/diagnostics/logs/IndexedLogs/BufferIndexHelpers.ts +29 -2
  9. package/src/diagnostics/logs/IndexedLogs/BufferUnitIndex.ts +61 -20
  10. package/src/diagnostics/logs/IndexedLogs/BufferUnitSet.ts +2 -2
  11. package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts +7 -7
  12. package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx +250 -243
  13. package/src/diagnostics/logs/IndexedLogs/LogViewerParams.ts +21 -0
  14. package/src/diagnostics/logs/IndexedLogs/{bufferMatcher.ts → bufferSearchFindMatcher.ts} +9 -4
  15. package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts +3 -3
  16. package/src/diagnostics/logs/diskLogger.ts +0 -38
  17. package/src/diagnostics/logs/errorNotifications2/errorNotifications2.ts +9 -0
  18. package/src/diagnostics/logs/injectFileLocationToConsole.ts +3 -0
  19. package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx +24 -22
  20. package/src/diagnostics/managementPages.tsx +0 -18
  21. package/test.ts +0 -5
  22. package/bin/error-email.js +0 -8
  23. package/bin/error-im.js +0 -8
  24. package/src/diagnostics/logs/FastArchiveAppendable.ts +0 -843
  25. package/src/diagnostics/logs/FastArchiveController.ts +0 -573
  26. package/src/diagnostics/logs/FastArchiveViewer.tsx +0 -1090
  27. package/src/diagnostics/logs/LogViewer2.tsx +0 -552
  28. package/src/diagnostics/logs/errorNotifications/ErrorDigestPage.tsx +0 -409
  29. package/src/diagnostics/logs/errorNotifications/ErrorNotificationController.ts +0 -756
  30. package/src/diagnostics/logs/errorNotifications/ErrorSuppressionUI.tsx +0 -280
  31. package/src/diagnostics/logs/errorNotifications/ErrorWarning.tsx +0 -254
  32. package/src/diagnostics/logs/errorNotifications/errorDigestEmail.tsx +0 -233
  33. package/src/diagnostics/logs/errorNotifications/errorDigestEntry.tsx +0 -14
  34. package/src/diagnostics/logs/errorNotifications/errorDigests.tsx +0 -292
  35. package/src/diagnostics/logs/errorNotifications/errorWatchEntry.tsx +0 -209
  36. package/src/diagnostics/logs/importLogsEntry.ts +0 -38
  37. package/src/diagnostics/logs/lifeCycleAnalysis/LifeCyclePages.tsx +0 -150
  38. package/src/diagnostics/logs/logViewerExtractField.ts +0 -36
@@ -26,11 +26,10 @@ import { IndexedLogResults, createEmptyIndexedLogResults, mergeIndexedLogResults
26
26
  import { TimeFilePath } from "./TimeFileTree";
27
27
  import { isPublic } from "../../../config";
28
28
  import { Zip } from "../../../zip";
29
+ import { readProductionLogsURL, searchTextURL } from "./LogViewerParams";
29
30
 
30
- let searchText = new URLParam("searchText", "");
31
31
  let excludePendingResults = new URLParam("excludePendingResults", false);
32
32
  let limitURL = new URLParam("limit", 100);
33
- let readPublicLogs = new URLParam("readPublicLogs", false);
34
33
 
35
34
  let savedPathsURL = new URLParam("savedPaths", "");
36
35
  let selectedFieldsURL = new URLParam("selectedFields", {} as Record<string, boolean>);
@@ -105,237 +104,6 @@ export class LogViewer3 extends qreact.Component {
105
104
  await this.loadPaths();
106
105
  }
107
106
 
108
- renderSearchStats() {
109
- const stats = this.state.stats;
110
- if (!stats) return undefined;
111
-
112
- let totalSizeRead = 0;
113
- let cachedSize = 0;
114
- let cachedCount = 0;
115
- let uncachedSize = 0;
116
- let uncachedCount = 0;
117
- let uncachedRemoteSize = 0;
118
- let uncachedRemoteCount = 0;
119
- let totalSize = 0;
120
- let remoteTotalSize = 0;
121
- let localTotalSize = 0;
122
-
123
- for (let read of stats.reads) {
124
- totalSizeRead += read.size;
125
- if (read.cached) {
126
- cachedSize += read.size;
127
- cachedCount += read.count;
128
- } else {
129
- uncachedSize += read.size;
130
- uncachedCount += read.count;
131
- totalSize += read.totalSize;
132
- if (read.remote) {
133
- remoteTotalSize += read.totalSize;
134
- } else {
135
- localTotalSize += read.totalSize;
136
- }
137
- }
138
- if (read.remote && !read.cached) {
139
- uncachedRemoteSize += read.size;
140
- uncachedRemoteCount += read.count;
141
- }
142
- }
143
-
144
- const renderStage = (title: string, items: preact.ComponentChild[], hue: number | undefined, isFirst: boolean, isLast: boolean) => {
145
- const triangleWidth = 20;
146
-
147
- let color = hue !== undefined ? css.hsl(hue, 70, 80) : css.hsl(0, 0, 100);
148
- let colorhsl = hue !== undefined ? css.hslcolor(hue, 80, 30) : css;
149
-
150
- return (
151
- <div className={css.vbox(2).relative.minWidth(200).paddingTop(8).paddingBottom(8).paddingRight(10).paddingLeft(isFirst ? 10 : 10 + triangleWidth) + color}>
152
- <div className={css.fontWeight(600) + colorhsl}>{title}</div>
153
- {items.map((item, i) => (
154
- <div key={i}>{item}</div>
155
- ))}
156
- {!isLast && hue !== undefined && (
157
- <svg width={triangleWidth} height="100" viewBox="0 0 20 100" preserveAspectRatio="none" className={css.absolute.right(-triangleWidth).top(0).fillHeight.zIndex(1)}>
158
- <polygon points="0,0 20,50 0,100" fill={`hsl(${hue}, 70%, 80%)`} />
159
- </svg>
160
- )}
161
- </div>
162
- );
163
- };
164
-
165
- const fileItems = [
166
- `${formatTime(stats.fileFindTime)} | ${formatNumber(stats.localFilesSearched + stats.backblazeFilesSearched)} / ${formatNumber(stats.totalLocalFiles + stats.totalBackblazeFiles)} | ${formatNumber(totalSize)}B`,
167
- `remote ${formatNumber(stats.backblazeFilesSearched)} / ${formatNumber(stats.totalBackblazeFiles)} | ${formatNumber(remoteTotalSize)}B`,
168
- `pending ${formatNumber(stats.localFilesSearched)} / ${formatNumber(stats.totalLocalFiles)} | ${formatNumber(localTotalSize)}B`,
169
- ];
170
-
171
- const totalIndexesSearched = stats.remoteIndexesSearched + stats.localIndexesSearched;
172
- const totalIndexSize = stats.remoteIndexSize + stats.localIndexSize;
173
- const indexItems = [
174
- `${formatTime(stats.indexSearchTime)} | ${formatNumber(totalIndexesSearched)} | ${formatNumber(totalIndexSize)}B | ${formatNumber(totalSize / totalIndexSize)}X`,
175
- `remote ${formatNumber(stats.remoteIndexesSearched)} | ${formatNumber(stats.remoteIndexSize)}B | ${formatNumber(remoteTotalSize / stats.remoteIndexSize)}X`,
176
- `local ${formatNumber(stats.localIndexesSearched)} | ${formatNumber(stats.localIndexSize)}B | ${formatNumber(localTotalSize / stats.localIndexSize)}X`,
177
- ];
178
-
179
- const blockItems = [
180
- <div title={`Scanned size = ${formatNumber(stats.blocksCheckedCompressedSize)}B, Decompressed size = ${formatNumber(stats.blocksCheckedDecompressedSize)}B, Compression ratio = ${formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X`}>{formatTime(stats.blockSearchTime)} | {formatNumber(stats.blockCheckedCount)} | {formatNumber(stats.blocksCheckedCompressedSize)}B | {formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X</div>,
181
- `total scanned ${formatNumber(stats.totalBlockCount)} (${formatPercent(stats.blockCheckedCount / stats.totalBlockCount)})`,
182
- `remote ${formatNumber(stats.remoteBlockCheckedCount)} / ${formatNumber(stats.remoteBlockCount)} (${formatPercent(stats.remoteBlockCheckedCount / stats.remoteBlockCount)}) | local ${formatNumber(stats.localBlockCheckedCount)} / ${formatNumber(stats.localBlockCount)} (${formatPercent(stats.localBlockCheckedCount / stats.localBlockCount)})`,
183
- ];
184
- let cacheItems = [
185
- `disk read ${formatNumber(uncachedSize)}B (${formatNumber(uncachedCount)})`,
186
- `remote ${formatPercent(uncachedRemoteSize / totalSizeRead)} (${formatNumber(uncachedRemoteCount)})`,
187
- `cached ${formatNumber(cachedSize)} (${formatNumber(cachedCount)})`,
188
- ];
189
-
190
- const resultItems = [
191
- `${formatTime(stats.timeToFirstMatch)} until first result`,
192
- `${formatNumber(stats.matchCount)} results / ${limitURL.value} limit`,
193
- ];
194
-
195
- const doneItems = [
196
- `${formatTime(stats.totalSearchTime)} total search time`,
197
- ];
198
-
199
- let hasErrors = stats.fileErrors.length > 0 || stats.blockErrors.length > 0;
200
-
201
- return (
202
- <div className={css.hbox(0).alignItems("stretch")}>
203
- {renderStage("Files", fileItems, 290, true, false)}
204
- {renderStage("Indexes", indexItems, 250, false, false)}
205
- {renderStage("Blocks", blockItems, 210, false, false)}
206
- {renderStage("Reads", cacheItems, 160, false, false)}
207
- {renderStage("Results", resultItems, 120, false, this.state.searching && !hasErrors)}
208
- {!this.state.searching && renderStage("Done", doneItems, undefined, false, !hasErrors)}
209
- {(() => {
210
- if (!hasErrors) return undefined;
211
- let errorItems: preact.ComponentChild[] = [];
212
- if (stats.fileErrors.length > 0) {
213
- const errorTitle = stats.fileErrors.map(e => `${e.path}:\n${e.error}`).join("\n\n");
214
- errorItems.push(<div title={errorTitle}>{stats.fileErrors.length} files failed</div>);
215
- }
216
- if (stats.blockErrors.length > 0) {
217
- errorItems.push(<div title={stats.blockErrors.join("\n")}>{stats.blockErrors.length} blocks failed</div>);
218
- }
219
- return renderStage("Errors", errorItems, 0, false, true);
220
- })()}
221
- </div>
222
- );
223
- }
224
-
225
- renderPendingLogWarnings() {
226
- const paths = this.getPaths();
227
- const now = Date.now();
228
- const threshold = now - (2 * PUBLIC_MOVE_THRESHOLD);
229
-
230
- const machineWarnings: Record<string, { machineId: string; oldestTime: number; count: number; totalSize: number }> = {};
231
-
232
- for (const path of paths) {
233
- if (path.logCount !== undefined) continue;
234
-
235
- if (path.startTime >= threshold) continue;
236
-
237
- if (!path.machineId) continue;
238
-
239
- const existing = machineWarnings[path.machineId];
240
- if (!existing) {
241
- machineWarnings[path.machineId] = {
242
- machineId: path.machineId,
243
- oldestTime: path.startTime,
244
- count: 1,
245
- totalSize: path.size || 0,
246
- };
247
- } else {
248
- if (path.startTime < existing.oldestTime) {
249
- existing.oldestTime = path.startTime;
250
- }
251
- existing.count++;
252
- existing.totalSize += path.size || 0;
253
- }
254
- }
255
-
256
- const warnings = Object.values(machineWarnings);
257
- if (warnings.length === 0) return undefined;
258
-
259
- const isFrozen = !!savedPathsURL.value;
260
-
261
- return (
262
- <div className={css.vbox(10).pad2(10).hsl(0, 50, 50).colorhsl(60, 50, 100)}>
263
- {isFrozen && (
264
- <>
265
- <div className={css.fontWeight(600)}>Frozen files are too old:</div>
266
- {warnings.map((warning) => (
267
- <div key={warning.machineId} className={css.hbox(10)}>
268
- <MachineThreadInfo machineId={warning.machineId} />
269
- <span>has {formatNumber(warning.count)} pending files ({formatNumber(warning.totalSize)}B) that are {formatTime(now - warning.oldestTime)} old</span>
270
- </div>
271
- ))}
272
- <Button
273
- onClick={() => {
274
- void this.clearFrozenPaths();
275
- }}
276
- hue={180}
277
- >
278
- Clear Frozen Files and Reload
279
- </Button>
280
- </>
281
- )}
282
- {!isFrozen && (
283
- <>
284
- <div className={css.fontWeight(600)}>Pending logs not being merged!</div>
285
- {warnings.map((warning) => (
286
- <div key={warning.machineId} className={css.hbox(10)}>
287
- <MachineThreadInfo machineId={warning.machineId} />
288
- <span>is not moving logs to remote storage. {formatNumber(warning.count)} files ({formatNumber(warning.totalSize)}B) are {formatTime(now - warning.oldestTime)} old</span>
289
- </div>
290
- ))}
291
- {!isPublic() && !readPublicLogs.value && <Button
292
- onClick={async () => {
293
- Querysub.commitLocal(() => {
294
- this.state.forceMoveStartTime = Date.now();
295
- this.state.forceMoveEndTime = undefined;
296
- });
297
-
298
- let forceMoveInterval = setInterval(() => {
299
- this.forceUpdate();
300
- }, 100);
301
-
302
- try {
303
- let loggers = await getLoggers2Async();
304
- for (let logger of Object.values(loggers)) {
305
- await logger.clientForceMoveLogsToPublic();
306
- }
307
- await this.loadPaths();
308
- } finally {
309
- if (forceMoveInterval) {
310
- clearInterval(forceMoveInterval);
311
- }
312
-
313
- Querysub.commitLocal(() => {
314
- this.state.forceMoveEndTime = Date.now();
315
- });
316
- }
317
- }}
318
- hue={180}
319
- >
320
- Force Move Logs to Public
321
- </Button>}
322
- {(() => {
323
- if (this.state.forceMoveStartTime !== undefined && this.state.forceMoveEndTime === undefined) {
324
- const elapsed = Date.now() - this.state.forceMoveStartTime;
325
- return <div>Moving logs... {formatTime(elapsed)}</div>;
326
- }
327
- if (this.state.forceMoveStartTime !== undefined && this.state.forceMoveEndTime !== undefined) {
328
- const duration = this.state.forceMoveEndTime - this.state.forceMoveStartTime;
329
- return <div>Moved logs in {formatTime(duration)}</div>;
330
- }
331
- return undefined;
332
- })()}
333
- </>
334
- )}
335
- </div>
336
- );
337
- }
338
-
339
107
  async loadPaths() {
340
108
  if (savedPathsURL.value) {
341
109
  return;
@@ -372,7 +140,7 @@ export class LogViewer3 extends qreact.Component {
372
140
  startTime: range.startTime,
373
141
  endTime: range.endTime,
374
142
  only: excludePendingResults.value ? "public" : undefined,
375
- forceReadPublic: readPublicLogs.value,
143
+ forceReadProduction: readProductionLogsURL.value,
376
144
  });
377
145
  allPaths.push(...paths);
378
146
  }));
@@ -468,7 +236,7 @@ export class LogViewer3 extends qreact.Component {
468
236
  return;
469
237
  }
470
238
 
471
- let searchBuffer = Querysub.localRead(() => Buffer.from(searchText.value, "utf8"));
239
+ let searchBuffer = Querysub.localRead(() => Buffer.from(searchTextURL.value, "utf8"));
472
240
  let results: LogDatum[] = [];
473
241
  let stats: IndexedLogResults = createEmptyIndexedLogResults();
474
242
  stats.fileFindTime += getFilesTime;
@@ -512,11 +280,11 @@ export class LogViewer3 extends qreact.Component {
512
280
  findBuffer: searchBuffer,
513
281
  pathOverrides: paths,
514
282
  only: excludePendingResults.value ? "local" : undefined,
515
- forceReadPublic: readPublicLogs.value,
283
+ forceReadProduction: readProductionLogsURL.value,
516
284
  },
517
285
  onResult: (match: LogDatum) => {
518
- console.log("onResult", match);
519
286
  results.push(match);
287
+ sort(results, x => -x.time);
520
288
  void updateResults();
521
289
  },
522
290
  onResults: (loggerStats: IndexedLogResults) => {
@@ -548,6 +316,243 @@ export class LogViewer3 extends qreact.Component {
548
316
  });
549
317
  };
550
318
 
319
+ renderSearchStats() {
320
+ const stats = this.state.stats;
321
+ if (!stats) return undefined;
322
+
323
+ let totalSizeRead = 0;
324
+ let cachedSize = 0;
325
+ let cachedCount = 0;
326
+ let uncachedSize = 0;
327
+ let uncachedCount = 0;
328
+ let uncachedRemoteSize = 0;
329
+ let uncachedRemoteCount = 0;
330
+ let totalSize = 0;
331
+ let remoteTotalSize = 0;
332
+ let localTotalSize = 0;
333
+
334
+ for (let read of stats.reads) {
335
+ totalSizeRead += read.size;
336
+ if (read.cached) {
337
+ cachedSize += read.size;
338
+ cachedCount += read.count;
339
+ } else {
340
+ uncachedSize += read.size;
341
+ uncachedCount += read.count;
342
+ totalSize += read.totalSize;
343
+ if (read.remote) {
344
+ remoteTotalSize += read.totalSize;
345
+ } else {
346
+ localTotalSize += read.totalSize;
347
+ }
348
+ }
349
+ if (read.remote && !read.cached) {
350
+ uncachedRemoteSize += read.size;
351
+ uncachedRemoteCount += read.count;
352
+ }
353
+ }
354
+
355
+ const renderStage = (title: string, items: preact.ComponentChild[], hue: number | undefined, isFirst: boolean, isLast: boolean) => {
356
+ const triangleWidth = 20;
357
+
358
+ let color = hue !== undefined ? css.hsl(hue, 70, 80) : css.hsl(0, 0, 100);
359
+ let colorhsl = hue !== undefined ? css.hslcolor(hue, 80, 30) : css;
360
+
361
+ return (
362
+ <div className={css.vbox(2).relative.minWidth(200).paddingTop(8).paddingBottom(8).paddingRight(10).paddingLeft(isFirst ? 10 : 10 + triangleWidth) + color}>
363
+ <div className={css.fontWeight(600) + colorhsl}>{title}</div>
364
+ {items.map((item, i) => (
365
+ <div key={i}>{item}</div>
366
+ ))}
367
+ {!isLast && hue !== undefined && (
368
+ <svg width={triangleWidth} height="100" viewBox="0 0 20 100" preserveAspectRatio="none" className={css.absolute.right(-triangleWidth).top(0).fillHeight.zIndex(1)}>
369
+ <polygon points="0,0 20,50 0,100" fill={`hsl(${hue}, 70%, 80%)`} />
370
+ </svg>
371
+ )}
372
+ </div>
373
+ );
374
+ };
375
+
376
+ const fileItems = [
377
+ `${formatTime(stats.fileFindTime)} | ${formatNumber(stats.localFilesSearched + stats.backblazeFilesSearched)} / ${formatNumber(stats.totalLocalFiles + stats.totalBackblazeFiles)} | ${formatNumber(totalSize)}B`,
378
+ `remote ${formatNumber(stats.backblazeFilesSearched)} / ${formatNumber(stats.totalBackblazeFiles)} | ${formatNumber(remoteTotalSize)}B`,
379
+ `pending ${formatNumber(stats.localFilesSearched)} / ${formatNumber(stats.totalLocalFiles)} | ${formatNumber(localTotalSize)}B`,
380
+ ];
381
+
382
+ const totalIndexesSearched = stats.remoteIndexesSearched + stats.localIndexesSearched;
383
+ const totalIndexSize = stats.remoteIndexSize + stats.localIndexSize;
384
+ const indexItems = [
385
+ `${formatTime(stats.indexSearchTime)} | ${formatNumber(totalIndexesSearched)} | ${formatNumber(totalIndexSize)}B | ${formatNumber(totalSize / totalIndexSize)}X`,
386
+ `remote ${formatNumber(stats.remoteIndexesSearched)} | ${formatNumber(stats.remoteIndexSize)}B | ${formatNumber(remoteTotalSize / stats.remoteIndexSize)}X`,
387
+ `local ${formatNumber(stats.localIndexesSearched)} | ${formatNumber(stats.localIndexSize)}B | ${formatNumber(localTotalSize / stats.localIndexSize)}X`,
388
+ ];
389
+
390
+ const blockItems = [
391
+ <div title={`Scanned size = ${formatNumber(stats.blocksCheckedCompressedSize)}B, Decompressed size = ${formatNumber(stats.blocksCheckedDecompressedSize)}B, Compression ratio = ${formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X`}>{formatTime(stats.blockSearchTime)} | {formatNumber(stats.blockCheckedCount)} | {formatNumber(stats.blocksCheckedCompressedSize)}B | {formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X</div>,
392
+ `total scanned ${formatNumber(stats.totalBlockCount)} (${formatPercent(stats.blockCheckedCount / stats.totalBlockCount)})`,
393
+ `remote ${formatNumber(stats.remoteBlockCheckedCount)} / ${formatNumber(stats.remoteBlockCount)} (${formatPercent(stats.remoteBlockCheckedCount / stats.remoteBlockCount)}) | local ${formatNumber(stats.localBlockCheckedCount)} / ${formatNumber(stats.localBlockCount)} (${formatPercent(stats.localBlockCheckedCount / stats.localBlockCount)})`,
394
+ ];
395
+ let cacheItems = [
396
+ `disk read ${formatNumber(uncachedSize)}B (${formatNumber(uncachedCount)})`,
397
+ `remote ${formatPercent(uncachedRemoteSize / totalSizeRead)} (${formatNumber(uncachedRemoteCount)})`,
398
+ `cached ${formatNumber(cachedSize)} (${formatNumber(cachedCount)})`,
399
+ ];
400
+
401
+ const resultItems = [
402
+ `${formatTime(stats.timeToFirstMatch)} until first result`,
403
+ `${formatNumber(stats.matchCount)} results / ${limitURL.value} limit`,
404
+ ];
405
+
406
+ const doneItems = [
407
+ `${formatTime(stats.totalSearchTime)} total search time`,
408
+ ];
409
+
410
+ let hasErrors = stats.fileErrors.length > 0 || stats.blockErrors.length > 0;
411
+
412
+ return (
413
+ <div className={css.hbox(0).alignItems("stretch")}>
414
+ {renderStage("Files", fileItems, 290, true, false)}
415
+ {renderStage("Indexes", indexItems, 250, false, false)}
416
+ {renderStage("Blocks", blockItems, 210, false, false)}
417
+ {renderStage("Reads", cacheItems, 160, false, false)}
418
+ {renderStage("Results", resultItems, 120, false, this.state.searching && !hasErrors)}
419
+ {!this.state.searching && renderStage("Done", doneItems, undefined, false, !hasErrors)}
420
+ {(() => {
421
+ if (!hasErrors) return undefined;
422
+ let errorItems: preact.ComponentChild[] = [];
423
+ if (stats.fileErrors.length > 0) {
424
+ const errorTitle = stats.fileErrors.map(e => `${e.path}:\n${e.error}`).join("\n\n");
425
+ errorItems.push(<div title={errorTitle}>{stats.fileErrors.length} files failed</div>);
426
+ }
427
+ if (stats.blockErrors.length > 0) {
428
+ errorItems.push(<div title={stats.blockErrors.join("\n")}>{stats.blockErrors.length} blocks failed</div>);
429
+ }
430
+ return renderStage("Errors", errorItems, 0, false, true);
431
+ })()}
432
+ </div>
433
+ );
434
+ }
435
+
436
+ renderForceMoveLogs() {
437
+ return <><Button
438
+ onClick={async () => {
439
+ Querysub.commitLocal(() => {
440
+ this.state.forceMoveStartTime = Date.now();
441
+ this.state.forceMoveEndTime = undefined;
442
+ });
443
+
444
+ let forceMoveInterval = setInterval(() => {
445
+ this.forceUpdate();
446
+ }, 100);
447
+
448
+ try {
449
+ let loggers = await getLoggers2Async();
450
+ for (let logger of Object.values(loggers)) {
451
+ await logger.clientForceMoveLogsToPublic();
452
+ }
453
+ await this.loadPaths();
454
+ } finally {
455
+ if (forceMoveInterval) {
456
+ clearInterval(forceMoveInterval);
457
+ }
458
+
459
+ Querysub.commitLocal(() => {
460
+ this.state.forceMoveEndTime = Date.now();
461
+ });
462
+ }
463
+ }}
464
+ hue={180}
465
+ >
466
+ Force Move Logs to Public
467
+ </Button>
468
+ {(() => {
469
+ if (this.state.forceMoveStartTime !== undefined && this.state.forceMoveEndTime === undefined) {
470
+ const elapsed = Date.now() - this.state.forceMoveStartTime;
471
+ return <div>Moving logs... {formatTime(elapsed)}</div>;
472
+ }
473
+ if (this.state.forceMoveStartTime !== undefined && this.state.forceMoveEndTime !== undefined) {
474
+ const duration = this.state.forceMoveEndTime - this.state.forceMoveStartTime;
475
+ return <div>Moved logs in {formatTime(duration)}</div>;
476
+ }
477
+ return undefined;
478
+ })()}
479
+ </>;
480
+ }
481
+
482
+ renderPendingLogWarnings() {
483
+ const paths = this.getPaths();
484
+ const now = Date.now();
485
+ const threshold = now - (2 * PUBLIC_MOVE_THRESHOLD);
486
+
487
+ const machineWarnings: Record<string, { machineId: string; oldestTime: number; count: number; totalSize: number }> = {};
488
+
489
+ for (const path of paths) {
490
+ if (path.logCount !== undefined) continue;
491
+
492
+ if (path.startTime >= threshold) continue;
493
+
494
+ if (!path.machineId) continue;
495
+
496
+ const existing = machineWarnings[path.machineId];
497
+ if (!existing) {
498
+ machineWarnings[path.machineId] = {
499
+ machineId: path.machineId,
500
+ oldestTime: path.startTime,
501
+ count: 1,
502
+ totalSize: path.size || 0,
503
+ };
504
+ } else {
505
+ if (path.startTime < existing.oldestTime) {
506
+ existing.oldestTime = path.startTime;
507
+ }
508
+ existing.count++;
509
+ existing.totalSize += path.size || 0;
510
+ }
511
+ }
512
+
513
+ const warnings = Object.values(machineWarnings);
514
+ if (warnings.length === 0) return undefined;
515
+
516
+ const isFrozen = !!savedPathsURL.value;
517
+
518
+ return (
519
+ <div className={css.vbox(10).pad2(10).hsl(0, 50, 50).colorhsl(60, 50, 100)}>
520
+ {isFrozen && (
521
+ <>
522
+ <div className={css.fontWeight(600)}>Frozen files are too old:</div>
523
+ {warnings.map((warning) => (
524
+ <div key={warning.machineId} className={css.hbox(10)}>
525
+ <MachineThreadInfo machineId={warning.machineId} />
526
+ <span>has {formatNumber(warning.count)} pending files ({formatNumber(warning.totalSize)}B) that are {formatTime(now - warning.oldestTime)} old</span>
527
+ </div>
528
+ ))}
529
+ <Button
530
+ onClick={() => {
531
+ void this.clearFrozenPaths();
532
+ }}
533
+ hue={180}
534
+ >
535
+ Clear Frozen Files and Reload
536
+ </Button>
537
+ </>
538
+ )}
539
+ {!isFrozen && (
540
+ <>
541
+ <div className={css.fontWeight(600)}>Pending logs not being merged!</div>
542
+ {warnings.map((warning) => (
543
+ <div key={warning.machineId} className={css.hbox(10)}>
544
+ <MachineThreadInfo machineId={warning.machineId} />
545
+ <span>is not moving logs to remote storage. {formatNumber(warning.count)} files ({formatNumber(warning.totalSize)}B) are {formatTime(now - warning.oldestTime)} old</span>
546
+ </div>
547
+ ))}
548
+ {!isPublic() && !readProductionLogsURL.value && this.renderForceMoveLogs()}
549
+ </>
550
+ )}
551
+ </div>
552
+ );
553
+ }
554
+
555
+
551
556
  renderResults() {
552
557
  let fieldNames = new Set<string>();
553
558
  for (let i = 0; i < Math.min(1000, this.state.results.length); i++) {
@@ -695,8 +700,8 @@ export class LogViewer3 extends qreact.Component {
695
700
  />
696
701
  {!isPublic() && <InputLabelURL
697
702
  checkbox
698
- label="Read Public Logs"
699
- url={readPublicLogs}
703
+ label="Read Production Logs"
704
+ url={readProductionLogsURL}
700
705
  onChangeValue={(newValue) => {
701
706
  if (newValue) {
702
707
  savedPathsURL.value = "";
@@ -709,6 +714,7 @@ export class LogViewer3 extends qreact.Component {
709
714
  number
710
715
  url={limitURL}
711
716
  />
717
+ {!isPublic() && this.renderForceMoveLogs()}
712
718
  </div>
713
719
 
714
720
  <div className={css.hbox(10)}>
@@ -754,7 +760,7 @@ export class LogViewer3 extends qreact.Component {
754
760
  </Button>
755
761
  <TimeRangeSelector />
756
762
  </div>
757
- {!isPublic() && readPublicLogs.value && <div className={css.fontSize(40).pad2(12, 6).hsl(280, 40, 50).colorhsl(0, 0, 100).boldStyle}>
763
+ {!isPublic() && readProductionLogsURL.value && <div className={css.fontSize(40).pad2(12, 6).hsl(280, 40, 50).colorhsl(0, 0, 100).boldStyle}>
758
764
  Reading public logs
759
765
  </div>}
760
766
 
@@ -763,10 +769,10 @@ export class LogViewer3 extends qreact.Component {
763
769
  flavor="large"
764
770
  fillWidth
765
771
  focusOnMount
766
- url={searchText}
772
+ url={searchTextURL}
767
773
  onKeyDown={(e) => {
768
774
  if (e.key === "Enter") {
769
- searchText.value = e.currentTarget.value;
775
+ searchTextURL.value = e.currentTarget.value;
770
776
  void this.search();
771
777
  }
772
778
  }}
@@ -776,6 +782,7 @@ export class LogViewer3 extends qreact.Component {
776
782
  flavor="large"
777
783
  onClick={() => void this.loadPaths()}
778
784
  hue={this.state.paths.length ? 200 : 120}
785
+ className={css + (this.state.loadingPaths && css.opacity(0.5))}
779
786
  >
780
787
  {this.getPaths().length && "Reset Files" || "Preview Files"}
781
788
  </Button>
@@ -835,7 +842,7 @@ export class LogViewer3 extends qreact.Component {
835
842
  >
836
843
  {savedPathsURL.value ? `Clear Frozen Files (${this.getPaths().length})` : `Freeze Files (${this.state.paths.length})`}
837
844
  </Button>
838
- {!savedPathsURL.value && (
845
+ {savedPathsURL.value && (
839
846
  <div className={css.pad2(8, 4).hsl(40, 60, 50).colorhsl(40, 0, 100)}>
840
847
  Files frozen in memory. Click Preview Files to refresh.
841
848
  </div>
@@ -0,0 +1,21 @@
1
+ import { URLParam } from "../../../library-components/URLParam";
2
+ import { managementPageURL } from "../../managementPages";
3
+ import { LogDatum } from "../diskLogger";
4
+
5
+ export let searchTextURL = new URLParam("searchText", "");
6
+ export let readProductionLogsURL = new URLParam("readProductionLogs", false);
7
+
8
+ export function formatSearchString(obj: Record<string, unknown>): string {
9
+ return Object.entries(obj).map(([key, value]) => {
10
+ return `${JSON.stringify(key)}:${JSON.stringify(value)}`;
11
+ }).join("&");
12
+ }
13
+
14
+ export function getLogViewerParams(logObj: Partial<LogDatum>) {
15
+ return [
16
+ managementPageURL.getOverride("LogViewer3"),
17
+ searchTextURL.getOverride(formatSearchString(logObj)),
18
+ // If we're linking to the logs, it's almost certainly for production logs.
19
+ readProductionLogsURL.getOverride(true),
20
+ ];
21
+ }
@@ -1,5 +1,5 @@
1
1
  import { measureWrap } from "socket-function/src/profiling/measure";
2
- import { Unit, getAllUnits } from "./BufferIndexHelpers";
2
+ import { Unit, getAllUnits, remapSearchBuffer } from "./BufferIndexHelpers";
3
3
 
4
4
  const WILD_CARD_BYTE = 42;
5
5
  const WILD_CARD_CHAR = "*";
@@ -26,7 +26,7 @@ export type MatchStructure = {
26
26
  function parseMatchStructure(pattern: Buffer): MatchStructure {
27
27
  let text = pattern.toString("utf-8");
28
28
 
29
- let orParts = text.split(SEARCH_OR_CHAR);
29
+ let orParts = text.split(SEARCH_OR_CHAR).map(x => x.trim()).filter(x => x.length > 0);
30
30
  if (orParts.length > 1) {
31
31
  return {
32
32
  type: "or",
@@ -34,7 +34,7 @@ function parseMatchStructure(pattern: Buffer): MatchStructure {
34
34
  };
35
35
  }
36
36
 
37
- let andParts = text.split(SEARCH_AND_CHAR);
37
+ let andParts = text.split(SEARCH_AND_CHAR).map(x => x.trim()).filter(x => x.length > 0);
38
38
  if (andParts.length > 1) {
39
39
  return {
40
40
  type: "and",
@@ -42,7 +42,7 @@ function parseMatchStructure(pattern: Buffer): MatchStructure {
42
42
  };
43
43
  }
44
44
 
45
- let wildcardParts = text.split(WILD_CARD_CHAR).filter(p => p.length > 0);
45
+ let wildcardParts = text.split(WILD_CARD_CHAR).map(x => x.trim()).filter(x => x.length > 0);
46
46
  if (wildcardParts.length > 1) {
47
47
  return {
48
48
  type: "wildcard",
@@ -54,6 +54,8 @@ function parseMatchStructure(pattern: Buffer): MatchStructure {
54
54
  }
55
55
  // inside is and, outside is outer
56
56
  export function getSearchUnits(pattern: Buffer, disableSpecialCharacters: boolean): Unit[][] {
57
+ pattern = remapSearchBuffer(pattern);
58
+
57
59
  if (disableSpecialCharacters || !pattern.includes(SEARCH_OR_BYTE) && !pattern.includes(SEARCH_AND_BYTE) && !pattern.includes(WILD_CARD_BYTE)) {
58
60
  return [getAllUnits({ buffer: pattern, bufferIndex: 0, block: 0 }).map(u => u.unit)];
59
61
  }
@@ -107,9 +109,11 @@ function getSimplerStructure(structure: MatchStructure): Buffer[][] {
107
109
  // side must appear in order somewhere within buffer.
108
110
  // Returns a function that matches buffers against the pre-processed pattern.
109
111
  export function createMatchesPattern(pattern: Buffer, disableSpecialCharacters: boolean): (buffer: Buffer) => boolean {
112
+ pattern = remapSearchBuffer(pattern);
110
113
 
111
114
  if (disableSpecialCharacters || !pattern.includes(SEARCH_OR_BYTE) && !pattern.includes(SEARCH_AND_BYTE) && !pattern.includes(WILD_CARD_BYTE)) {
112
115
  return measureWrap(function matchesPattern(buffer: Buffer): boolean {
116
+ buffer = remapSearchBuffer(buffer);
113
117
  return buffer.indexOf(pattern) !== -1;
114
118
  }, "BufferIndex|matchesPatternExact");
115
119
  }
@@ -120,6 +124,7 @@ export function createMatchesPattern(pattern: Buffer, disableSpecialCharacters:
120
124
  let hasAnyWildcard = pattern.includes(WILD_CARD_BYTE);
121
125
 
122
126
  return measureWrap(function matchesPattern(buffer: Buffer): boolean {
127
+ buffer = remapSearchBuffer(buffer);
123
128
  // 1) First, use simplerStructure to exclude any false cases
124
129
  // simplerStructure is Buffer[][] where outer array is OR, inner array is AND
125
130
  let anyOrClausePassed = false;