querysub 0.374.0 → 0.376.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -4
- package/src/deployManager/components/MachineDetailPage.tsx +2 -5
- package/src/deployManager/components/ServiceDetailPage.tsx +2 -5
- package/src/deployManager/machineApplyMainCode.ts +7 -0
- package/src/diagnostics/NodeViewer.tsx +4 -5
- package/src/diagnostics/logs/IndexedLogs/BufferIndex.ts +10 -5
- package/src/diagnostics/logs/IndexedLogs/BufferIndexCPP.cpp +20 -0
- package/src/diagnostics/logs/IndexedLogs/BufferIndexHelpers.ts +29 -2
- package/src/diagnostics/logs/IndexedLogs/BufferUnitIndex.ts +61 -20
- package/src/diagnostics/logs/IndexedLogs/BufferUnitSet.ts +2 -2
- package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts +7 -7
- package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx +250 -243
- package/src/diagnostics/logs/IndexedLogs/LogViewerParams.ts +21 -0
- package/src/diagnostics/logs/IndexedLogs/{bufferMatcher.ts → bufferSearchFindMatcher.ts} +9 -4
- package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts +3 -3
- package/src/diagnostics/logs/diskLogger.ts +0 -38
- package/src/diagnostics/logs/errorNotifications2/errorNotifications2.ts +9 -0
- package/src/diagnostics/logs/injectFileLocationToConsole.ts +3 -0
- package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx +24 -22
- package/src/diagnostics/managementPages.tsx +0 -18
- package/test.ts +0 -5
- package/bin/error-email.js +0 -8
- package/bin/error-im.js +0 -8
- package/src/diagnostics/logs/FastArchiveAppendable.ts +0 -843
- package/src/diagnostics/logs/FastArchiveController.ts +0 -573
- package/src/diagnostics/logs/FastArchiveViewer.tsx +0 -1090
- package/src/diagnostics/logs/LogViewer2.tsx +0 -552
- package/src/diagnostics/logs/errorNotifications/ErrorDigestPage.tsx +0 -409
- package/src/diagnostics/logs/errorNotifications/ErrorNotificationController.ts +0 -756
- package/src/diagnostics/logs/errorNotifications/ErrorSuppressionUI.tsx +0 -280
- package/src/diagnostics/logs/errorNotifications/ErrorWarning.tsx +0 -254
- package/src/diagnostics/logs/errorNotifications/errorDigestEmail.tsx +0 -233
- package/src/diagnostics/logs/errorNotifications/errorDigestEntry.tsx +0 -14
- package/src/diagnostics/logs/errorNotifications/errorDigests.tsx +0 -292
- package/src/diagnostics/logs/errorNotifications/errorWatchEntry.tsx +0 -209
- package/src/diagnostics/logs/importLogsEntry.ts +0 -38
- package/src/diagnostics/logs/lifeCycleAnalysis/LifeCyclePages.tsx +0 -150
- package/src/diagnostics/logs/logViewerExtractField.ts +0 -36
|
@@ -26,11 +26,10 @@ import { IndexedLogResults, createEmptyIndexedLogResults, mergeIndexedLogResults
|
|
|
26
26
|
import { TimeFilePath } from "./TimeFileTree";
|
|
27
27
|
import { isPublic } from "../../../config";
|
|
28
28
|
import { Zip } from "../../../zip";
|
|
29
|
+
import { readProductionLogsURL, searchTextURL } from "./LogViewerParams";
|
|
29
30
|
|
|
30
|
-
let searchText = new URLParam("searchText", "");
|
|
31
31
|
let excludePendingResults = new URLParam("excludePendingResults", false);
|
|
32
32
|
let limitURL = new URLParam("limit", 100);
|
|
33
|
-
let readPublicLogs = new URLParam("readPublicLogs", false);
|
|
34
33
|
|
|
35
34
|
let savedPathsURL = new URLParam("savedPaths", "");
|
|
36
35
|
let selectedFieldsURL = new URLParam("selectedFields", {} as Record<string, boolean>);
|
|
@@ -105,237 +104,6 @@ export class LogViewer3 extends qreact.Component {
|
|
|
105
104
|
await this.loadPaths();
|
|
106
105
|
}
|
|
107
106
|
|
|
108
|
-
renderSearchStats() {
|
|
109
|
-
const stats = this.state.stats;
|
|
110
|
-
if (!stats) return undefined;
|
|
111
|
-
|
|
112
|
-
let totalSizeRead = 0;
|
|
113
|
-
let cachedSize = 0;
|
|
114
|
-
let cachedCount = 0;
|
|
115
|
-
let uncachedSize = 0;
|
|
116
|
-
let uncachedCount = 0;
|
|
117
|
-
let uncachedRemoteSize = 0;
|
|
118
|
-
let uncachedRemoteCount = 0;
|
|
119
|
-
let totalSize = 0;
|
|
120
|
-
let remoteTotalSize = 0;
|
|
121
|
-
let localTotalSize = 0;
|
|
122
|
-
|
|
123
|
-
for (let read of stats.reads) {
|
|
124
|
-
totalSizeRead += read.size;
|
|
125
|
-
if (read.cached) {
|
|
126
|
-
cachedSize += read.size;
|
|
127
|
-
cachedCount += read.count;
|
|
128
|
-
} else {
|
|
129
|
-
uncachedSize += read.size;
|
|
130
|
-
uncachedCount += read.count;
|
|
131
|
-
totalSize += read.totalSize;
|
|
132
|
-
if (read.remote) {
|
|
133
|
-
remoteTotalSize += read.totalSize;
|
|
134
|
-
} else {
|
|
135
|
-
localTotalSize += read.totalSize;
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
if (read.remote && !read.cached) {
|
|
139
|
-
uncachedRemoteSize += read.size;
|
|
140
|
-
uncachedRemoteCount += read.count;
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
const renderStage = (title: string, items: preact.ComponentChild[], hue: number | undefined, isFirst: boolean, isLast: boolean) => {
|
|
145
|
-
const triangleWidth = 20;
|
|
146
|
-
|
|
147
|
-
let color = hue !== undefined ? css.hsl(hue, 70, 80) : css.hsl(0, 0, 100);
|
|
148
|
-
let colorhsl = hue !== undefined ? css.hslcolor(hue, 80, 30) : css;
|
|
149
|
-
|
|
150
|
-
return (
|
|
151
|
-
<div className={css.vbox(2).relative.minWidth(200).paddingTop(8).paddingBottom(8).paddingRight(10).paddingLeft(isFirst ? 10 : 10 + triangleWidth) + color}>
|
|
152
|
-
<div className={css.fontWeight(600) + colorhsl}>{title}</div>
|
|
153
|
-
{items.map((item, i) => (
|
|
154
|
-
<div key={i}>{item}</div>
|
|
155
|
-
))}
|
|
156
|
-
{!isLast && hue !== undefined && (
|
|
157
|
-
<svg width={triangleWidth} height="100" viewBox="0 0 20 100" preserveAspectRatio="none" className={css.absolute.right(-triangleWidth).top(0).fillHeight.zIndex(1)}>
|
|
158
|
-
<polygon points="0,0 20,50 0,100" fill={`hsl(${hue}, 70%, 80%)`} />
|
|
159
|
-
</svg>
|
|
160
|
-
)}
|
|
161
|
-
</div>
|
|
162
|
-
);
|
|
163
|
-
};
|
|
164
|
-
|
|
165
|
-
const fileItems = [
|
|
166
|
-
`${formatTime(stats.fileFindTime)} | ${formatNumber(stats.localFilesSearched + stats.backblazeFilesSearched)} / ${formatNumber(stats.totalLocalFiles + stats.totalBackblazeFiles)} | ${formatNumber(totalSize)}B`,
|
|
167
|
-
`remote ${formatNumber(stats.backblazeFilesSearched)} / ${formatNumber(stats.totalBackblazeFiles)} | ${formatNumber(remoteTotalSize)}B`,
|
|
168
|
-
`pending ${formatNumber(stats.localFilesSearched)} / ${formatNumber(stats.totalLocalFiles)} | ${formatNumber(localTotalSize)}B`,
|
|
169
|
-
];
|
|
170
|
-
|
|
171
|
-
const totalIndexesSearched = stats.remoteIndexesSearched + stats.localIndexesSearched;
|
|
172
|
-
const totalIndexSize = stats.remoteIndexSize + stats.localIndexSize;
|
|
173
|
-
const indexItems = [
|
|
174
|
-
`${formatTime(stats.indexSearchTime)} | ${formatNumber(totalIndexesSearched)} | ${formatNumber(totalIndexSize)}B | ${formatNumber(totalSize / totalIndexSize)}X`,
|
|
175
|
-
`remote ${formatNumber(stats.remoteIndexesSearched)} | ${formatNumber(stats.remoteIndexSize)}B | ${formatNumber(remoteTotalSize / stats.remoteIndexSize)}X`,
|
|
176
|
-
`local ${formatNumber(stats.localIndexesSearched)} | ${formatNumber(stats.localIndexSize)}B | ${formatNumber(localTotalSize / stats.localIndexSize)}X`,
|
|
177
|
-
];
|
|
178
|
-
|
|
179
|
-
const blockItems = [
|
|
180
|
-
<div title={`Scanned size = ${formatNumber(stats.blocksCheckedCompressedSize)}B, Decompressed size = ${formatNumber(stats.blocksCheckedDecompressedSize)}B, Compression ratio = ${formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X`}>{formatTime(stats.blockSearchTime)} | {formatNumber(stats.blockCheckedCount)} | {formatNumber(stats.blocksCheckedCompressedSize)}B | {formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X</div>,
|
|
181
|
-
`total scanned ${formatNumber(stats.totalBlockCount)} (${formatPercent(stats.blockCheckedCount / stats.totalBlockCount)})`,
|
|
182
|
-
`remote ${formatNumber(stats.remoteBlockCheckedCount)} / ${formatNumber(stats.remoteBlockCount)} (${formatPercent(stats.remoteBlockCheckedCount / stats.remoteBlockCount)}) | local ${formatNumber(stats.localBlockCheckedCount)} / ${formatNumber(stats.localBlockCount)} (${formatPercent(stats.localBlockCheckedCount / stats.localBlockCount)})`,
|
|
183
|
-
];
|
|
184
|
-
let cacheItems = [
|
|
185
|
-
`disk read ${formatNumber(uncachedSize)}B (${formatNumber(uncachedCount)})`,
|
|
186
|
-
`remote ${formatPercent(uncachedRemoteSize / totalSizeRead)} (${formatNumber(uncachedRemoteCount)})`,
|
|
187
|
-
`cached ${formatNumber(cachedSize)} (${formatNumber(cachedCount)})`,
|
|
188
|
-
];
|
|
189
|
-
|
|
190
|
-
const resultItems = [
|
|
191
|
-
`${formatTime(stats.timeToFirstMatch)} until first result`,
|
|
192
|
-
`${formatNumber(stats.matchCount)} results / ${limitURL.value} limit`,
|
|
193
|
-
];
|
|
194
|
-
|
|
195
|
-
const doneItems = [
|
|
196
|
-
`${formatTime(stats.totalSearchTime)} total search time`,
|
|
197
|
-
];
|
|
198
|
-
|
|
199
|
-
let hasErrors = stats.fileErrors.length > 0 || stats.blockErrors.length > 0;
|
|
200
|
-
|
|
201
|
-
return (
|
|
202
|
-
<div className={css.hbox(0).alignItems("stretch")}>
|
|
203
|
-
{renderStage("Files", fileItems, 290, true, false)}
|
|
204
|
-
{renderStage("Indexes", indexItems, 250, false, false)}
|
|
205
|
-
{renderStage("Blocks", blockItems, 210, false, false)}
|
|
206
|
-
{renderStage("Reads", cacheItems, 160, false, false)}
|
|
207
|
-
{renderStage("Results", resultItems, 120, false, this.state.searching && !hasErrors)}
|
|
208
|
-
{!this.state.searching && renderStage("Done", doneItems, undefined, false, !hasErrors)}
|
|
209
|
-
{(() => {
|
|
210
|
-
if (!hasErrors) return undefined;
|
|
211
|
-
let errorItems: preact.ComponentChild[] = [];
|
|
212
|
-
if (stats.fileErrors.length > 0) {
|
|
213
|
-
const errorTitle = stats.fileErrors.map(e => `${e.path}:\n${e.error}`).join("\n\n");
|
|
214
|
-
errorItems.push(<div title={errorTitle}>{stats.fileErrors.length} files failed</div>);
|
|
215
|
-
}
|
|
216
|
-
if (stats.blockErrors.length > 0) {
|
|
217
|
-
errorItems.push(<div title={stats.blockErrors.join("\n")}>{stats.blockErrors.length} blocks failed</div>);
|
|
218
|
-
}
|
|
219
|
-
return renderStage("Errors", errorItems, 0, false, true);
|
|
220
|
-
})()}
|
|
221
|
-
</div>
|
|
222
|
-
);
|
|
223
|
-
}
|
|
224
|
-
|
|
225
|
-
renderPendingLogWarnings() {
|
|
226
|
-
const paths = this.getPaths();
|
|
227
|
-
const now = Date.now();
|
|
228
|
-
const threshold = now - (2 * PUBLIC_MOVE_THRESHOLD);
|
|
229
|
-
|
|
230
|
-
const machineWarnings: Record<string, { machineId: string; oldestTime: number; count: number; totalSize: number }> = {};
|
|
231
|
-
|
|
232
|
-
for (const path of paths) {
|
|
233
|
-
if (path.logCount !== undefined) continue;
|
|
234
|
-
|
|
235
|
-
if (path.startTime >= threshold) continue;
|
|
236
|
-
|
|
237
|
-
if (!path.machineId) continue;
|
|
238
|
-
|
|
239
|
-
const existing = machineWarnings[path.machineId];
|
|
240
|
-
if (!existing) {
|
|
241
|
-
machineWarnings[path.machineId] = {
|
|
242
|
-
machineId: path.machineId,
|
|
243
|
-
oldestTime: path.startTime,
|
|
244
|
-
count: 1,
|
|
245
|
-
totalSize: path.size || 0,
|
|
246
|
-
};
|
|
247
|
-
} else {
|
|
248
|
-
if (path.startTime < existing.oldestTime) {
|
|
249
|
-
existing.oldestTime = path.startTime;
|
|
250
|
-
}
|
|
251
|
-
existing.count++;
|
|
252
|
-
existing.totalSize += path.size || 0;
|
|
253
|
-
}
|
|
254
|
-
}
|
|
255
|
-
|
|
256
|
-
const warnings = Object.values(machineWarnings);
|
|
257
|
-
if (warnings.length === 0) return undefined;
|
|
258
|
-
|
|
259
|
-
const isFrozen = !!savedPathsURL.value;
|
|
260
|
-
|
|
261
|
-
return (
|
|
262
|
-
<div className={css.vbox(10).pad2(10).hsl(0, 50, 50).colorhsl(60, 50, 100)}>
|
|
263
|
-
{isFrozen && (
|
|
264
|
-
<>
|
|
265
|
-
<div className={css.fontWeight(600)}>Frozen files are too old:</div>
|
|
266
|
-
{warnings.map((warning) => (
|
|
267
|
-
<div key={warning.machineId} className={css.hbox(10)}>
|
|
268
|
-
<MachineThreadInfo machineId={warning.machineId} />
|
|
269
|
-
<span>has {formatNumber(warning.count)} pending files ({formatNumber(warning.totalSize)}B) that are {formatTime(now - warning.oldestTime)} old</span>
|
|
270
|
-
</div>
|
|
271
|
-
))}
|
|
272
|
-
<Button
|
|
273
|
-
onClick={() => {
|
|
274
|
-
void this.clearFrozenPaths();
|
|
275
|
-
}}
|
|
276
|
-
hue={180}
|
|
277
|
-
>
|
|
278
|
-
Clear Frozen Files and Reload
|
|
279
|
-
</Button>
|
|
280
|
-
</>
|
|
281
|
-
)}
|
|
282
|
-
{!isFrozen && (
|
|
283
|
-
<>
|
|
284
|
-
<div className={css.fontWeight(600)}>Pending logs not being merged!</div>
|
|
285
|
-
{warnings.map((warning) => (
|
|
286
|
-
<div key={warning.machineId} className={css.hbox(10)}>
|
|
287
|
-
<MachineThreadInfo machineId={warning.machineId} />
|
|
288
|
-
<span>is not moving logs to remote storage. {formatNumber(warning.count)} files ({formatNumber(warning.totalSize)}B) are {formatTime(now - warning.oldestTime)} old</span>
|
|
289
|
-
</div>
|
|
290
|
-
))}
|
|
291
|
-
{!isPublic() && !readPublicLogs.value && <Button
|
|
292
|
-
onClick={async () => {
|
|
293
|
-
Querysub.commitLocal(() => {
|
|
294
|
-
this.state.forceMoveStartTime = Date.now();
|
|
295
|
-
this.state.forceMoveEndTime = undefined;
|
|
296
|
-
});
|
|
297
|
-
|
|
298
|
-
let forceMoveInterval = setInterval(() => {
|
|
299
|
-
this.forceUpdate();
|
|
300
|
-
}, 100);
|
|
301
|
-
|
|
302
|
-
try {
|
|
303
|
-
let loggers = await getLoggers2Async();
|
|
304
|
-
for (let logger of Object.values(loggers)) {
|
|
305
|
-
await logger.clientForceMoveLogsToPublic();
|
|
306
|
-
}
|
|
307
|
-
await this.loadPaths();
|
|
308
|
-
} finally {
|
|
309
|
-
if (forceMoveInterval) {
|
|
310
|
-
clearInterval(forceMoveInterval);
|
|
311
|
-
}
|
|
312
|
-
|
|
313
|
-
Querysub.commitLocal(() => {
|
|
314
|
-
this.state.forceMoveEndTime = Date.now();
|
|
315
|
-
});
|
|
316
|
-
}
|
|
317
|
-
}}
|
|
318
|
-
hue={180}
|
|
319
|
-
>
|
|
320
|
-
Force Move Logs to Public
|
|
321
|
-
</Button>}
|
|
322
|
-
{(() => {
|
|
323
|
-
if (this.state.forceMoveStartTime !== undefined && this.state.forceMoveEndTime === undefined) {
|
|
324
|
-
const elapsed = Date.now() - this.state.forceMoveStartTime;
|
|
325
|
-
return <div>Moving logs... {formatTime(elapsed)}</div>;
|
|
326
|
-
}
|
|
327
|
-
if (this.state.forceMoveStartTime !== undefined && this.state.forceMoveEndTime !== undefined) {
|
|
328
|
-
const duration = this.state.forceMoveEndTime - this.state.forceMoveStartTime;
|
|
329
|
-
return <div>Moved logs in {formatTime(duration)}</div>;
|
|
330
|
-
}
|
|
331
|
-
return undefined;
|
|
332
|
-
})()}
|
|
333
|
-
</>
|
|
334
|
-
)}
|
|
335
|
-
</div>
|
|
336
|
-
);
|
|
337
|
-
}
|
|
338
|
-
|
|
339
107
|
async loadPaths() {
|
|
340
108
|
if (savedPathsURL.value) {
|
|
341
109
|
return;
|
|
@@ -372,7 +140,7 @@ export class LogViewer3 extends qreact.Component {
|
|
|
372
140
|
startTime: range.startTime,
|
|
373
141
|
endTime: range.endTime,
|
|
374
142
|
only: excludePendingResults.value ? "public" : undefined,
|
|
375
|
-
|
|
143
|
+
forceReadProduction: readProductionLogsURL.value,
|
|
376
144
|
});
|
|
377
145
|
allPaths.push(...paths);
|
|
378
146
|
}));
|
|
@@ -468,7 +236,7 @@ export class LogViewer3 extends qreact.Component {
|
|
|
468
236
|
return;
|
|
469
237
|
}
|
|
470
238
|
|
|
471
|
-
let searchBuffer = Querysub.localRead(() => Buffer.from(
|
|
239
|
+
let searchBuffer = Querysub.localRead(() => Buffer.from(searchTextURL.value, "utf8"));
|
|
472
240
|
let results: LogDatum[] = [];
|
|
473
241
|
let stats: IndexedLogResults = createEmptyIndexedLogResults();
|
|
474
242
|
stats.fileFindTime += getFilesTime;
|
|
@@ -512,11 +280,11 @@ export class LogViewer3 extends qreact.Component {
|
|
|
512
280
|
findBuffer: searchBuffer,
|
|
513
281
|
pathOverrides: paths,
|
|
514
282
|
only: excludePendingResults.value ? "local" : undefined,
|
|
515
|
-
|
|
283
|
+
forceReadProduction: readProductionLogsURL.value,
|
|
516
284
|
},
|
|
517
285
|
onResult: (match: LogDatum) => {
|
|
518
|
-
console.log("onResult", match);
|
|
519
286
|
results.push(match);
|
|
287
|
+
sort(results, x => -x.time);
|
|
520
288
|
void updateResults();
|
|
521
289
|
},
|
|
522
290
|
onResults: (loggerStats: IndexedLogResults) => {
|
|
@@ -548,6 +316,243 @@ export class LogViewer3 extends qreact.Component {
|
|
|
548
316
|
});
|
|
549
317
|
};
|
|
550
318
|
|
|
319
|
+
renderSearchStats() {
|
|
320
|
+
const stats = this.state.stats;
|
|
321
|
+
if (!stats) return undefined;
|
|
322
|
+
|
|
323
|
+
let totalSizeRead = 0;
|
|
324
|
+
let cachedSize = 0;
|
|
325
|
+
let cachedCount = 0;
|
|
326
|
+
let uncachedSize = 0;
|
|
327
|
+
let uncachedCount = 0;
|
|
328
|
+
let uncachedRemoteSize = 0;
|
|
329
|
+
let uncachedRemoteCount = 0;
|
|
330
|
+
let totalSize = 0;
|
|
331
|
+
let remoteTotalSize = 0;
|
|
332
|
+
let localTotalSize = 0;
|
|
333
|
+
|
|
334
|
+
for (let read of stats.reads) {
|
|
335
|
+
totalSizeRead += read.size;
|
|
336
|
+
if (read.cached) {
|
|
337
|
+
cachedSize += read.size;
|
|
338
|
+
cachedCount += read.count;
|
|
339
|
+
} else {
|
|
340
|
+
uncachedSize += read.size;
|
|
341
|
+
uncachedCount += read.count;
|
|
342
|
+
totalSize += read.totalSize;
|
|
343
|
+
if (read.remote) {
|
|
344
|
+
remoteTotalSize += read.totalSize;
|
|
345
|
+
} else {
|
|
346
|
+
localTotalSize += read.totalSize;
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
if (read.remote && !read.cached) {
|
|
350
|
+
uncachedRemoteSize += read.size;
|
|
351
|
+
uncachedRemoteCount += read.count;
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
const renderStage = (title: string, items: preact.ComponentChild[], hue: number | undefined, isFirst: boolean, isLast: boolean) => {
|
|
356
|
+
const triangleWidth = 20;
|
|
357
|
+
|
|
358
|
+
let color = hue !== undefined ? css.hsl(hue, 70, 80) : css.hsl(0, 0, 100);
|
|
359
|
+
let colorhsl = hue !== undefined ? css.hslcolor(hue, 80, 30) : css;
|
|
360
|
+
|
|
361
|
+
return (
|
|
362
|
+
<div className={css.vbox(2).relative.minWidth(200).paddingTop(8).paddingBottom(8).paddingRight(10).paddingLeft(isFirst ? 10 : 10 + triangleWidth) + color}>
|
|
363
|
+
<div className={css.fontWeight(600) + colorhsl}>{title}</div>
|
|
364
|
+
{items.map((item, i) => (
|
|
365
|
+
<div key={i}>{item}</div>
|
|
366
|
+
))}
|
|
367
|
+
{!isLast && hue !== undefined && (
|
|
368
|
+
<svg width={triangleWidth} height="100" viewBox="0 0 20 100" preserveAspectRatio="none" className={css.absolute.right(-triangleWidth).top(0).fillHeight.zIndex(1)}>
|
|
369
|
+
<polygon points="0,0 20,50 0,100" fill={`hsl(${hue}, 70%, 80%)`} />
|
|
370
|
+
</svg>
|
|
371
|
+
)}
|
|
372
|
+
</div>
|
|
373
|
+
);
|
|
374
|
+
};
|
|
375
|
+
|
|
376
|
+
const fileItems = [
|
|
377
|
+
`${formatTime(stats.fileFindTime)} | ${formatNumber(stats.localFilesSearched + stats.backblazeFilesSearched)} / ${formatNumber(stats.totalLocalFiles + stats.totalBackblazeFiles)} | ${formatNumber(totalSize)}B`,
|
|
378
|
+
`remote ${formatNumber(stats.backblazeFilesSearched)} / ${formatNumber(stats.totalBackblazeFiles)} | ${formatNumber(remoteTotalSize)}B`,
|
|
379
|
+
`pending ${formatNumber(stats.localFilesSearched)} / ${formatNumber(stats.totalLocalFiles)} | ${formatNumber(localTotalSize)}B`,
|
|
380
|
+
];
|
|
381
|
+
|
|
382
|
+
const totalIndexesSearched = stats.remoteIndexesSearched + stats.localIndexesSearched;
|
|
383
|
+
const totalIndexSize = stats.remoteIndexSize + stats.localIndexSize;
|
|
384
|
+
const indexItems = [
|
|
385
|
+
`${formatTime(stats.indexSearchTime)} | ${formatNumber(totalIndexesSearched)} | ${formatNumber(totalIndexSize)}B | ${formatNumber(totalSize / totalIndexSize)}X`,
|
|
386
|
+
`remote ${formatNumber(stats.remoteIndexesSearched)} | ${formatNumber(stats.remoteIndexSize)}B | ${formatNumber(remoteTotalSize / stats.remoteIndexSize)}X`,
|
|
387
|
+
`local ${formatNumber(stats.localIndexesSearched)} | ${formatNumber(stats.localIndexSize)}B | ${formatNumber(localTotalSize / stats.localIndexSize)}X`,
|
|
388
|
+
];
|
|
389
|
+
|
|
390
|
+
const blockItems = [
|
|
391
|
+
<div title={`Scanned size = ${formatNumber(stats.blocksCheckedCompressedSize)}B, Decompressed size = ${formatNumber(stats.blocksCheckedDecompressedSize)}B, Compression ratio = ${formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X`}>{formatTime(stats.blockSearchTime)} | {formatNumber(stats.blockCheckedCount)} | {formatNumber(stats.blocksCheckedCompressedSize)}B | {formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X</div>,
|
|
392
|
+
`total scanned ${formatNumber(stats.totalBlockCount)} (${formatPercent(stats.blockCheckedCount / stats.totalBlockCount)})`,
|
|
393
|
+
`remote ${formatNumber(stats.remoteBlockCheckedCount)} / ${formatNumber(stats.remoteBlockCount)} (${formatPercent(stats.remoteBlockCheckedCount / stats.remoteBlockCount)}) | local ${formatNumber(stats.localBlockCheckedCount)} / ${formatNumber(stats.localBlockCount)} (${formatPercent(stats.localBlockCheckedCount / stats.localBlockCount)})`,
|
|
394
|
+
];
|
|
395
|
+
let cacheItems = [
|
|
396
|
+
`disk read ${formatNumber(uncachedSize)}B (${formatNumber(uncachedCount)})`,
|
|
397
|
+
`remote ${formatPercent(uncachedRemoteSize / totalSizeRead)} (${formatNumber(uncachedRemoteCount)})`,
|
|
398
|
+
`cached ${formatNumber(cachedSize)} (${formatNumber(cachedCount)})`,
|
|
399
|
+
];
|
|
400
|
+
|
|
401
|
+
const resultItems = [
|
|
402
|
+
`${formatTime(stats.timeToFirstMatch)} until first result`,
|
|
403
|
+
`${formatNumber(stats.matchCount)} results / ${limitURL.value} limit`,
|
|
404
|
+
];
|
|
405
|
+
|
|
406
|
+
const doneItems = [
|
|
407
|
+
`${formatTime(stats.totalSearchTime)} total search time`,
|
|
408
|
+
];
|
|
409
|
+
|
|
410
|
+
let hasErrors = stats.fileErrors.length > 0 || stats.blockErrors.length > 0;
|
|
411
|
+
|
|
412
|
+
return (
|
|
413
|
+
<div className={css.hbox(0).alignItems("stretch")}>
|
|
414
|
+
{renderStage("Files", fileItems, 290, true, false)}
|
|
415
|
+
{renderStage("Indexes", indexItems, 250, false, false)}
|
|
416
|
+
{renderStage("Blocks", blockItems, 210, false, false)}
|
|
417
|
+
{renderStage("Reads", cacheItems, 160, false, false)}
|
|
418
|
+
{renderStage("Results", resultItems, 120, false, this.state.searching && !hasErrors)}
|
|
419
|
+
{!this.state.searching && renderStage("Done", doneItems, undefined, false, !hasErrors)}
|
|
420
|
+
{(() => {
|
|
421
|
+
if (!hasErrors) return undefined;
|
|
422
|
+
let errorItems: preact.ComponentChild[] = [];
|
|
423
|
+
if (stats.fileErrors.length > 0) {
|
|
424
|
+
const errorTitle = stats.fileErrors.map(e => `${e.path}:\n${e.error}`).join("\n\n");
|
|
425
|
+
errorItems.push(<div title={errorTitle}>{stats.fileErrors.length} files failed</div>);
|
|
426
|
+
}
|
|
427
|
+
if (stats.blockErrors.length > 0) {
|
|
428
|
+
errorItems.push(<div title={stats.blockErrors.join("\n")}>{stats.blockErrors.length} blocks failed</div>);
|
|
429
|
+
}
|
|
430
|
+
return renderStage("Errors", errorItems, 0, false, true);
|
|
431
|
+
})()}
|
|
432
|
+
</div>
|
|
433
|
+
);
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
renderForceMoveLogs() {
|
|
437
|
+
return <><Button
|
|
438
|
+
onClick={async () => {
|
|
439
|
+
Querysub.commitLocal(() => {
|
|
440
|
+
this.state.forceMoveStartTime = Date.now();
|
|
441
|
+
this.state.forceMoveEndTime = undefined;
|
|
442
|
+
});
|
|
443
|
+
|
|
444
|
+
let forceMoveInterval = setInterval(() => {
|
|
445
|
+
this.forceUpdate();
|
|
446
|
+
}, 100);
|
|
447
|
+
|
|
448
|
+
try {
|
|
449
|
+
let loggers = await getLoggers2Async();
|
|
450
|
+
for (let logger of Object.values(loggers)) {
|
|
451
|
+
await logger.clientForceMoveLogsToPublic();
|
|
452
|
+
}
|
|
453
|
+
await this.loadPaths();
|
|
454
|
+
} finally {
|
|
455
|
+
if (forceMoveInterval) {
|
|
456
|
+
clearInterval(forceMoveInterval);
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
Querysub.commitLocal(() => {
|
|
460
|
+
this.state.forceMoveEndTime = Date.now();
|
|
461
|
+
});
|
|
462
|
+
}
|
|
463
|
+
}}
|
|
464
|
+
hue={180}
|
|
465
|
+
>
|
|
466
|
+
Force Move Logs to Public
|
|
467
|
+
</Button>
|
|
468
|
+
{(() => {
|
|
469
|
+
if (this.state.forceMoveStartTime !== undefined && this.state.forceMoveEndTime === undefined) {
|
|
470
|
+
const elapsed = Date.now() - this.state.forceMoveStartTime;
|
|
471
|
+
return <div>Moving logs... {formatTime(elapsed)}</div>;
|
|
472
|
+
}
|
|
473
|
+
if (this.state.forceMoveStartTime !== undefined && this.state.forceMoveEndTime !== undefined) {
|
|
474
|
+
const duration = this.state.forceMoveEndTime - this.state.forceMoveStartTime;
|
|
475
|
+
return <div>Moved logs in {formatTime(duration)}</div>;
|
|
476
|
+
}
|
|
477
|
+
return undefined;
|
|
478
|
+
})()}
|
|
479
|
+
</>;
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
renderPendingLogWarnings() {
|
|
483
|
+
const paths = this.getPaths();
|
|
484
|
+
const now = Date.now();
|
|
485
|
+
const threshold = now - (2 * PUBLIC_MOVE_THRESHOLD);
|
|
486
|
+
|
|
487
|
+
const machineWarnings: Record<string, { machineId: string; oldestTime: number; count: number; totalSize: number }> = {};
|
|
488
|
+
|
|
489
|
+
for (const path of paths) {
|
|
490
|
+
if (path.logCount !== undefined) continue;
|
|
491
|
+
|
|
492
|
+
if (path.startTime >= threshold) continue;
|
|
493
|
+
|
|
494
|
+
if (!path.machineId) continue;
|
|
495
|
+
|
|
496
|
+
const existing = machineWarnings[path.machineId];
|
|
497
|
+
if (!existing) {
|
|
498
|
+
machineWarnings[path.machineId] = {
|
|
499
|
+
machineId: path.machineId,
|
|
500
|
+
oldestTime: path.startTime,
|
|
501
|
+
count: 1,
|
|
502
|
+
totalSize: path.size || 0,
|
|
503
|
+
};
|
|
504
|
+
} else {
|
|
505
|
+
if (path.startTime < existing.oldestTime) {
|
|
506
|
+
existing.oldestTime = path.startTime;
|
|
507
|
+
}
|
|
508
|
+
existing.count++;
|
|
509
|
+
existing.totalSize += path.size || 0;
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
const warnings = Object.values(machineWarnings);
|
|
514
|
+
if (warnings.length === 0) return undefined;
|
|
515
|
+
|
|
516
|
+
const isFrozen = !!savedPathsURL.value;
|
|
517
|
+
|
|
518
|
+
return (
|
|
519
|
+
<div className={css.vbox(10).pad2(10).hsl(0, 50, 50).colorhsl(60, 50, 100)}>
|
|
520
|
+
{isFrozen && (
|
|
521
|
+
<>
|
|
522
|
+
<div className={css.fontWeight(600)}>Frozen files are too old:</div>
|
|
523
|
+
{warnings.map((warning) => (
|
|
524
|
+
<div key={warning.machineId} className={css.hbox(10)}>
|
|
525
|
+
<MachineThreadInfo machineId={warning.machineId} />
|
|
526
|
+
<span>has {formatNumber(warning.count)} pending files ({formatNumber(warning.totalSize)}B) that are {formatTime(now - warning.oldestTime)} old</span>
|
|
527
|
+
</div>
|
|
528
|
+
))}
|
|
529
|
+
<Button
|
|
530
|
+
onClick={() => {
|
|
531
|
+
void this.clearFrozenPaths();
|
|
532
|
+
}}
|
|
533
|
+
hue={180}
|
|
534
|
+
>
|
|
535
|
+
Clear Frozen Files and Reload
|
|
536
|
+
</Button>
|
|
537
|
+
</>
|
|
538
|
+
)}
|
|
539
|
+
{!isFrozen && (
|
|
540
|
+
<>
|
|
541
|
+
<div className={css.fontWeight(600)}>Pending logs not being merged!</div>
|
|
542
|
+
{warnings.map((warning) => (
|
|
543
|
+
<div key={warning.machineId} className={css.hbox(10)}>
|
|
544
|
+
<MachineThreadInfo machineId={warning.machineId} />
|
|
545
|
+
<span>is not moving logs to remote storage. {formatNumber(warning.count)} files ({formatNumber(warning.totalSize)}B) are {formatTime(now - warning.oldestTime)} old</span>
|
|
546
|
+
</div>
|
|
547
|
+
))}
|
|
548
|
+
{!isPublic() && !readProductionLogsURL.value && this.renderForceMoveLogs()}
|
|
549
|
+
</>
|
|
550
|
+
)}
|
|
551
|
+
</div>
|
|
552
|
+
);
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
|
|
551
556
|
renderResults() {
|
|
552
557
|
let fieldNames = new Set<string>();
|
|
553
558
|
for (let i = 0; i < Math.min(1000, this.state.results.length); i++) {
|
|
@@ -695,8 +700,8 @@ export class LogViewer3 extends qreact.Component {
|
|
|
695
700
|
/>
|
|
696
701
|
{!isPublic() && <InputLabelURL
|
|
697
702
|
checkbox
|
|
698
|
-
label="Read
|
|
699
|
-
url={
|
|
703
|
+
label="Read Production Logs"
|
|
704
|
+
url={readProductionLogsURL}
|
|
700
705
|
onChangeValue={(newValue) => {
|
|
701
706
|
if (newValue) {
|
|
702
707
|
savedPathsURL.value = "";
|
|
@@ -709,6 +714,7 @@ export class LogViewer3 extends qreact.Component {
|
|
|
709
714
|
number
|
|
710
715
|
url={limitURL}
|
|
711
716
|
/>
|
|
717
|
+
{!isPublic() && this.renderForceMoveLogs()}
|
|
712
718
|
</div>
|
|
713
719
|
|
|
714
720
|
<div className={css.hbox(10)}>
|
|
@@ -754,7 +760,7 @@ export class LogViewer3 extends qreact.Component {
|
|
|
754
760
|
</Button>
|
|
755
761
|
<TimeRangeSelector />
|
|
756
762
|
</div>
|
|
757
|
-
{!isPublic() &&
|
|
763
|
+
{!isPublic() && readProductionLogsURL.value && <div className={css.fontSize(40).pad2(12, 6).hsl(280, 40, 50).colorhsl(0, 0, 100).boldStyle}>
|
|
758
764
|
Reading public logs
|
|
759
765
|
</div>}
|
|
760
766
|
|
|
@@ -763,10 +769,10 @@ export class LogViewer3 extends qreact.Component {
|
|
|
763
769
|
flavor="large"
|
|
764
770
|
fillWidth
|
|
765
771
|
focusOnMount
|
|
766
|
-
url={
|
|
772
|
+
url={searchTextURL}
|
|
767
773
|
onKeyDown={(e) => {
|
|
768
774
|
if (e.key === "Enter") {
|
|
769
|
-
|
|
775
|
+
searchTextURL.value = e.currentTarget.value;
|
|
770
776
|
void this.search();
|
|
771
777
|
}
|
|
772
778
|
}}
|
|
@@ -776,6 +782,7 @@ export class LogViewer3 extends qreact.Component {
|
|
|
776
782
|
flavor="large"
|
|
777
783
|
onClick={() => void this.loadPaths()}
|
|
778
784
|
hue={this.state.paths.length ? 200 : 120}
|
|
785
|
+
className={css + (this.state.loadingPaths && css.opacity(0.5))}
|
|
779
786
|
>
|
|
780
787
|
{this.getPaths().length && "Reset Files" || "Preview Files"}
|
|
781
788
|
</Button>
|
|
@@ -835,7 +842,7 @@ export class LogViewer3 extends qreact.Component {
|
|
|
835
842
|
>
|
|
836
843
|
{savedPathsURL.value ? `Clear Frozen Files (${this.getPaths().length})` : `Freeze Files (${this.state.paths.length})`}
|
|
837
844
|
</Button>
|
|
838
|
-
{
|
|
845
|
+
{savedPathsURL.value && (
|
|
839
846
|
<div className={css.pad2(8, 4).hsl(40, 60, 50).colorhsl(40, 0, 100)}>
|
|
840
847
|
Files frozen in memory. Click Preview Files to refresh.
|
|
841
848
|
</div>
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { URLParam } from "../../../library-components/URLParam";
|
|
2
|
+
import { managementPageURL } from "../../managementPages";
|
|
3
|
+
import { LogDatum } from "../diskLogger";
|
|
4
|
+
|
|
5
|
+
export let searchTextURL = new URLParam("searchText", "");
|
|
6
|
+
export let readProductionLogsURL = new URLParam("readProductionLogs", false);
|
|
7
|
+
|
|
8
|
+
export function formatSearchString(obj: Record<string, unknown>): string {
|
|
9
|
+
return Object.entries(obj).map(([key, value]) => {
|
|
10
|
+
return `${JSON.stringify(key)}:${JSON.stringify(value)}`;
|
|
11
|
+
}).join("&");
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function getLogViewerParams(logObj: Partial<LogDatum>) {
|
|
15
|
+
return [
|
|
16
|
+
managementPageURL.getOverride("LogViewer3"),
|
|
17
|
+
searchTextURL.getOverride(formatSearchString(logObj)),
|
|
18
|
+
// If we're linking to the logs, it's almost certainly for production logs.
|
|
19
|
+
readProductionLogsURL.getOverride(true),
|
|
20
|
+
];
|
|
21
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { measureWrap } from "socket-function/src/profiling/measure";
|
|
2
|
-
import { Unit, getAllUnits } from "./BufferIndexHelpers";
|
|
2
|
+
import { Unit, getAllUnits, remapSearchBuffer } from "./BufferIndexHelpers";
|
|
3
3
|
|
|
4
4
|
const WILD_CARD_BYTE = 42;
|
|
5
5
|
const WILD_CARD_CHAR = "*";
|
|
@@ -26,7 +26,7 @@ export type MatchStructure = {
|
|
|
26
26
|
function parseMatchStructure(pattern: Buffer): MatchStructure {
|
|
27
27
|
let text = pattern.toString("utf-8");
|
|
28
28
|
|
|
29
|
-
let orParts = text.split(SEARCH_OR_CHAR);
|
|
29
|
+
let orParts = text.split(SEARCH_OR_CHAR).map(x => x.trim()).filter(x => x.length > 0);
|
|
30
30
|
if (orParts.length > 1) {
|
|
31
31
|
return {
|
|
32
32
|
type: "or",
|
|
@@ -34,7 +34,7 @@ function parseMatchStructure(pattern: Buffer): MatchStructure {
|
|
|
34
34
|
};
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
-
let andParts = text.split(SEARCH_AND_CHAR);
|
|
37
|
+
let andParts = text.split(SEARCH_AND_CHAR).map(x => x.trim()).filter(x => x.length > 0);
|
|
38
38
|
if (andParts.length > 1) {
|
|
39
39
|
return {
|
|
40
40
|
type: "and",
|
|
@@ -42,7 +42,7 @@ function parseMatchStructure(pattern: Buffer): MatchStructure {
|
|
|
42
42
|
};
|
|
43
43
|
}
|
|
44
44
|
|
|
45
|
-
let wildcardParts = text.split(WILD_CARD_CHAR).filter(
|
|
45
|
+
let wildcardParts = text.split(WILD_CARD_CHAR).map(x => x.trim()).filter(x => x.length > 0);
|
|
46
46
|
if (wildcardParts.length > 1) {
|
|
47
47
|
return {
|
|
48
48
|
type: "wildcard",
|
|
@@ -54,6 +54,8 @@ function parseMatchStructure(pattern: Buffer): MatchStructure {
|
|
|
54
54
|
}
|
|
55
55
|
// inside is and, outside is outer
|
|
56
56
|
export function getSearchUnits(pattern: Buffer, disableSpecialCharacters: boolean): Unit[][] {
|
|
57
|
+
pattern = remapSearchBuffer(pattern);
|
|
58
|
+
|
|
57
59
|
if (disableSpecialCharacters || !pattern.includes(SEARCH_OR_BYTE) && !pattern.includes(SEARCH_AND_BYTE) && !pattern.includes(WILD_CARD_BYTE)) {
|
|
58
60
|
return [getAllUnits({ buffer: pattern, bufferIndex: 0, block: 0 }).map(u => u.unit)];
|
|
59
61
|
}
|
|
@@ -107,9 +109,11 @@ function getSimplerStructure(structure: MatchStructure): Buffer[][] {
|
|
|
107
109
|
// side must appear in order somewhere within buffer.
|
|
108
110
|
// Returns a function that matches buffers against the pre-processed pattern.
|
|
109
111
|
export function createMatchesPattern(pattern: Buffer, disableSpecialCharacters: boolean): (buffer: Buffer) => boolean {
|
|
112
|
+
pattern = remapSearchBuffer(pattern);
|
|
110
113
|
|
|
111
114
|
if (disableSpecialCharacters || !pattern.includes(SEARCH_OR_BYTE) && !pattern.includes(SEARCH_AND_BYTE) && !pattern.includes(WILD_CARD_BYTE)) {
|
|
112
115
|
return measureWrap(function matchesPattern(buffer: Buffer): boolean {
|
|
116
|
+
buffer = remapSearchBuffer(buffer);
|
|
113
117
|
return buffer.indexOf(pattern) !== -1;
|
|
114
118
|
}, "BufferIndex|matchesPatternExact");
|
|
115
119
|
}
|
|
@@ -120,6 +124,7 @@ export function createMatchesPattern(pattern: Buffer, disableSpecialCharacters:
|
|
|
120
124
|
let hasAnyWildcard = pattern.includes(WILD_CARD_BYTE);
|
|
121
125
|
|
|
122
126
|
return measureWrap(function matchesPattern(buffer: Buffer): boolean {
|
|
127
|
+
buffer = remapSearchBuffer(buffer);
|
|
123
128
|
// 1) First, use simplerStructure to exclude any false cases
|
|
124
129
|
// simplerStructure is Buffer[][] where outer array is OR, inner array is AND
|
|
125
130
|
let anyOrClausePassed = false;
|