querysub 0.398.0 → 0.399.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.398.0",
+  "version": "0.399.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-a-archives/archivesBackBlaze.ts

@@ -517,7 +517,7 @@ export class ArchivesBackblaze {
 
             // If it's a 503 and it's been a minute since we last reset, then Wait and reset. 
             if (
-                (err.stack.includes(`503`)
+                (err.stack.includes(`"status": 503`)
                     || err.stack.includes(`"service_unavailable"`)
                     || err.stack.includes(`"internal_error"`)
                     || err.stack.includes(`ENOBUFS`)

package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts

@@ -1,4 +1,4 @@
-import { list, sort, timeInDay, timeInHour, timeInMinute } from "socket-function/src/misc";
+import { list, sort, timeInDay, timeInHour, timeInMinute, timeInSecond } from "socket-function/src/misc";
 import { MaybePromise } from "socket-function/src/types";
 import { decodeNodeId } from "../../-a-auth/certs";
 import { getOwnNodeId, getOwnNodeIdAssert } from "../../-f-node-discovery/NodeDiscovery";
@@ -18,7 +18,7 @@ import { logDisk } from "../../diagnostics/logs/diskLogger";
 
 /** Clean up old files after a while */
 const DEAD_CREATE_THRESHOLD = timeInHour * 12;
-const ARCHIVE_PROPAGATION_TIME = 5000;
+const ARCHIVE_PROPAGATION_TIME = timeInSecond * 5;
 // NOTE: There isn't a reason for transaction to not apply. If applying throws, we throw,
 //  and can throw infinite times, but if it doesn't throw, it SHOULD be applied. We retry
 //  a few times though, just in case the storage system was having some issues. After enough
@@ -374,6 +374,7 @@ class TransactionLocker {
     }
 
     private lastFilesRead: FileInfo[] | undefined;
+    private lastFilesReadTime: number | undefined;
     private async readDataState(): Promise<{
         rawDataFiles: FileInfo[];
         /** Confirmed FileInfos are === the FileInfos in rawDataFiles */
@@ -387,14 +388,15 @@ class TransactionLocker {
     }> {
         let bufferCache = new Map<string, Buffer>();
         const tryToRead = async () => {
+            let time = Date.now();
             let files = await this.storage.getKeys();
-            if (this.lastFilesRead) {
+            if (this.lastFilesRead && this.lastFilesReadTime) {
                 let prevFiles = new Set(this.lastFilesRead.map(a => a.file));
+                let suspiciousThreshold = this.lastFilesReadTime - ARCHIVE_PROPAGATION_TIME * 10;
                 let newFiles = files.filter(a => !prevFiles.has(a.file));
-                let pastTime = Date.now() - ARCHIVE_PROPAGATION_TIME * 10;
-                let veryBadFiles = newFiles.filter(x => x.createTime < pastTime);
+                let veryBadFiles = newFiles.filter(x => x.createTime < suspiciousThreshold);
                 if (veryBadFiles.length > 0) {
-                    console.error(`Old files suddenly appeared. This isn't possible, if they are old, they should have appeared when they were created! This likely means that our getKeys() failed to actually read all of the files. This is bad and can result in us deleting seemingly broken files for missing a confirmation, when they in fact had a confirmation.`, {
+                    console.error(`Old read was not exhaustive, because we just found files which existed well before it, but that it did not read. This likely means our propagation assumptions are wrong, and propagation is much slower than anticipated. OR, getKeys is flakey and randomly misses files? Either way, this will BREAK THE DATABASE! Any node experiencing this might start deleting valid database files, which is very bad!`, {
                         newlyAppearingOldFiles: veryBadFiles.map(x => x.file),
                         prevFiles: this.lastFilesRead.map(x => x.file),
                         newFiles: files.map(x => x.file),
@@ -402,6 +404,7 @@ class TransactionLocker {
                 }
             }
             this.lastFilesRead = files;
+            this.lastFilesReadTime = time;
 
             let transactions: (Transaction & {
                 seqNum: number;

package/src/0-path-value-core/pathValueCore.ts

@@ -980,7 +980,6 @@ class AuthorityPathValueStorage {
             if (!isCoreQuiet) {
                 console.log(`GCing everything older than ${debugPathValuePath(values[0])}`);
                 for (let value of removed) {
-                    console.log(`    ${debugPathValuePath(value)}`);
                     auditLog("GCing", { path: value.path });
                 }
             }
@@ -990,7 +989,6 @@ class AuthorityPathValueStorage {
                 for (let i = values.length - 1; i > firstGoldenIndex; i--) {
                     let gced = values.pop();
                     if (!isCoreQuiet && gced) {
-                        console.log(`GCing ${debugPathValuePath(gced)}`);
                         auditLog("GCing", { path: gced.path });
                     }
                 }
@@ -1935,21 +1933,22 @@ class WriteValidStorage {
                             debugger;
                         }
                         let timeToReject = now - valueGroup[0].time.time;
-                        console.log(red(`!!! VALUE REJECTED DUE TO USING MISSING / REJECTED READ!!!(rejected after ${timeToReject}ms at ${Date.now()})`));
-                        console.log(red(`${debugTime(valueGroup[0].time)} (write)`));
-                        console.log(red(`    rejected as the server could not find: `));
+                        let message = `!!! VALUE REJECTED DUE TO USING MISSING / REJECTED READ!!!(rejected after ${timeToReject}ms at ${Date.now()})`;
+                        message += `\n${debugTime(valueGroup[0].time)} (write)`;
+                        message += `\n    rejected as the server could not find: `;
                         if (lock.readIsTransparent) {
-                            console.log(red(`${debugTime(lock.startTime)} to ${debugTime(lock.endTime)} ${getPathFromStr(lock.path).join(".")} `));
+                            message += `\n${debugTime(lock.startTime)} to ${debugTime(lock.endTime)} ${getPathFromStr(lock.path).join(".")} `;
                         } else {
-                            console.log(red(`${debugTime(lock.startTime)} ${getPathFromStr(lock.path).join(".")} `));
+                            message += `\n${debugTime(lock.startTime)} ${getPathFromStr(lock.path).join(".")} `;
                         }
                         if (lock.readIsTransparent) {
-                            console.log(red(`    (read was undefined, so presumably a value exists which the writer missed)`));
+                            message += `\n    (read was undefined, so presumably a value exists which the writer missed)`;
                         }
-                        console.log(yellow(`Full list of writes rejected: `));
+                        message += `\nFull list of writes rejected: `;
                         for (let pathValue of valueGroup) {
-                            console.log(yellow(debugPathValue(pathValue)));
+                            message += `\n${debugPathValue(pathValue)}`;
                         }
+                        console.log(red(message));
 
                         if (debugRejections) {
                             debugbreak(2);
@@ -1978,17 +1977,18 @@ class WriteValidStorage {
                         let changed = valueGroup.filter(x => x.valid);
                         if (changed.length > 0) {
                             let timeToReject = now - changed[0].time.time;
-                            console.log(red(`!!! LOCK CONTENTION FIXED VIA REJECTION OF VALUE!!!(rejected after ${timeToReject}ms)`));
+                            let redMessage = `!!! LOCK CONTENTION FIXED VIA REJECTION OF VALUE!!!(rejected after ${timeToReject}ms)`;
                             for (let pathValue of changed) {
-                                console.log(red(debugPathValue(pathValue)));
+                                redMessage += `\n${debugPathValue(pathValue)}`;
                             }
-                            console.log(red(`    (write rejected due to original read not noticing value at: ${lock.path})`));
-                            console.log(red(`    (original read from: ${debugTime(lock.startTime)}`));
-                            console.log(red(`    (at time: ${debugTime(lock.endTime)}`));
-                            console.log(red(`    (current time: ${Date.now()})`));
+                            redMessage += `\n    (write rejected due to original read not noticing value at: ${lock.path})`;
+                            redMessage += `\n    (original read from: ${debugTime(lock.startTime)}`;
+                            redMessage += `\n    (at time: ${debugTime(lock.endTime)}`;
+                            redMessage += `\n    (current time: ${Date.now()})`;
                             for (let lockFailed of inRange) {
-                                console.log(red(`    (conflict write at: ${debugTime(lockFailed.time)}`));
+                                redMessage += `\n    (conflict write at: ${debugTime(lockFailed.time)}`;
                             }
+                            console.log(red(redMessage));
                             if (debugRejections) {
                                 debugbreak(2);
                                 debugger;

package/src/deployManager/machineApplyMainCode.ts

@@ -593,10 +593,10 @@ const ensureGitSynced = measureWrap(async function ensureGitSynced(config: {
     repoUrl: string;
     gitRef: string;
 }) {
-    if (!await fsExistsAsync(config.gitFolder + ".git")) {
-        await runPromise(`git clone ${config.repoUrl} ${config.gitFolder}`);
-    }
     try {
+        if (!await fsExistsAsync(config.gitFolder + ".git")) {
+            await runPromise(`git clone ${config.repoUrl} ${config.gitFolder}`);
+        }
         await setGitRef(config);
     } catch (e: any) {
         console.warn(`Failed to set git ref, trying to delete and clone again (${config.gitFolder}): ${e.stack}`);

package/src/diagnostics/logs/IndexedLogs/BufferIndex.ts

@@ -419,14 +419,26 @@ export class BufferIndex {
         keepIterating: () => boolean;
         onResult: (match: Buffer) => void;
         results: IndexedLogResults;
-    }): Promise<void> {
+    }): Promise<{
+        blockSearchTime: number;
+        indexSearchTime: number;
+    }> {
         let { index, dataReader, params, results } = config;
+        let startTime = Date.now();
+        let indexEndTime = startTime;
+        function createResult() {
+            let endTime = Date.now();
+            return {
+                indexSearchTime: indexEndTime - startTime,
+                blockSearchTime: endTime - indexEndTime,
+            };
+        }
 
         const matchesPattern = createMatchesPattern(params.findBuffer, !!params.disableWildCards);
 
         let allSearchUnits = getSearchUnits(params.findBuffer, !!params.disableWildCards);
         if (allSearchUnits.length === 0) {
-            return;
+            return createResult();
         }
 
         let type = index[0];
@@ -441,12 +453,13 @@ export class BufferIndex {
             if (index.length === 0) {
                 index = await BufferIndex.rebuildLocalIndexFromData(dataReader);
                 if (index.length === 0) {
-                    return;
+                    return createResult();
                 }
             }
 
             // Fix partial index before processing
             index = await BufferIndex.fixPartialIndex({ index, dataReader, results });
+            indexEndTime = Date.now();
 
             await BufferIndex.findLocal({ index, dataReader, params, keepIterating: config.keepIterating, onResult: config.onResult, results, allSearchUnits, matchesPattern });
         } else if (type === BULK_TYPE) {
@@ -465,9 +478,11 @@ export class BufferIndex {
             let dataLength = await dataReader.getLength();
             if (dataLength < 32) {
                 // If it's a small file, nothing of value will be lost by ignoring it. And almost all of our errors are coming from extremely small files. 
-                return;
+                return createResult();
             }
             throw new Error(`Unknown type in index file: ${type}`);
         }
+
+        return createResult();
     }
 }

package/src/diagnostics/logs/IndexedLogs/BufferIndexHelpers.ts

@@ -134,6 +134,8 @@ export type IndexedLogResults = {
     totalBackblazeLogs: number;
     backblazeLogsSearched: number;
 
+    backblazeUncompressedSize: number;
+
     totalBlockCount: number;
     blockCheckedCount: number;
 
@@ -165,7 +167,7 @@ export type IndexedLogResults = {
 };
 export function createEmptyIndexedLogResults(): IndexedLogResults {
     return {
-        matchCount: 0, reads: [], totalLocalFiles: 0, totalBackblazeFiles: 0, localFilesSearched: 0, backblazeFilesSearched: 0, totalBackblazeLogs: 0, backblazeLogsSearched: 0, totalBlockCount: 0, blockCheckedCount: 0, remoteBlockCount: 0, localBlockCount: 0, remoteBlockCheckedCount: 0, localBlockCheckedCount: 0, blocksCheckedCompressedSize: 0, blocksCheckedDecompressedSize: 0, blockErrors: [], fileErrors: [], remoteIndexesSearched: 0, remoteIndexSize: 0, localIndexesSearched: 0, localIndexSize: 0, timeToFirstMatch: -1, fileFindTime: 0, indexSearchTime: 0, blockSearchTime: 0, totalSearchTime: 0, cancel: undefined, limitGroup: undefined,
+        matchCount: 0, reads: [], totalLocalFiles: 0, totalBackblazeFiles: 0, localFilesSearched: 0, backblazeFilesSearched: 0, totalBackblazeLogs: 0, backblazeLogsSearched: 0, totalBlockCount: 0, blockCheckedCount: 0, remoteBlockCount: 0, localBlockCount: 0, remoteBlockCheckedCount: 0, localBlockCheckedCount: 0, blocksCheckedCompressedSize: 0, blocksCheckedDecompressedSize: 0, backblazeUncompressedSize: 0, blockErrors: [], fileErrors: [], remoteIndexesSearched: 0, remoteIndexSize: 0, localIndexesSearched: 0, localIndexSize: 0, timeToFirstMatch: -1, fileFindTime: 0, indexSearchTime: 0, blockSearchTime: 0, totalSearchTime: 0, cancel: undefined, limitGroup: undefined,
     };
 }
 
@@ -211,6 +213,7 @@ export function mergeIndexedLogResults(existing: IndexedLogResults, incoming: In
         blockCheckedCount: existing.blockCheckedCount + incoming.blockCheckedCount,
         blocksCheckedCompressedSize: existing.blocksCheckedCompressedSize + incoming.blocksCheckedCompressedSize,
         blocksCheckedDecompressedSize: existing.blocksCheckedDecompressedSize + incoming.blocksCheckedDecompressedSize,
+        backblazeUncompressedSize: existing.backblazeUncompressedSize + incoming.backblazeUncompressedSize,
         blockErrors: [...existing.blockErrors, ...incoming.blockErrors],
         fileErrors: [...existing.fileErrors, ...incoming.fileErrors],
         remoteIndexesSearched: existing.remoteIndexesSearched + incoming.remoteIndexesSearched,

package/src/diagnostics/logs/IndexedLogs/BufferIndexLogsOptimizationConstants.ts

@@ -1,7 +1,9 @@
 import { timeInHour, timeInMinute, timeInSecond } from "socket-function/src/misc";
 
 // NOTE: This is the block size BEFORE compression. The actual download size is probably going to be around 1/20th of this. 
-export const DEFAULT_BLOCK_SIZE = 1024 * 1024;
+// BUT, This is also going to be the amount of data we need to scan generally per result, as the results are often, at least in the cases that we're optimizing for, spread out over many files.
+// NOTE: This used to be 1024 * 1024, which in practice seemed to give us about a 170X factor from the full data to the index size. If this change to 1024 * 256 moves it below a 100X factor we should increase the size again (small block sizes mean more blocks, which means the index file has to store more data, making it less efficient).
+export const DEFAULT_BLOCK_SIZE = 1024 * 256;
 export const DEFAULT_TARGET_UNITS_PER_BUCKET = 16;
 
 // Worst case scenario, the logs the user wants are very spread out, in which case having too high of a threshold will require decoding more data (the threshold amount per result).

package/src/diagnostics/logs/IndexedLogs/BufferUnitIndex.ts

@@ -512,7 +512,11 @@ export class BufferUnitIndex {
     }): Promise<void> {
         const { params, index, reader, keepIterating, results, allSearchUnits } = config;
 
+        let indexTime = Date.now();
         let candidateBlocksList = allSearchUnits.map(units => this.findBlocks({ units, index })).flat();
+        indexTime = Date.now() - indexTime;
+        results.indexSearchTime += indexTime;
+        results.blockSearchTime -= indexTime;
         let candidateBlocksSet = new Set(candidateBlocksList);
 
 

package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts

@@ -10,7 +10,7 @@ import { getArchivesBackblaze, getArchivesBackblazePrivateImmutable } from "../.
 import { getMachineId, getOwnThreadId } from "../../../-a-auth/certs";
 import { ArchivesMemoryCacheStats, createArchivesMemoryCache } from "../../../-a-archives/archivesMemoryCache";
 import { registerShutdownHandler } from "../../periodic";
-import { measureBlock, measureFnc } from "socket-function/src/profiling/measure";
+import { measureBlock, measureFnc, measureWrap } from "socket-function/src/profiling/measure";
 import { isNode } from "typesafecss";
 import { getOwnMachineId, isNodeIdOnOwnMachineId, isOwnNodeId } from "../../../-f-node-discovery/NodeDiscovery";
 import { TimeFilePath, TimeFileTree } from "./TimeFileTree";
@@ -27,7 +27,8 @@ import { getAllNodeIds } from "../../../-f-node-discovery/NodeDiscovery";
 import { NodeCapabilitiesController } from "../../../-g-core-values/NodeCapabilities";
 import { getLoggers2Async } from "../diskLogger";
 import { watchAllValues } from "../errorNotifications2/logWatcher";
-// 
+
+// Ensure it's available so that the controller is listening on all servers
 watchAllValues;
 
 export type TimeFilePathWithSize = TimeFilePath & {
@@ -92,11 +93,11 @@ export class IndexedLogs<T> {
         });
         return archives;
     });
-    private getPublicLogs = lazy((): Archives => {
+    private getPublicLogsBase(loadPublic: boolean): Archives {
         let basePublic: Archives = getArchivesHome(getDomain());
         // NOTE: The local disk is so fast that reading in 10 megabytes is nothing, And if we read in too small of a value, the overhead per read ends up making this take forever. 
         let extraReadSize = 1024 * 1024 * 10;
-        if (isPublic()) {
+        if (loadPublic) {
             basePublic = getArchivesBackblaze(getDomain());
             // NOTE: While the latency to back plays is high, now we're reading in parallel, so it shouldn't be as big of an issue. 
             extraReadSize = 1024 * 1024 * 1;
@@ -111,6 +112,12 @@ export class IndexedLogs<T> {
             sizeCache: this.fileSizeCache,
         });
         return archives;
+    };
+    private getPublicLogs = lazy((): Archives => {
+        return this.getPublicLogsBase(isPublic());
+    });
+    private getRealPublicLogs = lazy((): Archives => {
+        return this.getPublicLogsBase(true);
     });
 
     private getTimeBlock(time: number): { startTime: number, endTime: number } {
@@ -374,7 +381,10 @@ export class IndexedLogs<T> {
         onResults?: (results: IndexedLogResults) => Promise<boolean>;
     }): Promise<IndexedLogResults> {
 
-        if (config.params.forceReadProduction && !isPublic()) {
+        let { params } = config;
+        if (params.pathOverrides && config.params.only === "public") {
+            // Fine, if they provided path overrides, and they are all public, just read them, as they'll resolve the same no matter where we read from
+        } else if (config.params.forceReadProduction && !isPublic()) {
             let machineNodes = await this.getMachineNodes();
             if (machineNodes.length === 0) throw new Error(`Cannot find any public nodes to read from`);
             return await this.clientFind({
@@ -391,13 +401,14 @@ export class IndexedLogs<T> {
             for (let result of allResults.values()) {
                 results = mergeIndexedLogResults(results, result);
             }
+
             return results;
         };
         let allResults = new Map<string, IndexedLogResults>();
         let allDone: Promise<void>[] = [];
 
         // Read other nodes if on a public server
-        if (config.params.only !== "local" && isPublic()) {
+        if (config.params.only !== "public" && isPublic()) {
             let machineNodes = await this.getMachineNodes();
             allDone.push(...machineNodes.map(async (machineNode) => {
                 try {
@@ -501,9 +512,13 @@ export class IndexedLogs<T> {
         onResult: (match: T) => void;
         results: IndexedLogResults;
     }): Promise<void> {
+        let results = config.results;
         let startTime = Date.now();
         let localLogs = this.getLocalLogs();
         let backblazeLogs = this.getPublicLogs();
+        if (!isPublic() && config.params.forceReadProduction) {
+            backblazeLogs = this.getRealPublicLogs();
+        }
 
         let fileFindTime = Date.now();
         let paths = config.params.pathOverrides ?? await this.getPaths({
@@ -518,6 +533,10 @@ export class IndexedLogs<T> {
         }
         fileFindTime = Date.now() - fileFindTime;
 
+        for (let path of paths) {
+            results.backblazeUncompressedSize += path.uncompressedSize ?? 0;
+        }
+
         // Sort by time based on search direction
         if (config.params.searchFromStart) {
             sort(paths, x => x.startTime);
@@ -525,7 +544,6 @@ export class IndexedLogs<T> {
             sort(paths, x => - x.startTime);
         }
 
-        let results = config.results;
         let localPaths = paths.filter(x => x.logCount === undefined);
         let remotePaths = paths.filter(x => x.logCount !== undefined);
         results.totalLocalFiles = localPaths.length;
@@ -547,10 +565,10 @@ export class IndexedLogs<T> {
         });
 
 
-        const searchPath = async (path: TimeFilePathWithSize) => {
+        const searchPath = measureWrap(async (path: TimeFilePathWithSize) => {
             if (!progressTracker.isSourceRelevant(path)) return;
             // Wait, so we don't lock up the main thread?
-            await delay(0);
+            //await delay(0);
 
             let remote = path.logCount !== undefined;
             let archives = remote ? backblazeLogs : localLogs;
@@ -578,9 +596,7 @@ export class IndexedLogs<T> {
                         return data;
                     },
                 };
-                let totalSearchTime = Date.now();
-                let blockSearchTimeBefore = results.blockSearchTime;
-                await BufferIndex.find({
+                let timeResult = await BufferIndex.find({
                     index,
                     dataReader,
                     params: {
@@ -594,22 +610,21 @@ export class IndexedLogs<T> {
                     },
                     results,
                 });
-                totalSearchTime = Date.now() - totalSearchTime;
-                let blockSearchTimeAdded = results.blockSearchTime - blockSearchTimeBefore;
-                results.indexSearchTime += totalSearchTime - blockSearchTimeAdded;
+                results.indexSearchTime += timeResult.indexSearchTime;
+                results.blockSearchTime += timeResult.blockSearchTime;
 
             } catch (e: any) {
                 results.fileErrors.push({ error: String(e?.stack || e), path: path.fullPath });
                 console.warn(`Error in reading log file ${path.fullPath} logs: ${e}`);
             }
-        };
+        });
+
         let localDone = (async () => {
             for (let path of localPaths) {
                 await searchPath(path);
             }
         })();
-        // No parallel count when running locally, as running it in parallel makes timing more difficult. 
-        let parallelCount = isPublic() ? 32 : 1;
+        let parallelCount = 32;
         let remoteParallel = runInParallel({ parallelCount }, searchPath);
         await Promise.all(remotePaths.map(remoteParallel));
         await localDone;

package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx

@@ -38,7 +38,7 @@ import { startTimeParam, endTimeParam } from "../TimeRangeSelector";
 import { formatSearchString } from "./LogViewerParams";
 
 let excludePendingResults = new URLParam("excludePendingResults", false);
-let limitURL = new URLParam("limit", 100);
+let limitURL = new URLParam("limit", 16);
 
 let savedPathsURL = new URLParam("savedPaths", "");
 let selectedFieldsURL = new URLParam("selectedFields", {} as Record<string, boolean>);
@@ -298,7 +298,7 @@ export class LogViewer3 extends qreact.Component {
                     limit: limitURL.value,
                     findBuffer: searchBuffer,
                     pathOverrides: paths,
-                    only: excludePendingResults.value ? "local" : undefined,
+                    only: excludePendingResults.value ? "public" : undefined,
                     forceReadProduction: readProductionLogsURL.value,
                     searchFromStart: range.searchFromStart,
                 },

package/src/diagnostics/logs/IndexedLogs/RenderSearchStats.tsx

@@ -67,21 +67,23 @@ export class RenderSearchStats extends qreact.Component<{
 
         const fileItems = [
             `${formatTime(stats.fileFindTime)} | ${formatNumber(stats.localFilesSearched + stats.backblazeFilesSearched)} / ${formatNumber(stats.totalLocalFiles + stats.totalBackblazeFiles)} | ${formatNumber(totalSize)}B`,
-            `remote ${formatNumber(stats.backblazeFilesSearched)} / ${formatNumber(stats.totalBackblazeFiles)} | ${formatNumber(remoteTotalSize)}B | ${formatNumber(stats.backblazeLogsSearched)} logs`,
+            `remote ${formatNumber(stats.backblazeFilesSearched)} / ${formatNumber(stats.totalBackblazeFiles)} | ${formatNumber(stats.backblazeUncompressedSize)}B uncomp | ${formatNumber(stats.backblazeUncompressedSize / remoteTotalSize)}X | ${formatNumber(stats.backblazeLogsSearched)} logs`,
             `pending ${formatNumber(stats.localFilesSearched)} / ${formatNumber(stats.totalLocalFiles)} | ${formatNumber(localTotalSize)}B`,
         ];
 
         const totalIndexesSearched = stats.remoteIndexesSearched + stats.localIndexesSearched;
         const totalIndexSize = stats.remoteIndexSize + stats.localIndexSize;
         const indexItems = [
-            `${formatTime(stats.indexSearchTime)} | ${formatNumber(totalIndexesSearched)} | ${formatNumber(totalIndexSize)}B | ${formatNumber(totalSize / totalIndexSize)}X`,
-            `remote ${formatNumber(stats.remoteIndexesSearched)} | ${formatNumber(stats.remoteIndexSize)}B | ${formatNumber(remoteTotalSize / stats.remoteIndexSize)}X`,
-            `local ${formatNumber(stats.localIndexesSearched)} | ${formatNumber(stats.localIndexSize)}B | ${formatNumber(localTotalSize / stats.localIndexSize)}X`,
+            `${formatTime(stats.indexSearchTime)} | ${formatNumber(totalIndexesSearched)} | ${formatNumber(totalIndexSize)}B`,
+            `remote ${formatNumber(stats.remoteIndexSize)}`,
+            `local ${formatNumber(stats.localIndexesSearched)} | ${formatNumber(stats.localIndexSize)}B`,
         ];
 
         const blockItems = [
-            <div title={`Scanned size = ${formatNumber(stats.blocksCheckedCompressedSize)}B, Decompressed size = ${formatNumber(stats.blocksCheckedDecompressedSize)}B, Compression ratio = ${formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X`}>{formatTime(stats.blockSearchTime)} | {formatNumber(stats.blockCheckedCount)} | {formatNumber(stats.blocksCheckedCompressedSize)}B | {formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X</div>,
-            `total scanned ${formatNumber(stats.totalBlockCount)} (${formatPercent(stats.blockCheckedCount / stats.totalBlockCount)})`,
+            <div title={`Scanned size = ${formatNumber(stats.blocksCheckedCompressedSize)}B, Decompressed size = ${formatNumber(stats.blocksCheckedDecompressedSize)}B, Compression ratio = ${formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X`}>
+                {formatTime(stats.blockSearchTime)} | {formatNumber(stats.blockCheckedCount)} | {formatNumber(stats.blocksCheckedDecompressedSize)}B uncomp | {formatNumber(stats.blocksCheckedDecompressedSize / stats.blocksCheckedCompressedSize)}X
+            </div>,
+            `scanned ${formatNumber(stats.blockCheckedCount)} (${formatPercent(stats.blockCheckedCount / stats.totalBlockCount)})`,
             `remote ${formatNumber(stats.remoteBlockCheckedCount)} / ${formatNumber(stats.remoteBlockCount)} (${formatPercent(stats.remoteBlockCheckedCount / stats.remoteBlockCount)}) | local ${formatNumber(stats.localBlockCheckedCount)} / ${formatNumber(stats.localBlockCount)} (${formatPercent(stats.localBlockCheckedCount / stats.localBlockCount)})`,
         ];
         let cacheItems = [

package/src/diagnostics/logs/ObjectDisplay.tsx

@@ -137,16 +137,16 @@ export class PrimitiveDisplay extends qreact.Component<{
     render() {
         let value = this.props.value;
         if (typeof value === "number") {
-            return <span className={css.boldStyle.hslcolor(265, 70, 60)}>{value}</span>;
+            return <span className={css.boldStyle.hslcolor(265, 70, 70)}>{value}</span>;
         }
         if (typeof value === "boolean") {
-            return <span className={css.boldStyle.hslcolor(207, 70, 60)}>{JSON.stringify(value)}</span>;
+            return <span className={css.boldStyle.hslcolor(207, 70, 70)}>{JSON.stringify(value)}</span>;
         }
 
         if (typeof value === "string") {
             let colors = parseAnsiColors(value);
             if (colors.length === 1 && !colors[0].color) {
-                return <span className={css.boldStyle.hslcolor(30, 70, 50)}>"{value}"</span>;
+                return <span className={css.boldStyle.hslcolor(30, 70, 70)}>"{value}"</span>;
             }
             return (
                 <span className={css.display("inline-flex").rowGap(4).columnGap(1).wrap}>
@@ -155,7 +155,7 @@ export class PrimitiveDisplay extends qreact.Component<{
                         return <span
                             className={
                                 (hue !== undefined && css.boldStyle.pad2(4, 1).hsla(hue, 70, 50, 0.5) || "")
-                                + (hue === undefined && css.boldStyle.hslcolor(30, 70, 50))
+                                + (hue === undefined && css.boldStyle.hslcolor(30, 70, 70))
                             }
                         >{text}</span>;
                     })}

package/src/diagnostics/logs/lifeCycleAnalysis/LifeCyclePage.tsx

@@ -27,8 +27,10 @@ import { StartEllipsis } from "../../../library-components/StartEllipsis";
 import { LifeCycleEntryReadMode } from "./LifeCycleEntryReadMode";
 import { LifeCycleEntryEditor } from "./LifeCycleEntryEditor";
 import { LifeCycleRenderer, LifeCycleInstanceRenderer } from "./LifeCycleRenderer";
+import { readProductionLogsURL } from "../IndexedLogs/LogViewerParams";
+import { isPublic } from "../../../config";
 export let lifecycleIdURL = new URLParam("lifecycleid", "");
-export let limitURL = new URLParam("lifecyclelimit", 100);
+export let limitURL = new URLParam("lifecyclelimit", 16);
 export let additionalSearchURL = new URLParam("lifecyclesearch", "");
 
 export class LifeCyclePage extends qreact.Component {
@@ -94,6 +96,12 @@ export class LifeCyclePage extends qreact.Component {
                         number
                         url={limitURL}
                     />
+                    {!isPublic() && <InputLabelURL
+                        checkbox
+                        label="Read Production Logs"
+                        url={readProductionLogsURL}
+                    />}
+
                     <TimeRangeSelector />
                 </div>
 
@@ -221,6 +229,11 @@ export class LifeCyclePage extends qreact.Component {
                                         <span key={key} className={css.colorhsl(0, 70, 40)}>{key}</span>
                                     ))}
                                 </div>
+                                <Button hue={200} onClick={() => {
+                                    console.log(invalid.datum);
+                                }}>
+                                    Log
+                                </Button>
                             </div>
                         ))}
                     </div>

package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycleSearch.tsx

@@ -5,7 +5,7 @@ import { getLoggers2Async, LogDatum } from "../diskLogger";
 import { IndexedLogResults, createEmptyIndexedLogResults, mergeIndexedLogResults } from "../IndexedLogs/BufferIndexHelpers";
 import { errorToUndefined } from "../../../errors";
 import { unique, maybeUndefined } from "../../../misc";
-import { formatSearchString } from "../IndexedLogs/LogViewerParams";
+import { formatSearchString, readProductionLogsURL } from "../IndexedLogs/LogViewerParams";
 import { createMatchesPatternCached } from "../IndexedLogs/bufferSearchFindMatcher";
 import { getPathStr } from "../../../path";
 import { LifeCycle, LifeCycleEntry, LifeCyclesController } from "./lifeCycles";
@@ -229,6 +229,7 @@ export function createLifeCycleSearch(
                     limit: limit,
                     findBuffer: searchBuffer,
                     searchFromStart: range.searchFromStart,
+                    forceReadProduction: readProductionLogsURL.value,
                 },
                 onResult: (match: LogDatum) => {
                     results.push(match);
@@ -423,6 +424,7 @@ export function createLifeCycleSearch(
                     limit: searchLimit,
                     findBuffer: searchBuffer,
                     searchFromStart: range.searchFromStart,
+                    forceReadProduction: readProductionLogsURL.value,
                 },
                 onResult: (match: LogDatum) => {
                     results.push(match);