querysub 0.326.0 → 0.328.0

package/src/diagnostics/logs/FastArchiveAppendable.ts

@@ -21,7 +21,7 @@ import { errorToUndefined, ignoreErrors, timeoutToUndefinedSilent } from "../../
 import { getCallObj } from "socket-function/src/nodeProxy";
 import { getSyncedController } from "../../library-components/SyncedController";
 import { getBrowserUrlNode, getOwnNodeId } from "../../-f-node-discovery/NodeDiscovery";
-import { secureRandom } from "../../misc/random";
+import { secureRandom, shuffle } from "../../misc/random";
 import { getPathIndex, getPathStr2 } from "../../path";
 import { onNextPaint } from "../../functional/onNextPaint";
 import { getArchivesBackblazePrivateImmutable, getArchivesBackblazePublicImmutable } from "../../-a-archives/archivesBackBlaze";
@@ -34,6 +34,7 @@ import { sha256 } from "js-sha256";
 import { assertIsNetworkTrusted } from "../../-d-trust/NetworkTrust2";
 import { blue, magenta } from "socket-function/src/formatting/logColors";
 import { FileMetadata, FastArchiveAppendableControllerBase, FastArchiveAppendableController, getFileMetadataHash } from "./FastArchiveController";
+import { fsExistsAsync } from "../../fs";
 
 // NOTE: In a single command line micro-test it looks like we can write about 40K writes of 500 per once, when using 10X parallel, on a fairly potato server. We should probably batch though, and only do 1X parallel.
 /*
@@ -140,7 +141,7 @@ export class FastArchiveAppendable<Datum> {
                 await this.flushNow();
             });
             // Random, to try to prevent the dead file detection code from getting in sync. It's fine if it gets in sync, it's just inefficient as we'll have multiple services uploading the same file to the same location at the same time. 
-            void runInfinitePollCallAtStart(timeInMinute * 20 + Math.random() * timeInMinute * 5, async () => {
+            void runInfinitePoll(timeInMinute * 20 + Math.random() * timeInMinute * 5, async () => {
                 await this.moveLogsToBackblaze();
             });
         }
@@ -151,13 +152,16 @@ export class FastArchiveAppendable<Datum> {
         return nestArchives("fast-logs/" + this.rootPath, archives);
     });
 
-    public getLocalPathRoot = lazy(() => {
-        let path = (
+    public baseGetLocalPathRoot = () => {
+        return (
             os.homedir()
             + "/fast-log-cache/"
             + getDomain() + "/"
             + this.rootPath
         );
+    };
+    public getLocalPathRoot = lazy(() => {
+        let path = this.baseGetLocalPathRoot();
         if (!fs.existsSync(path)) {
             fs.mkdirSync(path, { recursive: true });
         }
@@ -199,7 +203,10 @@ export class FastArchiveAppendable<Datum> {
     // NOTE: This is disk writing, which should be fast, but if it's slow we might be able to remove the measureWrap (as technically spending 50% of our time writing to the disk is fine, and won't lag anything).
     @measureFnc
     public async flushNow(now = Date.now()) {
+
         await appendableSerialLock(async () => {
+            if (this.pendingWriteQueue.length === 0) return;
+
             // 2025-09-06T07
             let hourFile = new Date(now).toISOString().slice(0, 13) + ".log";
             let localCacheFolder = this.getLocalPathRoot() + getOwnThreadId() + "/";
@@ -208,8 +215,6 @@ export class FastArchiveAppendable<Datum> {
             // Always heartbeat
             await fs.promises.writeFile(localCacheFolder + "heartbeat", Buffer.from(now + ""));
 
-            if (this.pendingWriteQueue.length === 0) return;
-
             try {
                 let beforeSize = await fs.promises.stat(localCachePath);
                 if (beforeSize.size > UNCOMPRESSED_LOG_FILE_STOP_THRESHOLD) {
@@ -277,18 +282,22 @@ export class FastArchiveAppendable<Datum> {
 
     public async moveLogsToBackblaze() {
         await appendableSerialLock(async () => {
-            let rootCacheFolder = this.getLocalPathRoot();
+            let rootCacheFolder = this.baseGetLocalPathRoot();
+            if (!await fsExistsAsync(rootCacheFolder)) return;
             console.log(magenta(`Moving old logs to Backblaze from ${rootCacheFolder}`));
 
             let archives = this.getArchives();
             async function moveLogsForFolder(threadId: string) {
                 let threadDir = rootCacheFolder + threadId + "/";
-                if (!fs.existsSync(threadDir)) return;
+                if (!await fsExistsAsync(threadDir)) return;
                 let files = await fs.promises.readdir(threadDir);
+                // Shuffle, so if we do run multiple at the same time, we are less likely to use the same files at the same time.
+                files = shuffle(files, Math.random());
                 for (let file of files) {
                     if (file === "heartbeat") continue;
                     let fullPath = threadDir + file;
                     try {
+                        if (!await fsExistsAsync(fullPath)) continue;
                         // We could use modified time here? Although, this is nice if we move files around, and then manually have them moved, although even then... this could cause problem be tripping while we are copying the file, so... maybe this is just wrong?
                         let timeStamp = getFileTimeStamp(fullPath);
                         if (timeStamp.endTime > Date.now() - UPLOAD_THRESHOLD) continue;
@@ -419,6 +428,7 @@ export class FastArchiveAppendable<Datum> {
         });
 
         let createProgress = async (section: string, max: number) => {
+            section = `${this.rootPath}|${section}`;
             let cancelled: Error | undefined;
             let lastValue = 0;
             let baseBatch = batchFunction({ delay: 150, },
@@ -499,7 +509,7 @@ export class FastArchiveAppendable<Datum> {
             synchronizeObj.parametersResult.resolve(syncResult);
 
 
-            let downloadProgress = await createProgress("Downloading (bytes)", 0);
+            let downloadProgress = await createProgress(`Downloading (bytes)`, 0);
             let decompressProgress = await createProgress("Decompressing (bytes)", 0);
             let scanProgress = await createProgress("Scanning (datums)", 0);
             let decodeProgress = await createProgress("Decoding (datums)", 0);
@@ -509,13 +519,14 @@ export class FastArchiveAppendable<Datum> {
             const self = this;
 
 
-            async function downloadAndParseFile(file: FileMetadata) {
+            async function downloadAndParseFile(file: FileMetadata, runInner: (code: () => Promise<void>) => Promise<void>) {
                 const onFetchedData = runInSerial(async (data: Buffer) => {
                     await downloadProgress(data.length, data.length, true);
                     await decompressWriter.write(data);
                 });
 
                 const onDecompressedData = createLogScanner({
+                    debugName: file.path,
                     onParsedData,
                 });
                 let batchedData: Buffer[] = [];
@@ -610,65 +621,79 @@ export class FastArchiveAppendable<Datum> {
 
                 // TODO: Stream from the local cache instead? It should be possible, we can get the total size, and read chunks.
                 let hash = getFileMetadataHash(file) + ".file";
-                let info = await localCache.getInfo(hash);
-                if (info?.size) {
-                    const CHUNK_SIZE = 1000 * 1000 * 10;
-                    for (let i = 0; i < info.size; i += CHUNK_SIZE) {
-                        let data = await localCache.get(hash, { range: { start: i, end: i + CHUNK_SIZE } });
-                        if (!data) {
-                            throw new Error(`Cached data disappeared (the info was there?) for ${hash} at ${i}`);
+                let contents = await localCache.get(hash);
+                if (stoppedPromise.resolveCalled) return;
+                await runInner(async () => {
+                    if (contents?.length) {
+                        const CHUNK_SIZE = 1000 * 1000 * 10;
+                        for (let i = 0; i < contents.length; i += CHUNK_SIZE) {
+                            let data = contents.slice(i, i + CHUNK_SIZE);
+                            await onFetchedData(data);
+                        }
+                    } else {
+                        const response = await fetch(url);
+                        if (!response.ok) {
+                            throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
                         }
-                        await onFetchedData(data);
-                    }
-                } else {
-                    const response = await fetch(url);
-                    if (!response.ok) {
-                        throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
-                    }
 
-                    if (!response.body) {
-                        throw new Error(`Response body is undefined for ${url}`);
-                    }
-                    let values: Buffer[] = [];
-                    const reader = response.body.getReader();
-                    void stoppedPromise.promise.finally(() => {
-                        void response.body?.cancel();
-                    });
-                    try {
-                        for await (let value of streamToIteratable(reader)) {
-                            // Cancel entirely
-                            if (stoppedPromise.resolveCalled) return;
-                            if (!value) continue;
-                            let buffer = Buffer.from(value);
-                            values.push(buffer);
-                            await onFetchedData(buffer);
+                        if (!response.body) {
+                            throw new Error(`Response body is undefined for ${url}`);
+                        }
+                        let values: Buffer[] = [];
+                        const reader = response.body.getReader();
+                        void stoppedPromise.promise.finally(() => {
+                            void response.body?.cancel();
+                        });
+                        try {
+                            for await (let value of streamToIteratable(reader)) {
+                                // Cancel entirely
+                                if (stoppedPromise.resolveCalled) return;
+                                if (!value) continue;
+                                let buffer = Buffer.from(value);
+                                values.push(buffer);
+                                await onFetchedData(buffer);
+                            }
+                        } finally {
+                            reader.releaseLock();
                         }
-                    } finally {
-                        reader.releaseLock();
-                    }
 
-                    await localCache.set(hash, Buffer.concat(values));
-                }
-                await decompressWriter.close();
-                await decompressPromise;
+                        await localCache.set(hash, Buffer.concat(values));
+                    }
+                    await decompressWriter.close();
+                    await decompressPromise;
+                });
             }
 
             // Fork off the processing
             void (async () => {
                 try {
                     // Iterate over all files and process them
-                    let fileProgress = await createProgress("Files", 0);
+                    let fileProgress = await createProgress("Files", syncResult.files.length);
+                    let fileInnerProgress = await createProgress("Files Inner", syncResult.files.length);
                     let failedFiles = await createProgress("Failed Files", 0);
-                    for (let file of syncResult.files) {
+                    let runSerial = runInSerial(async (fnc: () => Promise<void>) => {
+                        try {
+                            await fnc();
+                        } finally {
+                            await fileInnerProgress(1, 0, true);
+                        }
+                    });
+                    async function downloadFileWrapper(file: FileMetadata) {
                         if (stoppedPromise.resolveCalled) return;
                         try {
-                            await downloadAndParseFile(file);
+                            await downloadAndParseFile(file, runSerial);
                         } catch (e: any) {
                             console.warn(`Failed to download and parse file ${file.path}:\n${e.stack}`);
                             await failedFiles(1, 1, true);
                         }
-                        await fileProgress(1, 1, true);
+                        if (stoppedPromise.resolveCalled) return;
+                        await fileProgress(1, 0, true);
+                    }
+                    for (let file of syncResult.files) {
+                        await downloadFileWrapper(file);
                     }
+                    await runInSerial(async () => {
+                    });
 
                     await (await createProgress("Done", 0))(1, 1, true);
                 } catch (e: any) {
@@ -697,14 +722,18 @@ export class FastArchiveAppendable<Datum> {
 
 
 export function createLogScanner(config: {
+    debugName: string;
     onParsedData: (posStart: number, posEnd: number, buffer: Buffer | "done") => MaybePromise<void>;
 }): (data: Buffer | "done") => Promise<void> {
     const { onParsedData } = config;
     let pendingData: Buffer[] = [];
 
+    let finished = false;
+
     let delimitterMatchIndex = 0;
-    return runInSerial(async (data: Buffer | "done") => {
+    return (async (data: Buffer | "done") => {
         if (data === "done") {
+            finished = true;
             // Flush any pending data, even though we have no delimitter. It will probably fail to parse, but... maybe it will work?
             if (pendingData.length > 0) {
                 let combinedBuffer = Buffer.concat(pendingData);
@@ -714,6 +743,9 @@ export function createLogScanner(config: {
             await onParsedData(0, 0, "done");
             return;
         }
+        if (finished) {
+            throw new Error(`Finished scan, but we received more data: ${data.length}, sample is: ${data.slice(0, 100).toString("hex")}, ${config.debugName}`);
+        }
 
         let lastStart = 0;
         await measureBlock(async () => {
@@ -734,6 +766,7 @@ export function createLogScanner(config: {
                             ...pendingData,
                             data.slice(lastStart, i + 1),
                         ]).slice(0, -objectDelimitterBuffer.length);
+                        pendingData = [];
                         posStart = 0;
                         posEnd = buffer.length;
                     } else {
@@ -741,19 +774,12 @@ export function createLogScanner(config: {
                         posStart = lastStart;
                         posEnd = i + 1 - objectDelimitterBuffer.length;
                     }
-                    // Delimitter was the start of the chunk, and it's the first chunk. Just skip it.
-                    if (posStart === posEnd && i === 0) {
-                        lastStart = i + 1;
-                        continue;
-                    }
-
                     // Only sometimes awaiting here makes scanning almost 2X faster, in the normal case, somehow?
                     let maybePromise = onParsedData(posStart, posEnd, buffer);
                     if (maybePromise) {
                         await maybePromise;
                     }
 
-                    pendingData = [];
                     lastStart = i + 1;
                 }
             }

package/src/diagnostics/logs/FastArchiveController.ts

@@ -35,6 +35,7 @@ import { assertIsNetworkTrusted } from "../../-d-trust/NetworkTrust2";
 import { blue, magenta } from "socket-function/src/formatting/logColors";
 import { FastArchiveAppendable, getFileTimeStamp } from "./FastArchiveAppendable";
 import { IdentityController_getMachineId, IdentityController_getReconnectNodeId } from "../../-c-identity/IdentityController";
+import { fsExistsAsync } from "../../fs";
 
 export type FileMetadata = {
     nodeId?: string;
@@ -84,7 +85,8 @@ export class FastArchiveAppendableControllerBase {
     /** Get all pending local files for this rootPath that haven't been uploaded to Backblaze yet */
     public async getPendingFiles(rootPath: string, timeRange: { startTime: number; endTime: number; }): Promise<InternalFileMetadata[]> {
 
-        let rootCacheFolder = new FastArchiveAppendable(rootPath).getLocalPathRoot();
+        let rootCacheFolder = new FastArchiveAppendable(rootPath).baseGetLocalPathRoot();
+        if (!await fsExistsAsync(rootCacheFolder)) return [];
 
         //console.log(`Searching for pending files in ${rootCacheFolder} for time range ${new Date(timeRange.startTime).toISOString()} to ${new Date(timeRange.endTime).toISOString()}`);
 
@@ -377,6 +379,7 @@ export class FastArchiveAppendableControllerBase {
             let remoteProgress = this.updateProgress(syncId, "Discovering remote files", byMachineId.size);
             let remoteValue = 0;
 
+
             await Promise.all(Array.from(byMachineId).map(async ([machineId, nodeObjs]) => {
                 let firstAliveNode = new PromiseObj<string>();
                 let allFinished = Promise.all(nodeObjs.map(async ({ nodeId, entryPoint }) => {
@@ -396,7 +399,7 @@ export class FastArchiveAppendableControllerBase {
 
                 let pendingFiles = await errorToUndefined(
                     controller.getPendingFiles(config.rootPath, config.range));
-                console.log(blue(`Found ${pendingFiles?.length} pending files on node ${aliveNodeId}`), nodeObjs);
+                console.log(blue(`Found ${pendingFiles?.length} pending files on node ${aliveNodeId}`));
 
                 remoteValue++;
                 remoteProgress(remoteValue);