querysub 0.356.0 → 0.357.0

package/src/diagnostics/logs/IndexedLogs/TimeFileTree.ts

@@ -0,0 +1,236 @@
+import { Archives } from "../../../-a-archives/archives";
+import { getOwnThreadId, getOwnMachineId } from "../../../-f-node-discovery/NodeDiscovery";
+
+export type TimeFilePath = {
+    fullPath: string;
+
+    // If not pending, this isn't the source, this is just the writer. 
+    threadId?: string;
+    // Presently machine should always be set. But it might not if we start merging logs in backblaze.
+    machineId?: string;
+
+    // If logCount is undefined it means it's a pending file
+    logCount?: number;
+    uncompressedSize?: number;
+    compressedSize?: number;
+
+    startTime: number;
+    endTime: number;
+
+    // Probably just random?
+    dedupe: number;
+};
+
+const LOG_FILE_EXTENSION = ".logfile";
+
+function encodeLogFilePath(path: Omit<TimeFilePath, "fullPath">): string {
+    // Create folder structure: year/month/day/
+    const date = new Date(path.startTime);
+    const year = date.getUTCFullYear();
+    const month = String(date.getUTCMonth() + 1).padStart(2, "0");
+    const day = String(date.getUTCDate()).padStart(2, "0");
+
+    const folderPath = `${year}/${month}/${day}/`;
+
+    // Build file name with dashed parts
+    const parts: string[] = [];
+
+    if (path.threadId !== undefined) {
+        parts.push("threadid", path.threadId);
+    }
+    if (path.machineId !== undefined) {
+        parts.push("machineid", path.machineId);
+    }
+    parts.push("start", String(path.startTime));
+    parts.push("end", String(path.endTime));
+    parts.push("dedupe", String(path.dedupe));
+    if (path.logCount !== undefined) {
+        parts.push("logcount", String(path.logCount));
+    }
+    if (path.uncompressedSize !== undefined) {
+        parts.push("uncompressed", String(path.uncompressedSize));
+    }
+    if (path.compressedSize !== undefined) {
+        parts.push("compressed", String(path.compressedSize));
+    }
+
+    const fileName = parts.join("-") + LOG_FILE_EXTENSION;
+
+    return folderPath + fileName;
+}
+
+const KEY_MAPPING: Record<string, { field: keyof Omit<TimeFilePath, "fullPath">; parseAsNumber: boolean }> = {
+    "threadid": { field: "threadId", parseAsNumber: false },
+    "machineid": { field: "machineId", parseAsNumber: false },
+    "start": { field: "startTime", parseAsNumber: true },
+    "end": { field: "endTime", parseAsNumber: true },
+    "dedupe": { field: "dedupe", parseAsNumber: true },
+    "logcount": { field: "logCount", parseAsNumber: true },
+    "uncompressed": { field: "uncompressedSize", parseAsNumber: true },
+    "compressed": { field: "compressedSize", parseAsNumber: true },
+};
+
+function decodeLogFilePath(path: string): TimeFilePath | undefined {
+    if (!path.endsWith(LOG_FILE_EXTENSION)) {
+        return undefined;
+    }
+
+    // Extract the file name from the path
+    const lastSlash = path.lastIndexOf("/");
+    const fileName = lastSlash >= 0 ? path.substring(lastSlash + 1) : path;
+
+    // Remove extension
+    const nameWithoutExt = fileName.substring(0, fileName.length - LOG_FILE_EXTENSION.length);
+
+    // Parse the dashed parts
+    const parts = nameWithoutExt.split("-");
+
+    const result: TimeFilePath = {
+        fullPath: path,
+        startTime: 0,
+        endTime: 0,
+        dedupe: 0,
+    };
+
+    for (let i = 0; i < parts.length; i++) {
+        const key = parts[i];
+        const mapping = KEY_MAPPING[key];
+
+        if (mapping && i + 1 < parts.length) {
+            const value = parts[i + 1];
+            (result as any)[mapping.field] = mapping.parseAsNumber ? parseInt(value, 10) : value;
+            i++;
+        }
+    }
+
+    return result;
+}
+
+// NOTE: This class will add its own extensions to the files, and it'll use it to detect these files. You can append your own extension and then we will ignore those files. 
+export class TimeFileTree {
+    public constructor(public archives: Archives) { }
+
+    public async findAllPaths(config: {
+        startTime: number;
+        endTime: number;
+    }): Promise<TimeFilePath[]> {
+        const results: TimeFilePath[] = [];
+
+        // Generate all year/month/day combinations in the range
+        const startDate = new Date(config.startTime);
+        const endDate = new Date(config.endTime);
+
+        const startYear = startDate.getUTCFullYear();
+        const endYear = endDate.getUTCFullYear();
+
+        // Find all years in range
+        const years = await this.archives.find("", { shallow: true, type: "folders" });
+
+        for (const yearFolder of years) {
+            // yearFolder is a full path like "2024"
+            const year = parseInt(yearFolder.split("/").at(-1) || "", 10);
+
+            if (isNaN(year) || year < startYear || year > endYear) {
+                continue;
+            }
+
+            // Find all months in this year
+            const months = await this.archives.find(yearFolder + "/", { shallow: true, type: "folders" });
+
+            for (const monthFolder of months) {
+                // monthFolder is a full path like "2024/01"
+                const month = parseInt(monthFolder.split("/").at(-1) || "", 10);
+
+                if (isNaN(month) || month < 1 || month > 12) {
+                    continue;
+                }
+
+                // Check if this month is in range
+                const monthDate = new Date(Date.UTC(year, month - 1, 1));
+                const monthEndDate = new Date(Date.UTC(year, month, 0, 23, 59, 59, 999));
+
+                if (monthEndDate.getTime() < config.startTime || monthDate.getTime() > config.endTime) {
+                    continue;
+                }
+
+                // Find all days in this month
+                const days = await this.archives.find(monthFolder + "/", { shallow: true, type: "folders" });
+
+                for (const dayFolder of days) {
+                    // dayFolder is a full path like "2024/01/15"
+                    const day = parseInt(dayFolder.split("/").at(-1) || "", 10);
+
+                    if (isNaN(day) || day < 1 || day > 31) {
+                        continue;
+                    }
+
+                    // Check if this day is in range
+                    const dayDate = new Date(Date.UTC(year, month - 1, day, 0, 0, 0, 0));
+                    const dayEndDate = new Date(Date.UTC(year, month - 1, day, 23, 59, 59, 999));
+
+                    if (dayEndDate.getTime() < config.startTime || dayDate.getTime() > config.endTime) {
+                        continue;
+                    }
+
+                    // Find all log files in this day
+                    const files = await this.archives.find(dayFolder + "/", { shallow: true, type: "files" });
+
+                    for (const file of files) {
+                        const decoded = decodeLogFilePath(file);
+
+                        if (decoded === undefined) {
+                            continue;
+                        }
+
+                        // Check if file's time range overlaps with requested range
+                        if (decoded.endTime >= config.startTime && decoded.startTime <= config.endTime) {
+                            results.push(decoded);
+                        }
+                    }
+                }
+            }
+        }
+
+        return results;
+    }
+
+    public async deleteAll() {
+        let rootFolders = await this.archives.find("", { shallow: true, type: "folders" });
+        for (let rootFolder of rootFolders) {
+            await this.archives.del(rootFolder);
+        }
+    }
+
+    public getNewPendingPath(config: {
+        startTime: number;
+        endTime: number;
+    }): string {
+        return encodeLogFilePath({
+            startTime: config.startTime,
+            endTime: config.endTime,
+            threadId: getOwnThreadId(),
+            machineId: getOwnMachineId(),
+            dedupe: dedupeSeqNum++,
+        });
+    }
+
+    public getNewFinalPath(config: {
+        logCount: number;
+        uncompressedSize: number;
+        compressedSize: number;
+        startTime: number;
+        endTime: number;
+    }): string {
+        return encodeLogFilePath({
+            startTime: config.startTime,
+            endTime: config.endTime,
+            machineId: getOwnMachineId(),
+            threadId: getOwnThreadId(),
+            dedupe: dedupeSeqNum++,
+            logCount: config.logCount,
+            uncompressedSize: config.uncompressedSize,
+            compressedSize: config.compressedSize,
+        });
+    }
+}
+let dedupeSeqNum = 0;

package/src/diagnostics/logs/IndexedLogs/binding.gyp

@@ -0,0 +1,23 @@
+{
+  "targets": [
+    {
+      "target_name": "BufferIndexCPP",
+      "sources": [ "BufferIndexCPP.cpp" ],
+      "cflags": [ "-O3", "-march=native" ],
+      "cflags_cc": [ "-O3", "-march=native" ],
+      "msvs_settings": {
+        "VCCLCompilerTool": {
+          "Optimization": 3,
+          "FavorSizeOrSpeed": 1,
+          "InlineFunctionExpansion": 2,
+          "EnableIntrinsicFunctions": "true",
+          "RuntimeLibrary": 0
+        }
+      },
+      "xcode_settings": {
+        "GCC_OPTIMIZATION_LEVEL": "3",
+        "OTHER_CFLAGS": [ "-march=native" ]
+      }
+    }
+  ]
+}

package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts

@@ -0,0 +1,221 @@
+import { delay } from "socket-function/src/batching";
+import { sort, keyByArray } from "socket-function/src/misc";
+import { Archives } from "../../../-a-archives/archives";
+import { getOwnThreadId } from "../../../-f-node-discovery/NodeDiscovery";
+import { BufferIndex } from "./BufferIndex";
+import { LogStreamer } from "./LogStreamer";
+import { TimeFileTree } from "./TimeFileTree";
+import { green, magenta } from "socket-function/src/formatting/logColors";
+import { formatNumber, formatTime } from "socket-function/src/formatting/format";
+
+
+
+export async function moveLogsToPublic(config: {
+    publicMoveThreshold: number;
+    maxSingleFileData: number;
+    movingTimeout: number;
+    // Unsafe, but useful for testing
+    forceAll: boolean;
+    localLogs: Archives;
+    publicLogs: Archives;
+    getIndexPath: (path: string) => string;
+}) {
+    let { forceAll, localLogs, publicLogs, publicMoveThreshold, maxSingleFileData, getIndexPath, movingTimeout } = config;
+    let now = Date.now();
+    let threadId = getOwnThreadId();
+    let ourMovingFileName = `${now}-${threadId}.moving`;
+
+    async function tryToGetMoveLock(): Promise<boolean> {
+
+        // Helper function to parse moving file names and extract timestamp
+        const parseMovingFileName = (fileName: string): number | undefined => {
+            if (!fileName.endsWith(".moving")) return undefined;
+            let parts = fileName.split("-");
+            if (parts.length < 2) return undefined;
+            let timestamp = parseInt(parts[0]);
+            if (isNaN(timestamp)) return undefined;
+            return timestamp;
+        };
+
+        // Helper function to check and clean up stale moving files
+        const checkAndCleanMovingFiles = async (): Promise<string[]> => {
+            // 1) We do a shallow read of the local logs directory and then find any files that have the moving extension and then we see when they've been written.
+            let allFiles = await localLogs.find("", { shallow: true, type: "files" });
+            let movingFiles = allFiles.filter(x => x.endsWith(".moving"));
+
+            let now = Date.now();
+            let validMovingFiles: string[] = [];
+            let stalePaths: string[] = [];
+
+            for (let movingFile of movingFiles) {
+                let timestamp = parseMovingFileName(movingFile);
+                if (timestamp === undefined) continue;
+
+                let age = now - timestamp;
+                // 2) If they're not past our timeout threshold at the top of the file, Then we just return (NOOP).
+                // 3) If any are past their timeout threshold, we delete them.
+                if (age > movingTimeout) {
+                    stalePaths.push(movingFile);
+                } else {
+                    validMovingFiles.push(movingFile);
+                }
+            }
+
+            // Delete stale moving files
+            for (let stalePath of stalePaths) {
+                await localLogs.del(stalePath);
+            }
+
+            // Sort by timestamp (oldest first)
+            sort(validMovingFiles, x => parseMovingFileName(x) || Infinity);
+
+            return validMovingFiles;
+        };
+
+
+        // Initial check for existing moving files
+        let validMovingFiles = await checkAndCleanMovingFiles();
+        // 2) If they're not past our timeout threshold at the top of the file, Then we just return (NOOP).
+        if (validMovingFiles.length > 0) {
+            return false;
+        }
+
+        // 4) We then write our own moving file flag
+        await localLogs.set(ourMovingFileName, Buffer.from("moving"));
+
+        if (!forceAll) {
+            // 5) Wait 15 seconds, to let things settle, so we aren't racing
+            await delay(15000);
+        }
+
+        // 6) Check again if anything is moving and it's past the threshold (This code should be in a function that's local to this function, so we can easily run it again)
+        validMovingFiles = await checkAndCleanMovingFiles();
+
+        // 7) Also, out of all of the moving files which aren't beyond the timeout, we need to make sure ours is the oldest, but not older than half the timeout. If this isn't true, then we just return.
+        let oldestMovingFile = validMovingFiles[0];
+        if (oldestMovingFile !== ourMovingFileName) {
+            // Clean up our moving file since we didn't get the lock
+            await localLogs.del(ourMovingFileName);
+            return false;
+        }
+
+        let ourTimestamp = parseMovingFileName(ourMovingFileName);
+        if (ourTimestamp !== undefined) {
+            let ourAge = Date.now() - ourTimestamp;
+            // If we got even close to the threshold, we need to abort. Because if we're close to the threshold, someone else might have thought we exceeded the threshold, and starting moving themselves.
+            if (ourAge > movingTimeout / 2) {
+                // Clean up our moving file since it's too old
+                await localLogs.del(ourMovingFileName);
+                return false;
+            }
+        }
+
+        // 8) Then, if all this passes, return true
+        return true;
+    }
+
+    let localPaths = await new TimeFileTree(localLogs).findAllPaths({
+        startTime: 0,
+        endTime: Date.now(),
+    });
+
+    if (!forceAll) {
+        let threshold = Date.now() - publicMoveThreshold;
+        localPaths = localPaths.filter(x => x.startTime < threshold);
+    }
+
+    if (!localPaths.length) return;
+
+    if (!await tryToGetMoveLock()) return;
+
+    let byStartTime = keyByArray(localPaths, x => x.startTime);
+
+    for (let group of byStartTime.values()) {
+        let time = Date.now();
+        let buffers: Buffer[] = [];
+        await Promise.all(group.map(async x => {
+            let buffer = await localLogs.get(x.fullPath);
+            if (buffer) {
+                buffers.push(buffer);
+            }
+        }));
+        if (buffers.length === 0) continue;
+
+        console.log(magenta(`Merging and moving ${group.length} log files backblaze`));
+
+        let allBuffers: Buffer[][] = [];
+
+        for (let buffer of buffers) {
+            let decoded = await BufferIndex.decodeAll(buffer);
+            allBuffers.push(decoded);
+        }
+
+        let buffersFlat = allBuffers.flat();
+        // Reverse them, so the newest come first.
+        buffersFlat.reverse();
+        // Group buffersFlat so we are under MAX_SINGLE_FILE_DATA
+        let groupedBuffers: Buffer[][] = [];
+        let currentGroup: Buffer[] = [];
+        let currentGroupSize = maxSingleFileData + 1;
+        for (let buffer of buffersFlat) {
+            if (currentGroupSize > maxSingleFileData) {
+                currentGroup = [];
+                groupedBuffers.push(currentGroup);
+                currentGroupSize = 0;
+            }
+            currentGroup.push(buffer);
+            currentGroupSize += buffer.length;
+        }
+
+        let startTime = Math.min(...group.map(x => x.startTime));
+        let endTime = Math.max(...group.map(x => x.endTime));
+
+        let encoded: {
+            data: Buffer;
+            index: Buffer;
+            compressedSize: number;
+            uncompressedSize: number;
+            logCount: number;
+        }[] = [];
+        for (let group of groupedBuffers) {
+            let uncompressedSize = 0;
+            for (let buffer of group) {
+                uncompressedSize += buffer.length;
+            }
+            let obj = BufferIndex.encodeAll({
+                data: group,
+            });
+            encoded.push({
+                data: obj.data,
+                index: obj.index,
+                compressedSize: obj.data.length,
+                uncompressedSize,
+                logCount: group.length,
+            });
+            // Wait, so if we have to encode a lot of data, we don't block for too long? 
+            await delay(0);
+        }
+
+        let backblazeManager = new TimeFileTree(publicLogs);
+        for (let obj of encoded) {
+            let path = backblazeManager.getNewFinalPath({
+                logCount: obj.logCount,
+                uncompressedSize: obj.uncompressedSize,
+                compressedSize: obj.compressedSize,
+                startTime,
+                endTime,
+            });
+            let indexPath = getIndexPath(path);
+            await publicLogs.set(indexPath, obj.index);
+            await publicLogs.set(path, obj.data);
+        }
+
+        console.log(green(`Wrote ${encoded.length} log files to backblaze (${formatNumber(encoded.reduce((acc, x) => acc + x.uncompressedSize, 0))}B compressed to ${formatNumber(encoded.reduce((acc, x) => acc + x.compressedSize, 0))}B + ${formatNumber(encoded.reduce((acc, x) => acc + x.index.length, 0))}B index) in ${formatTime(Date.now() - time)}`));
+
+        for (let path of group) {
+            await localLogs.del(path.fullPath);
+        }
+    }
+
+    await localLogs.del(ourMovingFileName);
+}

package/src/diagnostics/logs/IndexedLogs/moveLogsEntry.ts

@@ -0,0 +1,10 @@
+import "../../../inject";
+
+import { logErrors } from "../../../errors";
+import { IndexedLogs } from "./IndexedLogs";
+
+async function main() {
+    IndexedLogs.runLogMoveLoop();
+}
+
+main().catch(logErrors);