querysub 0.327.0 → 0.329.0

package/src/deployManager/machineApplyMainCode.ts

@@ -24,9 +24,11 @@ import { shutdown } from "../diagnostics/periodic";
 import { onServiceConfigChange, triggerRollingUpdate } from "./machineController";
 import { PromiseObj } from "../promise";
 import path from "path";
+import { fsExistsAsync } from "../fs";
 
 const PIPE_FILE_LINE_LIMIT = 10_000;
 
+
 const getMemoryInfo = measureWrap(async function getMemoryInfo(): Promise<{ value: number; max: number } | undefined> {
     if (os.platform() === "win32") {
         throw new Error("Windows is not supported for machine resource monitoring");
@@ -379,7 +381,7 @@ const getScreenState = measureWrap(async function getScreenState(populateIsProce
 });
 async function removeOldNodeId(screenName: string) {
     let nodeIdFile = os.homedir() + "/" + SERVICE_FOLDER + screenName + "/" + SERVICE_NODE_FILE_NAME;
-    if (fs.existsSync(nodeIdFile)) {
+    if (await fsExistsAsync(nodeIdFile)) {
         let nodeId = await fs.promises.readFile(nodeIdFile, "utf8");
         console.log(green(`Removing node if for dead service on ${nodeIdFile}, node id ${nodeId}`));
         await fs.promises.unlink(nodeIdFile);
@@ -416,7 +418,7 @@ const runScreenCommand = measureWrap(async function runScreenCommand(config: {
         if (existingScreen && !rollingObj && await isScreenRunningProcess(existingScreen.pid)) {
             let nodeIdPath = os.homedir() + "/" + SERVICE_FOLDER + screenName + "/" + SERVICE_NODE_FILE_NAME;
             let rollingFinalTime = Date.now() + config.rollingWindow;
-            if (fs.existsSync(nodeIdPath)) {
+            if (await fsExistsAsync(nodeIdPath)) {
                 let nodeId = await fs.promises.readFile(nodeIdPath, "utf8");
                 // REMOVE the nodeId file, so we the node isn't terminated!
                 await fs.promises.unlink(nodeIdPath);
@@ -551,7 +553,7 @@ const ensureGitSynced = measureWrap(async function ensureGitSynced(config: {
     repoUrl: string;
     gitRef: string;
 }) {
-    if (!await fs.existsSync(config.gitFolder + ".git")) {
+    if (!await fsExistsAsync(config.gitFolder + ".git")) {
         await runPromise(`git clone ${config.repoUrl} ${config.gitFolder}`);
     }
     try {
@@ -595,7 +597,7 @@ async function quickIsOutdated() {
             let folder = root + screenName + "/";
             await fs.promises.mkdir(folder, { recursive: true });
             let parameterPath = folder + "/parameters.json";
-            if (!fs.existsSync(parameterPath)) return true;
+            if (!await fsExistsAsync(parameterPath)) return true;
             let prevParameters = await fs.promises.readFile(parameterPath, "utf8");
             if (prevParameters !== newParametersString) return true;
         }
@@ -654,7 +656,7 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
                 }
                 let parameterPath = folder + "/parameters.json";
                 let prevParameters = "";
-                if (fs.existsSync(parameterPath)) {
+                if (await fsExistsAsync(parameterPath)) {
                     prevParameters = await fs.promises.readFile(parameterPath, "utf8");
                 }
                 let newParametersString = JSON.stringify(config.parameters);
@@ -673,7 +675,7 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
                 await fs.promises.writeFile(parameterPath, newParametersString);
 
                 let nodePathId = folder + SERVICE_NODE_FILE_NAME;
-                if (fs.existsSync(nodePathId)) {
+                if (await fsExistsAsync(nodePathId)) {
                     let nodeId = await fs.promises.readFile(nodePathId, "utf8");
                     machineInfo.services[config.serviceId].nodeId = nodeId;
                 }
@@ -788,7 +790,7 @@ export async function machineApplyMain() {
 
     // NOTE: Error's don't get logged unless we host, so... we can't do this earlier than this...
     let lastErrorPath = os.homedir() + "/lastAlwaysUpError.txt";
-    if (fs.existsSync(lastErrorPath)) {
+    if (await fsExistsAsync(lastErrorPath)) {
         let lastError = await fs.promises.readFile(lastErrorPath, "utf8");
         await fs.promises.unlink(lastErrorPath);
         console.error(`Always up error: ${lastError}`);
@@ -796,7 +798,7 @@ export async function machineApplyMain() {
 
     // Kill the last running one
     let isRunningPath = os.homedir() + "/machineApplyPID.txt";
-    if (fs.existsSync(isRunningPath)) {
+    if (await fsExistsAsync(isRunningPath)) {
         let pid = await fs.promises.readFile(isRunningPath, "utf8");
         try {
             process.kill(parseInt(pid), "SIGKILL");

package/src/deployManager/machineSchema.ts

@@ -23,6 +23,7 @@ import { DeployProgress, deployFunctions, deployGetFunctions } from "../4-deploy
 import { FunctionSpec, functionSchema } from "../3-path-functions/PathFunctionRunner";
 import { Querysub } from "../4-querysub/QuerysubController";
 import { green, red } from "socket-function/src/formatting/logColors";
+import { fsExistsAsync } from "../fs";
 
 const SERVICE_FOLDER_NAME = "machine-services";
 export const SERVICE_FOLDER = `${SERVICE_FOLDER_NAME}/`;
@@ -240,7 +241,7 @@ export class MachineServiceControllerBase {
         let querysubFolder = path.resolve("../querysub");
         let querysubRef = "";
         let querysubUncommitted: string[] = [];
-        if (fs.existsSync(querysubFolder)) {
+        if (await fsExistsAsync(querysubFolder)) {
             querysubRef = await getGitRefLive(querysubFolder);
             querysubUncommitted = await getGitUncommitted(querysubFolder);
         }
@@ -276,7 +277,7 @@ export class MachineServiceControllerBase {
     public async commitPushService(commitMessage: string) {
         if (commitMessage.toLowerCase().includes("querysub")) {
             let querysubFolder = path.resolve("../querysub");
-            if (fs.existsSync(querysubFolder)) {
+            if (await fsExistsAsync(querysubFolder)) {
                 let querysubLastCommit = await getGitRefInfo({
                     gitDir: querysubFolder,
                     ref: "origin/main",
@@ -293,7 +294,7 @@ export class MachineServiceControllerBase {
     }
     public async commitPushAndPublishQuerysub(commitMessage: string) {
         let querysubFolder = path.resolve("../querysub");
-        if (!fs.existsSync(querysubFolder)) {
+        if (!await fsExistsAsync(querysubFolder)) {
             throw new Error(`Querysub folder does not exist at ${querysubFolder}`);
         }
 

package/src/deployManager/setupMachineMain.ts

@@ -6,6 +6,7 @@ import fs from "fs";
 import os from "os";
 import readline from "readline";
 import open from "open";
+import { fsExistsAsync } from "../fs";
 // Import querysub, to fix missing dependencies
 Querysub;
 
@@ -32,7 +33,7 @@ async function getGitHubApiKey(repoUrl: string, sshRemote: string): Promise<stri
     const cacheFile = os.homedir() + `/githubkey_${repoOwner}_${repoName}.json`;
 
     // Check if we have a cached key
-    if (fs.existsSync(cacheFile)) {
+    if (await fsExistsAsync(cacheFile)) {
         try {
             const cached = JSON.parse(fs.readFileSync(cacheFile, "utf8"));
             if (cached.apiKey) {
@@ -222,7 +223,7 @@ async function main() {
 
     // 1. Copy backblaze file to remote server (~/backblaze.json)
     console.log("Copying backblaze credentials...");
-    if (fs.existsSync(backblazePath)) {
+    if (await fsExistsAsync(backblazePath)) {
         await runPromise(`scp "${backblazePath}" ${sshRemote}:~/backblaze.json`);
         console.log("✅ Backblaze credentials copied");
     } else {

package/src/diagnostics/logs/FastArchiveAppendable.ts

@@ -5,7 +5,7 @@ import { getMachineId, getOwnMachineId } from "../../-a-auth/certs";
 import { isDefined, parseFileNameKVP, parsePath, partialCopyObject, streamToIteratable, sum, toFileNameKVP } from "../../misc";
 import { registerShutdownHandler } from "../periodic";
 import { batchFunction, delay, runInSerial, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
-import { PromiseObj, isNode, keyByArray, nextId, sort, timeInDay, timeInHour, timeInMinute } from "socket-function/src/misc";
+import { PromiseObj, isNode, keyByArray, list, nextId, sort, timeInDay, timeInHour, timeInMinute } from "socket-function/src/misc";
 import os from "os";
 import { getOwnThreadId } from "../../-f-node-discovery/NodeDiscovery";
 import fs from "fs";
@@ -21,7 +21,7 @@ import { errorToUndefined, ignoreErrors, timeoutToUndefinedSilent } from "../../
 import { getCallObj } from "socket-function/src/nodeProxy";
 import { getSyncedController } from "../../library-components/SyncedController";
 import { getBrowserUrlNode, getOwnNodeId } from "../../-f-node-discovery/NodeDiscovery";
-import { secureRandom } from "../../misc/random";
+import { secureRandom, shuffle } from "../../misc/random";
 import { getPathIndex, getPathStr2 } from "../../path";
 import { onNextPaint } from "../../functional/onNextPaint";
 import { getArchivesBackblazePrivateImmutable, getArchivesBackblazePublicImmutable } from "../../-a-archives/archivesBackBlaze";
@@ -34,6 +34,7 @@ import { sha256 } from "js-sha256";
 import { assertIsNetworkTrusted } from "../../-d-trust/NetworkTrust2";
 import { blue, magenta } from "socket-function/src/formatting/logColors";
 import { FileMetadata, FastArchiveAppendableControllerBase, FastArchiveAppendableController, getFileMetadataHash } from "./FastArchiveController";
+import { fsExistsAsync } from "../../fs";
 
 // NOTE: In a single command line micro-test it looks like we can write about 40K writes of 500 per once, when using 10X parallel, on a fairly potato server. We should probably batch though, and only do 1X parallel.
 /*
@@ -140,7 +141,7 @@ export class FastArchiveAppendable<Datum> {
                 await this.flushNow();
             });
             // Random, to try to prevent the dead file detection code from getting in sync. It's fine if it gets in sync, it's just inefficient as we'll have multiple services uploading the same file to the same location at the same time. 
-            void runInfinitePollCallAtStart(timeInMinute * 20 + Math.random() * timeInMinute * 5, async () => {
+            void runInfinitePoll(timeInMinute * 20 + Math.random() * timeInMinute * 5, async () => {
                 await this.moveLogsToBackblaze();
             });
         }
@@ -151,13 +152,16 @@ export class FastArchiveAppendable<Datum> {
         return nestArchives("fast-logs/" + this.rootPath, archives);
     });
 
-    public getLocalPathRoot = lazy(() => {
-        let path = (
+    public baseGetLocalPathRoot = () => {
+        return (
             os.homedir()
             + "/fast-log-cache/"
             + getDomain() + "/"
             + this.rootPath
         );
+    };
+    public getLocalPathRoot = lazy(() => {
+        let path = this.baseGetLocalPathRoot();
         if (!fs.existsSync(path)) {
             fs.mkdirSync(path, { recursive: true });
         }
@@ -199,7 +203,10 @@ export class FastArchiveAppendable<Datum> {
     // NOTE: This is disk writing, which should be fast, but if it's slow we might be able to remove the measureWrap (as technically spending 50% of our time writing to the disk is fine, and won't lag anything).
     @measureFnc
     public async flushNow(now = Date.now()) {
+
         await appendableSerialLock(async () => {
+            if (this.pendingWriteQueue.length === 0) return;
+
             // 2025-09-06T07
             let hourFile = new Date(now).toISOString().slice(0, 13) + ".log";
             let localCacheFolder = this.getLocalPathRoot() + getOwnThreadId() + "/";
@@ -208,8 +215,6 @@ export class FastArchiveAppendable<Datum> {
             // Always heartbeat
             await fs.promises.writeFile(localCacheFolder + "heartbeat", Buffer.from(now + ""));
 
-            if (this.pendingWriteQueue.length === 0) return;
-
             try {
                 let beforeSize = await fs.promises.stat(localCachePath);
                 if (beforeSize.size > UNCOMPRESSED_LOG_FILE_STOP_THRESHOLD) {
@@ -277,18 +282,22 @@ export class FastArchiveAppendable<Datum> {
 
     public async moveLogsToBackblaze() {
         await appendableSerialLock(async () => {
-            let rootCacheFolder = this.getLocalPathRoot();
+            let rootCacheFolder = this.baseGetLocalPathRoot();
+            if (!await fsExistsAsync(rootCacheFolder)) return;
             console.log(magenta(`Moving old logs to Backblaze from ${rootCacheFolder}`));
 
             let archives = this.getArchives();
             async function moveLogsForFolder(threadId: string) {
                 let threadDir = rootCacheFolder + threadId + "/";
-                if (!fs.existsSync(threadDir)) return;
+                if (!await fsExistsAsync(threadDir)) return;
                 let files = await fs.promises.readdir(threadDir);
+                // Shuffle, so if we do run multiple at the same time, we are less likely to use the same files at the same time.
+                files = shuffle(files, Math.random());
                 for (let file of files) {
                     if (file === "heartbeat") continue;
                     let fullPath = threadDir + file;
                     try {
+                        if (!await fsExistsAsync(fullPath)) continue;
                         // We could use modified time here? Although, this is nice if we move files around, and then manually have them moved, although even then... this could cause problem be tripping while we are copying the file, so... maybe this is just wrong?
                         let timeStamp = getFileTimeStamp(fullPath);
                         if (timeStamp.endTime > Date.now() - UPLOAD_THRESHOLD) continue;
@@ -301,7 +310,10 @@ export class FastArchiveAppendable<Datum> {
                         let compressed = await measureBlock(async () => Zip.gzip(data), "FastArchiveAppendable|compress");
                         console.log(`Uploading ${formatNumber(data.length)}B (compressed to ${formatNumber(compressed.length)}B) logs to ${backblazePath} from ${fullPath}`);
                         await archives.set(backblazePath, compressed);
-                        await fs.promises.unlink(fullPath);
+                        // Ignore unlink errors to reduce excess logging. This races on startup, so it is likely we'll hit this a fair amount (especially because archives.set is so slow)
+                        try {
+                            await fs.promises.unlink(fullPath);
+                        } catch { }
                     } catch (e: any) {
                         // Just skip it, if the first file in the directory is broken we don't want to never move any files
                         console.error(`Error moving log file ${fullPath}: ${e.stack}`);
@@ -419,6 +431,7 @@ export class FastArchiveAppendable<Datum> {
         });
 
         let createProgress = async (section: string, max: number) => {
+            section = `${this.rootPath}|${section}`;
             let cancelled: Error | undefined;
             let lastValue = 0;
             let baseBatch = batchFunction({ delay: 150, },
@@ -499,7 +512,7 @@ export class FastArchiveAppendable<Datum> {
             synchronizeObj.parametersResult.resolve(syncResult);
 
 
-            let downloadProgress = await createProgress("Downloading (bytes)", 0);
+            let downloadProgress = await createProgress(`Downloading (bytes)`, 0);
             let decompressProgress = await createProgress("Decompressing (bytes)", 0);
             let scanProgress = await createProgress("Scanning (datums)", 0);
             let decodeProgress = await createProgress("Decoding (datums)", 0);
@@ -509,7 +522,7 @@ export class FastArchiveAppendable<Datum> {
             const self = this;
 
 
-            async function downloadAndParseFile(file: FileMetadata) {
+            async function downloadAndParseFile(file: FileMetadata, runInner: (code: () => Promise<void>) => Promise<void>) {
                 const onFetchedData = runInSerial(async (data: Buffer) => {
                     await downloadProgress(data.length, data.length, true);
                     await decompressWriter.write(data);
@@ -611,65 +624,85 @@ export class FastArchiveAppendable<Datum> {
 
                 // TODO: Stream from the local cache instead? It should be possible, we can get the total size, and read chunks.
                 let hash = getFileMetadataHash(file) + ".file";
-                let info = await localCache.getInfo(hash);
-                if (info?.size) {
-                    const CHUNK_SIZE = 1000 * 1000 * 10;
-                    for (let i = 0; i < info.size; i += CHUNK_SIZE) {
-                        let data = await localCache.get(hash, { range: { start: i, end: i + CHUNK_SIZE } });
-                        if (!data) {
-                            throw new Error(`Cached data disappeared (the info was there?) for ${hash} at ${i}`);
+                let contents = await localCache.get(hash);
+                if (stoppedPromise.resolveCalled) return;
+                await runInner(async () => {
+                    if (contents?.length) {
+                        const CHUNK_SIZE = 1000 * 1000 * 10;
+                        for (let i = 0; i < contents.length; i += CHUNK_SIZE) {
+                            let data = contents.slice(i, i + CHUNK_SIZE);
+                            await onFetchedData(data);
+                        }
+                    } else {
+                        const response = await fetch(url);
+                        if (!response.ok) {
+                            throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
                         }
-                        await onFetchedData(data);
-                    }
-                } else {
-                    const response = await fetch(url);
-                    if (!response.ok) {
-                        throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
-                    }
 
-                    if (!response.body) {
-                        throw new Error(`Response body is undefined for ${url}`);
-                    }
-                    let values: Buffer[] = [];
-                    const reader = response.body.getReader();
-                    void stoppedPromise.promise.finally(() => {
-                        void response.body?.cancel();
-                    });
-                    try {
-                        for await (let value of streamToIteratable(reader)) {
-                            // Cancel entirely
-                            if (stoppedPromise.resolveCalled) return;
-                            if (!value) continue;
-                            let buffer = Buffer.from(value);
-                            values.push(buffer);
-                            await onFetchedData(buffer);
+                        if (!response.body) {
+                            throw new Error(`Response body is undefined for ${url}`);
+                        }
+                        let values: Buffer[] = [];
+                        const reader = response.body.getReader();
+                        void stoppedPromise.promise.finally(() => {
+                            void response.body?.cancel();
+                        });
+                        try {
+                            for await (let value of streamToIteratable(reader)) {
+                                // Cancel entirely
+                                if (stoppedPromise.resolveCalled) return;
+                                if (!value) continue;
+                                let buffer = Buffer.from(value);
+                                values.push(buffer);
+                                await onFetchedData(buffer);
+                            }
+                        } finally {
+                            reader.releaseLock();
                         }
-                    } finally {
-                        reader.releaseLock();
-                    }
 
-                    await localCache.set(hash, Buffer.concat(values));
-                }
-                await decompressWriter.close();
-                await decompressPromise;
+                        await localCache.set(hash, Buffer.concat(values));
+                    }
+                    await decompressWriter.close();
+                    await decompressPromise;
+                });
             }
 
             // Fork off the processing
             void (async () => {
                 try {
                     // Iterate over all files and process them
-                    let fileProgress = await createProgress("Files", 0);
+                    let fileProgress = await createProgress("Files", syncResult.files.length);
+                    let fileInnerProgress = await createProgress("Files Inner", syncResult.files.length);
                     let failedFiles = await createProgress("Failed Files", 0);
-                    for (let file of syncResult.files) {
+                    let runSerial = runInSerial(async (fnc: () => Promise<void>) => {
+                        try {
+                            await fnc();
+                        } finally {
+                            await fileInnerProgress(1, 0, true);
+                        }
+                    });
+                    async function downloadFileWrapper(file: FileMetadata) {
                         if (stoppedPromise.resolveCalled) return;
                         try {
-                            await downloadAndParseFile(file);
+                            await downloadAndParseFile(file, runSerial);
                         } catch (e: any) {
                             console.warn(`Failed to download and parse file ${file.path}:\n${e.stack}`);
                             await failedFiles(1, 1, true);
                         }
-                        await fileProgress(1, 1, true);
+                        if (stoppedPromise.resolveCalled) return;
+                        await fileProgress(1, 0, true);
+                    }
+                    let remaining = syncResult.files.slice();
+                    async function runThread() {
+                        while (true) {
+                            let file = remaining.shift();
+                            if (!file) {
+                                return;
+                            }
+                            await downloadFileWrapper(file);
+                        }
                     }
+                    await Promise.all(list(32).map(() => runThread()));
 
                     await (await createProgress("Done", 0))(1, 1, true);
                 } catch (e: any) {

package/src/diagnostics/logs/FastArchiveController.ts

@@ -35,6 +35,8 @@ import { assertIsNetworkTrusted } from "../../-d-trust/NetworkTrust2";
 import { blue, magenta } from "socket-function/src/formatting/logColors";
 import { FastArchiveAppendable, getFileTimeStamp } from "./FastArchiveAppendable";
 import { IdentityController_getMachineId, IdentityController_getReconnectNodeId } from "../../-c-identity/IdentityController";
+import { fsExistsAsync } from "../../fs";
+import { Querysub } from "../../4-querysub/QuerysubController";
 
 export type FileMetadata = {
     nodeId?: string;
@@ -84,7 +86,8 @@ export class FastArchiveAppendableControllerBase {
     /** Get all pending local files for this rootPath that haven't been uploaded to Backblaze yet */
     public async getPendingFiles(rootPath: string, timeRange: { startTime: number; endTime: number; }): Promise<InternalFileMetadata[]> {
 
-        let rootCacheFolder = new FastArchiveAppendable(rootPath).getLocalPathRoot();
+        let rootCacheFolder = new FastArchiveAppendable(rootPath).baseGetLocalPathRoot();
+        if (!await fsExistsAsync(rootCacheFolder)) return [];
 
         //console.log(`Searching for pending files in ${rootCacheFolder} for time range ${new Date(timeRange.startTime).toISOString()} to ${new Date(timeRange.endTime).toISOString()}`);
 
@@ -273,6 +276,9 @@ export class FastArchiveAppendableControllerBase {
     }): Promise<{
         files: FileMetadata[];
     }> {
+        if (!SocketFunction.mountedNodeId) {
+            throw new Error(`Cannot use FastArchiveAppendableController before SocketFunction is mounted`);
+        }
         let syncId = config.syncId ?? "";
 
         // Define inline functions for parallel execution
@@ -377,6 +383,7 @@ export class FastArchiveAppendableControllerBase {
             let remoteProgress = this.updateProgress(syncId, "Discovering remote files", byMachineId.size);
             let remoteValue = 0;
 
+
             await Promise.all(Array.from(byMachineId).map(async ([machineId, nodeObjs]) => {
                 let firstAliveNode = new PromiseObj<string>();
                 let allFinished = Promise.all(nodeObjs.map(async ({ nodeId, entryPoint }) => {
@@ -396,7 +403,7 @@ export class FastArchiveAppendableControllerBase {
 
                 let pendingFiles = await errorToUndefined(
                     controller.getPendingFiles(config.rootPath, config.range));
-                console.log(blue(`Found ${pendingFiles?.length} pending files on node ${aliveNodeId}`), nodeObjs);
+                console.log(blue(`Found ${pendingFiles?.length} pending files on node ${aliveNodeId}`));
 
                 remoteValue++;
                 remoteProgress(remoteValue);
@@ -418,6 +425,7 @@ export class FastArchiveAppendableControllerBase {
                     let urlObj = new URL(url);
                     urlObj.hostname = ipDomain;
                     url = urlObj.toString();
+
                     let timeStamp = getFileTimeStamp(file.path);
                     let startTime = timeStamp.startTime;
                     let endTime = timeStamp.endTime;
@@ -443,6 +451,7 @@ export class FastArchiveAppendableControllerBase {
 
         let allFilesList = await Promise.all(filePromises);
         let allFiles = allFilesList.flat();
+        // Newest first, so recent errors are found quickly
         sort(allFiles, x => -x.startTime);
 
         return {