querysub 0.713.0 → 0.714.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.713.0",
+  "version": "0.714.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -79,7 +79,7 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
-    "sliftutils": "^1.8.20",
+    "sliftutils": "^1.9.5",
     "socket-function": "^1.3.1",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",

package/src/-a-archives/archives2.ts

@@ -40,9 +40,9 @@ function createSourceWindows(
 }
 
 function archiveBuilder(bucket: string, overrides: Partial<RemoteConfigBase>) {
-    const ONTARIO = `https://99-250-124-91.querysubtest.com:5234/file/${STORAGE_ACCOUNT}/${bucket}/storage/storagerouting.json`;
+    const ONTARIO = `https://99-255-70-23.querysubtest.com:5234/file/${STORAGE_ACCOUNT}/${bucket}/storage/storagerouting.json`;
     // NOTE: This is on an overly large extra disk that's on our 4090 machine. I prefer to not use it because it might slow down diffusion, but it's quite large (2tb, and diffusion will never use more than 1tb), so we could use it if we needed to.
-    const ONTARIO2 = `https://99-250-124-91.querysubtest.com:5235/file/${STORAGE_ACCOUNT}/${bucket}/storage/storagerouting.json`;
+    const ONTARIO2 = `https://99-255-70-23.querysubtest.com:5235/file/${STORAGE_ACCOUNT}/${bucket}/storage/storagerouting.json`;
     const HETZNER = `https://65-109-93-113.querysubtest.com:5234/file/${STORAGE_ACCOUNT}/${bucket}/storage/storagerouting.json`;
     const BACKBLAZE = `https://f002.backblazeb2.com/file/${bucket}/storage/storagerouting.json`;
 
@@ -184,7 +184,7 @@ export function getArchives2Debug(folder: string) {
         sources: [
             // IMPORTANT! If we start changing around routes, it could resurrect old data. We can fix this via incrementing the route version.
             {
-                type: "remote", url: `https://99-250-124-91.querysubtest.com:5239/file/${STORAGE_ACCOUNT}/${folder}/storage/storagerouting.json`,
+                type: "remote", url: `https://99-255-70-23.querysubtest.com:5239/file/${STORAGE_ACCOUNT}/${folder}/storage/storagerouting.json`,
                 validWindow: [0, Number.MAX_SAFE_INTEGER],
                 route: [0, 0.5],
                 name: "storagea_v" + routeVersion,

package/src/-a-archives/archivesDisk.ts

@@ -3,7 +3,6 @@ import os from "os";
 import { fsExistsAsync } from "../fs";
 import fs from "fs";
 import { blue, red, yellow } from "socket-function/src/formatting/logColors";
-import { measureFnc } from "socket-function/src/profiling/measure";
 import { delay } from "socket-function/src/batching";
 import debugbreak from "debugbreak";
 import { Archives } from "./archives";
@@ -147,7 +146,6 @@ class ArchivesDisk {
         }
     }
 
-    @measureFnc
     public async gcDir(dir: string) {
         await this.init();
         if (!dir.startsWith(this.LOCAL_ARCHIVE_FOLDER)) {
@@ -187,8 +185,6 @@ class ArchivesDisk {
      *      it all at once).
      * @fsErrorRetryCount Retries count for file system error (not user errors). Ex, too many open files, etc.
      */
-    // NOTE: I've commented out the gate measuring so we can determine what is taking the time. We might want to remove all the timing from this file, as if we find any timing this file is slow, it doesn't help us optimize at all, because we then need to figure out what is calling it. 
-    //@measureFnc
     public async get(fileNameInput: string, config?: { range?: { start: number; end: number; }; retryCount?: number }): Promise<Buffer | undefined> {
         await this.init();
         this.log(blue(`Start read file ${fileNameInput}`));
@@ -238,7 +234,6 @@ class ArchivesDisk {
         }
     }
 
-    @measureFnc
     public async find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]> {
         await this.init();
         prefix = escapeFileName(prefix);
@@ -293,7 +288,6 @@ class ArchivesDisk {
 
         return results;
     }
-    @measureFnc
     public async findInfo(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<{ path: string; createTime: number; size: number; }[]> {
         // TODO: Isn't there any api to read file names and infos at the same time?
         let files = await this.find(prefix, config);
@@ -308,7 +302,6 @@ class ArchivesDisk {
     }
 
 
-    @measureFnc
     public async getInfo(pathInput: string): Promise<{
         writeTime: number;
         size: number;
@@ -335,7 +328,6 @@ class ArchivesDisk {
         escapeFileName(this.LOCAL_ARCHIVE_FOLDER + path);
     }
 
-    @measureFnc
     public async move(config: {
         path: string;
         target: Archives;
@@ -366,7 +358,6 @@ class ArchivesDisk {
         }
     }
 
-    @measureFnc
     public async copy(config: {
         path: string;
         target: Archives;

package/src/deployManager/processLogs.ts

@@ -96,7 +96,28 @@ done`;
 export async function readProcessRecord(folder: string, pid: string, startTime: number): Promise<ProcessRecord | undefined> {
     let file = getRecordPath(folder, pid, startTime);
     if (!await fsExistsAsync(file)) return undefined;
-    return JSON.parse(await fs.promises.readFile(file, "utf8")) as ProcessRecord;
+    return await parseRecordFile(file);
+}
+
+const reportedUnreadableRecords = new Set<string>();
+
+async function parseRecordFile(file: string): Promise<ProcessRecord | undefined> {
+    let contents: string;
+    try {
+        contents = await fs.promises.readFile(file, "utf8");
+    } catch (e) {
+        if ((e as NodeJS.ErrnoException).code === "ENOENT") return undefined;
+        throw e;
+    }
+    try {
+        return JSON.parse(contents) as ProcessRecord;
+    } catch (e) {
+        if (!reportedUnreadableRecords.has(file)) {
+            reportedUnreadableRecords.add(file);
+            console.error(`[pid ${process.pid}] Ignoring unreadable process record ${file} (${contents.length} bytes). ${(e as Error).stack ?? e}`);
+        }
+        return undefined;
+    }
 }
 
 /** The live record for a pid in this folder. The pid is what identifies a process, so a start time that reads back a second off - the sources only resolve to the second - must not make us miss the record and write a duplicate. */
@@ -116,7 +137,10 @@ export async function markProcessDead(pid: string, now: number): Promise<void> {
 
 export async function writeProcessRecord(record: ProcessRecord): Promise<void> {
     await fs.promises.mkdir(path.join(record.folder, PROCESS_FOLDER), { recursive: true });
-    await fs.promises.writeFile(getRecordPath(record.folder, record.pid, record.startTime), JSON.stringify(record));
+    let file = getRecordPath(record.folder, record.pid, record.startTime);
+    let tempFile = `${file}.${process.pid}.tmp`;
+    await fs.promises.writeFile(tempFile, JSON.stringify(record));
+    await fs.promises.rename(tempFile, file);
 }
 
 async function readFolderRecords(folder: string): Promise<ProcessRecord[]> {
@@ -130,7 +154,10 @@ async function readFolderRecords(folder: string): Promise<ProcessRecord[]> {
     let records: ProcessRecord[] = [];
     for (let file of files) {
         if (!file.endsWith(RECORD_SUFFIX)) continue;
-        records.push(JSON.parse(await fs.promises.readFile(path.join(processFolder, file), "utf8")) as ProcessRecord);
+        let record = await parseRecordFile(path.join(processFolder, file));
+        if (record) {
+            records.push(record);
+        }
     }
     return records;
 }

package/src/diagnostics/NodeViewer.tsx

@@ -280,7 +280,6 @@ export class NodeViewer extends qreact.Component {
                                                 }
                                                 console.log(url);
                                                 console.log(url.replace("https://notdevtools.com/devtools", "devtools://devtools/bundled"));
-                                                // devtools://devtools/bundled/inspector.html?experiments=true&v8only=true&wss=99-250-124-91.querysub.com:2058/cc5b76b2-549f-4a66-8026-021811e9edd3
 
                                                 window.open(url, "_blank");
                                             }}>