querysub 0.328.0 → 0.329.0

package/bin/error-email.js

@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+
+// public, so we can connect to it from any node (for diagnostics, etc)
+process.argv.push("--public");
+process.argv.push("--nobreak");
+
+require("typenode");
+require("../src/diagnostics/logs/errorNotifications/errorDigestEmail.tsx");

package/bin/error-im.js

@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+
+// public, so we can connect to it from any node (for diagnostics, etc)
+process.argv.push("--public");
+process.argv.push("--nobreak");
+
+require("typenode");
+require("../src/diagnostics/logs/errorNotifications/errorDigests.tsx");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.328.0",
+  "version": "0.329.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -22,7 +22,7 @@
     "js-sha512": "^0.9.0",
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
-    "socket-function": "^0.138.0",
+    "socket-function": "^0.140.0",
     "terser": "^5.31.0",
     "typesafecss": "^0.22.0",
     "yaml": "^2.5.0",
@@ -55,7 +55,9 @@
     "join": "./bin/join.js",
     "join-public": "./bin/join-public.js",
     "merge": "./bin/merge.js",
-    "addsuperuser": "./bin/addsuperuser.js"
+    "addsuperuser": "./bin/addsuperuser.js",
+    "error-email": "./bin/error-email.js",
+    "error-im": "./bin/error-im.js"
   },
   "devDependencies": {
     "dependency-cruiser": "^12.11.0",

package/src/-a-archives/archivesCborT.ts

@@ -0,0 +1,52 @@
+import { lazy } from "socket-function/src/caching";
+import { Archives } from "./archives";
+import { decodeCborx, encodeCborx } from "../misc/cloneHelpers";
+import { ArchiveT } from "./archivesJSONT";
+
+export function archiveCborT<T>(archives: () => Archives): ArchiveT<T> {
+    archives = lazy(archives);
+    async function get(key: string) {
+        let buffer = await archives().get(key);
+        if (!buffer) return undefined;
+        return decodeCborx(buffer) as T;
+    }
+    async function set(key: string, value: T) {
+        await archives().set(key, encodeCborx(value));
+    }
+    async function deleteFnc(key: string) {
+        await archives().del(key);
+    }
+    async function keys() {
+        return (await archives().find("")).map(value => value.toString());
+    }
+    async function values() {
+        let keysArray = await keys();
+        let results: T[] = [];
+        await Promise.all(keysArray.map(async key => {
+            let value = await get(key);
+            if (value) {
+                results.push(value);
+            }
+        }));
+        return results;
+    }
+    async function entries(): Promise<[string, T][]> {
+        let keysArray = await keys();
+        let results: [string, T][] = [];
+        await Promise.all(keysArray.map(async key => {
+            let value = await get(key);
+            if (value) {
+                results.push([key, value]);
+            }
+        }));
+        return results;
+    }
+    async function* asyncIterator(): AsyncIterator<[string, T]> {
+        for (let [key, value] of await entries()) {
+            yield [key, value];
+        }
+    }
+    return {
+        get, set, delete: deleteFnc, keys, values, entries, [Symbol.asyncIterator]: asyncIterator
+    };
+}

package/src/-a-archives/archivesJSONT.ts

@@ -1,7 +1,7 @@
 import { lazy } from "socket-function/src/caching";
 import { Archives } from "./archives";
 
-export type ArchiveJSONT<T> = {
+export type ArchiveT<T> = {
     get(key: string): Promise<T | undefined>;
     set(key: string, value: T): Promise<void>;
     delete(key: string): Promise<void>;
@@ -11,12 +11,12 @@ export type ArchiveJSONT<T> = {
     [Symbol.asyncIterator](): AsyncIterator<[string, T]>;
 };
 
-export function archiveJSONT<T>(archives: () => Archives): ArchiveJSONT<T> {
+export function archiveJSONT<T>(archives: () => Archives): ArchiveT<T> {
     archives = lazy(archives);
     async function get(key: string) {
         let buffer = await archives().get(key);
         if (!buffer) return undefined;
-        return JSON.parse(buffer.toString());
+        return JSON.parse(buffer.toString()) as T;
     }
     async function set(key: string, value: T) {
         await archives().set(key, Buffer.from(JSON.stringify(value)));
@@ -29,11 +29,25 @@ export function archiveJSONT<T>(archives: () => Archives): ArchiveJSONT<T> {
     }
     async function values() {
         let keysArray = await keys();
-        return Promise.all(keysArray.map(key => get(key)));
+        let results: T[] = [];
+        await Promise.all(keysArray.map(async key => {
+            let value = await get(key);
+            if (value) {
+                results.push(value);
+            }
+        }));
+        return results;
     }
     async function entries(): Promise<[string, T][]> {
         let keysArray = await keys();
-        return Promise.all(keysArray.map(async key => [key, await get(key)]));
+        let results: [string, T][] = [];
+        await Promise.all(keysArray.map(async key => {
+            let value = await get(key);
+            if (value) {
+                results.push([key, value]);
+            }
+        }));
+        return results;
     }
     async function* asyncIterator(): AsyncIterator<[string, T]> {
         for (let [key, value] of await entries()) {

package/src/2-proxy/PathValueProxyWatcher.ts

@@ -35,6 +35,7 @@ import { deepCloneCborx } from "../misc/cloneHelpers";
 import { formatPercent, formatTime } from "socket-function/src/formatting/format";
 import { addStatPeriodic, interceptCalls, onAllPredictionsFinished, onTimeProfile } from "../-0-hooks/hooks";
 import { onNextPaint } from "../functional/onNextPaint";
+import { isAsyncFunction } from "../misc";
 
 // TODO: Break this into two parts:
 //  1) Run and get accesses
@@ -975,6 +976,9 @@ export class PathValueProxyWatcher {
     public createWatcher<Result = void>(
         options: WatcherOptions<Result>
     ): SyncWatcher {
+        if (isAsyncFunction(options.watchFunction)) {
+            throw new Error(`Async functions are not supported in watchers. They must run the caller synchronously. You are likely not using Await anyway, so just remove the async and make it a synchronous function. The caller will be called again whenever the data you access changes, And if you are running this to return a result, it will be rerun until all the data you want is synchronized. Watch function: ${options.watchFunction.toString()}`);
+        }
         // NOTE: Setting an order is needed for rendering, so parents render before children. I believe
         //  it is generally what we want, so event triggering is consistent, and fits with any tree based
         //  watching system. If this causes problems we COULD remove it from here and have just qreact.tsx set it.

package/src/config.ts

@@ -11,6 +11,7 @@ export const serverPort = 11748;
 let yargObj = parseArgsFactory()
     .option("nonetwork", { type: "boolean", desc: `Disables all network requirements. Reduces security, as this means we cannot use real certificates.` })
     .option("domain", { type: "string", desc: `Sets the domain` })
+    .option("emaildomain", { type: "string", desc: `Sets the domain to use for email` })
     .option("client", { type: "boolean", desc: `Drops permissions, acting as an unauthenticated node` })
     .option("authority", { type: "string", desc: `Defines the base paths we are an authority on (the domain is prepended to them). Either a file path to a JSON(AuthorityPath[]), or a base64 representation of the JSON(AuthorityPath[]).` })
     .option("nobreak", { type: "boolean", desc: "Do not break on errors. Safer to set this than to just not set debugbreak, as some places might break without checking debugbreak, but nobreak works at a level where it is always used." })
@@ -20,12 +21,15 @@ let yargObj = parseArgsFactory()
     .option("recovery", { type: "boolean", desc: `Allows any localhost connections to act like a superuser (and a trusted node), to help recover the database (as you need permission to access the snapshot page).` })
     // TODO: The bootstrapper is a single file. Maybe we shouldn't run the entire service just for that. Although... maybe it's fine, as services are light?
     .option("bootstraponly", { type: "boolean", desc: "Don't register as an edge node, so we serve the bootstrap files, but we don't need up to date code because we are not used for endpoints or the UI." })
+    .option("notifyemails", { type: "array", desc: "The emails to notify when errors occur." })
     .argv
     ;
-
-let querysubConfig = lazy((): {
+type QuerysubConfig = {
     domain?: string;
-} => {
+    emaildomain?: string;
+    notifyemails?: string[];
+};
+let querysubConfig = lazy((): QuerysubConfig => {
     if (!isNode()) throw new Error("querysubConfig is only available on the server");
     const path = "./querysub.json";
     if (!fs.existsSync(path)) {
@@ -54,6 +58,14 @@ export function getDomain() {
     return yargObj.domain || querysubConfig().domain || "querysub.com";
 }
 
+export function getEmailDomain() {
+    return yargObj.emaildomain || querysubConfig().emaildomain || getDomain();
+}
+
+export function getNotifyEmails() {
+    return Array.from(new Set([...(yargObj.notifyemails || []), ...(querysubConfig().notifyemails || [])]));
+}
+
 export function baseIsClient() {
     return !isNode() || yargObj.client;
 }

package/src/diagnostics/logs/FastArchiveAppendable.ts

@@ -5,7 +5,7 @@ import { getMachineId, getOwnMachineId } from "../../-a-auth/certs";
 import { isDefined, parseFileNameKVP, parsePath, partialCopyObject, streamToIteratable, sum, toFileNameKVP } from "../../misc";
 import { registerShutdownHandler } from "../periodic";
 import { batchFunction, delay, runInSerial, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
-import { PromiseObj, isNode, keyByArray, nextId, sort, timeInDay, timeInHour, timeInMinute } from "socket-function/src/misc";
+import { PromiseObj, isNode, keyByArray, list, nextId, sort, timeInDay, timeInHour, timeInMinute } from "socket-function/src/misc";
 import os from "os";
 import { getOwnThreadId } from "../../-f-node-discovery/NodeDiscovery";
 import fs from "fs";
@@ -310,7 +310,10 @@ export class FastArchiveAppendable<Datum> {
                         let compressed = await measureBlock(async () => Zip.gzip(data), "FastArchiveAppendable|compress");
                         console.log(`Uploading ${formatNumber(data.length)}B (compressed to ${formatNumber(compressed.length)}B) logs to ${backblazePath} from ${fullPath}`);
                         await archives.set(backblazePath, compressed);
-                        await fs.promises.unlink(fullPath);
+                        // Ignore unlink errors to reduce excess logging. This races on startup, so it is likely we'll hit this a fair amount (especially because archives.set is so slow)
+                        try {
+                            await fs.promises.unlink(fullPath);
+                        } catch { }
                     } catch (e: any) {
                         // Just skip it, if the first file in the directory is broken we don't want to never move any files
                         console.error(`Error moving log file ${fullPath}: ${e.stack}`);
@@ -689,11 +692,17 @@ export class FastArchiveAppendable<Datum> {
                         if (stoppedPromise.resolveCalled) return;
                         await fileProgress(1, 0, true);
                     }
-                    for (let file of syncResult.files) {
-                        await downloadFileWrapper(file);
+                    let remaining = syncResult.files.slice();
+                    async function runThread() {
+                        while (true) {
+                            let file = remaining.shift();
+                            if (!file) {
+                                return;
+                            }
+                            await downloadFileWrapper(file);
+                        }
                     }
-                    await runInSerial(async () => {
-                    });
+                    await Promise.all(list(32).map(() => runThread()));
 
                     await (await createProgress("Done", 0))(1, 1, true);
                 } catch (e: any) {

package/src/diagnostics/logs/FastArchiveController.ts

@@ -36,6 +36,7 @@ import { blue, magenta } from "socket-function/src/formatting/logColors";
 import { FastArchiveAppendable, getFileTimeStamp } from "./FastArchiveAppendable";
 import { IdentityController_getMachineId, IdentityController_getReconnectNodeId } from "../../-c-identity/IdentityController";
 import { fsExistsAsync } from "../../fs";
+import { Querysub } from "../../4-querysub/QuerysubController";
 
 export type FileMetadata = {
     nodeId?: string;
@@ -275,6 +276,9 @@ export class FastArchiveAppendableControllerBase {
     }): Promise<{
         files: FileMetadata[];
     }> {
+        if (!SocketFunction.mountedNodeId) {
+            throw new Error(`Cannot use FastArchiveAppendableController before SocketFunction is mounted`);
+        }
         let syncId = config.syncId ?? "";
 
         // Define inline functions for parallel execution
@@ -421,6 +425,7 @@ export class FastArchiveAppendableControllerBase {
                     let urlObj = new URL(url);
                     urlObj.hostname = ipDomain;
                     url = urlObj.toString();
+
                     let timeStamp = getFileTimeStamp(file.path);
                     let startTime = timeStamp.startTime;
                     let endTime = timeStamp.endTime;
@@ -446,6 +451,7 @@ export class FastArchiveAppendableControllerBase {
 
         let allFilesList = await Promise.all(filePromises);
         let allFiles = allFilesList.flat();
+        // Newest first, so recent errors are found quickly
         sort(allFiles, x => -x.startTime);
 
         return {

package/src/diagnostics/logs/diskLogGlobalContext.ts

@@ -6,11 +6,12 @@ import { getOwnNodeId } from "../../-f-node-discovery/NodeDiscovery";
 import { logErrors } from "../../errors";
 import { addGlobalContext } from "./diskLogger";
 import child_process from "child_process";
+import { getNodeIdLocation } from "socket-function/src/nodeCache";
 
 export function addBuiltInContext() {
     addGlobalContext(() => {
         let nodeId = getOwnNodeId();
-        let nodeParts = decodeNodeId(nodeId);
+        let nodeParts = getNodeIdLocation(nodeId);
         return {
             __machineId: getOwnMachineId(),
             __mountId: SocketFunction.mountedNodeId,