querysub 0.472.0 → 0.474.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.472.0",
+  "version": "0.474.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -66,7 +66,7 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
-    "socket-function": "^1.1.35",
+    "socket-function": "^1.1.36",
     "terser": "^5.31.0",
     "typesafecss": "^0.29.0",
     "yaml": "^2.5.0",

package/src/-a-archives/archivesBackBlaze.ts

@@ -505,6 +505,8 @@ export class ArchivesBackblaze {
         return api;
     });
 
+    private currentReset: Promise<void> | undefined;
+
     // Keep track of when we last reset because of a 503
     private last503Reset = 0;
     // IMPORTANT! We must always CATCH AROUND the apiRetryLogic, NEVER inside of fnc. Otherwise we won't
@@ -532,14 +534,17 @@ export class ArchivesBackblaze {
                 ) && Date.now() - this.last503Reset > 60 * 1000) {
                 console.error(`[${context}] 503 error, waiting and resetting: ${err.message}`);
                 this.log(`[${context}] 503 error, waiting and resetting: ${err.message}`);
-                await delay(10 * 1000);
-                // We check again in case, and in the very likely case that this is being run in parallel, we only want to reset once.
-                if (Date.now() - this.last503Reset > 60 * 1000) {
-                    this.log(`[${context}] Resetting getAPI and getBucketAPI: ${err.message}`);
-                    this.last503Reset = Date.now();
-                    getAPI.reset();
-                    this.getBucketAPI.reset();
-                }
+                this.currentReset = this.currentReset || (async () => {
+                    await delay(10 * 1000);
+                    // We check again in case, and in the very likely case that this is being run in parallel, we only want to reset once.
+                    if (Date.now() - this.last503Reset > 60 * 1000) {
+                        this.log(`[${context}] Resetting getAPI and getBucketAPI: ${err.message}`);
+                        this.last503Reset = Date.now();
+                        getAPI.reset();
+                        this.getBucketAPI.reset();
+                    }
+                })().finally(() => this.currentReset = undefined);
+                await this.currentReset;
                 return this.apiRetryLogic(context, fnc, retries - 1);
             }
 

package/src/-c-identity/IdentityController.ts

@@ -20,6 +20,7 @@ import { red } from "socket-function/src/formatting/logColors";
 import { isNode } from "typesafecss";
 import { areNodeIdsEqual, getOwnNodeId, getOwnThreadId } from "../-f-node-discovery/NodeDiscovery";
 import { timeInMinute } from "socket-function/src/misc";
+import { isClient, isServer } from "../config2";
 
 // NOTE: This used to be small, but we cache this, so it would mean a node on startup would time out, and then we would refuse to talk to it ever again. So... this can't be small
 const MAX_CHANGE_IDENTITY_TIMEOUT = timeInMinute * 5;
@@ -239,8 +240,8 @@ const changeIdentityOnce = cacheWeak(async function changeIdentityOnce(connectio
         cert: threadKeyCert.cert.toString(),
         certIssuer: issuer.cert.toString(),
         mountedPort: getNodeIdLocation(SocketFunction.mountedNodeId)?.port,
-        debugEntryPoint: isNode() ? process.argv[1] : "browser",
-        clientIsNode: isNode(),
+        debugEntryPoint: isServer() ? process.argv[1] : "browser",
+        clientIsNode: isServer(),
     };
     let signature = sign(threadKeyCert, payload);
     await timeoutToError(

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -229,6 +229,7 @@ function onNodesChanged() {
 }
 
 let rootDiscoveryNodeId = "";
+/** IMPORTANT! This domain import is the querysub server. Usually this is going to be a local development server. However, you can use Node Discovery and use whatever Node you want. You don't need to know the node's machine ID. You just need to know its address.  */
 export function configRootDiscoveryLocation(config: {
     domain: string;
     port: number;
@@ -337,11 +338,13 @@ async function syncArchives() {
         console.info(`Synced node ids from archives`, { nodeIds });
         await setNodeIds(nodeIds);
     } else {
-        if (isNoNetwork() || !isNode()) {
+        if (isNoNetwork() || !isNode() || rootDiscoveryNodeId) {
             // NOTE: If no network, our trust source might be different, so we can't talk to regular nodes,
             //  and instead have to only talk to HTTP nodes
             await setNodeIds([getBrowserUrlNode()]);
         } else {
+            // NOTE: I don't think this pathway works. But in theory it should it should be possible for us in nodejs to automatically choose the server we want to talk to. In practice, it probably doesn't matter right now, as we're mostly just using this for development, in which case we want to force it to use the local server anyway, which will set root discovery node ID. 
+
             // If on the network, NetworkTrust2 should sync the trusted machines from backblaze, so we should be
             //  able to talk to any nodes.
             //  - If they user is using --client they only want to talk to querysub nodes. There might be multiple,
@@ -575,17 +578,33 @@ if (isServer()) {
 } else {
 
     if (isNode()) {
-        discoveryReady.resolve();
-        nodeBroadcasted.resolve();
-        // Just get the archives, syncing again if we haven't synced in a while
-        let lastGetTime = 0;
-        beforeGetNodeAllId = async () => {
-            let lastGetThreshold = lastGetTime + CLIENTSIDE_POLL_RATE;
-            if (Date.now() > lastGetThreshold) {
-                lastGetTime = Date.now();
-                await syncArchives();
-            }
-        };
+        if (rootDiscoveryNodeId) {
+            let nodes = [rootDiscoveryNodeId];
+            allNodeIds2 = new Set(nodes);
+            discoveryReady.resolve();
+            nodeBroadcasted.resolve();
+
+            // NOTE: We run into TLS issues (as in, our servers use self signed certs), if we try to talk to just
+            //  any node, so... we better just talk to the edge node
+            //  - We COULD probably just use some special domain (maybe JUST the machine domain?), with limited wildcard
+            //      certs (I think we can only wildcard a single depth anyways), and A records for the machines too...
+            //      but... having all traffic route through an edge node is probably better anyways...
+            nodeOverrides = nodes;
+
+        } else {
+            // NOTE: I don't think this pathway works. But in theory it should it should be possible for us in nodejs to automatically choose the server we want to talk to. In practice, it probably doesn't matter right now, as we're mostly just using this for development, in which case we want to force it to use the local server anyway, which will set root discovery node ID. 
+            discoveryReady.resolve();
+            nodeBroadcasted.resolve();
+            // Just get the archives, syncing again if we haven't synced in a while
+            let lastGetTime = 0;
+            beforeGetNodeAllId = async () => {
+                let lastGetThreshold = lastGetTime + CLIENTSIDE_POLL_RATE;
+                if (Date.now() > lastGetThreshold) {
+                    lastGetTime = Date.now();
+                    await syncArchives();
+                }
+            };
+        }
     } else {
         setImmediate(() => {
             let edgeNode = getBootedEdgeNode();

package/src/1-path-client/RemoteWatcher.ts

@@ -111,7 +111,7 @@ export class RemoteWatcher {
                 // NOTE: We keep around the old authorities in routing (for a few minutes), so if a new node doesn't start,
                 //  we will match the disconnected node again. This means we have a few minutes to start another server.
                 // TODO: Retry instead of throwing
-                console.log(yellow(`Trying to find new authority for disconnected watches ${paths.length} paths and ${parentPaths.length} parent paths`));
+                console.log(yellow(`Trying to find new authority for disconnected watches ${paths.length} paths and ${parentPaths.length} parent paths`), { authorityId });
                 logErrors(this.tryToReconnectNow());
             })());
             // Reconnection is our loop running again, maybe matching the same nodeId, and then using it again.

package/src/2-proxy/PathValueProxyWatcher.ts

@@ -1664,7 +1664,7 @@ export class PathValueProxyWatcher {
 
                 let maxLocks = watcher.options.maxLocksOverride || DEFAULT_MAX_LOCKS;
                 if (locks.length > maxLocks) {
-                    throw new Error(`Too many locks for ${watcher.debugName} (${locks.length} > ${maxLocks}). Use Querysub.noLocks(() => ...) around code that is accessing too many values, assuming you don't want to lock them. You can override max locks with maxLocksOverride (in options / functionMetadata). Some locks are ${JSON.stringify(locks.slice(-10).map(x => ({ path: x.path, startTime: x.startTime, endTime: x.endTime })))}.`);
+                    throw new Error(`Too many locks for ${watcher.debugName} (${locks.length} > ${maxLocks}). Use Querysub.noLocks(() => ...) around code that is accessing too many values, assuming you don't want to lock them. You can override max locks with maxLocksOverride (in options / functionMetadata). Some write keys (which will be lost) are: ${JSON.stringify(Array.from(watcher.pendingWrites.keys()).slice(-10))}. Some locks are ${JSON.stringify(locks.slice(-10).map(x => ({ path: x.path, startTime: x.startTime, endTime: x.endTime })))}.`);
                 }
 
 

package/src/4-querysub/Querysub.ts

@@ -1053,7 +1053,7 @@ export class Querysub {
 
     public static async hostService(name: string, port = 0) {
         if (isClient()) {
-            throw new Error(`--client processes cannot host a service. Either stop passing --client and keep the process on the network and trusted, or stop calling hostServer and call Querysub.configRootDiscoveryLocation instead.`);
+            throw new Error(`--client processes cannot host a service. Either stop passing --client and keep the process on the network and trusted, or stop calling hostServer and call Querysub.configRootDiscoveryLocation instead. You MUST provide configRootDiscoveryLocation a valid nodeId. Which means either you do server selection manually, or if you are developing, just point it to "127-0-0-1.${getDomain()}:your local port here"`);
         }
         let times: {
             name: string;

package/src/4-querysub/querysubPrediction.ts

@@ -403,6 +403,7 @@ function getFunctionSpec(call: CallSpec): FunctionSpec | undefined {
             ModuleId: call.ModuleId,
             FunctionId: call.FunctionId,
         });
+        if (!obj && !Querysub.isAllSynced()) return undefined;
         if (!obj) throw new Error(`Function not referenced in deploy.ts ${call.DomainName}.${call.ModuleId}.${call.FunctionId}`);
         setGitURLMapping({ spec: obj.functionSpec, resolvedPath: obj.modulePath });
         return obj.functionSpec;

package/src/config.ts

@@ -5,6 +5,7 @@ import { MaybePromise } from "socket-function/src/types";
 import { parseArgsFactory } from "./misc/rawParams";
 import { lazy } from "socket-function/src/caching";
 import fs from "fs";
+import { SocketFunction } from "socket-function/SocketFunction";
 
 export const serverPort = 11748;
 
@@ -35,6 +36,12 @@ let yargObj = parseArgsFactory()
     .argv
     ;
 
+if (isNode()) {
+    if (yargObj.client) {
+        SocketFunction.ENABLE_CLIENT_MODE = true;
+    }
+}
+
 export function getRawFncFilter() {
     return yargObj.fncfilter;
 }

package/src/user-implementation/UserPage.tsx

@@ -70,7 +70,7 @@ export class UserPage extends qreact.Component {
                     `}
                 </style>
                 <div class={css.vbox(20)}>
-                    <div class={css.fontSize(28)}>User Configuration {isViewingOther && <span>({userId})</span>}</div>
+                    <div class={css.fontSize(28)}>User Configuration {<span>({userId})</span>}</div>
                     {userObj.userId !== userId && <div class={css.fontSize(16)}>(Used does not exist)</div>}
                     <div class={css.vbox(4)}>
                         <div><b>Email</b> {userObj.email}</div>

package/src/user-implementation/userData.ts

@@ -532,7 +532,7 @@ export function getCurrentUser(config?: { ignoreImpersonate?: boolean; }): strin
         let impersonate = !config?.ignoreImpersonate && impersonateURL.value;
         if (impersonate) return impersonate;
     }
-    if (isNode() && Querysub.getCallerIP() === "127.0.0.1" && isRecovery()) {
+    if (isNode() && Querysub.getCallerIPAllowUndefined() === "127.0.0.1" && isRecovery()) {
         return "local";
     }
     const machineId = Querysub.getCallerMachineId();
@@ -1121,13 +1121,12 @@ function unbanUser(config: { userId: string }) {
 }
 
 /** Adds the current machine to the shared "service" account, which otherwise is impossible to login as. */
-export async function registerServicePermissions() {
+export async function nodeJSLoginToUser(userId: string) {
     const machineId = Querysub.getOwnMachineId();
     const ips = [await getExternalIP(), "127.0.0.1", "10.0.0.1"];
     let now = Date.now();
 
     await Querysub.commitSynced(() => {
-        const userId = "service";
         if (!(userId in data().users)) {
             data().users[userId] = {
                 userId,