querysub 0.474.0 → 0.475.0

package/.claude/settings.local.json

@@ -24,7 +24,8 @@
       "mcp__node-debugger__listBreakpoints",
       "mcp__node-debugger__removeBreakpoint",
       "mcp__hottest__runTest",
-      "Bash(yarn test *)"
+      "Bash(yarn test *)",
+      "Bash(yarn ssh-a-claude *)"
     ]
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.474.0",
+  "version": "0.475.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -23,7 +23,8 @@
     "mcp": "yarn typenode ./src/diagnostics/logs/IndexedLogs/MCPIndexedLogsEntry.ts",
     "mc": "yarn typenode ./src/diagnostics/logs/IndexedLogs/MCPIndexedLogsEntry.ts --cwd D:/repos/qs-cyoa/",
     "mcp2": "yarn typenode ./src/diagnostics/debugger/mcp-server.ts",
-    "mc2": "yarn typenode ./src/diagnostics/debugger/mcp-server.ts --cwd D:/repos/qs-cyoa/"
+    "mc2": "yarn typenode ./src/diagnostics/debugger/mcp-server.ts --cwd D:/repos/qs-cyoa/",
+    "ssh-a-claude": "ssh root@a.querysubtest.com"
   },
   "bin": {
     "deploy": "./bin/deploy.js",

package/src/2-proxy/PathValueProxyWatcher.ts

@@ -1957,6 +1957,10 @@ export class PathValueProxyWatcher {
     public runOnce<Result = void>(
         options: Omit<WatcherOptions<Result>, "onResultUpdated" | "onWriteCommitted">
     ): Result {
+        // TODO: This is a recent change. We maybe should apply the user pass and options. However, we definitely don't want to call create watcher as that will apply our additional options, making it temporary, where this might actually be run inside of a non-temporary watcher like a render function. Which finally will break on commit finish because it'll think it's in a temporary function and subscribe to the wrong callback. 
+        if (this.inWatcher()) {
+            return options.watchFunction();
+        }
         let result: { result: Result } | { error: string } | undefined;
         let watcher = this.createWatcher({
             ...options,

package/src/4-deploy/git.ts

@@ -126,18 +126,52 @@ export const setGitRef = measureWrap(async function setGitRef(config: {
     gitRef: string;
 }) {
     await fs.promises.mkdir(config.gitFolder, { recursive: true });
-    let hostKey = await runPromise(`ssh-keyscan -t rsa bitbucket.org`);
-    hostKey = hostKey.split("\n").filter(x => !x.startsWith("#")).join("\n");
-    let knownHostsPath = os.homedir() + "/.ssh/known_hosts";
-    if (!fs.existsSync(knownHostsPath) || !fs.readFileSync(knownHostsPath).toString().includes(hostKey)) {
-        fs.appendFileSync(knownHostsPath, "\n" + hostKey + "\n");
+    // ssh-keyscan is a network call; a transient failure here must not abort the sync. The host key is normally already in known_hosts from machine setup, and callers only escalate to clobbering the checkout on *repo* failures — not on a momentary inability to refresh a host key.
+    try {
+        let hostKey = await runPromise(`ssh-keyscan -t rsa bitbucket.org`);
+        hostKey = hostKey.split("\n").filter(x => !x.startsWith("#")).join("\n");
+        let knownHostsPath = os.homedir() + "/.ssh/known_hosts";
+        if (hostKey && (!fs.existsSync(knownHostsPath) || !fs.readFileSync(knownHostsPath).toString().includes(hostKey))) {
+            fs.appendFileSync(knownHostsPath, "\n" + hostKey + "\n");
+        }
+    } catch (e: any) {
+        console.warn(`ssh-keyscan for bitbucket.org failed, continuing: ${e.stack ?? e}`);
     }
 
     await runPromise(`git remote update`, { cwd: config.gitFolder });
     await runPromise(`git add --all`, { cwd: config.gitFolder });
-    await runPromise(`git stash`, { cwd: config.gitFolder });
+    // nothrow: on a freshly re-initialized repo (rebuildGitFolder) there is no initial commit yet, so `git stash` errors. That's harmless — the `git reset --hard` below is what actually forces the working tree to the target ref.
+    await runPromise(`git stash`, { cwd: config.gitFolder, nothrow: true });
     await runPromise(`git fetch --all`, { cwd: config.gitFolder });
     await runPromise(`git reset --hard ${config.gitRef}`, { cwd: config.gitFolder });
     // Allows us to remove deleted objects from storage, ex, if we accidentally commit a 1GB, this deletes it, so we don't have to fix each server individually. Also I think the repo breaks if we go too long without pruning it?
     await runPromise(`git prune`, { cwd: config.gitFolder });
+});
+
+/** Recovers a broken repo WITHOUT disturbing the working-tree files. We delete only .git, re-init, re-add the remote, then let setGitRef's `git reset --hard` reconcile the tracked files to the ref. Untracked / gitignored files (notably node_modules, which is expensive to reinstall and may be open in a running service) are left exactly as they are. When the directory is empty this produces the same result as a fresh clone. */
+export const rebuildGitFolder = measureWrap(async function rebuildGitFolder(config: {
+    gitFolder: string;
+    repoUrl: string;
+    gitRef: string;
+}) {
+    await fs.promises.mkdir(config.gitFolder, { recursive: true });
+    await fs.promises.rm(config.gitFolder + ".git", { recursive: true, force: true });
+    await runPromise(`git init`, { cwd: config.gitFolder });
+    await runPromise(`git remote add origin ${config.repoUrl}`, { cwd: config.gitFolder });
+    await setGitRef({ gitFolder: config.gitFolder, gitRef: config.gitRef });
+});
+
+/** Last-resort recovery for a working tree git can't reconcile (rebuildGitFolder failed). Clones fresh into a sibling temp directory and swaps it in only after the clone and ref-set succeed, so a failed/interrupted clone can never leave the live folder empty — the failure mode that previously stranded running services with a deleted node_modules. This necessarily discards node_modules; the caller reinstalls it. */
+export const nuclearReclone = measureWrap(async function nuclearReclone(config: {
+    gitFolder: string;
+    repoUrl: string;
+    gitRef: string;
+}) {
+    let target = config.gitFolder.replace(/[\/\\]+$/, "");
+    let tempDir = target + ".reclone-" + Date.now();
+    await fs.promises.rm(tempDir, { recursive: true, force: true });
+    await runPromise(`git clone ${config.repoUrl} ${tempDir}`);
+    await setGitRef({ gitFolder: tempDir + "/", gitRef: config.gitRef });
+    await fs.promises.rm(target, { recursive: true, force: true });
+    await fs.promises.rename(tempDir, target);
 });

package/src/deployManager/machineApplyMainCode.ts

@@ -13,12 +13,12 @@ import { formatTime } from "socket-function/src/formatting/format";
 import { sort, timeInMinute, timeInSecond } from "socket-function/src/misc";
 import { isDefined } from "../misc";
 import { logLoadTime } from "../logModuleLoadTimes";
-import { delay, runInSerial, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { delay, retryFunctional, runInSerial, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
 import os from "os";
 import fs from "fs";
 import { spawn, ChildProcess } from "child_process";
 import { lazy } from "socket-function/src/caching";
-import { getGitRefLive, getGitURLLive, setGitRef } from "../4-deploy/git";
+import { getGitRefLive, getGitURLLive, setGitRef, rebuildGitFolder, nuclearReclone } from "../4-deploy/git";
 import { blue, green, magenta, red } from "socket-function/src/formatting/logColors";
 import { shutdown } from "../diagnostics/periodic";
 import { onServiceConfigChange, triggerRollingUpdate } from "./machineController";
@@ -607,24 +607,53 @@ const killScreen = measureWrap(async function killScreen(config: {
     await runPromise(`${prefix}tmux kill-session -t ${config.screenName}`);
     await removeOldNodeId(config.screenName);
 });
+// When a present repo fails to sync, retry a few times with backoff before deciding it is actually broken — most failures are transient network blips (ssh-keyscan / fetch), and reacting to those by clobbering the checkout is what stranded running services (they kept running but crashed on the next lazy `require("ws")` once node_modules was gone).
+const GIT_SYNC_MAX_RETRIES = 4;
+const GIT_SYNC_RETRY_MIN_DELAY = timeInSecond * 2;
+const GIT_SYNC_RETRY_MAX_DELAY = timeInSecond * 30;
+
 const ensureGitSynced = measureWrap(async function ensureGitSynced(config: {
-    // Ensures existingFolder is the repo, as in, cwd to it, and git clone ., if needed. Reset the hash, in whatever way we need to, to force it to gitRef (git add --all and stash, etc, etc).
-    //  - Maybe ask the AI the best way to forcefully switch a git repo.
+    // Forces the checkout at gitFolder to gitRef. Escalates through increasingly destructive recoveries, but only ever destroys the working tree as an absolute last resort — a network blip must never wipe node_modules out from under a running service.
     gitFolder: string;
     repoUrl: string;
     gitRef: string;
 }) {
-    try {
-        if (!await fsExistsAsync(config.gitFolder + ".git")) {
-            await runPromise(`git clone ${config.repoUrl} ${config.gitFolder}`);
+    let hasGit = await fsExistsAsync(config.gitFolder + ".git");
+
+    // Repo present: sync to the ref, retrying transient failures with backoff. We do NOT clobber the checkout just because a network call blipped.
+    if (hasGit) {
+        try {
+            await retryFunctional(() => setGitRef(config), {
+                maxRetries: GIT_SYNC_MAX_RETRIES,
+                minDelay: GIT_SYNC_RETRY_MIN_DELAY,
+                maxDelay: GIT_SYNC_RETRY_MAX_DELAY,
+            })();
+            return;
+        } catch (e: any) {
+            console.error(`Failed to sync git ref after ${GIT_SYNC_MAX_RETRIES} retries; rebuilding .git in place next (working tree, incl. node_modules, is preserved).`, {
+                gitFolder: config.gitFolder,
+                repoUrl: config.repoUrl,
+                gitRef: config.gitRef,
+                error: e?.stack ?? String(e),
+            });
         }
-        await setGitRef(config);
+    }
+
+    // Repo missing, or present-but-unsyncable: rebuild just the .git metadata and let `git reset --hard` reconcile the tracked files. The working-tree files (node_modules) are left untouched. Equivalent to a clone when the folder is empty.
+    try {
+        await rebuildGitFolder(config);
+        return;
     } catch (e: any) {
-        console.warn(`Failed to set git ref, trying to delete and clone again (${config.gitFolder}): ${e.stack}`);
-        await fs.promises.rm(config.gitFolder, { recursive: true });
-        await runPromise(`git clone ${config.repoUrl} ${config.gitFolder}`);
-        await setGitRef(config);
+        console.error(`Rebuilding .git in place failed; falling back to a full delete + clone (this wipes node_modules, which the caller reinstalls).`, {
+            gitFolder: config.gitFolder,
+            repoUrl: config.repoUrl,
+            gitRef: config.gitRef,
+            error: e?.stack ?? String(e),
+        });
     }
+
+    // Nuclear: the working tree itself is unusable. Clone fresh (into a temp dir, swapped in only on success).
+    await nuclearReclone(config);
 });
 
 
@@ -710,8 +739,10 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
                         gitRef: config.parameters.gitRef,
                     });
                     let afterGitRef = await getGitRefLive(gitFolder);
-                    if (afterGitRef !== prevGitRef) {
-                        console.log(green(`Yarn installing because git ref changed from ${prevGitRef} to ${afterGitRef} for ${magenta(screenName)}`));
+                    // Reinstall when the ref changed OR node_modules is missing. The latter is the real fix for the "Cannot find module 'ws'" crash: a recovery re-clone can land on the same commit, so a ref-only check would skip the install and leave the service with no node_modules. Restoring node_modules also self-heals an already-running process, since a failed `require` is never cached and the next reconnect re-resolves it.
+                    let nodeModulesMissing = !await fsExistsAsync(gitFolder + "node_modules");
+                    if (afterGitRef !== prevGitRef || nodeModulesMissing) {
+                        console.log(green(`Yarn installing for ${magenta(screenName)} (ref ${prevGitRef} -> ${afterGitRef}, nodeModulesMissing=${nodeModulesMissing})`));
                         await runPromise(`yarn install`, { cwd: gitFolder });
                     }
                 }

package/src/library-components/SyncedController.ts

@@ -1,4 +1,4 @@
-import { atomic, atomicObjectWrite, atomicObjectWriteNoFreeze, doAtomicWrites, isTransparentValue, proxyWatcher, specialObjectWriteValue } from "../../src/2-proxy/PathValueProxyWatcher";
+import { atomic, atomicObjectWrite, atomicObjectWriteNoFreeze, doAtomicWrites, doProxyOptions, isTransparentValue, proxyWatcher, specialObjectWriteValue } from "../../src/2-proxy/PathValueProxyWatcher";
 import { Querysub } from "../../src/4-querysub/Querysub";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { FullCallType, SocketRegistered } from "socket-function/SocketFunctionTypes";
@@ -91,6 +91,25 @@ const writeWatchers = new Map<string, {
     fncName: string;
 }[]>();
 
+export function asyncCache<T>(fnc: () => Promise<T>): (...args: any[]) => T | undefined {
+    return getSyncedController({
+        nodes: {
+            "": {
+                fnc,
+            },
+        },
+    })("").fnc;
+}
+export function asyncCacheObj<T>(fncs: T): {
+    [key in keyof T]: T[key] extends (...args: any[]) => Promise<infer Return> ? (...args: any[]) => Return | undefined : never;
+} {
+    return getSyncedController({
+        nodes: {
+            "": fncs as any,
+        },
+    })("") as any;
+}
+
 export function getSyncedController<T extends {
     nodes: {
         [key: string]: {
@@ -274,12 +293,15 @@ export function getSyncedController<T extends {
                         });
                         // We have to wait until we actually commit before making the call. Otherwise, if this commit is rejected because we need to do synchronized values, we'll call the function twice.
                         let fnc = controller.nodes[nodeId][fncName] as any;
+                        console.log(`Triggering call ${fncName} with args ${JSON.stringify(args)}`);
                         Querysub.onCommitFinished(() => {
-                            if (Querysub.isAllSynced()) {
-                                void Promise.resolve().then(() => {
-                                    doPromiseCall();
-                                });
-                            }
+                            console.log(`Commit finished for call ${fncName} with args ${JSON.stringify(args)}`);
+                            // Doesn't on commit finished also implicitly mean we're all synced? Pretty sure that isAll synced is making things break. 
+                            //if (Querysub.isAllSynced()) {
+                            void Promise.resolve().then(() => {
+                                doPromiseCall();
+                            });
+                            //}
                         });
                         function doPromiseCall() {
                             let promise = fnc(...args) as Promise<unknown>;
@@ -347,7 +369,10 @@ export function getSyncedController<T extends {
                                 }
                             }
                         }
+                        // Force it to commit, so even if we aren't synced, it runs.
+                        //doProxyOptions({ commitAllRuns: true }, () => {
                         call(...args);
+                        //});
                         // Assign to itself, to preset the type assumptions typescript makes (otherwise we get an error below)
                         obj = obj as any;
                         if (config?.cachePromiseCalls) {