querysub 0.200.0 → 0.201.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.200.0",
+  "version": "0.201.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-0-hooks/hooks.ts

@@ -12,13 +12,13 @@ function createHookFunction<Fnc extends (...args: any[]) => void>(debugName: str
 } {
     let queuedCalls = [] as Parameters<Fnc>[];
     let declaration: Fnc | undefined;
-    setImmediate(() => {
+    setTimeout(() => {
         if (!declaration) {
             setTimeout(flushQueued, 1000);
             return;
         }
         flushQueued();
-    });
+    }, 1000);
     function flushQueued() {
         if (!declaration) {
             throw new Error(`Hook function ${debugName} not declared`);

package/src/deployManager/machine.sh

@@ -1,6 +1,3 @@
 #! /bin/bash
-cd ~/machine-alwaysup
-yarn machine-alwaysup
-
 tmux new -s machine-alwaysup -d
 tmux send-keys -t machine-alwaysup "cd ~/machine-alwaysup && yarn machine-alwaysup" Enter

package/src/deployManager/setupMachineMain.ts

@@ -1,10 +1,158 @@
 import { getBackblazePath } from "../-a-archives/archivesBackBlaze";
-import { setGitRef, getGitURLLive, getGitRefLive } from "../4-deploy/git";
+import { getGitURLLive, getGitRefLive } from "../4-deploy/git";
+import { Querysub } from "../4-querysub/QuerysubController";
 import { runPromise } from "../functional/runCommand";
 import fs from "fs";
+import os from "os";
+import readline from "readline";
+import open from "open";
+// Import querysub, to fix missing dependencies
+Querysub;
 
 const pinnedNodeVersion = 22;
 
+async function getGitHubApiKey(repoUrl: string, sshRemote: string): Promise<string> {
+    // Parse repository info from URL
+    let repoOwner = "";
+    let repoName = "";
+    // Handle both SSH and HTTPS formats
+    // SSH: git@github.com:owner/repo.git
+    // HTTPS: https://github.com/owner/repo.git
+    const sshMatch = repoUrl.match(/git@github\.com:([^/]+)\/(.+)\.git$/);
+    const httpsMatch = repoUrl.match(/https:\/\/github\.com\/([^/]+)\/(.+)\.git$/);
+
+    if (sshMatch) {
+        repoOwner = sshMatch[1];
+        repoName = sshMatch[2];
+    } else if (httpsMatch) {
+        repoOwner = httpsMatch[1];
+        repoName = httpsMatch[2];
+    }
+
+    const cacheFile = os.homedir() + `/githubkey_${repoOwner}_${repoName}.json`;
+
+    // Check if we have a cached key
+    if (fs.existsSync(cacheFile)) {
+        try {
+            const cached = JSON.parse(fs.readFileSync(cacheFile, "utf8"));
+            if (cached.apiKey) {
+                console.log(`✅ Using cached GitHub API key from ${cacheFile}`);
+                return cached.apiKey;
+            }
+        } catch {
+            // Invalid cache file, we'll ask for a new key
+        }
+    }
+
+    // Need to get a new API key from user
+    console.log("\n🔑 GitHub API key required for private repository access");
+    console.log("Opening GitHub token creation page...");
+
+    // Construct URL for classic token (fine-grained tokens don't support deploy keys)
+    const repoInfo = repoOwner && repoName ? ` for repository ${repoOwner}/${repoName}` : "";
+    const instructions = `yarn setup-machine ${sshRemote}${repoInfo}
+1) Set expiration to 'No expiration' 
+2) Check the 'repo' scope (full repository access)
+3) Click 'Generate token'`;
+
+    let tokenUrl = `https://github.com/settings/tokens/new?description=${encodeURIComponent(instructions)}&scopes=repo`;
+    if (repoOwner && repoName) {
+        console.log(`Setting up access for repository: ${repoOwner}/${repoName}`);
+    }
+
+    await open(tokenUrl);
+
+    const rl = readline.createInterface({
+        input: process.stdin,
+        output: process.stdout
+    });
+
+    const apiKey = await new Promise<string>((resolve) => {
+        const repoInfo = repoOwner && repoName ? ` for repository ${repoOwner}/${repoName}` : "";
+        rl.question(`Please paste your GitHub API token (${repoInfo}): `, (answer) => {
+            rl.close();
+            resolve(answer.trim());
+        });
+    });
+
+    // Cache the key
+    fs.writeFileSync(cacheFile, JSON.stringify({ apiKey }));
+    console.log(`✅ Caching GitHub API key to ${cacheFile}`);
+
+    return apiKey;
+}
+
+async function addDeployKeyToGitHub(sshPublicKey: string, keyTitle: string, repoUrl: string, sshRemote: string): Promise<void> {
+    const apiKey = await getGitHubApiKey(repoUrl, sshRemote);
+
+    // Parse repository info from URL to get owner/repo for the API endpoint
+    let repoOwner = "";
+    let repoName = "";
+    const sshMatch = repoUrl.match(/git@github\.com:([^/]+)\/(.+)\.git$/);
+    const httpsMatch = repoUrl.match(/https:\/\/github\.com\/([^/]+)\/(.+)\.git$/);
+
+    if (sshMatch) {
+        repoOwner = sshMatch[1];
+        repoName = sshMatch[2];
+    } else if (httpsMatch) {
+        repoOwner = httpsMatch[1];
+        repoName = httpsMatch[2];
+    } else {
+        throw new Error(`Could not parse GitHub repository from URL: ${repoUrl}`);
+    }
+
+    let url = `https://api.github.com/repos/${repoOwner}/${repoName}/keys`;
+    console.log(url);
+    const response = await fetch(url, {
+        method: "POST",
+        headers: {
+            "Authorization": `Bearer ${apiKey}`,
+            "Accept": "application/vnd.github+json",
+            "Content-Type": "application/json"
+        },
+        body: JSON.stringify({
+            title: keyTitle,
+            key: sshPublicKey,
+            read_only: true
+        })
+    });
+
+    if (!response.ok) {
+        const errorText = await response.text();
+        throw new Error(`Failed to add deploy key to GitHub repository: ${response.status} ${errorText}`);
+    }
+
+    console.log("✅ Deploy key added to GitHub repository");
+}
+
+async function setupRepositoryOnRemote(sshRemote: string, gitURLLive: string, gitRefLive: string): Promise<void> {
+    // Create git folder on remote
+    await runPromise(`ssh ${sshRemote} "mkdir -p ~/machine-alwaysup"`);
+
+    // Add bitbucket.org host key to remote machine's known_hosts
+    await runPromise(`ssh ${sshRemote} "mkdir -p ~/.ssh"`);
+    await runPromise(`ssh ${sshRemote} "ssh-keyscan -t rsa bitbucket.org >> ~/.ssh/known_hosts 2>/dev/null || true"`);
+    await runPromise(`ssh ${sshRemote} "ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts 2>/dev/null || true"`);
+
+    // Check if repo already exists, if not clone it
+    try {
+        await runPromise(`ssh ${sshRemote} "cd ~/machine-alwaysup && git status"`);
+        console.log("Repository already exists, updating...");
+        // Repository exists, update it
+        await runPromise(`ssh ${sshRemote} "cd ~/machine-alwaysup && git remote update"`);
+        await runPromise(`ssh ${sshRemote} "cd ~/machine-alwaysup && git add --all"`);
+        await runPromise(`ssh ${sshRemote} "cd ~/machine-alwaysup && git stash"`);
+        await runPromise(`ssh ${sshRemote} "cd ~/machine-alwaysup && git fetch --all"`);
+        await runPromise(`ssh ${sshRemote} "cd ~/machine-alwaysup && git reset --hard ${gitRefLive}"`);
+        await runPromise(`ssh ${sshRemote} "cd ~/machine-alwaysup && git prune"`);
+    } catch {
+        console.log("Cloning repository...");
+        // Repository doesn't exist, clone it
+        await runPromise(`ssh ${sshRemote} "git clone ${gitURLLive} ~/machine-alwaysup"`);
+        await runPromise(`ssh ${sshRemote} "cd ~/machine-alwaysup && git reset --hard ${gitRefLive}"`);
+    }
+}
+
 async function main() {
     let sshRemote = process.argv.slice(2).join(" ");
     if (!sshRemote) {
@@ -39,6 +187,17 @@ async function main() {
         console.log("✅ Git installed");
     }
 
+    // 3. Ensure build tools are installed (needed for native modules)
+    console.log("Ensuring build tools are installed...");
+    try {
+        await runPromise(`ssh ${sshRemote} "which make"`);
+        console.log("✅ Build tools already installed");
+    } catch {
+        console.log("Installing build tools...");
+        await runPromise(`ssh ${sshRemote} "sudo apt install -y build-essential"`);
+        console.log("✅ Build tools installed");
+    }
+
     // 3. Ensure nodejs is installed on remote server
     console.log("Ensuring Node.js is installed...");
     try {
@@ -63,12 +222,47 @@ async function main() {
         console.log("✅ Yarn installed");
     }
 
-    // 5. Clone current repo into ~/machine-alwaysup with setGitRef
+    // 5. Clone current repo into ~/machine-alwaysup with SSH key handling for private repos
     console.log("Setting up repository...");
     let gitURLLive = await getGitURLLive();
     let gitRefLive = await getGitRefLive();
-    await setGitRef({ gitFolder: "~/machine-alwaysup", repoUrl: gitURLLive, gitRef: gitRefLive });
-    console.log("✅ Repository cloned and set to correct reference");
+
+    try {
+        await setupRepositoryOnRemote(sshRemote, gitURLLive, gitRefLive);
+        console.log("✅ Repository cloned and set to correct reference");
+    } catch (error) {
+        // Check if this is a private repository access issue
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        if (errorMessage.includes("Permission denied") || errorMessage.includes("Repository not found") || errorMessage.includes("fatal: Could not read from remote repository")) {
+            console.log("⚠️  Repository appears to be private, setting up SSH key access...");
+
+            // Ensure SSH key exists on remote machine
+            await runPromise(`ssh ${sshRemote} "mkdir -p ~/.ssh"`);
+            try {
+                await runPromise(`ssh ${sshRemote} "test -f ~/.ssh/id_rsa"`);
+                console.log("✅ SSH key already exists on remote machine");
+            } catch {
+                console.log("Generating SSH key on remote machine...");
+                await runPromise(`ssh ${sshRemote} "ssh-keygen -t rsa -b 4096 -f ~/.ssh/id_rsa -N ''"`);
+                console.log("✅ SSH key generated on remote machine");
+            }
+
+            // Get the public key from remote machine
+            const sshPublicKey = await runPromise(`ssh ${sshRemote} "cat ~/.ssh/id_rsa.pub"`);
+            const keyTitle = `Machine Setup - ${sshRemote} - ${new Date().toISOString()}`;
+
+            // Add the deploy key to GitHub repository
+            await addDeployKeyToGitHub(sshPublicKey.trim(), keyTitle, gitURLLive, sshRemote);
+
+            // Retry repository setup
+            console.log("Retrying repository setup with SSH key...");
+            await setupRepositoryOnRemote(sshRemote, gitURLLive, gitRefLive);
+            console.log("✅ Repository cloned and set to correct reference");
+        } else {
+            // Re-throw other errors
+            throw error;
+        }
+    }
 
     // Install dependencies
     console.log("Installing dependencies...");
@@ -79,11 +273,9 @@ async function main() {
     console.log("Setting up cron job...");
 
     // Create startup script directly on remote machine
-    await runPromise(`ssh ${sshRemote} "cat > ~/machine-startup.sh << 'EOF'
-#!/bin/bash
-cd ~/machine-alwaysup/src/deployManager
-bash machine.sh
-EOF"`);
+    await runPromise(`ssh ${sshRemote} "echo '#!/bin/bash' > ~/machine-startup.sh"`);
+    await runPromise(`ssh ${sshRemote} "echo 'cd ~/machine-alwaysup/node_modules/querysub/src/deployManager' >> ~/machine-startup.sh"`);
+    await runPromise(`ssh ${sshRemote} "echo 'bash machine.sh' >> ~/machine-startup.sh"`);
     await runPromise(`ssh ${sshRemote} "chmod +x ~/machine-startup.sh"`);
 
     // 7. Setup crontab to run ~/machine-startup.sh on startup

package/src/deployManager/spec.txt

@@ -19,6 +19,9 @@
 
 4) Test on a new server, that we run temporarily with some test scripts
 
+5) OH! Also setup swap usage on the machine, as it isn't by default often?
+
+
 
 
 5) Show command for our setup script in the web UI, in case we forget it (basically just instructions)

package/src/diagnostics/logs/diskShimConsoleLogs.ts

@@ -1,9 +1,19 @@
 import { canHaveChildren } from "socket-function/src/types";
-import { logDisk } from "./diskLogger";
 import { isNode } from "typesafecss";
 import { red, yellow } from "socket-function/src/formatting/logColors";
-import { addLocalLog } from "../errorLogs/ErrorLogController";
-import { LogType } from "../errorLogs/ErrorLogCore";
+
+let addLocalLogModule: (typeof addLocalLogPromise extends Promise<infer T> ? T : never) | undefined;
+const addLocalLogPromise = import("../errorLogs/ErrorLogController").then(x => {
+    (addLocalLogModule as any) = x;
+    return x;
+});
+
+let diskLoggerModule: typeof diskLoggerPromise extends Promise<infer T> ? T : never;
+const diskLoggerPromise = import("./diskLogger").then(x => {
+    (diskLoggerModule as any) = x;
+    return x;
+});
+
 
 let shimmed = false;
 export function shimConsoleLogs() {
@@ -26,10 +36,10 @@ export function shimConsoleLogs() {
                     args.length > 0
                     && String(args[0]).trim().length > 0
                 ) {
-                    if (typeof logDisk === "function") {
+                    if (typeof diskLoggerModule?.logDisk === "function") {
                         // Don't call it directly, so we don't get extra line debug context added to this call
                         //  (as it wouldn't be useful, as we really want the caller)
-                        let stopDoubleShim = logDisk;
+                        let stopDoubleShim = diskLoggerModule.logDisk;
                         stopDoubleShim(...args, { type: fncName });
                     }
                 }
@@ -39,10 +49,10 @@ export function shimConsoleLogs() {
                 // Filter out objects added by injectFileLocationToConsole
                 args = args.filter(x => !(canHaveChildren(x) && x["__FILE__"]));
 
-                if (typeof addLocalLog === "function") {
+                if (typeof addLocalLogModule?.addLocalLog === "function") {
                     if (fncName === "error" || fncName === "warn") {
                         // ALSO, track the logs in a file, for error notifications, etc
-                        addLocalLog({ message: args.join(" | ") + " | " + fileObj?.["__FILE__"], time: Date.now() }, fncName);
+                        addLocalLogModule.addLocalLog({ message: args.join(" | ") + " | " + fileObj?.["__FILE__"], time: Date.now() }, fncName);
                     }
                 }