querysub 0.675.0 → 0.677.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.675.0",
+  "version": "0.677.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/deployManager/githubRepoAccess.ts

@@ -0,0 +1,256 @@
+import fs from "fs";
+import os from "os";
+import readline from "readline";
+import open from "open";
+import { runOverSSH } from "sliftutils/security/helpers/remoteSSH";
+import { fsExistsAsync } from "../fs";
+
+const SSH_CONFIG_INCLUDE_LINE = "Include ~/.ssh/config.d/*";
+const MAX_ERROR_BODY_LENGTH = 500;
+// How often a pending question is re-printed, as other logging keeps running while we wait and buries it
+const QUESTION_REMIND_INTERVAL = 15 * 1000;
+
+/** Gives a remote machine read access to a private github repo, by giving that repo a deploy key of its own on that machine. Github will not take the same public key on two repos, so this key cannot be the machine's ~/.ssh/id_rsa - and ssh only chooses a key per host, not per repo, so the url is rewritten onto a host alias (global git config) which maps back to github.com with that key (ssh config). Both are global on the machine, so every clone on it picks this up, the service checkouts and their yarn installs included. */
+export async function ensureRemoteRepoAccess(config: { sshRemote: string; repoURL: string }) {
+    let { sshRemote, repoURL } = config;
+    let { owner, name } = parseGitHubRepo(repoURL);
+    let hostAlias = `github.com-${owner}-${name}`;
+    let keyPath = `~/.ssh/id_github_${owner}_${name}`;
+    let aliasPath = `~/.ssh/config.d/${owner}-${name}`;
+    let aliasURL = `git@${hostAlias}:${owner}/${name}.git`;
+
+    console.log(`Giving ${sshRemote} access to ${owner}/${name}, with the key at ${keyPath}`);
+    await runOverSSH({
+        host: sshRemote,
+        script: `set -e
+mkdir -p ~/.ssh/config.d
+chmod 700 ~/.ssh ~/.ssh/config.d
+touch ~/.ssh/config
+chmod 600 ~/.ssh/config
+ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null || true
+if [ ! -f ${keyPath} ]; then
+    ssh-keygen -t ed25519 -N '' -f ${keyPath} -C "${owner}/${name} deploy key"
+fi
+cat > ${aliasPath} <<'QUERYSUB_SSH_CONFIG_BLOCK'
+Host ${hostAlias}
+    HostName github.com
+    HostKeyAlias github.com
+    User git
+    IdentityFile ${keyPath}
+    IdentitiesOnly yes
+QUERYSUB_SSH_CONFIG_BLOCK
+chmod 600 ${aliasPath}
+if ! grep -qxF '${SSH_CONFIG_INCLUDE_LINE}' ~/.ssh/config; then
+    printf '${SSH_CONFIG_INCLUDE_LINE}\\n\\n' | cat - ~/.ssh/config > ~/.ssh/config.querysub-new
+    mv ~/.ssh/config.querysub-new ~/.ssh/config
+    chmod 600 ~/.ssh/config
+fi
+git config --global --replace-all "url.${aliasURL}.insteadOf" "git@github.com:${owner}/${name}.git"
+git config --global --replace-all "url.ssh://git@${hostAlias}/${owner}/${name}.git.insteadOf" "ssh://git@github.com/${owner}/${name}.git"`,
+    });
+
+    let probe = await runOverSSH({ host: sshRemote, script: `git ls-remote ${aliasURL} HEAD`, allowFailure: true });
+    if (probe.status === 0) {
+        console.log(`✅ ${sshRemote} can already read ${owner}/${name}, so its deploy key is left as it is`);
+        return;
+    }
+    console.log(`${sshRemote} cannot read ${owner}/${name} yet, adding its key as a deploy key`);
+
+    let publicKey = (await runOverSSH({ host: sshRemote, script: `cat ${keyPath}.pub` })).stdout.trim().split("\n").at(-1) || "";
+    if (!publicKey.startsWith("ssh-")) {
+        throw new Error(`Expected a public key at ${keyPath}.pub on ${sshRemote}, was ${publicKey.slice(0, MAX_ERROR_BODY_LENGTH) || "(nothing)"}`);
+    }
+    await addDeployKeyToGitHub({
+        sshPublicKey: publicKey,
+        keyTitle: `Machine Setup - ${sshRemote} - ${new Date().toISOString()}`,
+        repoURL,
+        sshRemote,
+    });
+
+    let recheck = await runOverSSH({ host: sshRemote, script: `git ls-remote ${aliasURL} HEAD`, allowFailure: true });
+    if (recheck.status !== 0) {
+        throw new Error(
+            `Expected ${sshRemote} to read ${owner}/${name} after its deploy key was added, git ls-remote ${aliasURL} exited ${recheck.status}.\n`
+            + `${(recheck.stdout + recheck.stderr).trim().slice(0, MAX_ERROR_BODY_LENGTH)}`
+        );
+    }
+    console.log(`✅ ${sshRemote} can now read ${owner}/${name}`);
+}
+
+export function parseGitHubRepo(repoURL: string): { owner: string; name: string } {
+    let cleaned = repoURL.replace(/^git\+/, "").split("#")[0];
+    let match = cleaned.match(/github\.com[:/]([^/]+)\/(.+?)(?:\.git)?\/?$/);
+    if (!match) {
+        throw new Error(`Expected a github repository url, was ${JSON.stringify(repoURL.slice(0, MAX_ERROR_BODY_LENGTH))}`);
+    }
+    return { owner: match[1], name: match[2] };
+}
+
+export async function addDeployKeyToGitHub(config: {
+    sshPublicKey: string;
+    keyTitle: string;
+    repoURL: string;
+    sshRemote: string;
+}): Promise<void> {
+    let { sshPublicKey, keyTitle, repoURL, sshRemote } = config;
+    let { owner, name } = parseGitHubRepo(repoURL);
+
+    let url = `https://api.github.com/repos/${owner}/${name}/keys`;
+    console.log(url);
+
+    let forceRefresh = false;
+    while (true) {
+        const apiKey = await getGitHubApiKey({ repoURL, sshRemote, forceRefresh });
+        const response = await fetch(url, {
+            method: "POST",
+            headers: {
+                "Authorization": `Bearer ${apiKey}`,
+                "Accept": "application/vnd.github+json",
+                "Content-Type": "application/json"
+            },
+            body: JSON.stringify({
+                title: keyTitle,
+                key: sshPublicKey,
+                read_only: true
+            })
+        });
+
+        if (response.ok) {
+            console.log("✅ Deploy key added to GitHub repository");
+            return;
+        }
+
+        const errorText = await response.text();
+        if (response.status === 401 && !forceRefresh) {
+            console.warn(`⚠️  GitHub API rejected credentials (401). Invalidating cached token and re-prompting.`);
+            try {
+                fs.unlinkSync(getGitHubKeyCachePath(repoURL));
+            } catch {
+                // Cache file may not exist; ignore
+            }
+            forceRefresh = true;
+            continue;
+        }
+        throw new Error(`Failed to add deploy key to GitHub repository: ${response.status} ${errorText}`);
+    }
+}
+
+export async function getGitHubApiKey(config: { repoURL: string; sshRemote: string; forceRefresh?: boolean }): Promise<string> {
+    let { repoURL, sshRemote, forceRefresh } = config;
+    let { owner, name } = parseGitHubRepo(repoURL);
+    const cacheFile = getGitHubKeyCachePath(repoURL);
+
+    // Check if we have a cached key
+    if (!forceRefresh && await fsExistsAsync(cacheFile)) {
+        try {
+            const cached = JSON.parse(fs.readFileSync(cacheFile, "utf8")) as { apiKey?: string };
+            if (cached.apiKey) {
+                if (await verifyGitHubApiKey(cached.apiKey)) {
+                    console.log(`✅ Using cached GitHub API key from ${cacheFile}`);
+                    return cached.apiKey;
+                }
+                console.warn(`⚠️  Cached GitHub API key at ${cacheFile} failed verification (401 Bad credentials) — requesting a new one`);
+            }
+        } catch {
+            // Invalid cache file, we'll ask for a new key
+        }
+    }
+
+    // A key that is already cached for another repo is very likely to work here too, as the scope is per account rather than per repo
+    if (!forceRefresh) {
+        let existing = await findApiKeyWithAccess({ owner, name, exceptPath: cacheFile });
+        if (existing) {
+            fs.writeFileSync(cacheFile, JSON.stringify({ apiKey: existing.apiKey }));
+            console.log(`✅ Reusing the GitHub API key from ${existing.path}, which has access to ${owner}/${name}, and caching it to ${cacheFile}`);
+            return existing.apiKey;
+        }
+    }
+
+    // Need to get a new API key from user
+    console.log("\n🔑 GitHub API key required for private repository access");
+    console.log("Opening GitHub token creation page...");
+
+    // Construct URL for classic token (fine-grained tokens don't support deploy keys)
+    const instructions = `yarn setup-machine ${sshRemote} for repository ${owner}/${name}
+1) Set expiration to 'No expiration'
+2) Check the 'repo' scope (full repository access)
+3) Click 'Generate token'`;
+
+    let tokenUrl = `https://github.com/settings/tokens/new?description=${encodeURIComponent(instructions)}&scopes=repo`;
+    console.log(`Setting up access for repository: ${owner}/${name}`);
+
+    await open(tokenUrl);
+
+    const apiKey = await askQuestion(`Please paste your GitHub API token (for repository ${owner}/${name}): `);
+
+    // Cache the key
+    fs.writeFileSync(cacheFile, JSON.stringify({ apiKey }));
+    console.log(`✅ Caching GitHub API key to ${cacheFile}`);
+
+    return apiKey;
+}
+
+async function findApiKeyWithAccess(config: { owner: string; name: string; exceptPath: string }): Promise<{ apiKey: string; path: string } | undefined> {
+    let { owner, name, exceptPath } = config;
+    let homeDir = os.homedir();
+    let fileNames = await fs.promises.readdir(homeDir);
+    for (let fileName of fileNames) {
+        if (!fileName.startsWith("githubkey_") || !fileName.endsWith(".json")) continue;
+        let path = `${homeDir}/${fileName}`;
+        if (path === exceptPath) continue;
+        let apiKey = "";
+        try {
+            apiKey = (JSON.parse(await fs.promises.readFile(path, "utf8")) as { apiKey?: string }).apiKey || "";
+        } catch {
+            continue;
+        }
+        if (!apiKey) continue;
+        let response = await fetch(`https://api.github.com/repos/${owner}/${name}`, {
+            headers: {
+                "Authorization": `Bearer ${apiKey}`,
+                "Accept": "application/vnd.github+json"
+            }
+        });
+        if (!response.ok) continue;
+        let repo = await response.json() as { permissions?: { admin?: boolean } };
+        if (!repo.permissions?.admin) continue;
+        return { apiKey, path };
+    }
+    return undefined;
+}
+
+export function getGitHubKeyCachePath(repoURL: string): string {
+    let { owner, name } = parseGitHubRepo(repoURL);
+    return os.homedir() + `/githubkey_${owner}_${name}.json`;
+}
+
+export async function verifyGitHubApiKey(apiKey: string): Promise<boolean> {
+    const response = await fetch("https://api.github.com/user", {
+        headers: {
+            "Authorization": `Bearer ${apiKey}`,
+            "Accept": "application/vnd.github+json"
+        }
+    });
+    return response.ok;
+}
+
+export async function askQuestion(prompt: string): Promise<string> {
+    const rl = readline.createInterface({
+        // Cast, as different @types/node versions disagree on the stream types
+        input: process.stdin as unknown as NodeJS.ReadableStream,
+        output: process.stdout,
+    });
+    let remind = setInterval(() => {
+        console.log(`\n(still waiting for an answer)\n${prompt}`);
+    }, QUESTION_REMIND_INTERVAL);
+    try {
+        return await new Promise<string>(resolve => {
+            rl.question(prompt, answer => {
+                resolve(answer.trim());
+            });
+        });
+    } finally {
+        clearInterval(remind);
+        rl.close();
+    }
+}

package/src/deployManager/setupMachineMain.ts

@@ -2,14 +2,11 @@ import { getBackblazePath } from "../misc/appPaths";
 import { getGitURLLive, getGitRefLive } from "../4-deploy/git";
 import { Querysub } from "../4-querysub/Querysub";
 import { runPromise } from "../functional/runCommand";
-import fs from "fs";
-import os from "os";
 import path from "path";
-import readline from "readline";
-import open from "open";
 import { fsExistsAsync } from "../fs";
 import { delay } from "socket-function/src/batching";
 import { SERVICE_NAME, SERVICE_UNIT_NAME } from "./machineDaemonShared";
+import { addDeployKeyToGitHub, askQuestion } from "./githubRepoAccess";
 // Import querysub, to fix missing dependencies
 Querysub;
 
@@ -145,174 +142,6 @@ async function installUnofficialNode(sshRemote: string, major: number): Promise<
     }
 }
 
-// How often a pending question is re-printed, as other logging keeps running while we wait and buries it
-const QUESTION_REMIND_INTERVAL = 15 * 1000;
-async function askQuestion(prompt: string): Promise<string> {
-    const rl = readline.createInterface({
-        // Cast, as different @types/node versions disagree on the stream types
-        input: process.stdin as unknown as NodeJS.ReadableStream,
-        output: process.stdout,
-    });
-    let remind = setInterval(() => {
-        console.log(`\n(still waiting for an answer)\n${prompt}`);
-    }, QUESTION_REMIND_INTERVAL);
-    try {
-        return await new Promise<string>(resolve => {
-            rl.question(prompt, answer => {
-                resolve(answer.trim());
-            });
-        });
-    } finally {
-        clearInterval(remind);
-        rl.close();
-    }
-}
-
-function getGitHubKeyCachePath(repoUrl: string): string {
-    let repoOwner = "";
-    let repoName = "";
-    const sshMatch = repoUrl.match(/git@github\.com:([^/]+)\/(.+)\.git$/);
-    const httpsMatch = repoUrl.match(/https:\/\/github\.com\/([^/]+)\/(.+)\.git$/);
-    if (sshMatch) {
-        repoOwner = sshMatch[1];
-        repoName = sshMatch[2];
-    } else if (httpsMatch) {
-        repoOwner = httpsMatch[1];
-        repoName = httpsMatch[2];
-    }
-    return os.homedir() + `/githubkey_${repoOwner}_${repoName}.json`;
-}
-
-async function verifyGitHubApiKey(apiKey: string): Promise<boolean> {
-    const response = await fetch("https://api.github.com/user", {
-        headers: {
-            "Authorization": `Bearer ${apiKey}`,
-            "Accept": "application/vnd.github+json"
-        }
-    });
-    return response.ok;
-}
-
-async function getGitHubApiKey(repoUrl: string, sshRemote: string, forceRefresh = false): Promise<string> {
-    // Parse repository info from URL
-    let repoOwner = "";
-    let repoName = "";
-    // Handle both SSH and HTTPS formats
-    // SSH: git@github.com:owner/repo.git
-    // HTTPS: https://github.com/owner/repo.git
-    const sshMatch = repoUrl.match(/git@github\.com:([^/]+)\/(.+)\.git$/);
-    const httpsMatch = repoUrl.match(/https:\/\/github\.com\/([^/]+)\/(.+)\.git$/);
-
-    if (sshMatch) {
-        repoOwner = sshMatch[1];
-        repoName = sshMatch[2];
-    } else if (httpsMatch) {
-        repoOwner = httpsMatch[1];
-        repoName = httpsMatch[2];
-    }
-
-    const cacheFile = getGitHubKeyCachePath(repoUrl);
-
-    // Check if we have a cached key
-    if (!forceRefresh && await fsExistsAsync(cacheFile)) {
-        try {
-            const cached = JSON.parse(fs.readFileSync(cacheFile, "utf8"));
-            if (cached.apiKey) {
-                if (await verifyGitHubApiKey(cached.apiKey)) {
-                    console.log(`✅ Using cached GitHub API key from ${cacheFile}`);
-                    return cached.apiKey;
-                }
-                console.warn(`⚠️  Cached GitHub API key at ${cacheFile} failed verification (401 Bad credentials) — requesting a new one`);
-            }
-        } catch {
-            // Invalid cache file, we'll ask for a new key
-        }
-    }
-
-    // Need to get a new API key from user
-    console.log("\n🔑 GitHub API key required for private repository access");
-    console.log("Opening GitHub token creation page...");
-
-    // Construct URL for classic token (fine-grained tokens don't support deploy keys)
-    const repoInfo = repoOwner && repoName ? ` for repository ${repoOwner}/${repoName}` : "";
-    const instructions = `yarn setup-machine ${sshRemote}${repoInfo}
-1) Set expiration to 'No expiration' 
-2) Check the 'repo' scope (full repository access)
-3) Click 'Generate token'`;
-
-    let tokenUrl = `https://github.com/settings/tokens/new?description=${encodeURIComponent(instructions)}&scopes=repo`;
-    if (repoOwner && repoName) {
-        console.log(`Setting up access for repository: ${repoOwner}/${repoName}`);
-    }
-
-    await open(tokenUrl);
-
-    const apiKey = await askQuestion(`Please paste your GitHub API token (${repoOwner && repoName && `for repository ${repoOwner}/${repoName}` || ""}): `);
-
-    // Cache the key
-    fs.writeFileSync(cacheFile, JSON.stringify({ apiKey }));
-    console.log(`✅ Caching GitHub API key to ${cacheFile}`);
-
-    return apiKey;
-}
-
-async function addDeployKeyToGitHub(sshPublicKey: string, keyTitle: string, repoUrl: string, sshRemote: string): Promise<void> {
-    // Parse repository info from URL to get owner/repo for the API endpoint
-    let repoOwner = "";
-    let repoName = "";
-    const sshMatch = repoUrl.match(/git@github\.com:([^/]+)\/(.+)\.git$/);
-    const httpsMatch = repoUrl.match(/https:\/\/github\.com\/([^/]+)\/(.+)\.git$/);
-
-    if (sshMatch) {
-        repoOwner = sshMatch[1];
-        repoName = sshMatch[2];
-    } else if (httpsMatch) {
-        repoOwner = httpsMatch[1];
-        repoName = httpsMatch[2];
-    } else {
-        throw new Error(`Could not parse GitHub repository from URL: ${repoUrl}`);
-    }
-
-    let url = `https://api.github.com/repos/${repoOwner}/${repoName}/keys`;
-    console.log(url);
-
-    let forceRefresh = false;
-    while (true) {
-        const apiKey = await getGitHubApiKey(repoUrl, sshRemote, forceRefresh);
-        const response = await fetch(url, {
-            method: "POST",
-            headers: {
-                "Authorization": `Bearer ${apiKey}`,
-                "Accept": "application/vnd.github+json",
-                "Content-Type": "application/json"
-            },
-            body: JSON.stringify({
-                title: keyTitle,
-                key: sshPublicKey,
-                read_only: true
-            })
-        });
-
-        if (response.ok) {
-            console.log("✅ Deploy key added to GitHub repository");
-            return;
-        }
-
-        const errorText = await response.text();
-        if (response.status === 401 && !forceRefresh) {
-            console.warn(`⚠️  GitHub API rejected credentials (401). Invalidating cached token and re-prompting.`);
-            try {
-                fs.unlinkSync(getGitHubKeyCachePath(repoUrl));
-            } catch {
-                // Cache file may not exist; ignore
-            }
-            forceRefresh = true;
-            continue;
-        }
-        throw new Error(`Failed to add deploy key to GitHub repository: ${response.status} ${errorText}`);
-    }
-}
-
 async function setupRepositoryOnRemote(sshRemote: string, gitURLLive: string, gitRefLive: string): Promise<void> {
     // Create git folder on remote
     await runPromise(`ssh ${sshRemote} "mkdir -p ~/machine-alwaysup"`);
@@ -543,6 +372,13 @@ async function main() {
     await runPromise(`ssh ${sshRemote} "grep -qxF 'set-option -g history-limit 100000' ~/.tmux.conf 2>/dev/null || echo 'set-option -g history-limit 100000' >> ~/.tmux.conf"`);
     console.log("✅ Tmux scrollback history set to 100000");
 
+    // Before anything clones or installs, so an application that needs credentials on the machine (a private dependency it cannot yarn install without) can put them there first. As early as it can be: git is what the hook has to work with, and it is only installed above.
+    await runApplicationHook({
+        sshRemote,
+        hookName: "machineSetupStart",
+        onFailure: "Anything it was meant to set up on the machine is missing, so the clone and install below may fail.",
+    });
+
     // 5. Clone current repo into ~/machine-alwaysup with SSH key handling for private repos
     console.log("Setting up repository...");
     let gitURLLive = await getGitURLLive();
@@ -574,7 +410,7 @@ async function main() {
             const keyTitle = `Machine Setup - ${sshRemote} - ${new Date().toISOString()}`;
 
             // Add the deploy key to GitHub repository
-            await addDeployKeyToGitHub(sshPublicKey.trim(), keyTitle, gitURLLive, sshRemote);
+            await addDeployKeyToGitHub({ sshPublicKey: sshPublicKey.trim(), keyTitle, repoURL: gitURLLive, sshRemote });
 
             // Retry repository setup
             console.log("Retrying repository setup with SSH key...");
@@ -655,13 +491,18 @@ sudo systemctl start ${SERVICE_UNIT_NAME}
     console.log(`   Screen:  ssh ${sshRemote} -t "tmux attach -t ${SERVICE_NAME}"`);
     console.log(`   Status:  ssh ${sshRemote} "systemctl status ${SERVICE_UNIT_NAME}"`);
 
-    await runMachineDeployHook(sshRemote);
+    await runApplicationHook({
+        sshRemote,
+        hookName: "machineDeploy",
+        onFailure: "This is fine if the synchronization database hasn't been set up yet, or isn't running.",
+    });
 
     console.log("\n🎉 Machine setup complete!");
 }
 
-async function runMachineDeployHook(sshRemote: string) {
-    let hookPathBase = path.resolve("./machineDeploy");
+async function runApplicationHook(config: { sshRemote: string; hookName: string; onFailure: string }) {
+    let { sshRemote, hookName, onFailure } = config;
+    let hookPathBase = path.resolve(`./${hookName}`);
     let hookExists = false;
     for (let extension of [".ts", ".tsx", ".js"]) {
         if (await fsExistsAsync(hookPathBase + extension)) {
@@ -670,20 +511,21 @@ async function runMachineDeployHook(sshRemote: string) {
         }
     }
     if (!hookExists) {
-        console.log(`No machineDeploy file at ${hookPathBase}, skipping the application deploy hook`);
+        console.log(`No ${hookName} file at ${hookPathBase}, skipping that application hook`);
         return;
     }
-    console.log(`Running the machineDeploy hook at ${hookPathBase}`);
+    console.log(`Running the ${hookName} hook at ${hookPathBase}`);
     try {
-        let hookModule = await import(hookPathBase) as { machineDeploy?: (config: { sshRemote: string }) => Promise<void> };
-        if (!hookModule.machineDeploy) {
-            throw new Error(`Expected ${hookPathBase} to export machineDeploy, exports are: ${Object.keys(hookModule).join(", ") || "(none)"}`);
+        let hookModule = await import(hookPathBase) as { [exportName: string]: ((config: { sshRemote: string }) => Promise<void>) | undefined };
+        let hook = hookModule[hookName];
+        if (!hook) {
+            throw new Error(`Expected ${hookPathBase} to export ${hookName}, exports are: ${Object.keys(hookModule).join(", ") || "(none)"}`);
         }
-        await hookModule.machineDeploy({ sshRemote });
-        console.log("✅ machineDeploy hook finished");
+        await hook({ sshRemote });
+        console.log(`✅ ${hookName} hook finished`);
     } catch (e) {
         console.error((e as Error).stack || String(e));
-        console.error(`⚠️  The machineDeploy hook failed (see the error above). This is fine if the synchronization database hasn't been set up yet, or isn't running.`);
+        console.error(`⚠️  The ${hookName} hook failed (see the error above). ${onFailure}`);
     }
 }