querysub 0.674.0 → 0.676.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.674.0",
+  "version": "0.676.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -79,8 +79,8 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
-    "sliftutils": "^1.7.140",
-    "socket-function": "^1.2.35",
+    "sliftutils": "^1.7.149",
+    "socket-function": "^1.2.36",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",
     "typesafecss": "^0.32.0",

package/src/deployManager/components/ServiceDetailPage.tsx

@@ -138,7 +138,9 @@ export class ServiceDetailPage extends qreact.Component {
                         {group.map(({ entryIndex, index, target }) => {
                             // Variables set on the entry but no longer in the command are still shown (flagged), so stale values can be cleared.
                             let variableNames = [...new Set([...templateVariables, ...Object.keys(target.variables)])];
-                            return <div key={entryIndex} className={css.hbox(8).alignItems("center").pad2(8, 4).hsl(0, 0, 100)}>
+                            return <div key={entryIndex} className={css.hbox(8).alignItems("center").pad2(8, 4).hsl(0, 0, 100)}
+                                title={applyCommandTemplate(config.parameters.command, target.variables)}
+                            >
                                 <div className={css.fontSize(11).colorhsl(0, 0, 40)}>{getScreenName({ serviceKey: config.parameters.key, index })}</div>
                                 {variableNames.map(name => {
                                     let inCommand = templateVariables.includes(name);

package/src/deployManager/githubRepoAccess.ts

@@ -0,0 +1,256 @@
+import fs from "fs";
+import os from "os";
+import readline from "readline";
+import open from "open";
+import { runOverSSH } from "sliftutils/security/helpers/remoteSSH";
+import { fsExistsAsync } from "../fs";
+
+const SSH_CONFIG_INCLUDE_LINE = "Include ~/.ssh/config.d/*";
+const MAX_ERROR_BODY_LENGTH = 500;
+// How often a pending question is re-printed, as other logging keeps running while we wait and buries it
+const QUESTION_REMIND_INTERVAL = 15 * 1000;
+
+/** Gives a remote machine read access to a private github repo, by giving that repo a deploy key of its own on that machine. Github will not take the same public key on two repos, so this key cannot be the machine's ~/.ssh/id_rsa - and ssh only chooses a key per host, not per repo, so the url is rewritten onto a host alias (global git config) which maps back to github.com with that key (ssh config). Both are global on the machine, so every clone on it picks this up, the service checkouts and their yarn installs included. */
+export async function ensureRemoteRepoAccess(config: { sshRemote: string; repoURL: string }) {
+    let { sshRemote, repoURL } = config;
+    let { owner, name } = parseGitHubRepo(repoURL);
+    let hostAlias = `github.com-${owner}-${name}`;
+    let keyPath = `~/.ssh/id_github_${owner}_${name}`;
+    let aliasPath = `~/.ssh/config.d/${owner}-${name}`;
+    let aliasURL = `git@${hostAlias}:${owner}/${name}.git`;
+
+    console.log(`Giving ${sshRemote} access to ${owner}/${name}, with the key at ${keyPath}`);
+    await runOverSSH({
+        host: sshRemote,
+        script: `set -e
+mkdir -p ~/.ssh/config.d
+chmod 700 ~/.ssh ~/.ssh/config.d
+touch ~/.ssh/config
+chmod 600 ~/.ssh/config
+ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null || true
+if [ ! -f ${keyPath} ]; then
+    ssh-keygen -t ed25519 -N '' -f ${keyPath} -C "${owner}/${name} deploy key"
+fi
+cat > ${aliasPath} <<'QUERYSUB_SSH_CONFIG_BLOCK'
+Host ${hostAlias}
+    HostName github.com
+    HostKeyAlias github.com
+    User git
+    IdentityFile ${keyPath}
+    IdentitiesOnly yes
+QUERYSUB_SSH_CONFIG_BLOCK
+chmod 600 ${aliasPath}
+if ! grep -qxF '${SSH_CONFIG_INCLUDE_LINE}' ~/.ssh/config; then
+    printf '${SSH_CONFIG_INCLUDE_LINE}\\n\\n' | cat - ~/.ssh/config > ~/.ssh/config.querysub-new
+    mv ~/.ssh/config.querysub-new ~/.ssh/config
+    chmod 600 ~/.ssh/config
+fi
+git config --global --replace-all "url.${aliasURL}.insteadOf" "git@github.com:${owner}/${name}.git"
+git config --global --replace-all "url.ssh://git@${hostAlias}/${owner}/${name}.git.insteadOf" "ssh://git@github.com/${owner}/${name}.git"`,
+    });
+
+    let probe = await runOverSSH({ host: sshRemote, script: `git ls-remote ${aliasURL} HEAD`, allowFailure: true });
+    if (probe.status === 0) {
+        console.log(`✅ ${sshRemote} can already read ${owner}/${name}, so its deploy key is left as it is`);
+        return;
+    }
+    console.log(`${sshRemote} cannot read ${owner}/${name} yet, adding its key as a deploy key`);
+
+    let publicKey = (await runOverSSH({ host: sshRemote, script: `cat ${keyPath}.pub` })).stdout.trim().split("\n").at(-1) || "";
+    if (!publicKey.startsWith("ssh-")) {
+        throw new Error(`Expected a public key at ${keyPath}.pub on ${sshRemote}, was ${publicKey.slice(0, MAX_ERROR_BODY_LENGTH) || "(nothing)"}`);
+    }
+    await addDeployKeyToGitHub({
+        sshPublicKey: publicKey,
+        keyTitle: `Machine Setup - ${sshRemote} - ${new Date().toISOString()}`,
+        repoURL,
+        sshRemote,
+    });
+
+    let recheck = await runOverSSH({ host: sshRemote, script: `git ls-remote ${aliasURL} HEAD`, allowFailure: true });
+    if (recheck.status !== 0) {
+        throw new Error(
+            `Expected ${sshRemote} to read ${owner}/${name} after its deploy key was added, git ls-remote ${aliasURL} exited ${recheck.status}.\n`
+            + `${(recheck.stdout + recheck.stderr).trim().slice(0, MAX_ERROR_BODY_LENGTH)}`
+        );
+    }
+    console.log(`✅ ${sshRemote} can now read ${owner}/${name}`);
+}
+
+export function parseGitHubRepo(repoURL: string): { owner: string; name: string } {
+    let cleaned = repoURL.replace(/^git\+/, "").split("#")[0];
+    let match = cleaned.match(/github\.com[:/]([^/]+)\/(.+?)(?:\.git)?\/?$/);
+    if (!match) {
+        throw new Error(`Expected a github repository url, was ${JSON.stringify(repoURL.slice(0, MAX_ERROR_BODY_LENGTH))}`);
+    }
+    return { owner: match[1], name: match[2] };
+}
+
+export async function addDeployKeyToGitHub(config: {
+    sshPublicKey: string;
+    keyTitle: string;
+    repoURL: string;
+    sshRemote: string;
+}): Promise<void> {
+    let { sshPublicKey, keyTitle, repoURL, sshRemote } = config;
+    let { owner, name } = parseGitHubRepo(repoURL);
+
+    let url = `https://api.github.com/repos/${owner}/${name}/keys`;
+    console.log(url);
+
+    let forceRefresh = false;
+    while (true) {
+        const apiKey = await getGitHubApiKey({ repoURL, sshRemote, forceRefresh });
+        const response = await fetch(url, {
+            method: "POST",
+            headers: {
+                "Authorization": `Bearer ${apiKey}`,
+                "Accept": "application/vnd.github+json",
+                "Content-Type": "application/json"
+            },
+            body: JSON.stringify({
+                title: keyTitle,
+                key: sshPublicKey,
+                read_only: true
+            })
+        });
+
+        if (response.ok) {
+            console.log("✅ Deploy key added to GitHub repository");
+            return;
+        }
+
+        const errorText = await response.text();
+        if (response.status === 401 && !forceRefresh) {
+            console.warn(`⚠️  GitHub API rejected credentials (401). Invalidating cached token and re-prompting.`);
+            try {
+                fs.unlinkSync(getGitHubKeyCachePath(repoURL));
+            } catch {
+                // Cache file may not exist; ignore
+            }
+            forceRefresh = true;
+            continue;
+        }
+        throw new Error(`Failed to add deploy key to GitHub repository: ${response.status} ${errorText}`);
+    }
+}
+
+export async function getGitHubApiKey(config: { repoURL: string; sshRemote: string; forceRefresh?: boolean }): Promise<string> {
+    let { repoURL, sshRemote, forceRefresh } = config;
+    let { owner, name } = parseGitHubRepo(repoURL);
+    const cacheFile = getGitHubKeyCachePath(repoURL);
+
+    // Check if we have a cached key
+    if (!forceRefresh && await fsExistsAsync(cacheFile)) {
+        try {
+            const cached = JSON.parse(fs.readFileSync(cacheFile, "utf8")) as { apiKey?: string };
+            if (cached.apiKey) {
+                if (await verifyGitHubApiKey(cached.apiKey)) {
+                    console.log(`✅ Using cached GitHub API key from ${cacheFile}`);
+                    return cached.apiKey;
+                }
+                console.warn(`⚠️  Cached GitHub API key at ${cacheFile} failed verification (401 Bad credentials) — requesting a new one`);
+            }
+        } catch {
+            // Invalid cache file, we'll ask for a new key
+        }
+    }
+
+    // A key that is already cached for another repo is very likely to work here too, as the scope is per account rather than per repo
+    if (!forceRefresh) {
+        let existing = await findApiKeyWithAccess({ owner, name, exceptPath: cacheFile });
+        if (existing) {
+            fs.writeFileSync(cacheFile, JSON.stringify({ apiKey: existing.apiKey }));
+            console.log(`✅ Reusing the GitHub API key from ${existing.path}, which has access to ${owner}/${name}, and caching it to ${cacheFile}`);
+            return existing.apiKey;
+        }
+    }
+
+    // Need to get a new API key from user
+    console.log("\n🔑 GitHub API key required for private repository access");
+    console.log("Opening GitHub token creation page...");
+
+    // Construct URL for classic token (fine-grained tokens don't support deploy keys)
+    const instructions = `yarn setup-machine ${sshRemote} for repository ${owner}/${name}
+1) Set expiration to 'No expiration'
+2) Check the 'repo' scope (full repository access)
+3) Click 'Generate token'`;
+
+    let tokenUrl = `https://github.com/settings/tokens/new?description=${encodeURIComponent(instructions)}&scopes=repo`;
+    console.log(`Setting up access for repository: ${owner}/${name}`);
+
+    await open(tokenUrl);
+
+    const apiKey = await askQuestion(`Please paste your GitHub API token (for repository ${owner}/${name}): `);
+
+    // Cache the key
+    fs.writeFileSync(cacheFile, JSON.stringify({ apiKey }));
+    console.log(`✅ Caching GitHub API key to ${cacheFile}`);
+
+    return apiKey;
+}
+
+async function findApiKeyWithAccess(config: { owner: string; name: string; exceptPath: string }): Promise<{ apiKey: string; path: string } | undefined> {
+    let { owner, name, exceptPath } = config;
+    let homeDir = os.homedir();
+    let fileNames = await fs.promises.readdir(homeDir);
+    for (let fileName of fileNames) {
+        if (!fileName.startsWith("githubkey_") || !fileName.endsWith(".json")) continue;
+        let path = `${homeDir}/${fileName}`;
+        if (path === exceptPath) continue;
+        let apiKey = "";
+        try {
+            apiKey = (JSON.parse(await fs.promises.readFile(path, "utf8")) as { apiKey?: string }).apiKey || "";
+        } catch {
+            continue;
+        }
+        if (!apiKey) continue;
+        let response = await fetch(`https://api.github.com/repos/${owner}/${name}`, {
+            headers: {
+                "Authorization": `Bearer ${apiKey}`,
+                "Accept": "application/vnd.github+json"
+            }
+        });
+        if (!response.ok) continue;
+        let repo = await response.json() as { permissions?: { admin?: boolean } };
+        if (!repo.permissions?.admin) continue;
+        return { apiKey, path };
+    }
+    return undefined;
+}
+
+export function getGitHubKeyCachePath(repoURL: string): string {
+    let { owner, name } = parseGitHubRepo(repoURL);
+    return os.homedir() + `/githubkey_${owner}_${name}.json`;
+}
+
+export async function verifyGitHubApiKey(apiKey: string): Promise<boolean> {
+    const response = await fetch("https://api.github.com/user", {
+        headers: {
+            "Authorization": `Bearer ${apiKey}`,
+            "Accept": "application/vnd.github+json"
+        }
+    });
+    return response.ok;
+}
+
+export async function askQuestion(prompt: string): Promise<string> {
+    const rl = readline.createInterface({
+        // Cast, as different @types/node versions disagree on the stream types
+        input: process.stdin as unknown as NodeJS.ReadableStream,
+        output: process.stdout,
+    });
+    let remind = setInterval(() => {
+        console.log(`\n(still waiting for an answer)\n${prompt}`);
+    }, QUESTION_REMIND_INTERVAL);
+    try {
+        return await new Promise<string>(resolve => {
+            rl.question(prompt, answer => {
+                resolve(answer.trim());
+            });
+        });
+    } finally {
+        clearInterval(remind);
+        rl.close();
+    }
+}

package/src/deployManager/setupMachineMain.ts

@@ -2,14 +2,11 @@ import { getBackblazePath } from "../misc/appPaths";
 import { getGitURLLive, getGitRefLive } from "../4-deploy/git";
 import { Querysub } from "../4-querysub/Querysub";
 import { runPromise } from "../functional/runCommand";
-import fs from "fs";
-import os from "os";
 import path from "path";
-import readline from "readline";
-import open from "open";
 import { fsExistsAsync } from "../fs";
 import { delay } from "socket-function/src/batching";
 import { SERVICE_NAME, SERVICE_UNIT_NAME } from "./machineDaemonShared";
+import { addDeployKeyToGitHub, askQuestion } from "./githubRepoAccess";
 // Import querysub, to fix missing dependencies
 Querysub;
 
@@ -145,174 +142,6 @@ async function installUnofficialNode(sshRemote: string, major: number): Promise<
     }
 }
 
-// How often a pending question is re-printed, as other logging keeps running while we wait and buries it
-const QUESTION_REMIND_INTERVAL = 15 * 1000;
-async function askQuestion(prompt: string): Promise<string> {
-    const rl = readline.createInterface({
-        // Cast, as different @types/node versions disagree on the stream types
-        input: process.stdin as unknown as NodeJS.ReadableStream,
-        output: process.stdout,
-    });
-    let remind = setInterval(() => {
-        console.log(`\n(still waiting for an answer)\n${prompt}`);
-    }, QUESTION_REMIND_INTERVAL);
-    try {
-        return await new Promise<string>(resolve => {
-            rl.question(prompt, answer => {
-                resolve(answer.trim());
-            });
-        });
-    } finally {
-        clearInterval(remind);
-        rl.close();
-    }
-}
-
-function getGitHubKeyCachePath(repoUrl: string): string {
-    let repoOwner = "";
-    let repoName = "";
-    const sshMatch = repoUrl.match(/git@github\.com:([^/]+)\/(.+)\.git$/);
-    const httpsMatch = repoUrl.match(/https:\/\/github\.com\/([^/]+)\/(.+)\.git$/);
-    if (sshMatch) {
-        repoOwner = sshMatch[1];
-        repoName = sshMatch[2];
-    } else if (httpsMatch) {
-        repoOwner = httpsMatch[1];
-        repoName = httpsMatch[2];
-    }
-    return os.homedir() + `/githubkey_${repoOwner}_${repoName}.json`;
-}
-
-async function verifyGitHubApiKey(apiKey: string): Promise<boolean> {
-    const response = await fetch("https://api.github.com/user", {
-        headers: {
-            "Authorization": `Bearer ${apiKey}`,
-            "Accept": "application/vnd.github+json"
-        }
-    });
-    return response.ok;
-}
-
-async function getGitHubApiKey(repoUrl: string, sshRemote: string, forceRefresh = false): Promise<string> {
-    // Parse repository info from URL
-    let repoOwner = "";
-    let repoName = "";
-    // Handle both SSH and HTTPS formats
-    // SSH: git@github.com:owner/repo.git
-    // HTTPS: https://github.com/owner/repo.git
-    const sshMatch = repoUrl.match(/git@github\.com:([^/]+)\/(.+)\.git$/);
-    const httpsMatch = repoUrl.match(/https:\/\/github\.com\/([^/]+)\/(.+)\.git$/);
-
-    if (sshMatch) {
-        repoOwner = sshMatch[1];
-        repoName = sshMatch[2];
-    } else if (httpsMatch) {
-        repoOwner = httpsMatch[1];
-        repoName = httpsMatch[2];
-    }
-
-    const cacheFile = getGitHubKeyCachePath(repoUrl);
-
-    // Check if we have a cached key
-    if (!forceRefresh && await fsExistsAsync(cacheFile)) {
-        try {
-            const cached = JSON.parse(fs.readFileSync(cacheFile, "utf8"));
-            if (cached.apiKey) {
-                if (await verifyGitHubApiKey(cached.apiKey)) {
-                    console.log(`✅ Using cached GitHub API key from ${cacheFile}`);
-                    return cached.apiKey;
-                }
-                console.warn(`⚠️  Cached GitHub API key at ${cacheFile} failed verification (401 Bad credentials) — requesting a new one`);
-            }
-        } catch {
-            // Invalid cache file, we'll ask for a new key
-        }
-    }
-
-    // Need to get a new API key from user
-    console.log("\n🔑 GitHub API key required for private repository access");
-    console.log("Opening GitHub token creation page...");
-
-    // Construct URL for classic token (fine-grained tokens don't support deploy keys)
-    const repoInfo = repoOwner && repoName ? ` for repository ${repoOwner}/${repoName}` : "";
-    const instructions = `yarn setup-machine ${sshRemote}${repoInfo}
-1) Set expiration to 'No expiration' 
-2) Check the 'repo' scope (full repository access)
-3) Click 'Generate token'`;
-
-    let tokenUrl = `https://github.com/settings/tokens/new?description=${encodeURIComponent(instructions)}&scopes=repo`;
-    if (repoOwner && repoName) {
-        console.log(`Setting up access for repository: ${repoOwner}/${repoName}`);
-    }
-
-    await open(tokenUrl);
-
-    const apiKey = await askQuestion(`Please paste your GitHub API token (${repoOwner && repoName && `for repository ${repoOwner}/${repoName}` || ""}): `);
-
-    // Cache the key
-    fs.writeFileSync(cacheFile, JSON.stringify({ apiKey }));
-    console.log(`✅ Caching GitHub API key to ${cacheFile}`);
-
-    return apiKey;
-}
-
-async function addDeployKeyToGitHub(sshPublicKey: string, keyTitle: string, repoUrl: string, sshRemote: string): Promise<void> {
-    // Parse repository info from URL to get owner/repo for the API endpoint
-    let repoOwner = "";
-    let repoName = "";
-    const sshMatch = repoUrl.match(/git@github\.com:([^/]+)\/(.+)\.git$/);
-    const httpsMatch = repoUrl.match(/https:\/\/github\.com\/([^/]+)\/(.+)\.git$/);
-
-    if (sshMatch) {
-        repoOwner = sshMatch[1];
-        repoName = sshMatch[2];
-    } else if (httpsMatch) {
-        repoOwner = httpsMatch[1];
-        repoName = httpsMatch[2];
-    } else {
-        throw new Error(`Could not parse GitHub repository from URL: ${repoUrl}`);
-    }
-
-    let url = `https://api.github.com/repos/${repoOwner}/${repoName}/keys`;
-    console.log(url);
-
-    let forceRefresh = false;
-    while (true) {
-        const apiKey = await getGitHubApiKey(repoUrl, sshRemote, forceRefresh);
-        const response = await fetch(url, {
-            method: "POST",
-            headers: {
-                "Authorization": `Bearer ${apiKey}`,
-                "Accept": "application/vnd.github+json",
-                "Content-Type": "application/json"
-            },
-            body: JSON.stringify({
-                title: keyTitle,
-                key: sshPublicKey,
-                read_only: true
-            })
-        });
-
-        if (response.ok) {
-            console.log("✅ Deploy key added to GitHub repository");
-            return;
-        }
-
-        const errorText = await response.text();
-        if (response.status === 401 && !forceRefresh) {
-            console.warn(`⚠️  GitHub API rejected credentials (401). Invalidating cached token and re-prompting.`);
-            try {
-                fs.unlinkSync(getGitHubKeyCachePath(repoUrl));
-            } catch {
-                // Cache file may not exist; ignore
-            }
-            forceRefresh = true;
-            continue;
-        }
-        throw new Error(`Failed to add deploy key to GitHub repository: ${response.status} ${errorText}`);
-    }
-}
-
 async function setupRepositoryOnRemote(sshRemote: string, gitURLLive: string, gitRefLive: string): Promise<void> {
     // Create git folder on remote
     await runPromise(`ssh ${sshRemote} "mkdir -p ~/machine-alwaysup"`);
@@ -574,7 +403,7 @@ async function main() {
             const keyTitle = `Machine Setup - ${sshRemote} - ${new Date().toISOString()}`;
 
             // Add the deploy key to GitHub repository
-            await addDeployKeyToGitHub(sshPublicKey.trim(), keyTitle, gitURLLive, sshRemote);
+            await addDeployKeyToGitHub({ sshPublicKey: sshPublicKey.trim(), keyTitle, repoURL: gitURLLive, sshRemote });
 
             // Retry repository setup
             console.log("Retrying repository setup with SSH key...");