ssh-client-wrapper 3.1.0 → 3.1.2

package/README.md

@@ -107,6 +107,17 @@ Downloads files or directories from the remote host using `rsync`.
 *   `replaceRetryableExitCodes` (boolean, optional): If `true`, `retryableExitCodes` replaces the built-in default instead of adding to it. Defaults to `false`.
 *   Returns: `Promise<void>`
 
+### Remote paths of `.send()` / `.recv()`
+
+A remote path (`dst` of `.send()`, `src` of `.recv()`) is interpreted by the login shell of the remote host, whatever the version or the implementation of the local `rsync` is:
+
+*   `~`, environment variables (`$HOME/work`), braces (`dir/{a,b}`; the remote shell must support them, e.g. bash), quotes (`'dir/a b'`) and backslash escapes (`dir/a\ b`) are expanded.
+*   A space which is neither quoted nor escaped separates two paths.
+*   Wildcards (`*`, `?`, `[...]`) are expanded by the remote side as usual.
+*   The destination of `.send()` must be expanded to exactly one path. Its directory is created if it does not exist.
+
+A path that consists only of letters, digits and `_ @ % + = : , . / -` and wildcards needs no expansion and costs no extra round trip. Otherwise the expansion is done by one extra command over the existing connection.
+
 ### Retry behavior for `.send()` / `.recv()`
 
 `.send()` and `.recv()` automatically retry (up to `hostInfo.maxRetry` times, default `3`, waiting `hostInfo.retryDuration` msec between attempts, default `1000`) when:
@@ -202,3 +213,36 @@ If no agent can be used, or `ssh-add` fails, the wrapper silently falls back to
 **Keyless / browser agents:** with no `keyFile`, set `identityAgent` to the socket of an already-populated agent (for example [`bssh-agent`](https://github.com/so5/browser-ssh-agent), which keeps the private key in a browser tab) and the wrapper will authenticate through it.
 
 **Windows:** an already-running agent (the OpenSSH Authentication Agent service, or a supplied `identityAgent`) is used, but the wrapper never spawns one; without a reachable agent it falls back to the legacy path.
+
+## Running the tests
+
+The tests are split into two groups:
+
+| command | what runs | needs test hosts |
+|---|---|---|
+| `npm run test:unit` | `test/unit`: option building, validation, agent handling, ... | no |
+| `npm run test:integration` | `test/integration`: real `ssh` and `rsync` against test hosts | yes |
+| `npm run mocha` | both | yes |
+| `npm test` | both, with coverage | yes |
+
+`npm run test:unit` prints a reminder that **the integration tests were not run**. A green unit run does not say anything about ssh or rsync.
+
+The integration tests check the test hosts first. If a required environment variable is missing, or a host can not be logged in, the whole run fails at once with a message that says what to check (it does not wait for timeouts of the tests).
+
+### Test hosts
+
+The easiest way is `./run_test_local.sh`. It starts two [`naoso5/test-sshd`](https://hub.docker.com/r/naoso5/test-sshd) containers with Docker Compose, installs a throw-away key, starts an ssh-agent, sets the variables below and runs `npm test`. (If you run it with a different `HOME`, keep the path short: the ssh-agent socket path is limited to about 100 characters.)
+
+To use your own hosts, set these environment variables:
+
+| variable | meaning |
+|---|---|
+| `TEST_HOST`, `TEST_PORT`, `TEST_USER` | the test host (required: `TEST_HOST`) |
+| `TEST_PW` or `TEST_KEYFILE` | password or private key of the test user (an ssh-agent also works) |
+| `TEST_HOST2`, `TEST_PORT2`, `TEST_USER2` | the second host used by `remoteToRemoteCopy`, **as seen from the first test host** (required: `TEST_HOST2`; `TEST_USER2` defaults to `TEST_USER`) |
+| `TEST_HOST2_VERIFY`, `TEST_PORT2_VERIFY` | the same second host, **as seen from the machine which runs the tests** (default: `TEST_HOST2`, `TEST_PORT2`) |
+| `TEST_KEEP_FILES` | if set, the files created on the test hosts are not removed |
+
+The two ways to name the second host differ when the hosts are Docker containers: the first container reaches the second one by its container name (`testbed2:22`), while the test runner reaches it through a published port (`localhost:4101`).
+
+The same tests run in GitHub Actions for Node.js 22/24 and for Debian (see `.github/workflows`). To run the Debian workflow locally, see the comment at the top of `run_test_debian.yml`.

package/lib/remotePath.js

@@ -0,0 +1,124 @@
+import crypto from "crypto";
+import Debug from "debug";
+
+import { sshExec } from "./sshExec.js";
+
+const debug = Debug("sshClientWrapper:debug:remotePath");
+
+//characters which need no shell expansion. wildcards are left to the remote side.
+const notSimpleChar = /[^\w@%+=:,./*?[\]-]/;
+const notSimpleCharGlobal = new RegExp(notSimpleChar.source, "g");
+
+/**
+ * check if the remote rsync arguments are taken literally by rsync.
+ * rsync 3.2.4 or later does not pass remote path to the remote shell by default.
+ * older rsync, openrsync (reports 2.6.9) and rsync4win (reports 0.x) let the remote shell evaluate it.
+ * @param {object} version - rsync version object ({major, minor, patch})
+ * @returns {boolean} - true if the remote path is taken literally
+ */
+export function hasLiteralRemoteArgs(version) {
+  if (!version
+    || !Number.isInteger(version.major)
+    || !Number.isInteger(version.minor)
+    || !Number.isInteger(version.patch)) {
+    return false;
+  }
+  const { major, minor, patch } = version;
+  if (major !== 3) {
+    return major > 3;
+  }
+  if (minor !== 2) {
+    return minor > 2;
+  }
+  return patch >= 4;
+}
+
+/**
+ * check if the remote path contains something which must be evaluated by the remote shell
+ * @param {string} pathSpec - remote path given by the caller
+ * @returns {boolean} - true if the remote path need to be expanded
+ */
+export function needsExpansion(pathSpec) {
+  return notSimpleChar.test(pathSpec);
+}
+
+/**
+ * escape all characters except wildcards with backslash, so that the remote shell passes it as is.
+ * @param {string} word - remote path
+ * @returns {string} - escaped remote path
+ */
+export function escapeRemoteArg(word) {
+  return word.replace(notSimpleCharGlobal, "\\$&");
+}
+
+/**
+ * quote a string with single quote for the remote shell. wildcards are NOT expanded.
+ * @param {string} word - string
+ * @returns {string} - quoted string
+ */
+export function shellQuote(word) {
+  return `'${word.replace(/'/g, "'\\''")}'`;
+}
+
+/**
+ * convert an expanded remote path to the form the local rsync expects.
+ * @param {string} word - expanded remote path
+ * @param {boolean} literal - true if the local rsync takes remote path literally
+ * @returns {string} - remote path for rsync command line
+ */
+export function formatRemoteArg(word, literal) {
+  return literal ? word : escapeRemoteArg(word);
+}
+
+/**
+ * make command line which print expanded remote path.
+ * filename expansion (wildcards) is disabled. it is left to the remote rsync.
+ * @param {string} pathSpec - remote path given by the caller
+ * @param {string} token - random string to find the result in output
+ * @returns {string} - command line for remote shell
+ */
+export function buildExpandCommand(pathSpec, token) {
+  return `set -f; echo ${token}B; for f in ${pathSpec}; do printf '%s\\n' "$f"; done; echo ${token}E`;
+}
+
+/**
+ * pick up expanded words from the output of command which is made by buildExpandCommand
+ * @param {string} output - whole output
+ * @param {string} token - random string which was given to buildExpandCommand
+ * @returns {string[] | null} - expanded words, or null if the output can not be parsed
+ */
+export function parseExpandOutput(output, token) {
+  const lines = output.split(/\r?\n/);
+  const begin = lines.indexOf(`${token}B`);
+  const end = lines.lastIndexOf(`${token}E`);
+  if (begin < 0 || end < begin) {
+    return null;
+  }
+  return lines.slice(begin + 1, end);
+}
+
+/**
+ * let the remote shell expand the remote path (tilde, variables, braces, quotes, and backslash escapes).
+ * if the remote shell can not do it, the path is returned as is.
+ * @param {object} hostInfo - hostinfo object
+ * @param {string} pathSpec - remote path given by the caller
+ * @param {number} timeout - timeout in seconds
+ * @returns {Promise<string[]>} - expanded paths
+ */
+export async function expandRemotePath(hostInfo, pathSpec, timeout = 0) {
+  if (!needsExpansion(pathSpec)) {
+    return [pathSpec];
+  }
+  const token = `SCW${crypto.randomBytes(8).toString("hex")}`;
+  const chunks = [];
+  const rt = await sshExec(hostInfo, buildExpandCommand(pathSpec, token), timeout, (data)=>{
+    chunks.push(data);
+  });
+  const words = parseExpandOutput(chunks.join(""), token);
+  if (rt !== 0 || words === null || words.length === 0) {
+    debug(`can not expand remote path on ${hostInfo.host}, use it as is: ${pathSpec}`);
+    return [pathSpec];
+  }
+  debug(`remote path ${pathSpec} is expanded to`, words);
+  return words;
+}

package/lib/rsyncExec.js

@@ -7,6 +7,7 @@ import { connect, retryWrapper, sshExec } from "./sshExec.js";
 import { fork } from "./fork.js";
 
 import { sshCmd, rsyncCmd, getSshOption, resolveRetryableExitCodes } from "./util.js";
+import { expandRemotePath, formatRemoteArg, hasLiteralRemoteArgs, shellQuote } from "./remotePath.js";
 
 const debug = Debug("sshClientWrapper:debug:rsyncExec");
 const debugVerbose = Debug("sshClientWrapper:verbose:rsyncExec");
@@ -67,39 +68,47 @@ export async function send(hostInfo, argSrc, dst, rsyncOpt, timeout, retryableEx
   const sshArgs = getSshOption(hostInfo, true);
   const rshOpt = ["-e", `${sshCmd} ${sshArgs.join(" ")}`];
 
-  const args = ["-avv", "--copy-unsafe-links", ...rshOpt, ...rsyncOpt, ...src, `${hostInfo.host}:${dst}`];
+  //the remote path is interpreted by the remote shell (~, $VAR, {a,b}, quotes and escapes)
+  const remoteDsts = await expandRemotePath(hostInfo, dst);
+  if (remoteDsts.length !== 1) {
+    throw new Error(`dst must be expanded to exactly one path: ${dst}`);
+  }
+  const [remoteDst] = remoteDsts;
+  const literal = await hasLiteralRemoteArgsOfLocalRsync(hostInfo);
+
+  const args = ["-avv", "--copy-unsafe-links", ...rshOpt, ...rsyncOpt, ...src, `${hostInfo.host}:${formatRemoteArg(remoteDst, literal)}`];
   let remoteDir;
-  if (dst.endsWith("/")) {
-    remoteDir = dst;
+  if (remoteDst.endsWith("/")) {
+    remoteDir = remoteDst;
   } else {
-    const { dir } = path.parse(dst);
+    const { dir } = path.parse(remoteDst);
     remoteDir = dir;
   }
   debug("remoteDir=", remoteDir);
 
   if (remoteDir !== "" && remoteDir !== "/") {
     debug(`mkdir ${remoteDir}`);
-    sshExec(hostInfo, `mkdir -p ${remoteDir}`, debugVerbose);
+    await sshExec(hostInfo, `mkdir -p ${shellQuote(remoteDir)}`, 0, debugVerbose);
   }
-  debug(`send ${src} to ${hostInfo.host}:${dst}`);
+  debug(`send ${src} to ${hostInfo.host}:${remoteDst}`);
   debugVerbose(`send: ${rsyncCmd} ${args.join(" ")}`);
   const exitCodes = resolveRetryableExitCodes(hostInfo, retryableExitCodes, replaceRetryableExitCodes);
   return retryWrapper(fork.bind(null, hostInfo, rsyncCmd, args, timeout, null, exitCodes), hostInfo);
 }
 
 /**
- * format src array to src string for recv
+ * check if the local rsync takes remote path literally (rsync 3.2.4 or later) or lets the remote shell evaluate it
  * @param {object} hostInfo - hostinfo object
- * @returns {boolean} - rsync command version is higher than 3.2.4 or not
+ * @returns {boolean} - true if the local rsync takes remote path literally
  */
-async function isNewRsync(hostInfo) {
+async function hasLiteralRemoteArgsOfLocalRsync(hostInfo) {
   if (!hostInfo.rsyncVersion
     || !Number.isInteger(hostInfo.rsyncVersion.major)
     || !Number.isInteger(hostInfo.rsyncVersion.minor)
     || !Number.isInteger(hostInfo.rsyncVersion.patch)) {
     hostInfo.rsyncVersion = await checkRsyncVersion(3);
   }
-  return hostInfo.rsyncVersion.major >= 3 && hostInfo.rsyncVersion.minor >= 2 && hostInfo.rsyncVersion.patch >= 4;
+  return hasLiteralRemoteArgs(hostInfo.rsyncVersion);
 }
 
 /**
@@ -116,19 +125,28 @@ async function isNewRsync(hostInfo) {
  */
 export async function recv(hostInfo, argSrc, dst, rsyncOpt, timeout, retryableExitCodes, replaceRetryableExitCodes) {
   await connect(hostInfo);
-  const src = `${hostInfo.host}:${argSrc.join(" ")}`;
+  //pass each source as its own "host:path" argument. Joining them into one
+  //space separated argument relies on the remote shell to split it, which
+  //rsync >= 3.2.4 (protected args) no longer does.
+  //the remote path is interpreted by the remote shell (~, $VAR, {a,b}, quotes and escapes).
+  //it is expanded here, so that the result does not depend on how the local rsync handles it.
+  const literal = await hasLiteralRemoteArgsOfLocalRsync(hostInfo);
+  const remoteSrcs = [];
+  for (const e of argSrc) {
+    remoteSrcs.push(...await expandRemotePath(hostInfo, e));
+  }
+  const src = remoteSrcs.map((e)=>{
+    return `${hostInfo.host}:${formatRemoteArg(e, literal)}`;
+  });
   const sshArgs = getSshOption(hostInfo, true);
   const rshOpt = ["-e", `${sshCmd} ${sshArgs.join(" ")}`];
-  const args = ["-avv", "--copy-unsafe-links", ...rshOpt, ...rsyncOpt, src, dst];
-  if (await isNewRsync(hostInfo)) {
-    args.unshift("--old-args");
-  }
+  const args = ["-avv", "--copy-unsafe-links", ...rshOpt, ...rsyncOpt, ...src, dst];
 
   //make destination directory
   const dstDir = argSrc.len > 1 || hasMagic(argSrc[0]) ? dst : path.dirname(dst);
   await fs.mkdir(dstDir, { recursive: true });
 
-  debug(`recv ${src} to ${dst}`);
+  debug(`recv ${src.join(" ")} to ${dst}`);
   debugVerbose(`recv: ${rsyncCmd} ${args.join(" ")}`);
   const exitCodes = resolveRetryableExitCodes(hostInfo, retryableExitCodes, replaceRetryableExitCodes);
   return retryWrapper(fork.bind(null, hostInfo, rsyncCmd, args, timeout, null, exitCodes), hostInfo);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ssh-client-wrapper",
-  "version": "3.1.0",
+  "version": "3.1.2",
   "description": "wrapper library for open ssh client and rsync",
   "keywords": [
     "ssh",
@@ -16,6 +16,8 @@
   "types": "lib/index.d.ts",
   "scripts": {
     "mocha": "mocha --recursive test",
+    "test:unit": "mocha --recursive test/unit && echo \"NOTE: only unit tests were run. The integration tests (ssh, rsync) were NOT run: use npm run test:integration with test hosts\"",
+    "test:integration": "mocha --recursive test/integration",
     "lint": "eslint --fix ./",
     "test": "c8 --reporter=html --reporter=text mocha --recursive test",
     "coverage": "c8 report --reporter=text-lcov > coverage-report.lcov",