msdev 0.10.1__tar.gz → 0.11.0__tar.gz

{msdev-0.10.1 → msdev-0.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: msdev
-Version: 0.10.1
+Version: 0.11.0
 Summary: Portable nodes and execution environments for msModelSlim development
 Author: msModelSlim Agent Contributors
 License-Expression: MulanPSL-2.0
@@ -62,6 +62,12 @@ implement ControlMaster multiplexing, so msdev:
   `env put`/`get` stay on scp/ssh streams (file bytes do not go through JSON RPC);
 - hides SSH console windows (`CREATE_NO_WINDOW`) and kills process trees with
   `taskkill /T` on timeout or cancel;
+- SSH is non-interactive: `BatchMode` unless a password is supplied, host keys
+  use `accept-new`, and there is no password/passphrase/yes-no prompt. When a
+  password is known, injects OpenSSH `SSH_ASKPASS` so `node connect` (and later
+  ssh/scp) can authenticate without a TTY; the local public key is then
+  installed on the Node so later logins are passwordless. The password is
+  never written to the node registry;
 - serves a TCP loopback endpoint for local `msdevd` when this Python build has
   no `AF_UNIX` sockets.
 
@@ -92,10 +98,23 @@ the remote user daemon:
 msdev node bootstrap dev-122
 ```
 
-Persistent OpenSSH masters are optional:
+Persistent OpenSSH masters are optional. All SSH from msdev is
+non-interactive: host keys use `accept-new`, and password/passphrase prompts
+are disabled. Supply a password via environment or file (never a prompt);
+connect then installs the local public key (`~/.ssh/id_ed25519`, generated
+if missing) so later connections are passwordless:
 
 ```bash
+# Environment (also used by exec / env put after connect if persist is off)
+export MSDEV_SSH_PASSWORD='...'
+export MSDEV_SSH_PASSWORD_DEV_122='...'   # per-node override
+
 msdev node connect dev-122
+msdev node connect dev-122 --password-file FILE
+msdev node connect dev-122 --password-env VAR
+msdev node connect dev-122 --password PASSWORD   # non-interactive; visible in ps
+msdev node connect dev-122 --identity FILE     # choose which key to install
+msdev node connect dev-122 --no-install-key    # password only, do not copy key
 msdev node disconnect dev-122
 ```
 
@@ -185,6 +204,13 @@ msdev exec \
   -- python3 run.py
 ```
 
+Quoted pipelines and `|` tokens are executed through a remote POSIX shell
+(`sh -c`), so they run on the Node instead of being treated as a single
+executable name. On a Windows `local` Env they use `cmd.exe /s /c`. Prefer
+`sh -c '…'` for complex scripts. Default streaming writes POSIX newlines as
+CRLF on a Windows console so remote line output does not staircase; piped
+CLI output keeps LF.
+
 `--timeout-seconds` defaults to `-1` (no deadline). Any positive finite duration
 is also accepted. A positive timeout kills the complete command
 process group and returns exit code 124. Cancelling the local CLI closes the RPC

{msdev-0.10.1 → msdev-0.11.0}/README.md

@@ -37,6 +37,12 @@ implement ControlMaster multiplexing, so msdev:
   `env put`/`get` stay on scp/ssh streams (file bytes do not go through JSON RPC);
 - hides SSH console windows (`CREATE_NO_WINDOW`) and kills process trees with
   `taskkill /T` on timeout or cancel;
+- SSH is non-interactive: `BatchMode` unless a password is supplied, host keys
+  use `accept-new`, and there is no password/passphrase/yes-no prompt. When a
+  password is known, injects OpenSSH `SSH_ASKPASS` so `node connect` (and later
+  ssh/scp) can authenticate without a TTY; the local public key is then
+  installed on the Node so later logins are passwordless. The password is
+  never written to the node registry;
 - serves a TCP loopback endpoint for local `msdevd` when this Python build has
   no `AF_UNIX` sockets.
 
@@ -67,10 +73,23 @@ the remote user daemon:
 msdev node bootstrap dev-122
 ```
 
-Persistent OpenSSH masters are optional:
+Persistent OpenSSH masters are optional. All SSH from msdev is
+non-interactive: host keys use `accept-new`, and password/passphrase prompts
+are disabled. Supply a password via environment or file (never a prompt);
+connect then installs the local public key (`~/.ssh/id_ed25519`, generated
+if missing) so later connections are passwordless:
 
 ```bash
+# Environment (also used by exec / env put after connect if persist is off)
+export MSDEV_SSH_PASSWORD='...'
+export MSDEV_SSH_PASSWORD_DEV_122='...'   # per-node override
+
 msdev node connect dev-122
+msdev node connect dev-122 --password-file FILE
+msdev node connect dev-122 --password-env VAR
+msdev node connect dev-122 --password PASSWORD   # non-interactive; visible in ps
+msdev node connect dev-122 --identity FILE     # choose which key to install
+msdev node connect dev-122 --no-install-key    # password only, do not copy key
 msdev node disconnect dev-122
 ```
 
@@ -160,6 +179,13 @@ msdev exec \
   -- python3 run.py
 ```
 
+Quoted pipelines and `|` tokens are executed through a remote POSIX shell
+(`sh -c`), so they run on the Node instead of being treated as a single
+executable name. On a Windows `local` Env they use `cmd.exe /s /c`. Prefer
+`sh -c '…'` for complex scripts. Default streaming writes POSIX newlines as
+CRLF on a Windows console so remote line output does not staircase; piped
+CLI output keeps LF.
+
 `--timeout-seconds` defaults to `-1` (no deadline). Any positive finite duration
 is also accepted. A positive timeout kills the complete command
 process group and returns exit code 124. Cancelling the local CLI closes the RPC

{msdev-0.10.1 → msdev-0.11.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "msdev"
-version = "0.10.1"
+version = "0.11.0"
 description = "Portable nodes and execution environments for msModelSlim development"
 readme = "README.md"
 requires-python = ">=3.10"

{msdev-0.10.1 → msdev-0.11.0}/src/msdev/__init__.py

@@ -1,3 +1,3 @@
 """Portable node and execution-environment tooling for msModelSlim."""
 
-__version__ = "0.10.1"
+__version__ = "0.11.0"

{msdev-0.10.1 → msdev-0.11.0}/src/msdev/cli.py

@@ -12,6 +12,7 @@ from dataclasses import asdict
 from pathlib import Path
 from typing import Any
 
+from .core.compat import ConsoleChunkWriter
 from .core.guides import (
     EnvironmentGuideStore,
     MAX_GUIDE_BYTES,
@@ -48,6 +49,7 @@ from .core.services import (
     NodeService,
 )
 from .core.resources import EnvironmentStore, NodeStore
+from .core.ssh_auth import apply_password_sources
 from .core.transport import (
     RpcError,
     SshRpcTransport,
@@ -205,9 +207,12 @@ def command_node(
             if item.error is not None:
                 print(f"{item.node}: error: {item.error}", file=sys.stderr)
             elif action == "connect":
+                suffix = f"(persist={item.persist})"
+                key = getattr(item, "key", None)
+                if key:
+                    suffix = f"(persist={item.persist}, key={key})"
                 print(
-                    f"{item.node}: {item.state} to {item.ssh_host} "
-                    f"(persist={item.persist})"
+                    f"{item.node}: {item.state} to {item.ssh_host} {suffix}"
                 )
             else:
                 print(f"{item.node}: {item.state}")
@@ -672,17 +677,13 @@ def command_exec(
         result = service.run(request)
         _print_json(result.to_dict())
     else:
+        writers = {
+            "stdout": ConsoleChunkWriter(sys.stdout),
+            "stderr": ConsoleChunkWriter(sys.stderr),
+        }
+
         def on_output(stream_name: str, chunk: bytes) -> None:
-            destination = (
-                sys.stdout if stream_name == "stdout" else sys.stderr
-            )
-            binary = getattr(destination, "buffer", None)
-            if binary is not None:
-                binary.write(chunk)
-                binary.flush()
-            else:
-                destination.write(chunk.decode("utf-8", errors="replace"))
-                destination.flush()
+            writers.get(stream_name, writers["stderr"]).write(chunk)
 
         result = service.stream(request, on_output)
     return result.returncode
@@ -748,10 +749,47 @@ def build_parser() -> argparse.ArgumentParser:
     node_bootstrap.add_argument("--package-path", type=Path)
     node_connect = node_sub.add_parser(
         "connect",
-        help="open a persistent SSH master and prompt once for authentication",
+        help="open a persistent SSH master without prompting",
     )
     node_connect.add_argument("name", nargs="?")
     node_connect.add_argument("--all", action="store_true")
+    node_connect.add_argument(
+        "--password",
+        default=None,
+        metavar="PASSWORD",
+        help=(
+            "SSH password for this process only (non-interactive). "
+            "Prefer --password-file or MSDEV_SSH_PASSWORD. "
+            "Not stored in the node registry"
+        ),
+    )
+    node_connect.add_argument(
+        "--password-file",
+        "--ssh-password-file",
+        type=Path,
+        dest="ssh_password_file",
+        metavar="FILE",
+        help="read SSH password from FILE (first line); not stored in the registry",
+    )
+    node_connect.add_argument(
+        "--password-env",
+        "--ssh-password-env",
+        dest="ssh_password_env",
+        metavar="VAR",
+        help="read SSH password from environment variable VAR",
+    )
+    node_connect.add_argument(
+        "--identity",
+        type=Path,
+        dest="ssh_identity",
+        metavar="FILE",
+        help="private key to install and use (default: ~/.ssh/id_ed25519)",
+    )
+    node_connect.add_argument(
+        "--no-install-key",
+        action="store_true",
+        help="do not install the local public key during password connect",
+    )
     node_disconnect = node_sub.add_parser(
         "disconnect",
         help="close a persistent SSH master",
@@ -1040,6 +1078,20 @@ def _watch_parent_exit(
 def main() -> None:
     parser = build_parser()
     args = parser.parse_args()
+    try:
+        apply_password_sources(
+            password=getattr(args, "password", None),
+            password_file=getattr(args, "ssh_password_file", None),
+            password_env=getattr(args, "ssh_password_env", None),
+        )
+    except ValueError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        raise SystemExit(2) from exc
+    identity = getattr(args, "ssh_identity", None)
+    if identity is not None:
+        os.environ["MSDEV_SSH_IDENTITY"] = str(identity)
+    if getattr(args, "no_install_key", False):
+        os.environ["MSDEV_SSH_INSTALL_KEY"] = "0"
     node_store = NodeStore(args.config)
     environment_store = EnvironmentStore(args.config)
     previous_sigterm = signal.getsignal(signal.SIGTERM)

{msdev-0.10.1 → msdev-0.11.0}/src/msdev/core/compat.py

@@ -49,6 +49,14 @@ def scp_executable() -> str:
     return fallback or "scp"
 
 
+def ssh_keygen_executable() -> str:
+    """Resolve ssh-keygen without requiring OpenSSH to be on PATH."""
+    if shutil.which("ssh-keygen"):
+        return "ssh-keygen"
+    fallback = _windows_openssh("ssh-keygen.exe") if IS_WINDOWS else None
+    return fallback or "ssh-keygen"
+
+
 def ssh_control_master_enabled() -> bool:
     """OpenSSH ControlMaster is unsupported by native Windows OpenSSH."""
     override = os.environ.get("MSDEV_SSH_CONTROL_MASTER", "").strip().lower()
@@ -87,6 +95,38 @@ def normalize_captured_text(value: str) -> str:
     return value.replace("\r\n", "\n").replace("\r", "\n")
 
 
+def translate_lf_to_crlf(chunk: bytes, carry_cr: bool) -> tuple[bytes, bool]:
+    """Convert POSIX newlines to Windows console CRLF without doubling CR."""
+    if carry_cr:
+        chunk = b"\r" + chunk
+    if chunk.endswith(b"\r"):
+        return (
+            chunk[:-1].replace(b"\r\n", b"\n").replace(b"\n", b"\r\n"),
+            True,
+        )
+    return chunk.replace(b"\r\n", b"\n").replace(b"\n", b"\r\n"), False
+
+
+class ConsoleChunkWriter:
+    """Write exec chunks to a text stream, fixing Windows TTY staircase output."""
+
+    def __init__(self, stream: Any) -> None:
+        self.stream = stream
+        self.binary = getattr(stream, "buffer", None)
+        self._translate = IS_WINDOWS and bool(getattr(stream, "isatty", lambda: False)())
+        self._carry_cr = False
+
+    def write(self, chunk: bytes) -> None:
+        if self._translate:
+            chunk, self._carry_cr = translate_lf_to_crlf(chunk, self._carry_cr)
+        if self.binary is not None:
+            self.binary.write(chunk)
+            self.binary.flush()
+            return
+        self.stream.write(chunk.decode("utf-8", errors="replace"))
+        self.stream.flush()
+
+
 def is_unix_socket(path: Path) -> bool:
     try:
         return stat.S_ISSOCK(path.lstat().st_mode)

{msdev-0.10.1 → msdev-0.11.0}/src/msdev/core/env_transfer.py

@@ -14,6 +14,13 @@ from typing import Any, BinaryIO
 from .compat import popen_kwargs
 
 
+def _run_kw(env: dict[str, str] | None = None) -> dict[str, Any]:
+    kwargs = popen_kwargs(hidden=True, new_group=False)
+    if env is not None:
+        kwargs["env"] = env
+    return kwargs
+
+
 def validate_absolute_local(path: str, *, label: str) -> str:
     """Accept a host-local absolute path (POSIX or Windows)."""
     if not isinstance(path, str) or not path or "\0" in path:
@@ -100,14 +107,21 @@ def local_get_file(
     return result
 
 
-def _run(argv: list[str], *, label: str, timeout: float | None) -> None:
+def _run(
+    argv: list[str],
+    *,
+    label: str,
+    timeout: float | None,
+    env: dict[str, str] | None = None,
+) -> None:
     try:
         completed = subprocess.run(
             argv,
             capture_output=True,
             check=False,
             timeout=timeout,
-            **popen_kwargs(hidden=True, new_group=False),
+            stdin=subprocess.DEVNULL,
+            **_run_kw(env),
         )
     except subprocess.TimeoutExpired as exc:
         raise RuntimeError(f"{label} timed out") from exc
@@ -134,6 +148,7 @@ def ssh_host_put_file(
     local_path: Path,
     remote_path: str,
     timeout: float | None,
+    env: dict[str, str] | None = None,
 ) -> dict[str, Any]:
     destination = validate_absolute_remote(remote_path, label="remote path")
     parent = str(PurePosixPath(destination).parent)
@@ -142,11 +157,13 @@ def ssh_host_put_file(
         [*ssh_argv, f"mkdir -p -- {shlex.quote(parent)}"],
         label="remote mkdir for env put",
         timeout=timeout,
+        env=env,
     )
     _run(
         [*scp_argv, str(local_path), _remote_spec(ssh_host, staging)],
         label="scp env put",
         timeout=timeout,
+        env=env,
     )
     _run(
         [
@@ -155,6 +172,7 @@ def ssh_host_put_file(
         ],
         label="remote finalize env put",
         timeout=timeout,
+        env=env,
     )
     return {
         "path": destination,
@@ -174,6 +192,7 @@ def ssh_host_get_file(
     output: Path | None,
     sink: BinaryIO | None,
     timeout: float | None,
+    env: dict[str, str] | None = None,
 ) -> dict[str, Any]:
     source = validate_absolute_remote(remote_path, label="remote path")
     if output is not None:
@@ -183,6 +202,7 @@ def ssh_host_get_file(
             [*scp_argv, _remote_spec(ssh_host, source), str(output)],
             label="scp env get",
             timeout=timeout,
+            env=env,
         )
         return file_result(output, runtime="host")
     if sink is not None:
@@ -190,7 +210,8 @@ def ssh_host_get_file(
             [*ssh_argv, f"cat -- {shlex.quote(source)}"],
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
-            **popen_kwargs(hidden=True, new_group=False),
+            stdin=subprocess.DEVNULL,
+            **_run_kw(env),
         )
         assert process.stdout is not None
         try:
@@ -210,7 +231,8 @@ def ssh_host_get_file(
         capture_output=True,
         check=False,
         timeout=timeout,
-        **popen_kwargs(hidden=True, new_group=False),
+        stdin=subprocess.DEVNULL,
+        **_run_kw(env),
     )
     if completed.returncode != 0:
         detail = (completed.stderr or b"").decode("utf-8", errors="replace").strip()
@@ -237,6 +259,7 @@ def ssh_docker_put_file(
     local_path: Path,
     remote_path: str,
     timeout: float | None,
+    env: dict[str, str] | None = None,
 ) -> dict[str, Any]:
     destination = validate_absolute_remote(remote_path, label="remote path")
     parent = str(PurePosixPath(destination).parent)
@@ -245,6 +268,7 @@ def ssh_docker_put_file(
         [*scp_argv, str(local_path), _remote_spec(ssh_host, staging)],
         label="scp env docker put",
         timeout=timeout,
+        env=env,
     )
     script = (
         f"docker exec {shlex.quote(container)} "
@@ -253,7 +277,7 @@ def ssh_docker_put_file(
         f"{shlex.quote(container)}:{shlex.quote(destination)} && "
         f"rm -f -- {shlex.quote(staging)}"
     )
-    _run([*ssh_argv, script], label="remote docker cp env put", timeout=timeout)
+    _run([*ssh_argv, script], label="remote docker cp env put", timeout=timeout, env=env)
     return {
         "path": destination,
         "type": "file",
@@ -273,6 +297,7 @@ def ssh_docker_get_file(
     output: Path | None,
     sink: BinaryIO | None,
     timeout: float | None,
+    env: dict[str, str] | None = None,
 ) -> dict[str, Any]:
     source = validate_absolute_remote(remote_path, label="remote path")
     if output is not None:
@@ -281,7 +306,12 @@ def ssh_docker_get_file(
             f"docker cp -- {shlex.quote(container)}:{shlex.quote(source)} "
             f"{shlex.quote(staging)}"
         )
-        _run([*ssh_argv, script], label="remote docker cp env get", timeout=timeout)
+        _run(
+            [*ssh_argv, script],
+            label="remote docker cp env get",
+            timeout=timeout,
+            env=env,
+        )
         try:
             output = output.expanduser()
             output.parent.mkdir(parents=True, exist_ok=True)
@@ -289,6 +319,7 @@ def ssh_docker_get_file(
                 [*scp_argv, _remote_spec(ssh_host, staging), str(output)],
                 label="scp env docker get",
                 timeout=timeout,
+                env=env,
             )
             return file_result(output, runtime="docker")
         finally:
@@ -296,6 +327,7 @@ def ssh_docker_get_file(
                 [*ssh_argv, f"rm -f -- {shlex.quote(staging)}"],
                 label="remote cleanup env docker get",
                 timeout=timeout,
+                env=env,
             )
     if sink is not None:
         process = subprocess.Popen(
@@ -310,7 +342,8 @@ def ssh_docker_get_file(
             ],
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
-            **popen_kwargs(hidden=True, new_group=False),
+            stdin=subprocess.DEVNULL,
+            **_run_kw(env),
         )
         assert process.stdout is not None
         try:
@@ -336,7 +369,8 @@ def ssh_docker_get_file(
         capture_output=True,
         check=False,
         timeout=timeout,
-        **popen_kwargs(hidden=True, new_group=False),
+        stdin=subprocess.DEVNULL,
+        **_run_kw(env),
     )
     if completed.returncode != 0:
         detail = (completed.stderr or b"").decode("utf-8", errors="replace").strip()

msdev-0.11.0/src/msdev/core/exec_argv.py

@@ -0,0 +1,82 @@
+"""Wrap user exec argv that uses shell pipelines or operators."""
+
+from __future__ import annotations
+
+import re
+import shlex
+import subprocess
+from pathlib import Path
+
+OPERATOR_TOKENS = frozenset({"|", "||", "&&", ";", "&", ">", "<", ">>"})
+_SHELL_SCRIPT_RE = re.compile(r"[|;&<>()`]|\$\(|\$\{|&&|\|\|")
+
+
+def _command_name(command: list[str]) -> str:
+    return Path(command[0]).name.lower()
+
+
+def already_shelled(command: list[str]) -> bool:
+    if len(command) < 3:
+        return False
+    name = _command_name(command)
+    if name in {"sh", "bash", "zsh", "dash"}:
+        return command[1] in {"-c", "-lc", "-ic"}
+    if name in {"cmd", "cmd.exe"}:
+        return any(part.lower() in {"/c", "/k"} for part in command[1:3])
+    if name in {"powershell", "powershell.exe", "pwsh", "pwsh.exe"}:
+        return any(
+            part.lower() in {"-c", "-command", "-encodedcommand"}
+            for part in command[1:4]
+        )
+    return False
+
+
+def needs_shell(command: list[str]) -> bool:
+    if not command or already_shelled(command):
+        return False
+    if len(command) == 1:
+        return bool(_SHELL_SCRIPT_RE.search(command[0]))
+    return any(item in OPERATOR_TOKENS for item in command)
+
+
+def join_shell_argv(command: list[str], *, windows: bool) -> str:
+    if len(command) == 1:
+        return command[0]
+    parts: list[str] = []
+    for item in command:
+        if item in OPERATOR_TOKENS:
+            parts.append(item)
+        elif windows:
+            parts.append(subprocess.list2cmdline([item]))
+        else:
+            parts.append(shlex.quote(item))
+    return " ".join(parts)
+
+
+def wrap_exec_argv(command: list[str], *, windows: bool) -> list[str]:
+    """Run pipelines through a shell; leave plain argv unchanged."""
+    wrapped = list(command)
+    if not needs_shell(wrapped):
+        return wrapped
+    script = join_shell_argv(wrapped, windows=windows)
+    if windows:
+        return ["cmd.exe", "/s", "/c", script]
+    return ["sh", "-c", script]
+
+
+def popen_exec_command(command: list[str]) -> str | list[str]:
+    """Avoid a second list2cmdline pass around ``cmd /c`` scripts.
+
+    ``subprocess.Popen(list)`` on Windows quotes each argument again, which
+    breaks nested quotes and ``|`` inside the script. Build the CreateProcess
+    string as ``cmd.exe /s /c "script"`` so cmd strips only the outer quotes.
+    """
+    if len(command) < 4:
+        return command
+    if _command_name(command) not in {"cmd", "cmd.exe"}:
+        return command
+    if not any(part.lower() in {"/c", "/k"} for part in command[1:3]):
+        return command
+    switches = command[1:-1]
+    script = command[-1]
+    return subprocess.list2cmdline([command[0], *switches]) + ' "' + script + '"'

{msdev-0.10.1 → msdev-0.11.0}/src/msdev/core/services/execution.py

@@ -5,6 +5,8 @@ from __future__ import annotations
 from dataclasses import dataclass, field
 from typing import Any, Callable, Mapping
 
+from ..compat import IS_WINDOWS
+from ..exec_argv import wrap_exec_argv
 from ..limits import (
     DEFAULT_EXEC_TIMEOUT_SECONDS,
     validate_exec_timeout_seconds,
@@ -57,6 +59,12 @@ class ExecutionService:
         argv = tuple(request.argv)
         if not argv or not all(isinstance(item, str) and item for item in argv):
             raise ValueError("exec argv must be a non-empty string array")
+        argv = tuple(
+            wrap_exec_argv(
+                list(argv),
+                windows=environment == "local" and IS_WINDOWS,
+            )
+        )
         env: dict[str, str] = {}
         for key, value in request.env.items():
             if not isinstance(key, str) or not key:

{msdev-0.10.1 → msdev-0.11.0}/src/msdev/core/services/node.py

@@ -35,6 +35,7 @@ class NodeConnectionResult:
     state: str | None = None
     persist: str | None = None
     error: str | None = None
+    key: str | None = None
 
 
 @dataclass(frozen=True)
@@ -156,6 +157,7 @@ class NodeService:
                         node.ssh_host,
                         state,
                         persist,
+                        key=result.get("key") if action == "connect" else None,
                     )
                 )
             except (RpcError, OSError) as exc: