msdev 0.15.1__tar.gz → 0.16.0__tar.gz

{msdev-0.15.1 → msdev-0.16.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: msdev
-Version: 0.15.1
+Version: 0.16.0
 Summary: Portable nodes and execution environments for msModelSlim development
 Author: msModelSlim Agent Contributors
 License-Expression: MulanPSL-2.0
@@ -44,6 +44,7 @@ git clone https://gitcode.com/rookie_hongchuan/msdev.git
 cd msdev
 python3 -m pip install -e .
 msdev --help
+msdev --version
 ```
 
 The package installs the `msdev` CLI.
@@ -198,9 +199,32 @@ msdev env get dev-122 /srv/project/config.yaml > ./config.yaml
 ```
 
 Editing uses content-oriented read/write. Paths inside the Env must be
-absolute. Host Envs use direct filesystem copy locally and ``sftp``/``ssh``
-streams remotely. Docker Envs stage through the node with ``docker cp``.
-File bytes never pass through JSON RPC.
+absolute. Host Envs copy directly on the local filesystem; remote Envs stream
+over SSH. Docker Envs stage through the node with ``docker cp``. File bytes
+never pass through JSON RPC.
+
+### Transfer backends
+
+Transfers go through one pluggable component with two backends:
+
+- `sftp` — OpenSSH SFTP batch, atomic `put staging` + `rename`.
+- `exec` — binary stdin/stdout over the SSH exec channel (`cat`, `mktemp`,
+  `mv`), for hosts where the SFTP subsystem is disabled or blocked.
+
+`auto` (the default) tries `sftp`, then falls back to `exec` only when SFTP is
+unavailable (subsystem rejected, connection closed, local `sftp` missing).
+Filesystem errors such as permission denied are never retried on another
+backend.
+
+```bash
+msdev env add dev-122 --node dev-122 --transfer auto
+msdev env put dev-122 ./payload.bin /srv/payload.bin --transfer exec
+msdev env get dev-122 /srv/payload.bin --output ./payload.bin --transfer sftp
+```
+
+`env add --transfer` stores the preference in the env profile; `--transfer` on
+put/get/read/write overrides it for one command. `MSDEV_TRANSFER` overrides
+both. Explicit `sftp`/`exec` never falls back.
 
 ```bash
 msdev env read dev-122 /srv/project/config.yaml
@@ -250,6 +274,11 @@ argument after `exec`; use `-c` for inline strings or `--file` for script
 content (path or `-` for stdin/heredoc). `-c -` is accepted but identical to
 `--file -`; prefer **`--file -`** only.
 
+Scripts up to 64 KiB of valid UTF-8 are inlined into the remote command.
+Larger or non-UTF-8 scripts are uploaded through the transfer component to a
+temporary file, executed, and removed afterwards. Set
+`MSDEV_EXEC_INLINE_BYTES` to change the threshold.
+
 Quoted pipelines and `|` tokens are executed through a remote POSIX shell
 (`sh -c`), so they run on the Node instead of being treated as a single
 executable name. On a Windows `local` Env they use `cmd.exe /s /c`. Prefer

{msdev-0.15.1 → msdev-0.16.0}/README.md

@@ -19,6 +19,7 @@ git clone https://gitcode.com/rookie_hongchuan/msdev.git
 cd msdev
 python3 -m pip install -e .
 msdev --help
+msdev --version
 ```
 
 The package installs the `msdev` CLI.
@@ -173,9 +174,32 @@ msdev env get dev-122 /srv/project/config.yaml > ./config.yaml
 ```
 
 Editing uses content-oriented read/write. Paths inside the Env must be
-absolute. Host Envs use direct filesystem copy locally and ``sftp``/``ssh``
-streams remotely. Docker Envs stage through the node with ``docker cp``.
-File bytes never pass through JSON RPC.
+absolute. Host Envs copy directly on the local filesystem; remote Envs stream
+over SSH. Docker Envs stage through the node with ``docker cp``. File bytes
+never pass through JSON RPC.
+
+### Transfer backends
+
+Transfers go through one pluggable component with two backends:
+
+- `sftp` — OpenSSH SFTP batch, atomic `put staging` + `rename`.
+- `exec` — binary stdin/stdout over the SSH exec channel (`cat`, `mktemp`,
+  `mv`), for hosts where the SFTP subsystem is disabled or blocked.
+
+`auto` (the default) tries `sftp`, then falls back to `exec` only when SFTP is
+unavailable (subsystem rejected, connection closed, local `sftp` missing).
+Filesystem errors such as permission denied are never retried on another
+backend.
+
+```bash
+msdev env add dev-122 --node dev-122 --transfer auto
+msdev env put dev-122 ./payload.bin /srv/payload.bin --transfer exec
+msdev env get dev-122 /srv/payload.bin --output ./payload.bin --transfer sftp
+```
+
+`env add --transfer` stores the preference in the env profile; `--transfer` on
+put/get/read/write overrides it for one command. `MSDEV_TRANSFER` overrides
+both. Explicit `sftp`/`exec` never falls back.
 
 ```bash
 msdev env read dev-122 /srv/project/config.yaml
@@ -225,6 +249,11 @@ argument after `exec`; use `-c` for inline strings or `--file` for script
 content (path or `-` for stdin/heredoc). `-c -` is accepted but identical to
 `--file -`; prefer **`--file -`** only.
 
+Scripts up to 64 KiB of valid UTF-8 are inlined into the remote command.
+Larger or non-UTF-8 scripts are uploaded through the transfer component to a
+temporary file, executed, and removed afterwards. Set
+`MSDEV_EXEC_INLINE_BYTES` to change the threshold.
+
 Quoted pipelines and `|` tokens are executed through a remote POSIX shell
 (`sh -c`), so they run on the Node instead of being treated as a single
 executable name. On a Windows `local` Env they use `cmd.exe /s /c`. Prefer

{msdev-0.15.1 → msdev-0.16.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "msdev"
-version = "0.15.1"
+version = "0.16.0"
 description = "Portable nodes and execution environments for msModelSlim development"
 readme = "README.md"
 requires-python = ">=3.10"

{msdev-0.15.1 → msdev-0.16.0}/src/msdev/__init__.py

@@ -1,3 +1,3 @@
 """Portable node and execution-environment tooling for msModelSlim."""
 
-__version__ = "0.15.1"
+__version__ = "0.16.0"

{msdev-0.15.1 → msdev-0.16.0}/src/msdev/cli.py

@@ -7,7 +7,9 @@ import json
 import os
 import signal
 import sys
+import tempfile
 import threading
+import uuid
 from dataclasses import asdict
 from pathlib import Path
 from typing import Any
@@ -321,6 +323,11 @@ def command_environment(
 ) -> int:
     service = EnvironmentService(_service_context(node_store, environment_store))
     action = args.environment_command
+    if (
+        action in {"put", "get", "read", "write"}
+        and getattr(args, "transfer", None)
+    ):
+        os.environ["MSDEV_TRANSFER"] = args.transfer
     if action == "guide":
         environment_store.get(args.name)
         return _command_guide(args, EnvironmentGuideStore(), "environment")
@@ -338,6 +345,7 @@ def command_environment(
                 setup=_read_setup(args.setup_file, args.setup),
                 wrap=tuple(args.wrap),
                 env_vars=_parse_env_vars(args.env_var),
+                transfer=args.transfer,
                 force=args.force,
             )
         )
@@ -416,12 +424,15 @@ def command_environment(
             sys.stdout.buffer.flush()
         return 0
     if action == "write":
-        content = _env_write_content(args)
+        content = None if args.file is not None else _env_write_content(args)
         result = service.write(
             EnvironmentWriteRequest(
                 environment=args.name,
                 remote_path=args.remote_path,
                 content=content,
+                source_path=(
+                    str(args.file) if args.file is not None else None
+                ),
                 expected_sha256=args.expected_sha256,
             )
         )
@@ -438,23 +449,119 @@ def _env_write_content(args: argparse.Namespace) -> bytes:
     return sys.stdin.buffer.read()
 
 
+def _exec_inline_limit() -> int:
+    raw = os.environ.get("MSDEV_EXEC_INLINE_BYTES", "65536").strip()
+    try:
+        value = int(raw)
+    except ValueError:
+        return 65536
+    return max(0, value)
+
+
+def _is_inline_script(raw: bytes, limit: int) -> bool:
+    if len(raw) > limit or b"\0" in raw:
+        return False
+    try:
+        raw.decode("utf-8")
+    except UnicodeDecodeError:
+        return False
+    return True
+
+
+def _upload_exec_script(
+    environment: str,
+    raw: bytes,
+    node_store: NodeStore,
+    environment_store: EnvironmentStore,
+) -> tuple[str, ...]:
+    """Upload a script through the transfer component and return its argv."""
+    context = _service_context(node_store, environment_store)
+    transport = context.transport_for_environment(environment)
+    remote = f"/tmp/msdev-exec-{uuid.uuid4().hex}.script"
+    with tempfile.NamedTemporaryFile(
+        prefix="msdev-exec-",
+        delete=False,
+    ) as handle:
+        temporary = Path(handle.name)
+        handle.write(raw)
+    try:
+        transport.put_file(temporary, remote)
+    finally:
+        temporary.unlink(missing_ok=True)
+    if raw.startswith(b"#!"):
+        line = raw.split(b"\n", 1)[0][2:].strip().decode("utf-8", "replace")
+        interpreter = line.split()
+        if interpreter:
+            # Run the interpreter explicitly: no exec bit and no noexec /tmp.
+            cleanup = (
+                'p=$1; shift; trap \'rm -f -- "$p"\' EXIT; "$@" "$p"'
+            )
+            return (
+                "sh",
+                "-c",
+                cleanup,
+                "msdev-exec",
+                remote,
+                *interpreter,
+            )
+    cleanup = 'p=$1; trap \'rm -f -- "$p"\' EXIT; sh "$p"'
+    return ("sh", "-c", cleanup, "msdev-exec", remote)
+
+
 def command_exec(
     args: argparse.Namespace,
     node_store: NodeStore,
     environment_store: EnvironmentStore,
 ) -> int:
     windows = args.environment == "local" and IS_WINDOWS
-    script_stdin: str | None = None
-    if args.shell_command == "-" or (
+    limit = _exec_inline_limit()
+    command: tuple[str, ...]
+    if args.command_file is not None and str(args.command_file) != "-":
+        source = args.command_file
+        if not source.is_file():
+            raise ValueError(f"script file not found: {source}")
+        raw = source.read_bytes()
+        if _is_inline_script(raw, limit):
+            command = resolve_exec_argv(
+                shell_command=None,
+                command_file=source,
+                windows=windows,
+            )
+        else:
+            if windows:
+                raise ValueError(
+                    "inline exec script exceeds the local Windows limit "
+                    f"({limit} bytes); use a smaller script or a remote env"
+                )
+            command = _upload_exec_script(
+                args.environment, raw, node_store, environment_store
+            )
+    elif args.shell_command == "-" or (
         args.command_file is not None and str(args.command_file) == "-"
     ):
-        script_stdin = sys.stdin.read()
-    command = resolve_exec_argv(
-        shell_command=args.shell_command,
-        command_file=args.command_file,
-        script_stdin=script_stdin,
-        windows=windows,
-    )
+        raw = sys.stdin.buffer.read()
+        if _is_inline_script(raw, limit):
+            command = resolve_exec_argv(
+                shell_command=None,
+                command_file=Path("-"),
+                script_stdin=raw.decode("utf-8"),
+                windows=windows,
+            )
+        else:
+            if windows:
+                raise ValueError(
+                    "inline exec script exceeds the local Windows limit "
+                    f"({limit} bytes); use a smaller script or a remote env"
+                )
+            command = _upload_exec_script(
+                args.environment, raw, node_store, environment_store
+            )
+    else:
+        command = resolve_exec_argv(
+            shell_command=args.shell_command,
+            command_file=None,
+            windows=windows,
+        )
     env: dict[str, str] = {}
     for value in args.env_var:
         key, separator, item = value.partition("=")
@@ -594,6 +701,12 @@ def build_parser() -> argparse.ArgumentParser:
         default=None,
         help="override local node/environment registry path",
     )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"%(prog)s {__version__}",
+        help="print the msdev version and exit",
+    )
     sub = parser.add_subparsers(dest="command", required=True)
 
     node = sub.add_parser("node", help="manage SSH connection nodes")
@@ -766,6 +879,12 @@ def build_parser() -> argparse.ArgumentParser:
         metavar="KEY=VALUE",
         help="static environment variable stored in the env profile",
     )
+    environment_add.add_argument(
+        "--transfer",
+        choices=("auto", "sftp", "exec", "scp"),
+        default="auto",
+        help="file transfer backend stored in the env profile (default: auto)",
+    )
     environment_sub.add_parser("list")
     environment_inspect = environment_sub.add_parser("inspect")
     environment_inspect.add_argument("name")
@@ -799,6 +918,12 @@ def build_parser() -> argparse.ArgumentParser:
         "remote_path",
         help="absolute destination path inside the environment",
     )
+    environment_put.add_argument(
+        "--transfer",
+        choices=("auto", "sftp", "exec"),
+        default=None,
+        help="override the transfer backend for this operation",
+    )
     environment_get = environment_sub.add_parser(
         "get",
         help="fetch a file from an environment filesystem",
@@ -819,6 +944,12 @@ def build_parser() -> argparse.ArgumentParser:
         action="store_true",
         help="print metadata JSON when --output is used",
     )
+    environment_get.add_argument(
+        "--transfer",
+        choices=("auto", "sftp", "exec"),
+        default=None,
+        help="override the transfer backend for this operation",
+    )
     environment_read = environment_sub.add_parser(
         "read",
         help="read a remote file for editing (stdout or --json with sha256)",
@@ -833,6 +964,12 @@ def build_parser() -> argparse.ArgumentParser:
         action="store_true",
         help="print metadata JSON including sha256 and content_base64",
     )
+    environment_read.add_argument(
+        "--transfer",
+        choices=("auto", "sftp", "exec"),
+        default=None,
+        help="override the transfer backend for this operation",
+    )
     environment_write = environment_sub.add_parser(
         "write",
         help="atomically replace a remote file from stdin, --file, or --content",
@@ -856,6 +993,12 @@ def build_parser() -> argparse.ArgumentParser:
         "--expected-sha256",
         help="compare-and-swap: require this digest before replacing",
     )
+    environment_write.add_argument(
+        "--transfer",
+        choices=("auto", "sftp", "exec"),
+        default=None,
+        help="override the transfer backend for this operation",
+    )
     environment_remove = environment_sub.add_parser("remove")
     environment_remove.add_argument("name")
 

msdev-0.16.0/src/msdev/core/env_transfer.py

@@ -0,0 +1,107 @@
+"""Host-local file transfer helpers used by the local transport."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import stat
+import uuid
+from pathlib import Path, PurePosixPath
+from typing import Any, BinaryIO
+
+
+
+def validate_absolute_local(path: str, *, label: str) -> str:
+    """Accept a host-local absolute path (POSIX or Windows)."""
+    if not isinstance(path, str) or not path or "\0" in path:
+        raise ValueError(f"{label} must be a non-empty NUL-free string")
+    candidate = Path(path)
+    if not candidate.is_absolute():
+        raise ValueError(f"{label} must be an absolute path")
+    return str(candidate)
+
+
+def validate_absolute_remote(path: str, *, label: str) -> str:
+    if not isinstance(path, str) or not path or "\0" in path:
+        raise ValueError(f"{label} must be a non-empty NUL-free string")
+    if not path.startswith("/"):
+        raise ValueError(f"{label} must be an absolute path")
+    normalized = PurePosixPath(path)
+    if str(normalized) != path:
+        raise ValueError(f"{label} must be a canonical absolute POSIX path")
+    if any(part in {"", ".", ".."} for part in normalized.parts[1:]):
+        raise ValueError(f"{label} must not contain '.' or '..' components")
+    return path
+
+
+def file_result(path: str | os.PathLike[str], *, runtime: str) -> dict[str, Any]:
+    target = Path(path)
+    metadata = target.lstat()
+    if not stat.S_ISREG(metadata.st_mode):
+        raise ValueError(f"path is not a regular file: {target}")
+    return {
+        "path": str(target),
+        "type": "file",
+        "size": metadata.st_size,
+        "runtime": runtime,
+        "atomic": True,
+    }
+
+
+def _atomic_copy(source: Path, destination: Path) -> None:
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    if destination.exists() and destination.is_symlink():
+        raise OSError(f"symlink is forbidden: {destination}")
+    if destination.exists() and destination.is_dir():
+        raise IsADirectoryError(f"path is a directory: {destination}")
+    temporary = destination.with_name(f".msdev-put-{uuid.uuid4().hex}")
+    try:
+        shutil.copyfile(source, temporary)
+        os.replace(temporary, destination)
+    finally:
+        temporary.unlink(missing_ok=True)
+
+
+def local_put_file(local_path: Path, remote_path: str) -> dict[str, Any]:
+    source = local_path.expanduser().resolve()
+    if not source.is_file():
+        raise FileNotFoundError(f"local file not found: {source}")
+    destination = Path(validate_absolute_local(remote_path, label="remote path"))
+    _atomic_copy(source, destination)
+    return file_result(destination, runtime="host")
+
+
+def local_get_file(
+    remote_path: str,
+    *,
+    output: Path | None = None,
+    sink: BinaryIO | None = None,
+) -> dict[str, Any]:
+    source = Path(validate_absolute_local(remote_path, label="remote path"))
+    if source.is_symlink():
+        raise OSError(f"symlink is forbidden: {source}")
+    if not source.is_file():
+        raise FileNotFoundError(f"remote file not found: {source}")
+    result = file_result(source, runtime="host")
+    if output is not None:
+        output = output.expanduser()
+        output.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copyfile(source, output)
+        return result
+    if sink is not None:
+        with source.open("rb") as handle:
+            shutil.copyfileobj(handle, sink)
+        return result
+    with source.open("rb") as handle:
+        result["content"] = handle.read()
+    return result
+
+
+def transfer_timeout_seconds() -> float | None:
+    raw = os.environ.get("MSDEV_TRANSFER_TIMEOUT_SECONDS", "-1").strip()
+    if raw in {"", "-1"}:
+        return None
+    value = float(raw)
+    if value <= 0:
+        return None
+    return value

{msdev-0.15.1 → msdev-0.16.0}/src/msdev/core/resources.py

@@ -12,7 +12,7 @@ from typing import Any
 from .config import _exclusive_file_lock, config_home, parse_environment_spec
 
 
-RESOURCE_CONFIG_VERSION = 3
+RESOURCE_CONFIG_VERSION = 4
 
 
 def _required(value: str, label: str) -> str:
@@ -44,6 +44,7 @@ class Environment:
     setup: str | None = None
     wrap: tuple[str, ...] = ()
     env_vars: dict[str, str] = field(default_factory=dict)
+    transfer: str = "auto"
 
     def __post_init__(self) -> None:
         _required(self.name, "environment name")
@@ -69,6 +70,8 @@ class Environment:
                 raise ValueError("wrap must be a non-empty string array")
         if self.runtime not in {"host", "docker"}:
             raise ValueError(f"unsupported environment runtime: {self.runtime}")
+        if self.transfer not in {"auto", "sftp", "exec", "scp"}:
+            raise ValueError(f"unsupported transfer backend: {self.transfer}")
         if self.runtime == "docker":
             _required(self.container or "", "docker container")
         elif self.container is not None:
@@ -114,7 +117,7 @@ class _ResourceStore:
         if not isinstance(value, dict) or not isinstance(value.get("nodes"), dict):
             raise ValueError(f"unsupported resource config schema in {self.path}")
         version = value.get("version")
-        if version not in {2, RESOURCE_CONFIG_VERSION}:
+        if version not in {2, 3, RESOURCE_CONFIG_VERSION}:
             raise ValueError(f"unsupported resource config schema in {self.path}")
         if not isinstance(value.get("environments"), dict):
             raise ValueError(f"unsupported resource config schema in {self.path}")