msdev 0.15.2__tar.gz → 0.16.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {msdev-0.15.2 → msdev-0.16.0}/PKG-INFO +32 -4
- {msdev-0.15.2 → msdev-0.16.0}/README.md +31 -3
- {msdev-0.15.2 → msdev-0.16.0}/pyproject.toml +1 -1
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/__init__.py +1 -1
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/cli.py +147 -10
- msdev-0.16.0/src/msdev/core/env_transfer.py +107 -0
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/resources.py +5 -2
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/services/environment.py +71 -19
- msdev-0.16.0/src/msdev/core/transfer/__init__.py +26 -0
- msdev-0.16.0/src/msdev/core/transfer/api.py +375 -0
- msdev-0.16.0/src/msdev/core/transfer/backends/__init__.py +7 -0
- msdev-0.16.0/src/msdev/core/transfer/backends/exec_stream.py +200 -0
- msdev-0.16.0/src/msdev/core/transfer/backends/local.py +52 -0
- msdev-0.16.0/src/msdev/core/transfer/backends/sftp.py +157 -0
- msdev-0.16.0/src/msdev/core/transfer/policy.py +38 -0
- msdev-0.16.0/src/msdev/core/transfer/types.py +97 -0
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/transport.py +33 -62
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/skills/msdev-cli/SKILL.md +11 -2
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev.egg-info/PKG-INFO +32 -4
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev.egg-info/SOURCES.txt +10 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_config.py +1 -1
- msdev-0.16.0/tests/test_env_transfer.py +75 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_node_env_config.py +1 -1
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_sftp_transfer_scenario.py +0 -39
- msdev-0.16.0/tests/test_transfer_capability_probe_scenario.py +1126 -0
- msdev-0.16.0/tests/test_transfer_component.py +244 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_transport.py +13 -9
- msdev-0.15.2/src/msdev/core/env_transfer.py +0 -399
- msdev-0.15.2/tests/test_env_transfer.py +0 -92
- {msdev-0.15.2 → msdev-0.16.0}/LICENSE +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/setup.cfg +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/__init__.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/compat.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/config.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/config_bundle.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/direct_exec.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/env_exec.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/exec_argv.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/guides.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/init_setup.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/inventory.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/limits.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/services/__init__.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/services/capabilities.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/services/context.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/services/execution.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/services/model.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/services/node.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/services/npu.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/services/results.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/sftp_transfer.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/skill_install.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/ssh_auth.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/ssh_config.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev.egg-info/dependency_links.txt +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev.egg-info/entry_points.txt +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/src/msdev.egg-info/top_level.txt +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_cli.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_cli_only_architecture.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_compat.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_config_bundle.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_connect_env_scenario.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_env_exec_scenario.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_env_put_get_scenario.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_env_read_write_scenario.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_exec_argv.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_exec_command_scenario.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_exec_default_timeout_scenario.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_init_skill.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_inventory.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_live_dev122_dev186_scenario.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_local_docker_ssh_smoke_scenario.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_model_cli.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_model_services.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_msdev_cli_skill.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_node_bootstrap_version_scenario.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_node_connect_provision_scenario.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_node_connect_save_password_scenario.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_node_env_services.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_node_sync_ssh_scenario.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_ssh_auth.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_ssh_config.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_target_exec_services.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_target_guides.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_version_scenario.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_windows_git_bash_ssh_scenario.py +0 -0
- {msdev-0.15.2 → msdev-0.16.0}/tests/test_windows_local_scenario.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: msdev
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.16.0
|
|
4
4
|
Summary: Portable nodes and execution environments for msModelSlim development
|
|
5
5
|
Author: msModelSlim Agent Contributors
|
|
6
6
|
License-Expression: MulanPSL-2.0
|
|
@@ -199,9 +199,32 @@ msdev env get dev-122 /srv/project/config.yaml > ./config.yaml
|
|
|
199
199
|
```
|
|
200
200
|
|
|
201
201
|
Editing uses content-oriented read/write. Paths inside the Env must be
|
|
202
|
-
absolute. Host Envs
|
|
203
|
-
|
|
204
|
-
|
|
202
|
+
absolute. Host Envs copy directly on the local filesystem; remote Envs stream
|
|
203
|
+
over SSH. Docker Envs stage through the node with ``docker cp``. File bytes
|
|
204
|
+
never pass through JSON RPC.
|
|
205
|
+
|
|
206
|
+
### Transfer backends
|
|
207
|
+
|
|
208
|
+
Transfers go through one pluggable component with two backends:
|
|
209
|
+
|
|
210
|
+
- `sftp` — OpenSSH SFTP batch, atomic `put staging` + `rename`.
|
|
211
|
+
- `exec` — binary stdin/stdout over the SSH exec channel (`cat`, `mktemp`,
|
|
212
|
+
`mv`), for hosts where the SFTP subsystem is disabled or blocked.
|
|
213
|
+
|
|
214
|
+
`auto` (the default) tries `sftp`, then falls back to `exec` only when SFTP is
|
|
215
|
+
unavailable (subsystem rejected, connection closed, local `sftp` missing).
|
|
216
|
+
Filesystem errors such as permission denied are never retried on another
|
|
217
|
+
backend.
|
|
218
|
+
|
|
219
|
+
```bash
|
|
220
|
+
msdev env add dev-122 --node dev-122 --transfer auto
|
|
221
|
+
msdev env put dev-122 ./payload.bin /srv/payload.bin --transfer exec
|
|
222
|
+
msdev env get dev-122 /srv/payload.bin --output ./payload.bin --transfer sftp
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
`env add --transfer` stores the preference in the env profile; `--transfer` on
|
|
226
|
+
put/get/read/write overrides it for one command. `MSDEV_TRANSFER` overrides
|
|
227
|
+
both. Explicit `sftp`/`exec` never falls back.
|
|
205
228
|
|
|
206
229
|
```bash
|
|
207
230
|
msdev env read dev-122 /srv/project/config.yaml
|
|
@@ -251,6 +274,11 @@ argument after `exec`; use `-c` for inline strings or `--file` for script
|
|
|
251
274
|
content (path or `-` for stdin/heredoc). `-c -` is accepted but identical to
|
|
252
275
|
`--file -`; prefer **`--file -`** only.
|
|
253
276
|
|
|
277
|
+
Scripts up to 64 KiB of valid UTF-8 are inlined into the remote command.
|
|
278
|
+
Larger or non-UTF-8 scripts are uploaded through the transfer component to a
|
|
279
|
+
temporary file, executed, and removed afterwards. Set
|
|
280
|
+
`MSDEV_EXEC_INLINE_BYTES` to change the threshold.
|
|
281
|
+
|
|
254
282
|
Quoted pipelines and `|` tokens are executed through a remote POSIX shell
|
|
255
283
|
(`sh -c`), so they run on the Node instead of being treated as a single
|
|
256
284
|
executable name. On a Windows `local` Env they use `cmd.exe /s /c`. Prefer
|
|
@@ -174,9 +174,32 @@ msdev env get dev-122 /srv/project/config.yaml > ./config.yaml
|
|
|
174
174
|
```
|
|
175
175
|
|
|
176
176
|
Editing uses content-oriented read/write. Paths inside the Env must be
|
|
177
|
-
absolute. Host Envs
|
|
178
|
-
|
|
179
|
-
|
|
177
|
+
absolute. Host Envs copy directly on the local filesystem; remote Envs stream
|
|
178
|
+
over SSH. Docker Envs stage through the node with ``docker cp``. File bytes
|
|
179
|
+
never pass through JSON RPC.
|
|
180
|
+
|
|
181
|
+
### Transfer backends
|
|
182
|
+
|
|
183
|
+
Transfers go through one pluggable component with two backends:
|
|
184
|
+
|
|
185
|
+
- `sftp` — OpenSSH SFTP batch, atomic `put staging` + `rename`.
|
|
186
|
+
- `exec` — binary stdin/stdout over the SSH exec channel (`cat`, `mktemp`,
|
|
187
|
+
`mv`), for hosts where the SFTP subsystem is disabled or blocked.
|
|
188
|
+
|
|
189
|
+
`auto` (the default) tries `sftp`, then falls back to `exec` only when SFTP is
|
|
190
|
+
unavailable (subsystem rejected, connection closed, local `sftp` missing).
|
|
191
|
+
Filesystem errors such as permission denied are never retried on another
|
|
192
|
+
backend.
|
|
193
|
+
|
|
194
|
+
```bash
|
|
195
|
+
msdev env add dev-122 --node dev-122 --transfer auto
|
|
196
|
+
msdev env put dev-122 ./payload.bin /srv/payload.bin --transfer exec
|
|
197
|
+
msdev env get dev-122 /srv/payload.bin --output ./payload.bin --transfer sftp
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
`env add --transfer` stores the preference in the env profile; `--transfer` on
|
|
201
|
+
put/get/read/write overrides it for one command. `MSDEV_TRANSFER` overrides
|
|
202
|
+
both. Explicit `sftp`/`exec` never falls back.
|
|
180
203
|
|
|
181
204
|
```bash
|
|
182
205
|
msdev env read dev-122 /srv/project/config.yaml
|
|
@@ -226,6 +249,11 @@ argument after `exec`; use `-c` for inline strings or `--file` for script
|
|
|
226
249
|
content (path or `-` for stdin/heredoc). `-c -` is accepted but identical to
|
|
227
250
|
`--file -`; prefer **`--file -`** only.
|
|
228
251
|
|
|
252
|
+
Scripts up to 64 KiB of valid UTF-8 are inlined into the remote command.
|
|
253
|
+
Larger or non-UTF-8 scripts are uploaded through the transfer component to a
|
|
254
|
+
temporary file, executed, and removed afterwards. Set
|
|
255
|
+
`MSDEV_EXEC_INLINE_BYTES` to change the threshold.
|
|
256
|
+
|
|
229
257
|
Quoted pipelines and `|` tokens are executed through a remote POSIX shell
|
|
230
258
|
(`sh -c`), so they run on the Node instead of being treated as a single
|
|
231
259
|
executable name. On a Windows `local` Env they use `cmd.exe /s /c`. Prefer
|
|
@@ -7,7 +7,9 @@ import json
|
|
|
7
7
|
import os
|
|
8
8
|
import signal
|
|
9
9
|
import sys
|
|
10
|
+
import tempfile
|
|
10
11
|
import threading
|
|
12
|
+
import uuid
|
|
11
13
|
from dataclasses import asdict
|
|
12
14
|
from pathlib import Path
|
|
13
15
|
from typing import Any
|
|
@@ -321,6 +323,11 @@ def command_environment(
|
|
|
321
323
|
) -> int:
|
|
322
324
|
service = EnvironmentService(_service_context(node_store, environment_store))
|
|
323
325
|
action = args.environment_command
|
|
326
|
+
if (
|
|
327
|
+
action in {"put", "get", "read", "write"}
|
|
328
|
+
and getattr(args, "transfer", None)
|
|
329
|
+
):
|
|
330
|
+
os.environ["MSDEV_TRANSFER"] = args.transfer
|
|
324
331
|
if action == "guide":
|
|
325
332
|
environment_store.get(args.name)
|
|
326
333
|
return _command_guide(args, EnvironmentGuideStore(), "environment")
|
|
@@ -338,6 +345,7 @@ def command_environment(
|
|
|
338
345
|
setup=_read_setup(args.setup_file, args.setup),
|
|
339
346
|
wrap=tuple(args.wrap),
|
|
340
347
|
env_vars=_parse_env_vars(args.env_var),
|
|
348
|
+
transfer=args.transfer,
|
|
341
349
|
force=args.force,
|
|
342
350
|
)
|
|
343
351
|
)
|
|
@@ -416,12 +424,15 @@ def command_environment(
|
|
|
416
424
|
sys.stdout.buffer.flush()
|
|
417
425
|
return 0
|
|
418
426
|
if action == "write":
|
|
419
|
-
content = _env_write_content(args)
|
|
427
|
+
content = None if args.file is not None else _env_write_content(args)
|
|
420
428
|
result = service.write(
|
|
421
429
|
EnvironmentWriteRequest(
|
|
422
430
|
environment=args.name,
|
|
423
431
|
remote_path=args.remote_path,
|
|
424
432
|
content=content,
|
|
433
|
+
source_path=(
|
|
434
|
+
str(args.file) if args.file is not None else None
|
|
435
|
+
),
|
|
425
436
|
expected_sha256=args.expected_sha256,
|
|
426
437
|
)
|
|
427
438
|
)
|
|
@@ -438,23 +449,119 @@ def _env_write_content(args: argparse.Namespace) -> bytes:
|
|
|
438
449
|
return sys.stdin.buffer.read()
|
|
439
450
|
|
|
440
451
|
|
|
452
|
+
def _exec_inline_limit() -> int:
|
|
453
|
+
raw = os.environ.get("MSDEV_EXEC_INLINE_BYTES", "65536").strip()
|
|
454
|
+
try:
|
|
455
|
+
value = int(raw)
|
|
456
|
+
except ValueError:
|
|
457
|
+
return 65536
|
|
458
|
+
return max(0, value)
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
def _is_inline_script(raw: bytes, limit: int) -> bool:
|
|
462
|
+
if len(raw) > limit or b"\0" in raw:
|
|
463
|
+
return False
|
|
464
|
+
try:
|
|
465
|
+
raw.decode("utf-8")
|
|
466
|
+
except UnicodeDecodeError:
|
|
467
|
+
return False
|
|
468
|
+
return True
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def _upload_exec_script(
|
|
472
|
+
environment: str,
|
|
473
|
+
raw: bytes,
|
|
474
|
+
node_store: NodeStore,
|
|
475
|
+
environment_store: EnvironmentStore,
|
|
476
|
+
) -> tuple[str, ...]:
|
|
477
|
+
"""Upload a script through the transfer component and return its argv."""
|
|
478
|
+
context = _service_context(node_store, environment_store)
|
|
479
|
+
transport = context.transport_for_environment(environment)
|
|
480
|
+
remote = f"/tmp/msdev-exec-{uuid.uuid4().hex}.script"
|
|
481
|
+
with tempfile.NamedTemporaryFile(
|
|
482
|
+
prefix="msdev-exec-",
|
|
483
|
+
delete=False,
|
|
484
|
+
) as handle:
|
|
485
|
+
temporary = Path(handle.name)
|
|
486
|
+
handle.write(raw)
|
|
487
|
+
try:
|
|
488
|
+
transport.put_file(temporary, remote)
|
|
489
|
+
finally:
|
|
490
|
+
temporary.unlink(missing_ok=True)
|
|
491
|
+
if raw.startswith(b"#!"):
|
|
492
|
+
line = raw.split(b"\n", 1)[0][2:].strip().decode("utf-8", "replace")
|
|
493
|
+
interpreter = line.split()
|
|
494
|
+
if interpreter:
|
|
495
|
+
# Run the interpreter explicitly: no exec bit and no noexec /tmp.
|
|
496
|
+
cleanup = (
|
|
497
|
+
'p=$1; shift; trap \'rm -f -- "$p"\' EXIT; "$@" "$p"'
|
|
498
|
+
)
|
|
499
|
+
return (
|
|
500
|
+
"sh",
|
|
501
|
+
"-c",
|
|
502
|
+
cleanup,
|
|
503
|
+
"msdev-exec",
|
|
504
|
+
remote,
|
|
505
|
+
*interpreter,
|
|
506
|
+
)
|
|
507
|
+
cleanup = 'p=$1; trap \'rm -f -- "$p"\' EXIT; sh "$p"'
|
|
508
|
+
return ("sh", "-c", cleanup, "msdev-exec", remote)
|
|
509
|
+
|
|
510
|
+
|
|
441
511
|
def command_exec(
|
|
442
512
|
args: argparse.Namespace,
|
|
443
513
|
node_store: NodeStore,
|
|
444
514
|
environment_store: EnvironmentStore,
|
|
445
515
|
) -> int:
|
|
446
516
|
windows = args.environment == "local" and IS_WINDOWS
|
|
447
|
-
|
|
448
|
-
|
|
517
|
+
limit = _exec_inline_limit()
|
|
518
|
+
command: tuple[str, ...]
|
|
519
|
+
if args.command_file is not None and str(args.command_file) != "-":
|
|
520
|
+
source = args.command_file
|
|
521
|
+
if not source.is_file():
|
|
522
|
+
raise ValueError(f"script file not found: {source}")
|
|
523
|
+
raw = source.read_bytes()
|
|
524
|
+
if _is_inline_script(raw, limit):
|
|
525
|
+
command = resolve_exec_argv(
|
|
526
|
+
shell_command=None,
|
|
527
|
+
command_file=source,
|
|
528
|
+
windows=windows,
|
|
529
|
+
)
|
|
530
|
+
else:
|
|
531
|
+
if windows:
|
|
532
|
+
raise ValueError(
|
|
533
|
+
"inline exec script exceeds the local Windows limit "
|
|
534
|
+
f"({limit} bytes); use a smaller script or a remote env"
|
|
535
|
+
)
|
|
536
|
+
command = _upload_exec_script(
|
|
537
|
+
args.environment, raw, node_store, environment_store
|
|
538
|
+
)
|
|
539
|
+
elif args.shell_command == "-" or (
|
|
449
540
|
args.command_file is not None and str(args.command_file) == "-"
|
|
450
541
|
):
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
542
|
+
raw = sys.stdin.buffer.read()
|
|
543
|
+
if _is_inline_script(raw, limit):
|
|
544
|
+
command = resolve_exec_argv(
|
|
545
|
+
shell_command=None,
|
|
546
|
+
command_file=Path("-"),
|
|
547
|
+
script_stdin=raw.decode("utf-8"),
|
|
548
|
+
windows=windows,
|
|
549
|
+
)
|
|
550
|
+
else:
|
|
551
|
+
if windows:
|
|
552
|
+
raise ValueError(
|
|
553
|
+
"inline exec script exceeds the local Windows limit "
|
|
554
|
+
f"({limit} bytes); use a smaller script or a remote env"
|
|
555
|
+
)
|
|
556
|
+
command = _upload_exec_script(
|
|
557
|
+
args.environment, raw, node_store, environment_store
|
|
558
|
+
)
|
|
559
|
+
else:
|
|
560
|
+
command = resolve_exec_argv(
|
|
561
|
+
shell_command=args.shell_command,
|
|
562
|
+
command_file=None,
|
|
563
|
+
windows=windows,
|
|
564
|
+
)
|
|
458
565
|
env: dict[str, str] = {}
|
|
459
566
|
for value in args.env_var:
|
|
460
567
|
key, separator, item = value.partition("=")
|
|
@@ -772,6 +879,12 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
772
879
|
metavar="KEY=VALUE",
|
|
773
880
|
help="static environment variable stored in the env profile",
|
|
774
881
|
)
|
|
882
|
+
environment_add.add_argument(
|
|
883
|
+
"--transfer",
|
|
884
|
+
choices=("auto", "sftp", "exec", "scp"),
|
|
885
|
+
default="auto",
|
|
886
|
+
help="file transfer backend stored in the env profile (default: auto)",
|
|
887
|
+
)
|
|
775
888
|
environment_sub.add_parser("list")
|
|
776
889
|
environment_inspect = environment_sub.add_parser("inspect")
|
|
777
890
|
environment_inspect.add_argument("name")
|
|
@@ -805,6 +918,12 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
805
918
|
"remote_path",
|
|
806
919
|
help="absolute destination path inside the environment",
|
|
807
920
|
)
|
|
921
|
+
environment_put.add_argument(
|
|
922
|
+
"--transfer",
|
|
923
|
+
choices=("auto", "sftp", "exec"),
|
|
924
|
+
default=None,
|
|
925
|
+
help="override the transfer backend for this operation",
|
|
926
|
+
)
|
|
808
927
|
environment_get = environment_sub.add_parser(
|
|
809
928
|
"get",
|
|
810
929
|
help="fetch a file from an environment filesystem",
|
|
@@ -825,6 +944,12 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
825
944
|
action="store_true",
|
|
826
945
|
help="print metadata JSON when --output is used",
|
|
827
946
|
)
|
|
947
|
+
environment_get.add_argument(
|
|
948
|
+
"--transfer",
|
|
949
|
+
choices=("auto", "sftp", "exec"),
|
|
950
|
+
default=None,
|
|
951
|
+
help="override the transfer backend for this operation",
|
|
952
|
+
)
|
|
828
953
|
environment_read = environment_sub.add_parser(
|
|
829
954
|
"read",
|
|
830
955
|
help="read a remote file for editing (stdout or --json with sha256)",
|
|
@@ -839,6 +964,12 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
839
964
|
action="store_true",
|
|
840
965
|
help="print metadata JSON including sha256 and content_base64",
|
|
841
966
|
)
|
|
967
|
+
environment_read.add_argument(
|
|
968
|
+
"--transfer",
|
|
969
|
+
choices=("auto", "sftp", "exec"),
|
|
970
|
+
default=None,
|
|
971
|
+
help="override the transfer backend for this operation",
|
|
972
|
+
)
|
|
842
973
|
environment_write = environment_sub.add_parser(
|
|
843
974
|
"write",
|
|
844
975
|
help="atomically replace a remote file from stdin, --file, or --content",
|
|
@@ -862,6 +993,12 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
862
993
|
"--expected-sha256",
|
|
863
994
|
help="compare-and-swap: require this digest before replacing",
|
|
864
995
|
)
|
|
996
|
+
environment_write.add_argument(
|
|
997
|
+
"--transfer",
|
|
998
|
+
choices=("auto", "sftp", "exec"),
|
|
999
|
+
default=None,
|
|
1000
|
+
help="override the transfer backend for this operation",
|
|
1001
|
+
)
|
|
865
1002
|
environment_remove = environment_sub.add_parser("remove")
|
|
866
1003
|
environment_remove.add_argument("name")
|
|
867
1004
|
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""Host-local file transfer helpers used by the local transport."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import shutil
|
|
7
|
+
import stat
|
|
8
|
+
import uuid
|
|
9
|
+
from pathlib import Path, PurePosixPath
|
|
10
|
+
from typing import Any, BinaryIO
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def validate_absolute_local(path: str, *, label: str) -> str:
|
|
15
|
+
"""Accept a host-local absolute path (POSIX or Windows)."""
|
|
16
|
+
if not isinstance(path, str) or not path or "\0" in path:
|
|
17
|
+
raise ValueError(f"{label} must be a non-empty NUL-free string")
|
|
18
|
+
candidate = Path(path)
|
|
19
|
+
if not candidate.is_absolute():
|
|
20
|
+
raise ValueError(f"{label} must be an absolute path")
|
|
21
|
+
return str(candidate)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def validate_absolute_remote(path: str, *, label: str) -> str:
|
|
25
|
+
if not isinstance(path, str) or not path or "\0" in path:
|
|
26
|
+
raise ValueError(f"{label} must be a non-empty NUL-free string")
|
|
27
|
+
if not path.startswith("/"):
|
|
28
|
+
raise ValueError(f"{label} must be an absolute path")
|
|
29
|
+
normalized = PurePosixPath(path)
|
|
30
|
+
if str(normalized) != path:
|
|
31
|
+
raise ValueError(f"{label} must be a canonical absolute POSIX path")
|
|
32
|
+
if any(part in {"", ".", ".."} for part in normalized.parts[1:]):
|
|
33
|
+
raise ValueError(f"{label} must not contain '.' or '..' components")
|
|
34
|
+
return path
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def file_result(path: str | os.PathLike[str], *, runtime: str) -> dict[str, Any]:
|
|
38
|
+
target = Path(path)
|
|
39
|
+
metadata = target.lstat()
|
|
40
|
+
if not stat.S_ISREG(metadata.st_mode):
|
|
41
|
+
raise ValueError(f"path is not a regular file: {target}")
|
|
42
|
+
return {
|
|
43
|
+
"path": str(target),
|
|
44
|
+
"type": "file",
|
|
45
|
+
"size": metadata.st_size,
|
|
46
|
+
"runtime": runtime,
|
|
47
|
+
"atomic": True,
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _atomic_copy(source: Path, destination: Path) -> None:
|
|
52
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
53
|
+
if destination.exists() and destination.is_symlink():
|
|
54
|
+
raise OSError(f"symlink is forbidden: {destination}")
|
|
55
|
+
if destination.exists() and destination.is_dir():
|
|
56
|
+
raise IsADirectoryError(f"path is a directory: {destination}")
|
|
57
|
+
temporary = destination.with_name(f".msdev-put-{uuid.uuid4().hex}")
|
|
58
|
+
try:
|
|
59
|
+
shutil.copyfile(source, temporary)
|
|
60
|
+
os.replace(temporary, destination)
|
|
61
|
+
finally:
|
|
62
|
+
temporary.unlink(missing_ok=True)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def local_put_file(local_path: Path, remote_path: str) -> dict[str, Any]:
|
|
66
|
+
source = local_path.expanduser().resolve()
|
|
67
|
+
if not source.is_file():
|
|
68
|
+
raise FileNotFoundError(f"local file not found: {source}")
|
|
69
|
+
destination = Path(validate_absolute_local(remote_path, label="remote path"))
|
|
70
|
+
_atomic_copy(source, destination)
|
|
71
|
+
return file_result(destination, runtime="host")
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def local_get_file(
|
|
75
|
+
remote_path: str,
|
|
76
|
+
*,
|
|
77
|
+
output: Path | None = None,
|
|
78
|
+
sink: BinaryIO | None = None,
|
|
79
|
+
) -> dict[str, Any]:
|
|
80
|
+
source = Path(validate_absolute_local(remote_path, label="remote path"))
|
|
81
|
+
if source.is_symlink():
|
|
82
|
+
raise OSError(f"symlink is forbidden: {source}")
|
|
83
|
+
if not source.is_file():
|
|
84
|
+
raise FileNotFoundError(f"remote file not found: {source}")
|
|
85
|
+
result = file_result(source, runtime="host")
|
|
86
|
+
if output is not None:
|
|
87
|
+
output = output.expanduser()
|
|
88
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
89
|
+
shutil.copyfile(source, output)
|
|
90
|
+
return result
|
|
91
|
+
if sink is not None:
|
|
92
|
+
with source.open("rb") as handle:
|
|
93
|
+
shutil.copyfileobj(handle, sink)
|
|
94
|
+
return result
|
|
95
|
+
with source.open("rb") as handle:
|
|
96
|
+
result["content"] = handle.read()
|
|
97
|
+
return result
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def transfer_timeout_seconds() -> float | None:
|
|
101
|
+
raw = os.environ.get("MSDEV_TRANSFER_TIMEOUT_SECONDS", "-1").strip()
|
|
102
|
+
if raw in {"", "-1"}:
|
|
103
|
+
return None
|
|
104
|
+
value = float(raw)
|
|
105
|
+
if value <= 0:
|
|
106
|
+
return None
|
|
107
|
+
return value
|
|
@@ -12,7 +12,7 @@ from typing import Any
|
|
|
12
12
|
from .config import _exclusive_file_lock, config_home, parse_environment_spec
|
|
13
13
|
|
|
14
14
|
|
|
15
|
-
RESOURCE_CONFIG_VERSION =
|
|
15
|
+
RESOURCE_CONFIG_VERSION = 4
|
|
16
16
|
|
|
17
17
|
|
|
18
18
|
def _required(value: str, label: str) -> str:
|
|
@@ -44,6 +44,7 @@ class Environment:
|
|
|
44
44
|
setup: str | None = None
|
|
45
45
|
wrap: tuple[str, ...] = ()
|
|
46
46
|
env_vars: dict[str, str] = field(default_factory=dict)
|
|
47
|
+
transfer: str = "auto"
|
|
47
48
|
|
|
48
49
|
def __post_init__(self) -> None:
|
|
49
50
|
_required(self.name, "environment name")
|
|
@@ -69,6 +70,8 @@ class Environment:
|
|
|
69
70
|
raise ValueError("wrap must be a non-empty string array")
|
|
70
71
|
if self.runtime not in {"host", "docker"}:
|
|
71
72
|
raise ValueError(f"unsupported environment runtime: {self.runtime}")
|
|
73
|
+
if self.transfer not in {"auto", "sftp", "exec", "scp"}:
|
|
74
|
+
raise ValueError(f"unsupported transfer backend: {self.transfer}")
|
|
72
75
|
if self.runtime == "docker":
|
|
73
76
|
_required(self.container or "", "docker container")
|
|
74
77
|
elif self.container is not None:
|
|
@@ -114,7 +117,7 @@ class _ResourceStore:
|
|
|
114
117
|
if not isinstance(value, dict) or not isinstance(value.get("nodes"), dict):
|
|
115
118
|
raise ValueError(f"unsupported resource config schema in {self.path}")
|
|
116
119
|
version = value.get("version")
|
|
117
|
-
if version not in {2, RESOURCE_CONFIG_VERSION}:
|
|
120
|
+
if version not in {2, 3, RESOURCE_CONFIG_VERSION}:
|
|
118
121
|
raise ValueError(f"unsupported resource config schema in {self.path}")
|
|
119
122
|
if not isinstance(value.get("environments"), dict):
|
|
120
123
|
raise ValueError(f"unsupported resource config schema in {self.path}")
|
|
@@ -27,6 +27,7 @@ class EnvironmentAddRequest:
|
|
|
27
27
|
setup: str | None = None
|
|
28
28
|
wrap: tuple[str, ...] = ()
|
|
29
29
|
env_vars: dict[str, str] | None = None
|
|
30
|
+
transfer: str = "auto"
|
|
30
31
|
force: bool = False
|
|
31
32
|
|
|
32
33
|
|
|
@@ -66,8 +67,9 @@ class EnvironmentReadRequest:
|
|
|
66
67
|
class EnvironmentWriteRequest:
|
|
67
68
|
environment: str
|
|
68
69
|
remote_path: str
|
|
69
|
-
content: bytes
|
|
70
|
+
content: bytes | None = None
|
|
70
71
|
expected_sha256: str | None = None
|
|
72
|
+
source_path: str | None = None
|
|
71
73
|
|
|
72
74
|
|
|
73
75
|
@dataclass(frozen=True)
|
|
@@ -129,6 +131,14 @@ class EnvironmentService:
|
|
|
129
131
|
def _sha256(content: bytes) -> str:
|
|
130
132
|
return hashlib.sha256(content).hexdigest()
|
|
131
133
|
|
|
134
|
+
@staticmethod
|
|
135
|
+
def _sha256_file(path: Path) -> str:
|
|
136
|
+
digest = hashlib.sha256()
|
|
137
|
+
with path.open("rb") as handle:
|
|
138
|
+
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
139
|
+
digest.update(chunk)
|
|
140
|
+
return digest.hexdigest()
|
|
141
|
+
|
|
132
142
|
def _environment(self, name: str) -> Environment:
|
|
133
143
|
environment_name = self._required(name, "environment name")
|
|
134
144
|
return self.context.environment_store.get(environment_name)
|
|
@@ -171,6 +181,7 @@ class EnvironmentService:
|
|
|
171
181
|
setup=request.setup,
|
|
172
182
|
wrap=tuple(request.wrap),
|
|
173
183
|
env_vars=dict(request.env_vars or {}),
|
|
184
|
+
transfer=request.transfer,
|
|
174
185
|
)
|
|
175
186
|
self.context.node_store.get(environment.node)
|
|
176
187
|
self.context.environment_store.add(environment, force=request.force)
|
|
@@ -235,7 +246,11 @@ class EnvironmentService:
|
|
|
235
246
|
path=str(raw.get("path") or remote_path),
|
|
236
247
|
size=int(raw.get("size") or local_path.stat().st_size),
|
|
237
248
|
runtime=str(raw.get("runtime") or environment.runtime),
|
|
238
|
-
sha256=
|
|
249
|
+
sha256=(
|
|
250
|
+
str(raw["sha256"])
|
|
251
|
+
if raw.get("sha256")
|
|
252
|
+
else self._sha256_file(local_path)
|
|
253
|
+
),
|
|
239
254
|
raw=raw,
|
|
240
255
|
)
|
|
241
256
|
|
|
@@ -256,11 +271,19 @@ class EnvironmentService:
|
|
|
256
271
|
raw = transport.get_file(remote_path, output=output)
|
|
257
272
|
content = None
|
|
258
273
|
digest = (
|
|
259
|
-
|
|
274
|
+
str(raw["sha256"])
|
|
275
|
+
if raw.get("sha256")
|
|
276
|
+
else (
|
|
277
|
+
self._sha256_file(output) if output.is_file() else None
|
|
278
|
+
)
|
|
260
279
|
)
|
|
261
280
|
else:
|
|
262
281
|
content, raw = self._read_bytes(transport, remote_path)
|
|
263
|
-
digest =
|
|
282
|
+
digest = (
|
|
283
|
+
str(raw["sha256"])
|
|
284
|
+
if raw.get("sha256")
|
|
285
|
+
else self._sha256(content)
|
|
286
|
+
)
|
|
264
287
|
if not isinstance(raw, dict):
|
|
265
288
|
raise RuntimeError("env get returned a non-object result")
|
|
266
289
|
size = raw.get("size")
|
|
@@ -288,7 +311,9 @@ class EnvironmentService:
|
|
|
288
311
|
self.context.transport_for_environment(environment.name)
|
|
289
312
|
)
|
|
290
313
|
content, raw = self._read_bytes(transport, remote_path)
|
|
291
|
-
digest =
|
|
314
|
+
digest = (
|
|
315
|
+
str(raw["sha256"]) if raw.get("sha256") else self._sha256(content)
|
|
316
|
+
)
|
|
292
317
|
return EnvironmentFileResult(
|
|
293
318
|
environment=environment.name,
|
|
294
319
|
path=str(raw.get("path") or remote_path),
|
|
@@ -307,9 +332,23 @@ class EnvironmentService:
|
|
|
307
332
|
"remote path",
|
|
308
333
|
local=environment.node == "local",
|
|
309
334
|
)
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
335
|
+
temporary: Path | None = None
|
|
336
|
+
if request.source_path is not None:
|
|
337
|
+
source = Path(request.source_path).expanduser()
|
|
338
|
+
if not source.is_file():
|
|
339
|
+
raise FileNotFoundError(f"source file not found: {source}")
|
|
340
|
+
content = None
|
|
341
|
+
else:
|
|
342
|
+
if not isinstance(request.content, (bytes, bytearray)):
|
|
343
|
+
raise TypeError("write content must be bytes")
|
|
344
|
+
content = bytes(request.content)
|
|
345
|
+
with tempfile.NamedTemporaryFile(
|
|
346
|
+
prefix="msdev-env-write-",
|
|
347
|
+
delete=False,
|
|
348
|
+
) as handle:
|
|
349
|
+
temporary = Path(handle.name)
|
|
350
|
+
handle.write(content)
|
|
351
|
+
source = temporary
|
|
313
352
|
expected = self._digest(request.expected_sha256, required=False)
|
|
314
353
|
transport = self._transfer(
|
|
315
354
|
self.context.transport_for_environment(environment.name)
|
|
@@ -321,28 +360,41 @@ class EnvironmentService:
|
|
|
321
360
|
except FileNotFoundError:
|
|
322
361
|
current_digest = None
|
|
323
362
|
if current_digest != expected:
|
|
363
|
+
if temporary is not None:
|
|
364
|
+
temporary.unlink(missing_ok=True)
|
|
324
365
|
raise RuntimeError(
|
|
325
366
|
f"env file changed before write: {remote_path} "
|
|
326
367
|
f"(expected {expected}, found {current_digest or 'missing'})"
|
|
327
368
|
)
|
|
328
|
-
with tempfile.NamedTemporaryFile(
|
|
329
|
-
prefix="msdev-env-write-",
|
|
330
|
-
delete=False,
|
|
331
|
-
) as handle:
|
|
332
|
-
temporary = Path(handle.name)
|
|
333
|
-
handle.write(content)
|
|
334
369
|
try:
|
|
335
|
-
raw = transport.put_file(
|
|
370
|
+
raw = transport.put_file(source, remote_path)
|
|
336
371
|
finally:
|
|
337
|
-
temporary
|
|
372
|
+
if temporary is not None:
|
|
373
|
+
temporary.unlink(missing_ok=True)
|
|
338
374
|
if not isinstance(raw, dict):
|
|
339
375
|
raise RuntimeError("env write returned a non-object result")
|
|
340
|
-
digest =
|
|
376
|
+
digest = (
|
|
377
|
+
str(raw["sha256"])
|
|
378
|
+
if raw.get("sha256")
|
|
379
|
+
else (
|
|
380
|
+
self._sha256(content)
|
|
381
|
+
if content is not None
|
|
382
|
+
else self._sha256_file(source)
|
|
383
|
+
)
|
|
384
|
+
)
|
|
385
|
+
size = int(
|
|
386
|
+
raw.get("size")
|
|
387
|
+
or (
|
|
388
|
+
len(content)
|
|
389
|
+
if content is not None
|
|
390
|
+
else source.stat().st_size
|
|
391
|
+
)
|
|
392
|
+
)
|
|
341
393
|
return EnvironmentFileResult(
|
|
342
394
|
environment=environment.name,
|
|
343
395
|
path=str(raw.get("path") or remote_path),
|
|
344
|
-
size=
|
|
396
|
+
size=size,
|
|
345
397
|
runtime=str(raw.get("runtime") or environment.runtime),
|
|
346
398
|
sha256=digest,
|
|
347
|
-
raw={**raw, "sha256": digest, "size":
|
|
399
|
+
raw={**raw, "sha256": digest, "size": size},
|
|
348
400
|
)
|