msdev 0.15.2__tar.gz → 0.16.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. {msdev-0.15.2 → msdev-0.16.0}/PKG-INFO +32 -4
  2. {msdev-0.15.2 → msdev-0.16.0}/README.md +31 -3
  3. {msdev-0.15.2 → msdev-0.16.0}/pyproject.toml +1 -1
  4. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/__init__.py +1 -1
  5. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/cli.py +147 -10
  6. msdev-0.16.0/src/msdev/core/env_transfer.py +107 -0
  7. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/resources.py +5 -2
  8. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/services/environment.py +71 -19
  9. msdev-0.16.0/src/msdev/core/transfer/__init__.py +26 -0
  10. msdev-0.16.0/src/msdev/core/transfer/api.py +375 -0
  11. msdev-0.16.0/src/msdev/core/transfer/backends/__init__.py +7 -0
  12. msdev-0.16.0/src/msdev/core/transfer/backends/exec_stream.py +200 -0
  13. msdev-0.16.0/src/msdev/core/transfer/backends/local.py +52 -0
  14. msdev-0.16.0/src/msdev/core/transfer/backends/sftp.py +157 -0
  15. msdev-0.16.0/src/msdev/core/transfer/policy.py +38 -0
  16. msdev-0.16.0/src/msdev/core/transfer/types.py +97 -0
  17. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/transport.py +33 -62
  18. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/skills/msdev-cli/SKILL.md +11 -2
  19. {msdev-0.15.2 → msdev-0.16.0}/src/msdev.egg-info/PKG-INFO +32 -4
  20. {msdev-0.15.2 → msdev-0.16.0}/src/msdev.egg-info/SOURCES.txt +10 -0
  21. {msdev-0.15.2 → msdev-0.16.0}/tests/test_config.py +1 -1
  22. msdev-0.16.0/tests/test_env_transfer.py +75 -0
  23. {msdev-0.15.2 → msdev-0.16.0}/tests/test_node_env_config.py +1 -1
  24. {msdev-0.15.2 → msdev-0.16.0}/tests/test_sftp_transfer_scenario.py +0 -39
  25. msdev-0.16.0/tests/test_transfer_capability_probe_scenario.py +1126 -0
  26. msdev-0.16.0/tests/test_transfer_component.py +244 -0
  27. {msdev-0.15.2 → msdev-0.16.0}/tests/test_transport.py +13 -9
  28. msdev-0.15.2/src/msdev/core/env_transfer.py +0 -399
  29. msdev-0.15.2/tests/test_env_transfer.py +0 -92
  30. {msdev-0.15.2 → msdev-0.16.0}/LICENSE +0 -0
  31. {msdev-0.15.2 → msdev-0.16.0}/setup.cfg +0 -0
  32. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/__init__.py +0 -0
  33. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/compat.py +0 -0
  34. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/config.py +0 -0
  35. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/config_bundle.py +0 -0
  36. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/direct_exec.py +0 -0
  37. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/env_exec.py +0 -0
  38. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/exec_argv.py +0 -0
  39. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/guides.py +0 -0
  40. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/init_setup.py +0 -0
  41. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/inventory.py +0 -0
  42. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/limits.py +0 -0
  43. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/services/__init__.py +0 -0
  44. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/services/capabilities.py +0 -0
  45. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/services/context.py +0 -0
  46. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/services/execution.py +0 -0
  47. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/services/model.py +0 -0
  48. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/services/node.py +0 -0
  49. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/services/npu.py +0 -0
  50. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/services/results.py +0 -0
  51. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/sftp_transfer.py +0 -0
  52. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/skill_install.py +0 -0
  53. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/ssh_auth.py +0 -0
  54. {msdev-0.15.2 → msdev-0.16.0}/src/msdev/core/ssh_config.py +0 -0
  55. {msdev-0.15.2 → msdev-0.16.0}/src/msdev.egg-info/dependency_links.txt +0 -0
  56. {msdev-0.15.2 → msdev-0.16.0}/src/msdev.egg-info/entry_points.txt +0 -0
  57. {msdev-0.15.2 → msdev-0.16.0}/src/msdev.egg-info/top_level.txt +0 -0
  58. {msdev-0.15.2 → msdev-0.16.0}/tests/test_cli.py +0 -0
  59. {msdev-0.15.2 → msdev-0.16.0}/tests/test_cli_only_architecture.py +0 -0
  60. {msdev-0.15.2 → msdev-0.16.0}/tests/test_compat.py +0 -0
  61. {msdev-0.15.2 → msdev-0.16.0}/tests/test_config_bundle.py +0 -0
  62. {msdev-0.15.2 → msdev-0.16.0}/tests/test_connect_env_scenario.py +0 -0
  63. {msdev-0.15.2 → msdev-0.16.0}/tests/test_env_exec_scenario.py +0 -0
  64. {msdev-0.15.2 → msdev-0.16.0}/tests/test_env_put_get_scenario.py +0 -0
  65. {msdev-0.15.2 → msdev-0.16.0}/tests/test_env_read_write_scenario.py +0 -0
  66. {msdev-0.15.2 → msdev-0.16.0}/tests/test_exec_argv.py +0 -0
  67. {msdev-0.15.2 → msdev-0.16.0}/tests/test_exec_command_scenario.py +0 -0
  68. {msdev-0.15.2 → msdev-0.16.0}/tests/test_exec_default_timeout_scenario.py +0 -0
  69. {msdev-0.15.2 → msdev-0.16.0}/tests/test_init_skill.py +0 -0
  70. {msdev-0.15.2 → msdev-0.16.0}/tests/test_inventory.py +0 -0
  71. {msdev-0.15.2 → msdev-0.16.0}/tests/test_live_dev122_dev186_scenario.py +0 -0
  72. {msdev-0.15.2 → msdev-0.16.0}/tests/test_local_docker_ssh_smoke_scenario.py +0 -0
  73. {msdev-0.15.2 → msdev-0.16.0}/tests/test_model_cli.py +0 -0
  74. {msdev-0.15.2 → msdev-0.16.0}/tests/test_model_services.py +0 -0
  75. {msdev-0.15.2 → msdev-0.16.0}/tests/test_msdev_cli_skill.py +0 -0
  76. {msdev-0.15.2 → msdev-0.16.0}/tests/test_node_bootstrap_version_scenario.py +0 -0
  77. {msdev-0.15.2 → msdev-0.16.0}/tests/test_node_connect_provision_scenario.py +0 -0
  78. {msdev-0.15.2 → msdev-0.16.0}/tests/test_node_connect_save_password_scenario.py +0 -0
  79. {msdev-0.15.2 → msdev-0.16.0}/tests/test_node_env_services.py +0 -0
  80. {msdev-0.15.2 → msdev-0.16.0}/tests/test_node_sync_ssh_scenario.py +0 -0
  81. {msdev-0.15.2 → msdev-0.16.0}/tests/test_ssh_auth.py +0 -0
  82. {msdev-0.15.2 → msdev-0.16.0}/tests/test_ssh_config.py +0 -0
  83. {msdev-0.15.2 → msdev-0.16.0}/tests/test_target_exec_services.py +0 -0
  84. {msdev-0.15.2 → msdev-0.16.0}/tests/test_target_guides.py +0 -0
  85. {msdev-0.15.2 → msdev-0.16.0}/tests/test_version_scenario.py +0 -0
  86. {msdev-0.15.2 → msdev-0.16.0}/tests/test_windows_git_bash_ssh_scenario.py +0 -0
  87. {msdev-0.15.2 → msdev-0.16.0}/tests/test_windows_local_scenario.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: msdev
3
- Version: 0.15.2
3
+ Version: 0.16.0
4
4
  Summary: Portable nodes and execution environments for msModelSlim development
5
5
  Author: msModelSlim Agent Contributors
6
6
  License-Expression: MulanPSL-2.0
@@ -199,9 +199,32 @@ msdev env get dev-122 /srv/project/config.yaml > ./config.yaml
199
199
  ```
200
200
 
201
201
  Editing uses content-oriented read/write. Paths inside the Env must be
202
- absolute. Host Envs use direct filesystem copy locally and ``sftp``/``ssh``
203
- streams remotely. Docker Envs stage through the node with ``docker cp``.
204
- File bytes never pass through JSON RPC.
202
+ absolute. Host Envs copy directly on the local filesystem; remote Envs stream
203
+ over SSH. Docker Envs stage through the node with ``docker cp``. File bytes
204
+ never pass through JSON RPC.
205
+
206
+ ### Transfer backends
207
+
208
+ Transfers go through one pluggable component with two backends:
209
+
210
+ - `sftp` — OpenSSH SFTP batch, atomic `put staging` + `rename`.
211
+ - `exec` — binary stdin/stdout over the SSH exec channel (`cat`, `mktemp`,
212
+ `mv`), for hosts where the SFTP subsystem is disabled or blocked.
213
+
214
+ `auto` (the default) tries `sftp`, then falls back to `exec` only when SFTP is
215
+ unavailable (subsystem rejected, connection closed, local `sftp` missing).
216
+ Filesystem errors such as permission denied are never retried on another
217
+ backend.
218
+
219
+ ```bash
220
+ msdev env add dev-122 --node dev-122 --transfer auto
221
+ msdev env put dev-122 ./payload.bin /srv/payload.bin --transfer exec
222
+ msdev env get dev-122 /srv/payload.bin --output ./payload.bin --transfer sftp
223
+ ```
224
+
225
+ `env add --transfer` stores the preference in the env profile; `--transfer` on
226
+ put/get/read/write overrides it for one command. `MSDEV_TRANSFER` overrides
227
+ both. Explicit `sftp`/`exec` never falls back.
205
228
 
206
229
  ```bash
207
230
  msdev env read dev-122 /srv/project/config.yaml
@@ -251,6 +274,11 @@ argument after `exec`; use `-c` for inline strings or `--file` for script
251
274
  content (path or `-` for stdin/heredoc). `-c -` is accepted but identical to
252
275
  `--file -`; prefer **`--file -`** only.
253
276
 
277
+ Scripts up to 64 KiB of valid UTF-8 are inlined into the remote command.
278
+ Larger or non-UTF-8 scripts are uploaded through the transfer component to a
279
+ temporary file, executed, and removed afterwards. Set
280
+ `MSDEV_EXEC_INLINE_BYTES` to change the threshold.
281
+
254
282
  Quoted pipelines and `|` tokens are executed through a remote POSIX shell
255
283
  (`sh -c`), so they run on the Node instead of being treated as a single
256
284
  executable name. On a Windows `local` Env they use `cmd.exe /s /c`. Prefer
@@ -174,9 +174,32 @@ msdev env get dev-122 /srv/project/config.yaml > ./config.yaml
174
174
  ```
175
175
 
176
176
  Editing uses content-oriented read/write. Paths inside the Env must be
177
- absolute. Host Envs use direct filesystem copy locally and ``sftp``/``ssh``
178
- streams remotely. Docker Envs stage through the node with ``docker cp``.
179
- File bytes never pass through JSON RPC.
177
+ absolute. Host Envs copy directly on the local filesystem; remote Envs stream
178
+ over SSH. Docker Envs stage through the node with ``docker cp``. File bytes
179
+ never pass through JSON RPC.
180
+
181
+ ### Transfer backends
182
+
183
+ Transfers go through one pluggable component with two backends:
184
+
185
+ - `sftp` — OpenSSH SFTP batch, atomic `put staging` + `rename`.
186
+ - `exec` — binary stdin/stdout over the SSH exec channel (`cat`, `mktemp`,
187
+ `mv`), for hosts where the SFTP subsystem is disabled or blocked.
188
+
189
+ `auto` (the default) tries `sftp`, then falls back to `exec` only when SFTP is
190
+ unavailable (subsystem rejected, connection closed, local `sftp` missing).
191
+ Filesystem errors such as permission denied are never retried on another
192
+ backend.
193
+
194
+ ```bash
195
+ msdev env add dev-122 --node dev-122 --transfer auto
196
+ msdev env put dev-122 ./payload.bin /srv/payload.bin --transfer exec
197
+ msdev env get dev-122 /srv/payload.bin --output ./payload.bin --transfer sftp
198
+ ```
199
+
200
+ `env add --transfer` stores the preference in the env profile; `--transfer` on
201
+ put/get/read/write overrides it for one command. `MSDEV_TRANSFER` overrides
202
+ both. Explicit `sftp`/`exec` never falls back.
180
203
 
181
204
  ```bash
182
205
  msdev env read dev-122 /srv/project/config.yaml
@@ -226,6 +249,11 @@ argument after `exec`; use `-c` for inline strings or `--file` for script
226
249
  content (path or `-` for stdin/heredoc). `-c -` is accepted but identical to
227
250
  `--file -`; prefer **`--file -`** only.
228
251
 
252
+ Scripts up to 64 KiB of valid UTF-8 are inlined into the remote command.
253
+ Larger or non-UTF-8 scripts are uploaded through the transfer component to a
254
+ temporary file, executed, and removed afterwards. Set
255
+ `MSDEV_EXEC_INLINE_BYTES` to change the threshold.
256
+
229
257
  Quoted pipelines and `|` tokens are executed through a remote POSIX shell
230
258
  (`sh -c`), so they run on the Node instead of being treated as a single
231
259
  executable name. On a Windows `local` Env they use `cmd.exe /s /c`. Prefer
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "msdev"
7
- version = "0.15.2"
7
+ version = "0.16.0"
8
8
  description = "Portable nodes and execution environments for msModelSlim development"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -1,3 +1,3 @@
1
1
  """Portable node and execution-environment tooling for msModelSlim."""
2
2
 
3
- __version__ = "0.15.2"
3
+ __version__ = "0.16.0"
@@ -7,7 +7,9 @@ import json
7
7
  import os
8
8
  import signal
9
9
  import sys
10
+ import tempfile
10
11
  import threading
12
+ import uuid
11
13
  from dataclasses import asdict
12
14
  from pathlib import Path
13
15
  from typing import Any
@@ -321,6 +323,11 @@ def command_environment(
321
323
  ) -> int:
322
324
  service = EnvironmentService(_service_context(node_store, environment_store))
323
325
  action = args.environment_command
326
+ if (
327
+ action in {"put", "get", "read", "write"}
328
+ and getattr(args, "transfer", None)
329
+ ):
330
+ os.environ["MSDEV_TRANSFER"] = args.transfer
324
331
  if action == "guide":
325
332
  environment_store.get(args.name)
326
333
  return _command_guide(args, EnvironmentGuideStore(), "environment")
@@ -338,6 +345,7 @@ def command_environment(
338
345
  setup=_read_setup(args.setup_file, args.setup),
339
346
  wrap=tuple(args.wrap),
340
347
  env_vars=_parse_env_vars(args.env_var),
348
+ transfer=args.transfer,
341
349
  force=args.force,
342
350
  )
343
351
  )
@@ -416,12 +424,15 @@ def command_environment(
416
424
  sys.stdout.buffer.flush()
417
425
  return 0
418
426
  if action == "write":
419
- content = _env_write_content(args)
427
+ content = None if args.file is not None else _env_write_content(args)
420
428
  result = service.write(
421
429
  EnvironmentWriteRequest(
422
430
  environment=args.name,
423
431
  remote_path=args.remote_path,
424
432
  content=content,
433
+ source_path=(
434
+ str(args.file) if args.file is not None else None
435
+ ),
425
436
  expected_sha256=args.expected_sha256,
426
437
  )
427
438
  )
@@ -438,23 +449,119 @@ def _env_write_content(args: argparse.Namespace) -> bytes:
438
449
  return sys.stdin.buffer.read()
439
450
 
440
451
 
452
+ def _exec_inline_limit() -> int:
453
+ raw = os.environ.get("MSDEV_EXEC_INLINE_BYTES", "65536").strip()
454
+ try:
455
+ value = int(raw)
456
+ except ValueError:
457
+ return 65536
458
+ return max(0, value)
459
+
460
+
461
+ def _is_inline_script(raw: bytes, limit: int) -> bool:
462
+ if len(raw) > limit or b"\0" in raw:
463
+ return False
464
+ try:
465
+ raw.decode("utf-8")
466
+ except UnicodeDecodeError:
467
+ return False
468
+ return True
469
+
470
+
471
+ def _upload_exec_script(
472
+ environment: str,
473
+ raw: bytes,
474
+ node_store: NodeStore,
475
+ environment_store: EnvironmentStore,
476
+ ) -> tuple[str, ...]:
477
+ """Upload a script through the transfer component and return its argv."""
478
+ context = _service_context(node_store, environment_store)
479
+ transport = context.transport_for_environment(environment)
480
+ remote = f"/tmp/msdev-exec-{uuid.uuid4().hex}.script"
481
+ with tempfile.NamedTemporaryFile(
482
+ prefix="msdev-exec-",
483
+ delete=False,
484
+ ) as handle:
485
+ temporary = Path(handle.name)
486
+ handle.write(raw)
487
+ try:
488
+ transport.put_file(temporary, remote)
489
+ finally:
490
+ temporary.unlink(missing_ok=True)
491
+ if raw.startswith(b"#!"):
492
+ line = raw.split(b"\n", 1)[0][2:].strip().decode("utf-8", "replace")
493
+ interpreter = line.split()
494
+ if interpreter:
495
+ # Run the interpreter explicitly: no exec bit and no noexec /tmp.
496
+ cleanup = (
497
+ 'p=$1; shift; trap \'rm -f -- "$p"\' EXIT; "$@" "$p"'
498
+ )
499
+ return (
500
+ "sh",
501
+ "-c",
502
+ cleanup,
503
+ "msdev-exec",
504
+ remote,
505
+ *interpreter,
506
+ )
507
+ cleanup = 'p=$1; trap \'rm -f -- "$p"\' EXIT; sh "$p"'
508
+ return ("sh", "-c", cleanup, "msdev-exec", remote)
509
+
510
+
441
511
  def command_exec(
442
512
  args: argparse.Namespace,
443
513
  node_store: NodeStore,
444
514
  environment_store: EnvironmentStore,
445
515
  ) -> int:
446
516
  windows = args.environment == "local" and IS_WINDOWS
447
- script_stdin: str | None = None
448
- if args.shell_command == "-" or (
517
+ limit = _exec_inline_limit()
518
+ command: tuple[str, ...]
519
+ if args.command_file is not None and str(args.command_file) != "-":
520
+ source = args.command_file
521
+ if not source.is_file():
522
+ raise ValueError(f"script file not found: {source}")
523
+ raw = source.read_bytes()
524
+ if _is_inline_script(raw, limit):
525
+ command = resolve_exec_argv(
526
+ shell_command=None,
527
+ command_file=source,
528
+ windows=windows,
529
+ )
530
+ else:
531
+ if windows:
532
+ raise ValueError(
533
+ "inline exec script exceeds the local Windows limit "
534
+ f"({limit} bytes); use a smaller script or a remote env"
535
+ )
536
+ command = _upload_exec_script(
537
+ args.environment, raw, node_store, environment_store
538
+ )
539
+ elif args.shell_command == "-" or (
449
540
  args.command_file is not None and str(args.command_file) == "-"
450
541
  ):
451
- script_stdin = sys.stdin.read()
452
- command = resolve_exec_argv(
453
- shell_command=args.shell_command,
454
- command_file=args.command_file,
455
- script_stdin=script_stdin,
456
- windows=windows,
457
- )
542
+ raw = sys.stdin.buffer.read()
543
+ if _is_inline_script(raw, limit):
544
+ command = resolve_exec_argv(
545
+ shell_command=None,
546
+ command_file=Path("-"),
547
+ script_stdin=raw.decode("utf-8"),
548
+ windows=windows,
549
+ )
550
+ else:
551
+ if windows:
552
+ raise ValueError(
553
+ "inline exec script exceeds the local Windows limit "
554
+ f"({limit} bytes); use a smaller script or a remote env"
555
+ )
556
+ command = _upload_exec_script(
557
+ args.environment, raw, node_store, environment_store
558
+ )
559
+ else:
560
+ command = resolve_exec_argv(
561
+ shell_command=args.shell_command,
562
+ command_file=None,
563
+ windows=windows,
564
+ )
458
565
  env: dict[str, str] = {}
459
566
  for value in args.env_var:
460
567
  key, separator, item = value.partition("=")
@@ -772,6 +879,12 @@ def build_parser() -> argparse.ArgumentParser:
772
879
  metavar="KEY=VALUE",
773
880
  help="static environment variable stored in the env profile",
774
881
  )
882
+ environment_add.add_argument(
883
+ "--transfer",
884
+ choices=("auto", "sftp", "exec", "scp"),
885
+ default="auto",
886
+ help="file transfer backend stored in the env profile (default: auto)",
887
+ )
775
888
  environment_sub.add_parser("list")
776
889
  environment_inspect = environment_sub.add_parser("inspect")
777
890
  environment_inspect.add_argument("name")
@@ -805,6 +918,12 @@ def build_parser() -> argparse.ArgumentParser:
805
918
  "remote_path",
806
919
  help="absolute destination path inside the environment",
807
920
  )
921
+ environment_put.add_argument(
922
+ "--transfer",
923
+ choices=("auto", "sftp", "exec"),
924
+ default=None,
925
+ help="override the transfer backend for this operation",
926
+ )
808
927
  environment_get = environment_sub.add_parser(
809
928
  "get",
810
929
  help="fetch a file from an environment filesystem",
@@ -825,6 +944,12 @@ def build_parser() -> argparse.ArgumentParser:
825
944
  action="store_true",
826
945
  help="print metadata JSON when --output is used",
827
946
  )
947
+ environment_get.add_argument(
948
+ "--transfer",
949
+ choices=("auto", "sftp", "exec"),
950
+ default=None,
951
+ help="override the transfer backend for this operation",
952
+ )
828
953
  environment_read = environment_sub.add_parser(
829
954
  "read",
830
955
  help="read a remote file for editing (stdout or --json with sha256)",
@@ -839,6 +964,12 @@ def build_parser() -> argparse.ArgumentParser:
839
964
  action="store_true",
840
965
  help="print metadata JSON including sha256 and content_base64",
841
966
  )
967
+ environment_read.add_argument(
968
+ "--transfer",
969
+ choices=("auto", "sftp", "exec"),
970
+ default=None,
971
+ help="override the transfer backend for this operation",
972
+ )
842
973
  environment_write = environment_sub.add_parser(
843
974
  "write",
844
975
  help="atomically replace a remote file from stdin, --file, or --content",
@@ -862,6 +993,12 @@ def build_parser() -> argparse.ArgumentParser:
862
993
  "--expected-sha256",
863
994
  help="compare-and-swap: require this digest before replacing",
864
995
  )
996
+ environment_write.add_argument(
997
+ "--transfer",
998
+ choices=("auto", "sftp", "exec"),
999
+ default=None,
1000
+ help="override the transfer backend for this operation",
1001
+ )
865
1002
  environment_remove = environment_sub.add_parser("remove")
866
1003
  environment_remove.add_argument("name")
867
1004
 
@@ -0,0 +1,107 @@
1
+ """Host-local file transfer helpers used by the local transport."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ import shutil
7
+ import stat
8
+ import uuid
9
+ from pathlib import Path, PurePosixPath
10
+ from typing import Any, BinaryIO
11
+
12
+
13
+
14
+ def validate_absolute_local(path: str, *, label: str) -> str:
15
+ """Accept a host-local absolute path (POSIX or Windows)."""
16
+ if not isinstance(path, str) or not path or "\0" in path:
17
+ raise ValueError(f"{label} must be a non-empty NUL-free string")
18
+ candidate = Path(path)
19
+ if not candidate.is_absolute():
20
+ raise ValueError(f"{label} must be an absolute path")
21
+ return str(candidate)
22
+
23
+
24
+ def validate_absolute_remote(path: str, *, label: str) -> str:
25
+ if not isinstance(path, str) or not path or "\0" in path:
26
+ raise ValueError(f"{label} must be a non-empty NUL-free string")
27
+ if not path.startswith("/"):
28
+ raise ValueError(f"{label} must be an absolute path")
29
+ normalized = PurePosixPath(path)
30
+ if str(normalized) != path:
31
+ raise ValueError(f"{label} must be a canonical absolute POSIX path")
32
+ if any(part in {"", ".", ".."} for part in normalized.parts[1:]):
33
+ raise ValueError(f"{label} must not contain '.' or '..' components")
34
+ return path
35
+
36
+
37
+ def file_result(path: str | os.PathLike[str], *, runtime: str) -> dict[str, Any]:
38
+ target = Path(path)
39
+ metadata = target.lstat()
40
+ if not stat.S_ISREG(metadata.st_mode):
41
+ raise ValueError(f"path is not a regular file: {target}")
42
+ return {
43
+ "path": str(target),
44
+ "type": "file",
45
+ "size": metadata.st_size,
46
+ "runtime": runtime,
47
+ "atomic": True,
48
+ }
49
+
50
+
51
+ def _atomic_copy(source: Path, destination: Path) -> None:
52
+ destination.parent.mkdir(parents=True, exist_ok=True)
53
+ if destination.exists() and destination.is_symlink():
54
+ raise OSError(f"symlink is forbidden: {destination}")
55
+ if destination.exists() and destination.is_dir():
56
+ raise IsADirectoryError(f"path is a directory: {destination}")
57
+ temporary = destination.with_name(f".msdev-put-{uuid.uuid4().hex}")
58
+ try:
59
+ shutil.copyfile(source, temporary)
60
+ os.replace(temporary, destination)
61
+ finally:
62
+ temporary.unlink(missing_ok=True)
63
+
64
+
65
+ def local_put_file(local_path: Path, remote_path: str) -> dict[str, Any]:
66
+ source = local_path.expanduser().resolve()
67
+ if not source.is_file():
68
+ raise FileNotFoundError(f"local file not found: {source}")
69
+ destination = Path(validate_absolute_local(remote_path, label="remote path"))
70
+ _atomic_copy(source, destination)
71
+ return file_result(destination, runtime="host")
72
+
73
+
74
+ def local_get_file(
75
+ remote_path: str,
76
+ *,
77
+ output: Path | None = None,
78
+ sink: BinaryIO | None = None,
79
+ ) -> dict[str, Any]:
80
+ source = Path(validate_absolute_local(remote_path, label="remote path"))
81
+ if source.is_symlink():
82
+ raise OSError(f"symlink is forbidden: {source}")
83
+ if not source.is_file():
84
+ raise FileNotFoundError(f"remote file not found: {source}")
85
+ result = file_result(source, runtime="host")
86
+ if output is not None:
87
+ output = output.expanduser()
88
+ output.parent.mkdir(parents=True, exist_ok=True)
89
+ shutil.copyfile(source, output)
90
+ return result
91
+ if sink is not None:
92
+ with source.open("rb") as handle:
93
+ shutil.copyfileobj(handle, sink)
94
+ return result
95
+ with source.open("rb") as handle:
96
+ result["content"] = handle.read()
97
+ return result
98
+
99
+
100
+ def transfer_timeout_seconds() -> float | None:
101
+ raw = os.environ.get("MSDEV_TRANSFER_TIMEOUT_SECONDS", "-1").strip()
102
+ if raw in {"", "-1"}:
103
+ return None
104
+ value = float(raw)
105
+ if value <= 0:
106
+ return None
107
+ return value
@@ -12,7 +12,7 @@ from typing import Any
12
12
  from .config import _exclusive_file_lock, config_home, parse_environment_spec
13
13
 
14
14
 
15
- RESOURCE_CONFIG_VERSION = 3
15
+ RESOURCE_CONFIG_VERSION = 4
16
16
 
17
17
 
18
18
  def _required(value: str, label: str) -> str:
@@ -44,6 +44,7 @@ class Environment:
44
44
  setup: str | None = None
45
45
  wrap: tuple[str, ...] = ()
46
46
  env_vars: dict[str, str] = field(default_factory=dict)
47
+ transfer: str = "auto"
47
48
 
48
49
  def __post_init__(self) -> None:
49
50
  _required(self.name, "environment name")
@@ -69,6 +70,8 @@ class Environment:
69
70
  raise ValueError("wrap must be a non-empty string array")
70
71
  if self.runtime not in {"host", "docker"}:
71
72
  raise ValueError(f"unsupported environment runtime: {self.runtime}")
73
+ if self.transfer not in {"auto", "sftp", "exec", "scp"}:
74
+ raise ValueError(f"unsupported transfer backend: {self.transfer}")
72
75
  if self.runtime == "docker":
73
76
  _required(self.container or "", "docker container")
74
77
  elif self.container is not None:
@@ -114,7 +117,7 @@ class _ResourceStore:
114
117
  if not isinstance(value, dict) or not isinstance(value.get("nodes"), dict):
115
118
  raise ValueError(f"unsupported resource config schema in {self.path}")
116
119
  version = value.get("version")
117
- if version not in {2, RESOURCE_CONFIG_VERSION}:
120
+ if version not in {2, 3, RESOURCE_CONFIG_VERSION}:
118
121
  raise ValueError(f"unsupported resource config schema in {self.path}")
119
122
  if not isinstance(value.get("environments"), dict):
120
123
  raise ValueError(f"unsupported resource config schema in {self.path}")
@@ -27,6 +27,7 @@ class EnvironmentAddRequest:
27
27
  setup: str | None = None
28
28
  wrap: tuple[str, ...] = ()
29
29
  env_vars: dict[str, str] | None = None
30
+ transfer: str = "auto"
30
31
  force: bool = False
31
32
 
32
33
 
@@ -66,8 +67,9 @@ class EnvironmentReadRequest:
66
67
  class EnvironmentWriteRequest:
67
68
  environment: str
68
69
  remote_path: str
69
- content: bytes
70
+ content: bytes | None = None
70
71
  expected_sha256: str | None = None
72
+ source_path: str | None = None
71
73
 
72
74
 
73
75
  @dataclass(frozen=True)
@@ -129,6 +131,14 @@ class EnvironmentService:
129
131
  def _sha256(content: bytes) -> str:
130
132
  return hashlib.sha256(content).hexdigest()
131
133
 
134
+ @staticmethod
135
+ def _sha256_file(path: Path) -> str:
136
+ digest = hashlib.sha256()
137
+ with path.open("rb") as handle:
138
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
139
+ digest.update(chunk)
140
+ return digest.hexdigest()
141
+
132
142
  def _environment(self, name: str) -> Environment:
133
143
  environment_name = self._required(name, "environment name")
134
144
  return self.context.environment_store.get(environment_name)
@@ -171,6 +181,7 @@ class EnvironmentService:
171
181
  setup=request.setup,
172
182
  wrap=tuple(request.wrap),
173
183
  env_vars=dict(request.env_vars or {}),
184
+ transfer=request.transfer,
174
185
  )
175
186
  self.context.node_store.get(environment.node)
176
187
  self.context.environment_store.add(environment, force=request.force)
@@ -235,7 +246,11 @@ class EnvironmentService:
235
246
  path=str(raw.get("path") or remote_path),
236
247
  size=int(raw.get("size") or local_path.stat().st_size),
237
248
  runtime=str(raw.get("runtime") or environment.runtime),
238
- sha256=self._sha256(local_path.read_bytes()),
249
+ sha256=(
250
+ str(raw["sha256"])
251
+ if raw.get("sha256")
252
+ else self._sha256_file(local_path)
253
+ ),
239
254
  raw=raw,
240
255
  )
241
256
 
@@ -256,11 +271,19 @@ class EnvironmentService:
256
271
  raw = transport.get_file(remote_path, output=output)
257
272
  content = None
258
273
  digest = (
259
- self._sha256(output.read_bytes()) if output.is_file() else None
274
+ str(raw["sha256"])
275
+ if raw.get("sha256")
276
+ else (
277
+ self._sha256_file(output) if output.is_file() else None
278
+ )
260
279
  )
261
280
  else:
262
281
  content, raw = self._read_bytes(transport, remote_path)
263
- digest = self._sha256(content)
282
+ digest = (
283
+ str(raw["sha256"])
284
+ if raw.get("sha256")
285
+ else self._sha256(content)
286
+ )
264
287
  if not isinstance(raw, dict):
265
288
  raise RuntimeError("env get returned a non-object result")
266
289
  size = raw.get("size")
@@ -288,7 +311,9 @@ class EnvironmentService:
288
311
  self.context.transport_for_environment(environment.name)
289
312
  )
290
313
  content, raw = self._read_bytes(transport, remote_path)
291
- digest = self._sha256(content)
314
+ digest = (
315
+ str(raw["sha256"]) if raw.get("sha256") else self._sha256(content)
316
+ )
292
317
  return EnvironmentFileResult(
293
318
  environment=environment.name,
294
319
  path=str(raw.get("path") or remote_path),
@@ -307,9 +332,23 @@ class EnvironmentService:
307
332
  "remote path",
308
333
  local=environment.node == "local",
309
334
  )
310
- if not isinstance(request.content, (bytes, bytearray)):
311
- raise TypeError("write content must be bytes")
312
- content = bytes(request.content)
335
+ temporary: Path | None = None
336
+ if request.source_path is not None:
337
+ source = Path(request.source_path).expanduser()
338
+ if not source.is_file():
339
+ raise FileNotFoundError(f"source file not found: {source}")
340
+ content = None
341
+ else:
342
+ if not isinstance(request.content, (bytes, bytearray)):
343
+ raise TypeError("write content must be bytes")
344
+ content = bytes(request.content)
345
+ with tempfile.NamedTemporaryFile(
346
+ prefix="msdev-env-write-",
347
+ delete=False,
348
+ ) as handle:
349
+ temporary = Path(handle.name)
350
+ handle.write(content)
351
+ source = temporary
313
352
  expected = self._digest(request.expected_sha256, required=False)
314
353
  transport = self._transfer(
315
354
  self.context.transport_for_environment(environment.name)
@@ -321,28 +360,41 @@ class EnvironmentService:
321
360
  except FileNotFoundError:
322
361
  current_digest = None
323
362
  if current_digest != expected:
363
+ if temporary is not None:
364
+ temporary.unlink(missing_ok=True)
324
365
  raise RuntimeError(
325
366
  f"env file changed before write: {remote_path} "
326
367
  f"(expected {expected}, found {current_digest or 'missing'})"
327
368
  )
328
- with tempfile.NamedTemporaryFile(
329
- prefix="msdev-env-write-",
330
- delete=False,
331
- ) as handle:
332
- temporary = Path(handle.name)
333
- handle.write(content)
334
369
  try:
335
- raw = transport.put_file(temporary, remote_path)
370
+ raw = transport.put_file(source, remote_path)
336
371
  finally:
337
- temporary.unlink(missing_ok=True)
372
+ if temporary is not None:
373
+ temporary.unlink(missing_ok=True)
338
374
  if not isinstance(raw, dict):
339
375
  raise RuntimeError("env write returned a non-object result")
340
- digest = self._sha256(content)
376
+ digest = (
377
+ str(raw["sha256"])
378
+ if raw.get("sha256")
379
+ else (
380
+ self._sha256(content)
381
+ if content is not None
382
+ else self._sha256_file(source)
383
+ )
384
+ )
385
+ size = int(
386
+ raw.get("size")
387
+ or (
388
+ len(content)
389
+ if content is not None
390
+ else source.stat().st_size
391
+ )
392
+ )
341
393
  return EnvironmentFileResult(
342
394
  environment=environment.name,
343
395
  path=str(raw.get("path") or remote_path),
344
- size=len(content),
396
+ size=size,
345
397
  runtime=str(raw.get("runtime") or environment.runtime),
346
398
  sha256=digest,
347
- raw={**raw, "sha256": digest, "size": len(content)},
399
+ raw={**raw, "sha256": digest, "size": size},
348
400
  )