dstack 0.19.25rc1__py3-none-any.whl → 0.19.26__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of dstack might be problematic. Click here for more details.

Files changed (128) hide show
  1. dstack/_internal/cli/commands/__init__.py +2 -2
  2. dstack/_internal/cli/commands/apply.py +3 -61
  3. dstack/_internal/cli/commands/attach.py +1 -1
  4. dstack/_internal/cli/commands/completion.py +1 -1
  5. dstack/_internal/cli/commands/delete.py +2 -2
  6. dstack/_internal/cli/commands/fleet.py +1 -1
  7. dstack/_internal/cli/commands/gateway.py +2 -2
  8. dstack/_internal/cli/commands/init.py +56 -24
  9. dstack/_internal/cli/commands/logs.py +1 -1
  10. dstack/_internal/cli/commands/metrics.py +1 -1
  11. dstack/_internal/cli/commands/offer.py +45 -7
  12. dstack/_internal/cli/commands/project.py +2 -2
  13. dstack/_internal/cli/commands/secrets.py +2 -2
  14. dstack/_internal/cli/commands/server.py +1 -1
  15. dstack/_internal/cli/commands/stop.py +1 -1
  16. dstack/_internal/cli/commands/volume.py +1 -1
  17. dstack/_internal/cli/main.py +2 -2
  18. dstack/_internal/cli/services/completion.py +2 -2
  19. dstack/_internal/cli/services/configurators/__init__.py +6 -2
  20. dstack/_internal/cli/services/configurators/base.py +6 -7
  21. dstack/_internal/cli/services/configurators/fleet.py +1 -3
  22. dstack/_internal/cli/services/configurators/gateway.py +2 -4
  23. dstack/_internal/cli/services/configurators/run.py +195 -58
  24. dstack/_internal/cli/services/configurators/volume.py +2 -4
  25. dstack/_internal/cli/services/profile.py +1 -1
  26. dstack/_internal/cli/services/repos.py +51 -47
  27. dstack/_internal/core/backends/aws/configurator.py +11 -7
  28. dstack/_internal/core/backends/azure/configurator.py +11 -7
  29. dstack/_internal/core/backends/base/configurator.py +25 -13
  30. dstack/_internal/core/backends/cloudrift/configurator.py +13 -7
  31. dstack/_internal/core/backends/cudo/configurator.py +11 -7
  32. dstack/_internal/core/backends/datacrunch/compute.py +5 -1
  33. dstack/_internal/core/backends/datacrunch/configurator.py +13 -7
  34. dstack/_internal/core/backends/gcp/configurator.py +11 -7
  35. dstack/_internal/core/backends/hotaisle/configurator.py +13 -7
  36. dstack/_internal/core/backends/kubernetes/configurator.py +13 -7
  37. dstack/_internal/core/backends/lambdalabs/configurator.py +11 -7
  38. dstack/_internal/core/backends/nebius/compute.py +1 -1
  39. dstack/_internal/core/backends/nebius/configurator.py +11 -7
  40. dstack/_internal/core/backends/nebius/resources.py +21 -11
  41. dstack/_internal/core/backends/oci/configurator.py +11 -7
  42. dstack/_internal/core/backends/runpod/configurator.py +11 -7
  43. dstack/_internal/core/backends/template/configurator.py.jinja +11 -7
  44. dstack/_internal/core/backends/tensordock/configurator.py +13 -7
  45. dstack/_internal/core/backends/vastai/configurator.py +11 -7
  46. dstack/_internal/core/backends/vultr/configurator.py +11 -4
  47. dstack/_internal/core/compatibility/gpus.py +13 -0
  48. dstack/_internal/core/compatibility/runs.py +1 -0
  49. dstack/_internal/core/models/common.py +3 -3
  50. dstack/_internal/core/models/configurations.py +172 -27
  51. dstack/_internal/core/models/files.py +1 -1
  52. dstack/_internal/core/models/fleets.py +5 -1
  53. dstack/_internal/core/models/profiles.py +41 -11
  54. dstack/_internal/core/models/resources.py +46 -42
  55. dstack/_internal/core/models/runs.py +4 -0
  56. dstack/_internal/core/services/configs/__init__.py +6 -3
  57. dstack/_internal/core/services/profiles.py +2 -2
  58. dstack/_internal/core/services/repos.py +5 -3
  59. dstack/_internal/core/services/ssh/ports.py +1 -1
  60. dstack/_internal/proxy/lib/deps.py +6 -2
  61. dstack/_internal/server/app.py +22 -17
  62. dstack/_internal/server/background/tasks/process_gateways.py +4 -1
  63. dstack/_internal/server/background/tasks/process_instances.py +10 -2
  64. dstack/_internal/server/background/tasks/process_probes.py +1 -1
  65. dstack/_internal/server/background/tasks/process_running_jobs.py +10 -4
  66. dstack/_internal/server/background/tasks/process_runs.py +1 -1
  67. dstack/_internal/server/background/tasks/process_submitted_jobs.py +54 -43
  68. dstack/_internal/server/background/tasks/process_terminating_jobs.py +2 -2
  69. dstack/_internal/server/background/tasks/process_volumes.py +1 -1
  70. dstack/_internal/server/db.py +8 -4
  71. dstack/_internal/server/models.py +1 -0
  72. dstack/_internal/server/routers/gpus.py +1 -6
  73. dstack/_internal/server/schemas/runner.py +10 -0
  74. dstack/_internal/server/services/backends/__init__.py +14 -8
  75. dstack/_internal/server/services/backends/handlers.py +6 -1
  76. dstack/_internal/server/services/docker.py +5 -5
  77. dstack/_internal/server/services/fleets.py +14 -13
  78. dstack/_internal/server/services/gateways/__init__.py +2 -0
  79. dstack/_internal/server/services/gateways/client.py +5 -2
  80. dstack/_internal/server/services/gateways/connection.py +1 -1
  81. dstack/_internal/server/services/gpus.py +50 -49
  82. dstack/_internal/server/services/instances.py +41 -1
  83. dstack/_internal/server/services/jobs/__init__.py +15 -4
  84. dstack/_internal/server/services/jobs/configurators/base.py +7 -11
  85. dstack/_internal/server/services/jobs/configurators/dev.py +5 -0
  86. dstack/_internal/server/services/jobs/configurators/extensions/cursor.py +3 -3
  87. dstack/_internal/server/services/jobs/configurators/extensions/vscode.py +3 -3
  88. dstack/_internal/server/services/jobs/configurators/service.py +1 -0
  89. dstack/_internal/server/services/jobs/configurators/task.py +3 -0
  90. dstack/_internal/server/services/locking.py +5 -5
  91. dstack/_internal/server/services/logging.py +10 -2
  92. dstack/_internal/server/services/logs/__init__.py +8 -6
  93. dstack/_internal/server/services/logs/aws.py +330 -327
  94. dstack/_internal/server/services/logs/filelog.py +7 -6
  95. dstack/_internal/server/services/logs/gcp.py +141 -139
  96. dstack/_internal/server/services/plugins.py +1 -1
  97. dstack/_internal/server/services/projects.py +2 -5
  98. dstack/_internal/server/services/proxy/repo.py +5 -1
  99. dstack/_internal/server/services/requirements/__init__.py +0 -0
  100. dstack/_internal/server/services/requirements/combine.py +259 -0
  101. dstack/_internal/server/services/runner/client.py +7 -0
  102. dstack/_internal/server/services/runs.py +1 -1
  103. dstack/_internal/server/services/services/__init__.py +8 -2
  104. dstack/_internal/server/services/services/autoscalers.py +2 -0
  105. dstack/_internal/server/services/ssh.py +2 -1
  106. dstack/_internal/server/services/storage/__init__.py +5 -6
  107. dstack/_internal/server/services/storage/gcs.py +49 -49
  108. dstack/_internal/server/services/storage/s3.py +52 -52
  109. dstack/_internal/server/statics/index.html +1 -1
  110. dstack/_internal/server/testing/common.py +1 -1
  111. dstack/_internal/server/utils/logging.py +3 -3
  112. dstack/_internal/server/utils/provisioning.py +3 -3
  113. dstack/_internal/utils/json_schema.py +3 -1
  114. dstack/_internal/utils/typing.py +14 -0
  115. dstack/api/_public/repos.py +21 -2
  116. dstack/api/_public/runs.py +5 -7
  117. dstack/api/server/__init__.py +17 -19
  118. dstack/api/server/_gpus.py +2 -1
  119. dstack/api/server/_group.py +4 -3
  120. dstack/api/server/_repos.py +20 -3
  121. dstack/plugins/builtin/rest_plugin/_plugin.py +1 -0
  122. dstack/version.py +1 -1
  123. {dstack-0.19.25rc1.dist-info → dstack-0.19.26.dist-info}/METADATA +1 -1
  124. {dstack-0.19.25rc1.dist-info → dstack-0.19.26.dist-info}/RECORD +127 -124
  125. dstack/api/huggingface/__init__.py +0 -73
  126. {dstack-0.19.25rc1.dist-info → dstack-0.19.26.dist-info}/WHEEL +0 -0
  127. {dstack-0.19.25rc1.dist-info → dstack-0.19.26.dist-info}/entry_points.txt +0 -0
  128. {dstack-0.19.25rc1.dist-info → dstack-0.19.26.dist-info}/licenses/LICENSE.md +0 -0
@@ -3,7 +3,7 @@ import subprocess
3
3
  import sys
4
4
  import time
5
5
  from pathlib import Path
6
- from typing import Dict, List, Optional, Set
6
+ from typing import Dict, List, Optional, Set, TypeVar
7
7
 
8
8
  import gpuhunt
9
9
  from pydantic import parse_obj_as
@@ -15,12 +15,14 @@ from dstack._internal.cli.services.configurators.base import (
15
15
  BaseApplyConfigurator,
16
16
  )
17
17
  from dstack._internal.cli.services.profile import apply_profile_args, register_profile_args
18
- from dstack._internal.cli.services.repos import init_default_virtual_repo
19
- from dstack._internal.cli.utils.common import (
20
- confirm_ask,
21
- console,
22
- warn,
18
+ from dstack._internal.cli.services.repos import (
19
+ get_repo_from_dir,
20
+ get_repo_from_url,
21
+ init_default_virtual_repo,
22
+ is_git_repo_url,
23
+ register_init_repo_args,
23
24
  )
25
+ from dstack._internal.cli.utils.common import confirm_ask, console, warn
24
26
  from dstack._internal.cli.utils.rich import MultiItemStatus
25
27
  from dstack._internal.cli.utils.run import get_runs_table, print_run_plan
26
28
  from dstack._internal.core.errors import (
@@ -33,8 +35,7 @@ from dstack._internal.core.models.common import ApplyAction, RegistryAuth
33
35
  from dstack._internal.core.models.configurations import (
34
36
  AnyRunConfiguration,
35
37
  ApplyConfigurationType,
36
- BaseRunConfiguration,
37
- BaseRunConfigurationWithPorts,
38
+ ConfigurationWithPortsParams,
38
39
  DevEnvironmentConfiguration,
39
40
  PortMapping,
40
41
  RunConfigurationType,
@@ -47,6 +48,7 @@ from dstack._internal.core.models.resources import CPUSpec
47
48
  from dstack._internal.core.models.runs import JobStatus, JobSubmission, RunSpec, RunStatus
48
49
  from dstack._internal.core.services.configs import ConfigManager
49
50
  from dstack._internal.core.services.diff import diff_models
51
+ from dstack._internal.core.services.repos import load_repo
50
52
  from dstack._internal.utils.common import local_time
51
53
  from dstack._internal.utils.interpolator import InterpolatorError, VariablesInterpolator
52
54
  from dstack._internal.utils.logging import get_logger
@@ -63,56 +65,34 @@ _BIND_ADDRESS_ARG = "bind_address"
63
65
 
64
66
  logger = get_logger(__name__)
65
67
 
68
+ RunConfigurationT = TypeVar("RunConfigurationT", bound=AnyRunConfiguration)
66
69
 
67
- class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
70
+
71
+ class BaseRunConfigurator(
72
+ ApplyEnvVarsConfiguratorMixin,
73
+ BaseApplyConfigurator[RunConfigurationT],
74
+ ):
68
75
  TYPE: ApplyConfigurationType
69
76
 
70
77
  def apply_configuration(
71
78
  self,
72
- conf: BaseRunConfiguration,
79
+ conf: RunConfigurationT,
73
80
  configuration_path: str,
74
81
  command_args: argparse.Namespace,
75
82
  configurator_args: argparse.Namespace,
76
83
  unknown_args: List[str],
77
- repo: Optional[Repo] = None,
78
84
  ):
85
+ if configurator_args.repo and configurator_args.no_repo:
86
+ raise CLIError("Either --repo or --no-repo can be specified")
87
+
79
88
  self.apply_args(conf, configurator_args, unknown_args)
80
89
  self.validate_gpu_vendor_and_image(conf)
81
90
  self.validate_cpu_arch_and_image(conf)
91
+
82
92
  config_manager = ConfigManager()
83
- if repo is None:
84
- repo_path = Path.cwd()
85
- repo_config = config_manager.get_repo_config(repo_path)
86
- if repo_config is None:
87
- warn(
88
- "The repo is not initialized. Starting from 0.19.25, repos are optional\n"
89
- "There are three options:\n"
90
- " - Run `dstack init` to initialize the current directory as a repo\n"
91
- " - Specify `--repo`\n"
92
- " - Specify `--no-repo` to not use any repo and supress this warning"
93
- " (this will be the default in the future versions)"
94
- )
95
- if not command_args.yes and not confirm_ask("Continue without the repo?"):
96
- console.print("\nExiting...")
97
- return
98
- repo = init_default_virtual_repo(self.api)
99
- else:
100
- # Unlikely, but may raise ConfigurationError if the repo does not exist
101
- # on the server side (stale entry in `config.yml`)
102
- repo = self.api.repos.load(repo_path)
103
- if isinstance(repo, LocalRepo):
104
- warn(
105
- f"{repo.repo_dir} is a local repo.\n"
106
- "Local repos are deprecated since 0.19.25"
107
- " and will be removed soon\n"
108
- "There are two options:\n"
109
- " - Migrate to `files`: https://dstack.ai/docs/concepts/tasks/#files\n"
110
- " - Specify `--no-repo` if you don't need the repo at all\n"
111
- "In either case, you can run `dstack init --remove` to remove the repo"
112
- " (only the record about the repo, not its files) and this warning"
113
- )
93
+ repo = self.get_repo(conf, configuration_path, configurator_args, config_manager)
114
94
  self.api.ssh_identity_file = get_ssh_keypair(
115
- command_args.ssh_identity_file,
95
+ configurator_args.ssh_identity_file,
116
96
  config_manager.dstack_key_path,
117
97
  )
118
98
  profile = load_profile(Path.cwd(), configurator_args.profile)
@@ -270,7 +250,7 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
270
250
 
271
251
  def delete_configuration(
272
252
  self,
273
- conf: AnyRunConfiguration,
253
+ conf: RunConfigurationT,
274
254
  configuration_path: str,
275
255
  command_args: argparse.Namespace,
276
256
  ):
@@ -296,7 +276,14 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
296
276
  console.print(f"Run [code]{conf.name}[/] deleted")
297
277
 
298
278
  @classmethod
299
- def register_args(cls, parser: argparse.ArgumentParser, default_max_offers: int = 3):
279
+ def register_args(cls, parser: argparse.ArgumentParser):
280
+ parser.add_argument(
281
+ "--ssh-identity",
282
+ metavar="SSH_PRIVATE_KEY",
283
+ help="The private SSH key path for SSH tunneling",
284
+ type=Path,
285
+ dest="ssh_identity_file",
286
+ )
300
287
  configuration_group = parser.add_argument_group(f"{cls.TYPE.value} Options")
301
288
  configuration_group.add_argument(
302
289
  "-n",
@@ -308,7 +295,7 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
308
295
  "--max-offers",
309
296
  help="Number of offers to show in the run plan",
310
297
  type=int,
311
- default=default_max_offers,
298
+ default=3,
312
299
  )
313
300
  cls.register_env_args(configuration_group)
314
301
  configuration_group.add_argument(
@@ -335,8 +322,32 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
335
322
  dest="disk_spec",
336
323
  )
337
324
  register_profile_args(parser)
325
+ repo_group = parser.add_argument_group("Repo Options")
326
+ repo_group.add_argument(
327
+ "-P",
328
+ "--repo",
329
+ help=("The repo to use for the run. Can be a local path or a Git repo URL."),
330
+ dest="repo",
331
+ )
332
+ repo_group.add_argument(
333
+ "--repo-branch",
334
+ help="The repo branch to use for the run",
335
+ dest="repo_branch",
336
+ )
337
+ repo_group.add_argument(
338
+ "--repo-hash",
339
+ help="The hash of the repo commit to use for the run",
340
+ dest="repo_hash",
341
+ )
342
+ repo_group.add_argument(
343
+ "--no-repo",
344
+ help="Do not use any repo for the run",
345
+ dest="no_repo",
346
+ action="store_true",
347
+ )
348
+ register_init_repo_args(repo_group)
338
349
 
339
- def apply_args(self, conf: BaseRunConfiguration, args: argparse.Namespace, unknown: List[str]):
350
+ def apply_args(self, conf: RunConfigurationT, args: argparse.Namespace, unknown: List[str]):
340
351
  apply_profile_args(args, conf)
341
352
  if args.run_name:
342
353
  conf.name = args.run_name
@@ -360,7 +371,7 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
360
371
  except InterpolatorError as e:
361
372
  raise ConfigurationError(e.args[0])
362
373
 
363
- def interpolate_env(self, conf: BaseRunConfiguration):
374
+ def interpolate_env(self, conf: RunConfigurationT):
364
375
  env_dict = conf.env.as_dict()
365
376
  interpolator = VariablesInterpolator({"env": env_dict}, skip=["secrets"])
366
377
  try:
@@ -380,7 +391,7 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
380
391
  except InterpolatorError as e:
381
392
  raise ConfigurationError(e.args[0])
382
393
 
383
- def validate_gpu_vendor_and_image(self, conf: BaseRunConfiguration) -> None:
394
+ def validate_gpu_vendor_and_image(self, conf: RunConfigurationT) -> None:
384
395
  """
385
396
  Infers and sets `resources.gpu.vendor` if not set, requires `image` if the vendor is AMD.
386
397
  """
@@ -441,7 +452,7 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
441
452
  "`image` is required if `resources.gpu.vendor` is `tenstorrent`"
442
453
  )
443
454
 
444
- def validate_cpu_arch_and_image(self, conf: BaseRunConfiguration) -> None:
455
+ def validate_cpu_arch_and_image(self, conf: RunConfigurationT) -> None:
445
456
  """
446
457
  Infers `resources.cpu.arch` if not set, requires `image` if the architecture is ARM.
447
458
  """
@@ -464,11 +475,122 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
464
475
  if arch == gpuhunt.CPUArchitecture.ARM and conf.image is None:
465
476
  raise ConfigurationError("`image` is required if `resources.cpu.arch` is `arm`")
466
477
 
478
+ def get_repo(
479
+ self,
480
+ conf: RunConfigurationT,
481
+ configuration_path: str,
482
+ configurator_args: argparse.Namespace,
483
+ config_manager: ConfigManager,
484
+ ) -> Repo:
485
+ if configurator_args.no_repo:
486
+ return init_default_virtual_repo(api=self.api)
487
+
488
+ repo: Optional[Repo] = None
489
+ repo_branch: Optional[str] = configurator_args.repo_branch
490
+ repo_hash: Optional[str] = configurator_args.repo_hash
491
+ # Should we (re)initialize the repo?
492
+ # If any Git credentials provided, we reinitialize the repo, as the user may have provided
493
+ # updated credentials.
494
+ init = (
495
+ configurator_args.git_identity_file is not None
496
+ or configurator_args.gh_token is not None
497
+ )
467
498
 
468
- class RunWithPortsConfigurator(BaseRunConfigurator):
499
+ url: Optional[str] = None
500
+ local_path: Optional[Path] = None
501
+ # dummy value, safe to join with any path
502
+ root_dir = Path(".")
503
+ # True if no repo specified, but we found one in `config.yml`
504
+ legacy_local_path = False
505
+ if repo_arg := configurator_args.repo:
506
+ if is_git_repo_url(repo_arg):
507
+ url = repo_arg
508
+ else:
509
+ local_path = Path(repo_arg)
510
+ # rel paths in `--repo` are resolved relative to the current working dir
511
+ root_dir = Path.cwd()
512
+ elif conf.repos:
513
+ repo_spec = conf.repos[0]
514
+ if repo_spec.url:
515
+ url = repo_spec.url
516
+ elif repo_spec.local_path:
517
+ local_path = Path(repo_spec.local_path)
518
+ # rel paths in the conf are resolved relative to the conf's parent dir
519
+ root_dir = Path(configuration_path).resolve().parent
520
+ else:
521
+ assert False, f"should not reach here: {repo_spec}"
522
+ if repo_branch is None:
523
+ repo_branch = repo_spec.branch
524
+ if repo_hash is None:
525
+ repo_hash = repo_spec.hash
526
+ else:
527
+ local_path = Path.cwd()
528
+ legacy_local_path = True
529
+ if url:
530
+ repo = get_repo_from_url(repo_url=url, repo_branch=repo_branch, repo_hash=repo_hash)
531
+ if not self.api.repos.is_initialized(repo, by_user=True):
532
+ init = True
533
+ elif local_path:
534
+ if legacy_local_path:
535
+ if repo_config := config_manager.get_repo_config(local_path):
536
+ repo = load_repo(repo_config)
537
+ # allow users with legacy configurations use shared repo creds
538
+ if self.api.repos.is_initialized(repo, by_user=False):
539
+ warn(
540
+ "The repo is not specified but found and will be used in the run\n"
541
+ "Future versions will not load repos automatically\n"
542
+ "To prepare for future versions and get rid of this warning:\n"
543
+ "- If you need the repo in the run, either specify [code]repos[/code]"
544
+ " in the configuration or use [code]--repo .[/code]\n"
545
+ "- If you don't need the repo in the run, either run"
546
+ " [code]dstack init --remove[/code] once (it removes only the record"
547
+ " about the repo, the repo files will remain intact)"
548
+ " or use [code]--no-repo[/code]"
549
+ )
550
+ else:
551
+ # ignore stale entries in `config.yml`
552
+ repo = None
553
+ init = False
554
+ else:
555
+ original_local_path = local_path
556
+ local_path = local_path.expanduser()
557
+ if not local_path.is_absolute():
558
+ local_path = (root_dir / local_path).resolve()
559
+ if not local_path.exists():
560
+ raise ConfigurationError(
561
+ f"Invalid repo path: {original_local_path} -> {local_path}"
562
+ )
563
+ local: bool = configurator_args.local
564
+ repo = get_repo_from_dir(local_path, local=local)
565
+ if not self.api.repos.is_initialized(repo, by_user=True):
566
+ init = True
567
+ else:
568
+ assert False, "should not reach here"
569
+
570
+ if repo is None:
571
+ return init_default_virtual_repo(api=self.api)
572
+
573
+ if init:
574
+ self.api.repos.init(
575
+ repo=repo,
576
+ git_identity_file=configurator_args.git_identity_file,
577
+ oauth_token=configurator_args.gh_token,
578
+ )
579
+ if isinstance(repo, LocalRepo):
580
+ warn(
581
+ f"{repo.repo_dir} is a local repo\n"
582
+ "Local repos are deprecated since 0.19.25 and will be removed soon\n"
583
+ "There are two options:\n"
584
+ "- Migrate to [code]files[/code]: https://dstack.ai/docs/concepts/tasks/#files\n"
585
+ "- Specify [code]--no-repo[/code] if you don't need the repo at all"
586
+ )
587
+
588
+ return repo
589
+
590
+
591
+ class RunWithPortsConfiguratorMixin:
469
592
  @classmethod
470
- def register_args(cls, parser: argparse.ArgumentParser):
471
- super().register_args(parser)
593
+ def register_ports_args(cls, parser: argparse.ArgumentParser):
472
594
  parser.add_argument(
473
595
  "-p",
474
596
  "--port",
@@ -485,29 +607,42 @@ class RunWithPortsConfigurator(BaseRunConfigurator):
485
607
  metavar="HOST",
486
608
  )
487
609
 
488
- def apply_args(
489
- self, conf: BaseRunConfigurationWithPorts, args: argparse.Namespace, unknown: List[str]
610
+ def apply_ports_args(
611
+ self,
612
+ conf: ConfigurationWithPortsParams,
613
+ args: argparse.Namespace,
490
614
  ):
491
- super().apply_args(conf, args, unknown)
492
615
  if args.ports:
493
616
  conf.ports = list(_merge_ports(conf.ports, args.ports).values())
494
617
 
495
618
 
496
- class TaskConfigurator(RunWithPortsConfigurator):
619
+ class TaskConfigurator(RunWithPortsConfiguratorMixin, BaseRunConfigurator):
497
620
  TYPE = ApplyConfigurationType.TASK
498
621
 
622
+ @classmethod
623
+ def register_args(cls, parser: argparse.ArgumentParser):
624
+ super().register_args(parser)
625
+ cls.register_ports_args(parser)
626
+
499
627
  def apply_args(self, conf: TaskConfiguration, args: argparse.Namespace, unknown: List[str]):
500
628
  super().apply_args(conf, args, unknown)
629
+ self.apply_ports_args(conf, args)
501
630
  self.interpolate_run_args(conf.commands, unknown)
502
631
 
503
632
 
504
- class DevEnvironmentConfigurator(RunWithPortsConfigurator):
633
+ class DevEnvironmentConfigurator(RunWithPortsConfiguratorMixin, BaseRunConfigurator):
505
634
  TYPE = ApplyConfigurationType.DEV_ENVIRONMENT
506
635
 
636
+ @classmethod
637
+ def register_args(cls, parser: argparse.ArgumentParser):
638
+ super().register_args(parser)
639
+ cls.register_ports_args(parser)
640
+
507
641
  def apply_args(
508
642
  self, conf: DevEnvironmentConfiguration, args: argparse.Namespace, unknown: List[str]
509
643
  ):
510
644
  super().apply_args(conf, args, unknown)
645
+ self.apply_ports_args(conf, args)
511
646
  if conf.ide == "vscode" and conf.version is None:
512
647
  conf.version = _detect_vscode_version()
513
648
  if conf.version is None:
@@ -677,6 +812,8 @@ def render_run_spec_diff(old_spec: RunSpec, new_spec: RunSpec) -> Optional[str]:
677
812
  if type(old_spec.profile) is not type(new_spec.profile):
678
813
  item = NestedListItem("Profile")
679
814
  else:
815
+ assert old_spec.profile is not None
816
+ assert new_spec.profile is not None
680
817
  item = NestedListItem(
681
818
  "Profile properties:",
682
819
  children=[
@@ -1,6 +1,6 @@
1
1
  import argparse
2
2
  import time
3
- from typing import List, Optional
3
+ from typing import List
4
4
 
5
5
  from rich.table import Table
6
6
 
@@ -14,7 +14,6 @@ from dstack._internal.cli.utils.rich import MultiItemStatus
14
14
  from dstack._internal.cli.utils.volume import get_volumes_table
15
15
  from dstack._internal.core.errors import ResourceNotExistsError
16
16
  from dstack._internal.core.models.configurations import ApplyConfigurationType
17
- from dstack._internal.core.models.repos.base import Repo
18
17
  from dstack._internal.core.models.volumes import (
19
18
  Volume,
20
19
  VolumeConfiguration,
@@ -26,7 +25,7 @@ from dstack._internal.utils.common import local_time
26
25
  from dstack.api._public import Client
27
26
 
28
27
 
29
- class VolumeConfigurator(BaseApplyConfigurator):
28
+ class VolumeConfigurator(BaseApplyConfigurator[VolumeConfiguration]):
30
29
  TYPE: ApplyConfigurationType = ApplyConfigurationType.VOLUME
31
30
 
32
31
  def apply_configuration(
@@ -36,7 +35,6 @@ class VolumeConfigurator(BaseApplyConfigurator):
36
35
  command_args: argparse.Namespace,
37
36
  configurator_args: argparse.Namespace,
38
37
  unknown_args: List[str],
39
- repo: Optional[Repo] = None,
40
38
  ):
41
39
  self.apply_args(conf, configurator_args, unknown_args)
42
40
  spec = VolumeSpec(
@@ -159,7 +159,7 @@ def apply_profile_args(
159
159
  if args.idle_duration is not None:
160
160
  profile_settings.idle_duration = args.idle_duration
161
161
  elif args.dont_destroy:
162
- profile_settings.idle_duration = "off"
162
+ profile_settings.idle_duration = -1
163
163
  if args.creation_policy_reuse:
164
164
  profile_settings.creation_policy = CreationPolicy.REUSE
165
165
 
@@ -1,10 +1,11 @@
1
1
  import argparse
2
- from pathlib import Path
3
- from typing import Optional
2
+ from typing import Literal, Optional, Union, overload
3
+
4
+ import git
4
5
 
5
6
  from dstack._internal.cli.services.configurators.base import ArgsParser
6
7
  from dstack._internal.core.errors import CLIError
7
- from dstack._internal.core.models.repos.base import Repo
8
+ from dstack._internal.core.models.repos.local import LocalRepo
8
9
  from dstack._internal.core.models.repos.remote import GitRepoURL, RemoteRepo, RepoError
9
10
  from dstack._internal.core.models.repos.virtual import VirtualRepo
10
11
  from dstack._internal.core.services.repos import get_default_branch
@@ -36,51 +37,54 @@ def register_init_repo_args(parser: ArgsParser):
36
37
  )
37
38
 
38
39
 
39
- def init_repo(
40
- api: Client,
41
- repo_path: PathLike,
42
- repo_branch: Optional[str],
43
- repo_hash: Optional[str],
44
- local: bool,
45
- git_identity_file: Optional[PathLike],
46
- oauth_token: Optional[str],
47
- ) -> Repo:
48
- if Path(repo_path).exists():
49
- repo = api.repos.load(
50
- repo_dir=repo_path,
51
- local=local,
52
- init=True,
53
- git_identity_file=git_identity_file,
54
- oauth_token=oauth_token,
55
- )
56
- elif isinstance(repo_path, str):
57
- try:
58
- GitRepoURL.parse(repo_path)
59
- except RepoError as e:
60
- raise CLIError("Invalid repo path") from e
61
- if repo_branch is None and repo_hash is None:
62
- repo_branch = get_default_branch(repo_path)
63
- if repo_branch is None:
64
- raise CLIError(
65
- "Failed to automatically detect remote repo branch."
66
- " Specify --repo-branch or --repo-hash."
67
- )
68
- repo = RemoteRepo.from_url(
69
- repo_url=repo_path,
70
- repo_branch=repo_branch,
71
- repo_hash=repo_hash,
72
- )
73
- api.repos.init(
74
- repo=repo,
75
- git_identity_file=git_identity_file,
76
- oauth_token=oauth_token,
77
- )
78
- else:
79
- raise CLIError("Invalid repo path")
80
- return repo
81
-
82
-
83
40
  def init_default_virtual_repo(api: Client) -> VirtualRepo:
84
41
  repo = VirtualRepo()
85
42
  api.repos.init(repo)
86
43
  return repo
44
+
45
+
46
+ def get_repo_from_url(
47
+ repo_url: str, repo_branch: Optional[str] = None, repo_hash: Optional[str] = None
48
+ ) -> RemoteRepo:
49
+ if repo_branch is None and repo_hash is None:
50
+ repo_branch = get_default_branch(repo_url)
51
+ if repo_branch is None:
52
+ raise CLIError(
53
+ "Failed to automatically detect remote repo branch. Specify branch or hash."
54
+ )
55
+ return RemoteRepo.from_url(
56
+ repo_url=repo_url,
57
+ repo_branch=repo_branch,
58
+ repo_hash=repo_hash,
59
+ )
60
+
61
+
62
+ @overload
63
+ def get_repo_from_dir(repo_dir: PathLike, local: Literal[False] = False) -> RemoteRepo: ...
64
+
65
+
66
+ @overload
67
+ def get_repo_from_dir(repo_dir: PathLike, local: Literal[True]) -> LocalRepo: ...
68
+
69
+
70
+ def get_repo_from_dir(repo_dir: PathLike, local: bool = False) -> Union[RemoteRepo, LocalRepo]:
71
+ if local:
72
+ return LocalRepo.from_dir(repo_dir)
73
+ try:
74
+ return RemoteRepo.from_dir(repo_dir)
75
+ except git.InvalidGitRepositoryError:
76
+ raise CLIError(
77
+ f"Git repo not found: {repo_dir}\n"
78
+ "Use `files` to mount an arbitrary directory:"
79
+ " https://dstack.ai/docs/concepts/tasks/#files"
80
+ )
81
+ except RepoError as e:
82
+ raise CLIError(str(e)) from e
83
+
84
+
85
+ def is_git_repo_url(value: str) -> bool:
86
+ try:
87
+ GitRepoURL.parse(value)
88
+ except RepoError:
89
+ return False
90
+ return True
@@ -7,7 +7,6 @@ from boto3.session import Session
7
7
  from dstack._internal.core.backends.aws import auth, compute, resources
8
8
  from dstack._internal.core.backends.aws.backend import AWSBackend
9
9
  from dstack._internal.core.backends.aws.models import (
10
- AnyAWSBackendConfig,
11
10
  AWSAccessKeyCreds,
12
11
  AWSBackendConfig,
13
12
  AWSBackendConfigWithCreds,
@@ -52,7 +51,12 @@ DEFAULT_REGIONS = REGION_VALUES
52
51
  MAIN_REGION = "us-east-1"
53
52
 
54
53
 
55
- class AWSConfigurator(Configurator):
54
+ class AWSConfigurator(
55
+ Configurator[
56
+ AWSBackendConfig,
57
+ AWSBackendConfigWithCreds,
58
+ ]
59
+ ):
56
60
  TYPE = BackendType.AWS
57
61
  BACKEND_CLASS = AWSBackend
58
62
 
@@ -87,12 +91,12 @@ class AWSConfigurator(Configurator):
87
91
  auth=AWSCreds.parse_obj(config.creds).json(),
88
92
  )
89
93
 
90
- def get_backend_config(
91
- self, record: BackendRecord, include_creds: bool
92
- ) -> AnyAWSBackendConfig:
94
+ def get_backend_config_with_creds(self, record: BackendRecord) -> AWSBackendConfigWithCreds:
95
+ config = self._get_config(record)
96
+ return AWSBackendConfigWithCreds.__response__.parse_obj(config)
97
+
98
+ def get_backend_config_without_creds(self, record: BackendRecord) -> AWSBackendConfig:
93
99
  config = self._get_config(record)
94
- if include_creds:
95
- return AWSBackendConfigWithCreds.__response__.parse_obj(config)
96
100
  return AWSBackendConfig.__response__.parse_obj(config)
97
101
 
98
102
  def get_backend(self, record: BackendRecord) -> AWSBackend:
@@ -24,7 +24,6 @@ from dstack._internal.core.backends.azure import auth, compute, resources
24
24
  from dstack._internal.core.backends.azure import utils as azure_utils
25
25
  from dstack._internal.core.backends.azure.backend import AzureBackend
26
26
  from dstack._internal.core.backends.azure.models import (
27
- AnyAzureBackendConfig,
28
27
  AzureBackendConfig,
29
28
  AzureBackendConfigWithCreds,
30
29
  AzureClientCreds,
@@ -71,7 +70,12 @@ DEFAULT_LOCATIONS = LOCATION_VALUES
71
70
  MAIN_LOCATION = "eastus"
72
71
 
73
72
 
74
- class AzureConfigurator(Configurator):
73
+ class AzureConfigurator(
74
+ Configurator[
75
+ AzureBackendConfig,
76
+ AzureBackendConfigWithCreds,
77
+ ]
78
+ ):
75
79
  TYPE = BackendType.AZURE
76
80
  BACKEND_CLASS = AzureBackend
77
81
 
@@ -130,12 +134,12 @@ class AzureConfigurator(Configurator):
130
134
  auth=AzureCreds.parse_obj(config.creds).__root__.json(),
131
135
  )
132
136
 
133
- def get_backend_config(
134
- self, record: BackendRecord, include_creds: bool
135
- ) -> AnyAzureBackendConfig:
137
+ def get_backend_config_with_creds(self, record: BackendRecord) -> AzureBackendConfigWithCreds:
138
+ config = self._get_config(record)
139
+ return AzureBackendConfigWithCreds.__response__.parse_obj(config)
140
+
141
+ def get_backend_config_without_creds(self, record: BackendRecord) -> AzureBackendConfig:
136
142
  config = self._get_config(record)
137
- if include_creds:
138
- return AzureBackendConfigWithCreds.__response__.parse_obj(config)
139
143
  return AzureBackendConfig.__response__.parse_obj(config)
140
144
 
141
145
  def get_backend(self, record: BackendRecord) -> AzureBackend: