dstack 0.19.25__py3-none-any.whl → 0.19.26__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of dstack might be problematic. Click here for more details.

Files changed (128) hide show
  1. dstack/_internal/cli/commands/__init__.py +2 -2
  2. dstack/_internal/cli/commands/apply.py +3 -61
  3. dstack/_internal/cli/commands/attach.py +1 -1
  4. dstack/_internal/cli/commands/completion.py +1 -1
  5. dstack/_internal/cli/commands/delete.py +2 -2
  6. dstack/_internal/cli/commands/fleet.py +1 -1
  7. dstack/_internal/cli/commands/gateway.py +2 -2
  8. dstack/_internal/cli/commands/init.py +56 -24
  9. dstack/_internal/cli/commands/logs.py +1 -1
  10. dstack/_internal/cli/commands/metrics.py +1 -1
  11. dstack/_internal/cli/commands/offer.py +45 -7
  12. dstack/_internal/cli/commands/project.py +2 -2
  13. dstack/_internal/cli/commands/secrets.py +2 -2
  14. dstack/_internal/cli/commands/server.py +1 -1
  15. dstack/_internal/cli/commands/stop.py +1 -1
  16. dstack/_internal/cli/commands/volume.py +1 -1
  17. dstack/_internal/cli/main.py +2 -2
  18. dstack/_internal/cli/services/completion.py +2 -2
  19. dstack/_internal/cli/services/configurators/__init__.py +6 -2
  20. dstack/_internal/cli/services/configurators/base.py +6 -7
  21. dstack/_internal/cli/services/configurators/fleet.py +1 -3
  22. dstack/_internal/cli/services/configurators/gateway.py +2 -4
  23. dstack/_internal/cli/services/configurators/run.py +195 -55
  24. dstack/_internal/cli/services/configurators/volume.py +2 -4
  25. dstack/_internal/cli/services/profile.py +1 -1
  26. dstack/_internal/cli/services/repos.py +51 -47
  27. dstack/_internal/core/backends/aws/configurator.py +11 -7
  28. dstack/_internal/core/backends/azure/configurator.py +11 -7
  29. dstack/_internal/core/backends/base/configurator.py +25 -13
  30. dstack/_internal/core/backends/cloudrift/configurator.py +13 -7
  31. dstack/_internal/core/backends/cudo/configurator.py +11 -7
  32. dstack/_internal/core/backends/datacrunch/compute.py +5 -1
  33. dstack/_internal/core/backends/datacrunch/configurator.py +13 -7
  34. dstack/_internal/core/backends/gcp/configurator.py +11 -7
  35. dstack/_internal/core/backends/hotaisle/configurator.py +13 -7
  36. dstack/_internal/core/backends/kubernetes/configurator.py +13 -7
  37. dstack/_internal/core/backends/lambdalabs/configurator.py +11 -7
  38. dstack/_internal/core/backends/nebius/compute.py +1 -1
  39. dstack/_internal/core/backends/nebius/configurator.py +11 -7
  40. dstack/_internal/core/backends/nebius/resources.py +21 -11
  41. dstack/_internal/core/backends/oci/configurator.py +11 -7
  42. dstack/_internal/core/backends/runpod/configurator.py +11 -7
  43. dstack/_internal/core/backends/template/configurator.py.jinja +11 -7
  44. dstack/_internal/core/backends/tensordock/configurator.py +13 -7
  45. dstack/_internal/core/backends/vastai/configurator.py +11 -7
  46. dstack/_internal/core/backends/vultr/configurator.py +11 -4
  47. dstack/_internal/core/compatibility/gpus.py +13 -0
  48. dstack/_internal/core/compatibility/runs.py +1 -0
  49. dstack/_internal/core/models/common.py +3 -3
  50. dstack/_internal/core/models/configurations.py +172 -27
  51. dstack/_internal/core/models/files.py +1 -1
  52. dstack/_internal/core/models/fleets.py +5 -1
  53. dstack/_internal/core/models/profiles.py +41 -11
  54. dstack/_internal/core/models/resources.py +46 -42
  55. dstack/_internal/core/models/runs.py +4 -0
  56. dstack/_internal/core/services/configs/__init__.py +2 -2
  57. dstack/_internal/core/services/profiles.py +2 -2
  58. dstack/_internal/core/services/repos.py +5 -3
  59. dstack/_internal/core/services/ssh/ports.py +1 -1
  60. dstack/_internal/proxy/lib/deps.py +6 -2
  61. dstack/_internal/server/app.py +22 -17
  62. dstack/_internal/server/background/tasks/process_gateways.py +4 -1
  63. dstack/_internal/server/background/tasks/process_instances.py +10 -2
  64. dstack/_internal/server/background/tasks/process_probes.py +1 -1
  65. dstack/_internal/server/background/tasks/process_running_jobs.py +10 -4
  66. dstack/_internal/server/background/tasks/process_runs.py +1 -1
  67. dstack/_internal/server/background/tasks/process_submitted_jobs.py +54 -43
  68. dstack/_internal/server/background/tasks/process_terminating_jobs.py +2 -2
  69. dstack/_internal/server/background/tasks/process_volumes.py +1 -1
  70. dstack/_internal/server/db.py +8 -4
  71. dstack/_internal/server/models.py +1 -0
  72. dstack/_internal/server/routers/gpus.py +1 -6
  73. dstack/_internal/server/schemas/runner.py +10 -0
  74. dstack/_internal/server/services/backends/__init__.py +14 -8
  75. dstack/_internal/server/services/backends/handlers.py +6 -1
  76. dstack/_internal/server/services/docker.py +5 -5
  77. dstack/_internal/server/services/fleets.py +14 -13
  78. dstack/_internal/server/services/gateways/__init__.py +2 -0
  79. dstack/_internal/server/services/gateways/client.py +5 -2
  80. dstack/_internal/server/services/gateways/connection.py +1 -1
  81. dstack/_internal/server/services/gpus.py +50 -49
  82. dstack/_internal/server/services/instances.py +41 -1
  83. dstack/_internal/server/services/jobs/__init__.py +15 -4
  84. dstack/_internal/server/services/jobs/configurators/base.py +7 -11
  85. dstack/_internal/server/services/jobs/configurators/dev.py +5 -0
  86. dstack/_internal/server/services/jobs/configurators/extensions/cursor.py +3 -3
  87. dstack/_internal/server/services/jobs/configurators/extensions/vscode.py +3 -3
  88. dstack/_internal/server/services/jobs/configurators/service.py +1 -0
  89. dstack/_internal/server/services/jobs/configurators/task.py +3 -0
  90. dstack/_internal/server/services/locking.py +5 -5
  91. dstack/_internal/server/services/logging.py +10 -2
  92. dstack/_internal/server/services/logs/__init__.py +8 -6
  93. dstack/_internal/server/services/logs/aws.py +330 -327
  94. dstack/_internal/server/services/logs/filelog.py +7 -6
  95. dstack/_internal/server/services/logs/gcp.py +141 -139
  96. dstack/_internal/server/services/plugins.py +1 -1
  97. dstack/_internal/server/services/projects.py +2 -5
  98. dstack/_internal/server/services/proxy/repo.py +5 -1
  99. dstack/_internal/server/services/requirements/__init__.py +0 -0
  100. dstack/_internal/server/services/requirements/combine.py +259 -0
  101. dstack/_internal/server/services/runner/client.py +7 -0
  102. dstack/_internal/server/services/runs.py +1 -1
  103. dstack/_internal/server/services/services/__init__.py +8 -2
  104. dstack/_internal/server/services/services/autoscalers.py +2 -0
  105. dstack/_internal/server/services/ssh.py +2 -1
  106. dstack/_internal/server/services/storage/__init__.py +5 -6
  107. dstack/_internal/server/services/storage/gcs.py +49 -49
  108. dstack/_internal/server/services/storage/s3.py +52 -52
  109. dstack/_internal/server/statics/index.html +1 -1
  110. dstack/_internal/server/testing/common.py +1 -1
  111. dstack/_internal/server/utils/logging.py +3 -3
  112. dstack/_internal/server/utils/provisioning.py +3 -3
  113. dstack/_internal/utils/json_schema.py +3 -1
  114. dstack/_internal/utils/typing.py +14 -0
  115. dstack/api/_public/repos.py +21 -2
  116. dstack/api/_public/runs.py +5 -7
  117. dstack/api/server/__init__.py +17 -19
  118. dstack/api/server/_gpus.py +2 -1
  119. dstack/api/server/_group.py +4 -3
  120. dstack/api/server/_repos.py +20 -3
  121. dstack/plugins/builtin/rest_plugin/_plugin.py +1 -0
  122. dstack/version.py +1 -1
  123. {dstack-0.19.25.dist-info → dstack-0.19.26.dist-info}/METADATA +1 -1
  124. {dstack-0.19.25.dist-info → dstack-0.19.26.dist-info}/RECORD +127 -124
  125. dstack/api/huggingface/__init__.py +0 -73
  126. {dstack-0.19.25.dist-info → dstack-0.19.26.dist-info}/WHEEL +0 -0
  127. {dstack-0.19.25.dist-info → dstack-0.19.26.dist-info}/entry_points.txt +0 -0
  128. {dstack-0.19.25.dist-info → dstack-0.19.26.dist-info}/licenses/LICENSE.md +0 -0
@@ -3,7 +3,7 @@ import subprocess
3
3
  import sys
4
4
  import time
5
5
  from pathlib import Path
6
- from typing import Dict, List, Optional, Set
6
+ from typing import Dict, List, Optional, Set, TypeVar
7
7
 
8
8
  import gpuhunt
9
9
  from pydantic import parse_obj_as
@@ -15,12 +15,14 @@ from dstack._internal.cli.services.configurators.base import (
15
15
  BaseApplyConfigurator,
16
16
  )
17
17
  from dstack._internal.cli.services.profile import apply_profile_args, register_profile_args
18
- from dstack._internal.cli.services.repos import init_default_virtual_repo
19
- from dstack._internal.cli.utils.common import (
20
- confirm_ask,
21
- console,
22
- warn,
18
+ from dstack._internal.cli.services.repos import (
19
+ get_repo_from_dir,
20
+ get_repo_from_url,
21
+ init_default_virtual_repo,
22
+ is_git_repo_url,
23
+ register_init_repo_args,
23
24
  )
25
+ from dstack._internal.cli.utils.common import confirm_ask, console, warn
24
26
  from dstack._internal.cli.utils.rich import MultiItemStatus
25
27
  from dstack._internal.cli.utils.run import get_runs_table, print_run_plan
26
28
  from dstack._internal.core.errors import (
@@ -33,8 +35,7 @@ from dstack._internal.core.models.common import ApplyAction, RegistryAuth
33
35
  from dstack._internal.core.models.configurations import (
34
36
  AnyRunConfiguration,
35
37
  ApplyConfigurationType,
36
- BaseRunConfiguration,
37
- BaseRunConfigurationWithPorts,
38
+ ConfigurationWithPortsParams,
38
39
  DevEnvironmentConfiguration,
39
40
  PortMapping,
40
41
  RunConfigurationType,
@@ -47,6 +48,7 @@ from dstack._internal.core.models.resources import CPUSpec
47
48
  from dstack._internal.core.models.runs import JobStatus, JobSubmission, RunSpec, RunStatus
48
49
  from dstack._internal.core.services.configs import ConfigManager
49
50
  from dstack._internal.core.services.diff import diff_models
51
+ from dstack._internal.core.services.repos import load_repo
50
52
  from dstack._internal.utils.common import local_time
51
53
  from dstack._internal.utils.interpolator import InterpolatorError, VariablesInterpolator
52
54
  from dstack._internal.utils.logging import get_logger
@@ -63,53 +65,34 @@ _BIND_ADDRESS_ARG = "bind_address"
63
65
 
64
66
  logger = get_logger(__name__)
65
67
 
68
+ RunConfigurationT = TypeVar("RunConfigurationT", bound=AnyRunConfiguration)
66
69
 
67
- class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
70
+
71
+ class BaseRunConfigurator(
72
+ ApplyEnvVarsConfiguratorMixin,
73
+ BaseApplyConfigurator[RunConfigurationT],
74
+ ):
68
75
  TYPE: ApplyConfigurationType
69
76
 
70
77
  def apply_configuration(
71
78
  self,
72
- conf: BaseRunConfiguration,
79
+ conf: RunConfigurationT,
73
80
  configuration_path: str,
74
81
  command_args: argparse.Namespace,
75
82
  configurator_args: argparse.Namespace,
76
83
  unknown_args: List[str],
77
- repo: Optional[Repo] = None,
78
84
  ):
85
+ if configurator_args.repo and configurator_args.no_repo:
86
+ raise CLIError("Either --repo or --no-repo can be specified")
87
+
79
88
  self.apply_args(conf, configurator_args, unknown_args)
80
89
  self.validate_gpu_vendor_and_image(conf)
81
90
  self.validate_cpu_arch_and_image(conf)
91
+
82
92
  config_manager = ConfigManager()
83
- if repo is None:
84
- repo_path = Path.cwd()
85
- repo_config = config_manager.get_repo_config(repo_path)
86
- if repo_config is None:
87
- warn(
88
- "Repo is not initialized. "
89
- "Use [code]--repo <dir>[/code] or [code]--no-repo[/code] to initialize it.\n"
90
- "Starting from 0.19.26, repos will be configured via YAML and this message won't appear."
91
- )
92
- if not command_args.yes and not confirm_ask("Continue without the repo?"):
93
- console.print("\nExiting...")
94
- return
95
- repo = init_default_virtual_repo(self.api)
96
- else:
97
- # Unlikely, but may raise ConfigurationError if the repo does not exist
98
- # on the server side (stale entry in `config.yml`)
99
- repo = self.api.repos.load(repo_path)
100
- if isinstance(repo, LocalRepo):
101
- warn(
102
- f"{repo.repo_dir} is a local repo.\n"
103
- "Local repos are deprecated since 0.19.25"
104
- " and will be removed soon\n"
105
- "There are two options:\n"
106
- " - Migrate to `files`: https://dstack.ai/docs/concepts/tasks/#files\n"
107
- " - Specify `--no-repo` if you don't need the repo at all\n"
108
- "In either case, you can run `dstack init --remove` to remove the repo"
109
- " (only the record about the repo, not its files) and this warning"
110
- )
93
+ repo = self.get_repo(conf, configuration_path, configurator_args, config_manager)
111
94
  self.api.ssh_identity_file = get_ssh_keypair(
112
- command_args.ssh_identity_file,
95
+ configurator_args.ssh_identity_file,
113
96
  config_manager.dstack_key_path,
114
97
  )
115
98
  profile = load_profile(Path.cwd(), configurator_args.profile)
@@ -267,7 +250,7 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
267
250
 
268
251
  def delete_configuration(
269
252
  self,
270
- conf: AnyRunConfiguration,
253
+ conf: RunConfigurationT,
271
254
  configuration_path: str,
272
255
  command_args: argparse.Namespace,
273
256
  ):
@@ -293,7 +276,14 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
293
276
  console.print(f"Run [code]{conf.name}[/] deleted")
294
277
 
295
278
  @classmethod
296
- def register_args(cls, parser: argparse.ArgumentParser, default_max_offers: int = 3):
279
+ def register_args(cls, parser: argparse.ArgumentParser):
280
+ parser.add_argument(
281
+ "--ssh-identity",
282
+ metavar="SSH_PRIVATE_KEY",
283
+ help="The private SSH key path for SSH tunneling",
284
+ type=Path,
285
+ dest="ssh_identity_file",
286
+ )
297
287
  configuration_group = parser.add_argument_group(f"{cls.TYPE.value} Options")
298
288
  configuration_group.add_argument(
299
289
  "-n",
@@ -305,7 +295,7 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
305
295
  "--max-offers",
306
296
  help="Number of offers to show in the run plan",
307
297
  type=int,
308
- default=default_max_offers,
298
+ default=3,
309
299
  )
310
300
  cls.register_env_args(configuration_group)
311
301
  configuration_group.add_argument(
@@ -332,8 +322,32 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
332
322
  dest="disk_spec",
333
323
  )
334
324
  register_profile_args(parser)
325
+ repo_group = parser.add_argument_group("Repo Options")
326
+ repo_group.add_argument(
327
+ "-P",
328
+ "--repo",
329
+ help=("The repo to use for the run. Can be a local path or a Git repo URL."),
330
+ dest="repo",
331
+ )
332
+ repo_group.add_argument(
333
+ "--repo-branch",
334
+ help="The repo branch to use for the run",
335
+ dest="repo_branch",
336
+ )
337
+ repo_group.add_argument(
338
+ "--repo-hash",
339
+ help="The hash of the repo commit to use for the run",
340
+ dest="repo_hash",
341
+ )
342
+ repo_group.add_argument(
343
+ "--no-repo",
344
+ help="Do not use any repo for the run",
345
+ dest="no_repo",
346
+ action="store_true",
347
+ )
348
+ register_init_repo_args(repo_group)
335
349
 
336
- def apply_args(self, conf: BaseRunConfiguration, args: argparse.Namespace, unknown: List[str]):
350
+ def apply_args(self, conf: RunConfigurationT, args: argparse.Namespace, unknown: List[str]):
337
351
  apply_profile_args(args, conf)
338
352
  if args.run_name:
339
353
  conf.name = args.run_name
@@ -357,7 +371,7 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
357
371
  except InterpolatorError as e:
358
372
  raise ConfigurationError(e.args[0])
359
373
 
360
- def interpolate_env(self, conf: BaseRunConfiguration):
374
+ def interpolate_env(self, conf: RunConfigurationT):
361
375
  env_dict = conf.env.as_dict()
362
376
  interpolator = VariablesInterpolator({"env": env_dict}, skip=["secrets"])
363
377
  try:
@@ -377,7 +391,7 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
377
391
  except InterpolatorError as e:
378
392
  raise ConfigurationError(e.args[0])
379
393
 
380
- def validate_gpu_vendor_and_image(self, conf: BaseRunConfiguration) -> None:
394
+ def validate_gpu_vendor_and_image(self, conf: RunConfigurationT) -> None:
381
395
  """
382
396
  Infers and sets `resources.gpu.vendor` if not set, requires `image` if the vendor is AMD.
383
397
  """
@@ -438,7 +452,7 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
438
452
  "`image` is required if `resources.gpu.vendor` is `tenstorrent`"
439
453
  )
440
454
 
441
- def validate_cpu_arch_and_image(self, conf: BaseRunConfiguration) -> None:
455
+ def validate_cpu_arch_and_image(self, conf: RunConfigurationT) -> None:
442
456
  """
443
457
  Infers `resources.cpu.arch` if not set, requires `image` if the architecture is ARM.
444
458
  """
@@ -461,11 +475,122 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
461
475
  if arch == gpuhunt.CPUArchitecture.ARM and conf.image is None:
462
476
  raise ConfigurationError("`image` is required if `resources.cpu.arch` is `arm`")
463
477
 
478
+ def get_repo(
479
+ self,
480
+ conf: RunConfigurationT,
481
+ configuration_path: str,
482
+ configurator_args: argparse.Namespace,
483
+ config_manager: ConfigManager,
484
+ ) -> Repo:
485
+ if configurator_args.no_repo:
486
+ return init_default_virtual_repo(api=self.api)
487
+
488
+ repo: Optional[Repo] = None
489
+ repo_branch: Optional[str] = configurator_args.repo_branch
490
+ repo_hash: Optional[str] = configurator_args.repo_hash
491
+ # Should we (re)initialize the repo?
492
+ # If any Git credentials provided, we reinitialize the repo, as the user may have provided
493
+ # updated credentials.
494
+ init = (
495
+ configurator_args.git_identity_file is not None
496
+ or configurator_args.gh_token is not None
497
+ )
464
498
 
465
- class RunWithPortsConfigurator(BaseRunConfigurator):
499
+ url: Optional[str] = None
500
+ local_path: Optional[Path] = None
501
+ # dummy value, safe to join with any path
502
+ root_dir = Path(".")
503
+ # True if no repo specified, but we found one in `config.yml`
504
+ legacy_local_path = False
505
+ if repo_arg := configurator_args.repo:
506
+ if is_git_repo_url(repo_arg):
507
+ url = repo_arg
508
+ else:
509
+ local_path = Path(repo_arg)
510
+ # rel paths in `--repo` are resolved relative to the current working dir
511
+ root_dir = Path.cwd()
512
+ elif conf.repos:
513
+ repo_spec = conf.repos[0]
514
+ if repo_spec.url:
515
+ url = repo_spec.url
516
+ elif repo_spec.local_path:
517
+ local_path = Path(repo_spec.local_path)
518
+ # rel paths in the conf are resolved relative to the conf's parent dir
519
+ root_dir = Path(configuration_path).resolve().parent
520
+ else:
521
+ assert False, f"should not reach here: {repo_spec}"
522
+ if repo_branch is None:
523
+ repo_branch = repo_spec.branch
524
+ if repo_hash is None:
525
+ repo_hash = repo_spec.hash
526
+ else:
527
+ local_path = Path.cwd()
528
+ legacy_local_path = True
529
+ if url:
530
+ repo = get_repo_from_url(repo_url=url, repo_branch=repo_branch, repo_hash=repo_hash)
531
+ if not self.api.repos.is_initialized(repo, by_user=True):
532
+ init = True
533
+ elif local_path:
534
+ if legacy_local_path:
535
+ if repo_config := config_manager.get_repo_config(local_path):
536
+ repo = load_repo(repo_config)
537
+ # allow users with legacy configurations use shared repo creds
538
+ if self.api.repos.is_initialized(repo, by_user=False):
539
+ warn(
540
+ "The repo is not specified but found and will be used in the run\n"
541
+ "Future versions will not load repos automatically\n"
542
+ "To prepare for future versions and get rid of this warning:\n"
543
+ "- If you need the repo in the run, either specify [code]repos[/code]"
544
+ " in the configuration or use [code]--repo .[/code]\n"
545
+ "- If you don't need the repo in the run, either run"
546
+ " [code]dstack init --remove[/code] once (it removes only the record"
547
+ " about the repo, the repo files will remain intact)"
548
+ " or use [code]--no-repo[/code]"
549
+ )
550
+ else:
551
+ # ignore stale entries in `config.yml`
552
+ repo = None
553
+ init = False
554
+ else:
555
+ original_local_path = local_path
556
+ local_path = local_path.expanduser()
557
+ if not local_path.is_absolute():
558
+ local_path = (root_dir / local_path).resolve()
559
+ if not local_path.exists():
560
+ raise ConfigurationError(
561
+ f"Invalid repo path: {original_local_path} -> {local_path}"
562
+ )
563
+ local: bool = configurator_args.local
564
+ repo = get_repo_from_dir(local_path, local=local)
565
+ if not self.api.repos.is_initialized(repo, by_user=True):
566
+ init = True
567
+ else:
568
+ assert False, "should not reach here"
569
+
570
+ if repo is None:
571
+ return init_default_virtual_repo(api=self.api)
572
+
573
+ if init:
574
+ self.api.repos.init(
575
+ repo=repo,
576
+ git_identity_file=configurator_args.git_identity_file,
577
+ oauth_token=configurator_args.gh_token,
578
+ )
579
+ if isinstance(repo, LocalRepo):
580
+ warn(
581
+ f"{repo.repo_dir} is a local repo\n"
582
+ "Local repos are deprecated since 0.19.25 and will be removed soon\n"
583
+ "There are two options:\n"
584
+ "- Migrate to [code]files[/code]: https://dstack.ai/docs/concepts/tasks/#files\n"
585
+ "- Specify [code]--no-repo[/code] if you don't need the repo at all"
586
+ )
587
+
588
+ return repo
589
+
590
+
591
+ class RunWithPortsConfiguratorMixin:
466
592
  @classmethod
467
- def register_args(cls, parser: argparse.ArgumentParser):
468
- super().register_args(parser)
593
+ def register_ports_args(cls, parser: argparse.ArgumentParser):
469
594
  parser.add_argument(
470
595
  "-p",
471
596
  "--port",
@@ -482,29 +607,42 @@ class RunWithPortsConfigurator(BaseRunConfigurator):
482
607
  metavar="HOST",
483
608
  )
484
609
 
485
- def apply_args(
486
- self, conf: BaseRunConfigurationWithPorts, args: argparse.Namespace, unknown: List[str]
610
+ def apply_ports_args(
611
+ self,
612
+ conf: ConfigurationWithPortsParams,
613
+ args: argparse.Namespace,
487
614
  ):
488
- super().apply_args(conf, args, unknown)
489
615
  if args.ports:
490
616
  conf.ports = list(_merge_ports(conf.ports, args.ports).values())
491
617
 
492
618
 
493
- class TaskConfigurator(RunWithPortsConfigurator):
619
+ class TaskConfigurator(RunWithPortsConfiguratorMixin, BaseRunConfigurator):
494
620
  TYPE = ApplyConfigurationType.TASK
495
621
 
622
+ @classmethod
623
+ def register_args(cls, parser: argparse.ArgumentParser):
624
+ super().register_args(parser)
625
+ cls.register_ports_args(parser)
626
+
496
627
  def apply_args(self, conf: TaskConfiguration, args: argparse.Namespace, unknown: List[str]):
497
628
  super().apply_args(conf, args, unknown)
629
+ self.apply_ports_args(conf, args)
498
630
  self.interpolate_run_args(conf.commands, unknown)
499
631
 
500
632
 
501
- class DevEnvironmentConfigurator(RunWithPortsConfigurator):
633
+ class DevEnvironmentConfigurator(RunWithPortsConfiguratorMixin, BaseRunConfigurator):
502
634
  TYPE = ApplyConfigurationType.DEV_ENVIRONMENT
503
635
 
636
+ @classmethod
637
+ def register_args(cls, parser: argparse.ArgumentParser):
638
+ super().register_args(parser)
639
+ cls.register_ports_args(parser)
640
+
504
641
  def apply_args(
505
642
  self, conf: DevEnvironmentConfiguration, args: argparse.Namespace, unknown: List[str]
506
643
  ):
507
644
  super().apply_args(conf, args, unknown)
645
+ self.apply_ports_args(conf, args)
508
646
  if conf.ide == "vscode" and conf.version is None:
509
647
  conf.version = _detect_vscode_version()
510
648
  if conf.version is None:
@@ -674,6 +812,8 @@ def render_run_spec_diff(old_spec: RunSpec, new_spec: RunSpec) -> Optional[str]:
674
812
  if type(old_spec.profile) is not type(new_spec.profile):
675
813
  item = NestedListItem("Profile")
676
814
  else:
815
+ assert old_spec.profile is not None
816
+ assert new_spec.profile is not None
677
817
  item = NestedListItem(
678
818
  "Profile properties:",
679
819
  children=[
@@ -1,6 +1,6 @@
1
1
  import argparse
2
2
  import time
3
- from typing import List, Optional
3
+ from typing import List
4
4
 
5
5
  from rich.table import Table
6
6
 
@@ -14,7 +14,6 @@ from dstack._internal.cli.utils.rich import MultiItemStatus
14
14
  from dstack._internal.cli.utils.volume import get_volumes_table
15
15
  from dstack._internal.core.errors import ResourceNotExistsError
16
16
  from dstack._internal.core.models.configurations import ApplyConfigurationType
17
- from dstack._internal.core.models.repos.base import Repo
18
17
  from dstack._internal.core.models.volumes import (
19
18
  Volume,
20
19
  VolumeConfiguration,
@@ -26,7 +25,7 @@ from dstack._internal.utils.common import local_time
26
25
  from dstack.api._public import Client
27
26
 
28
27
 
29
- class VolumeConfigurator(BaseApplyConfigurator):
28
+ class VolumeConfigurator(BaseApplyConfigurator[VolumeConfiguration]):
30
29
  TYPE: ApplyConfigurationType = ApplyConfigurationType.VOLUME
31
30
 
32
31
  def apply_configuration(
@@ -36,7 +35,6 @@ class VolumeConfigurator(BaseApplyConfigurator):
36
35
  command_args: argparse.Namespace,
37
36
  configurator_args: argparse.Namespace,
38
37
  unknown_args: List[str],
39
- repo: Optional[Repo] = None,
40
38
  ):
41
39
  self.apply_args(conf, configurator_args, unknown_args)
42
40
  spec = VolumeSpec(
@@ -159,7 +159,7 @@ def apply_profile_args(
159
159
  if args.idle_duration is not None:
160
160
  profile_settings.idle_duration = args.idle_duration
161
161
  elif args.dont_destroy:
162
- profile_settings.idle_duration = "off"
162
+ profile_settings.idle_duration = -1
163
163
  if args.creation_policy_reuse:
164
164
  profile_settings.creation_policy = CreationPolicy.REUSE
165
165
 
@@ -1,10 +1,11 @@
1
1
  import argparse
2
- from pathlib import Path
3
- from typing import Optional
2
+ from typing import Literal, Optional, Union, overload
3
+
4
+ import git
4
5
 
5
6
  from dstack._internal.cli.services.configurators.base import ArgsParser
6
7
  from dstack._internal.core.errors import CLIError
7
- from dstack._internal.core.models.repos.base import Repo
8
+ from dstack._internal.core.models.repos.local import LocalRepo
8
9
  from dstack._internal.core.models.repos.remote import GitRepoURL, RemoteRepo, RepoError
9
10
  from dstack._internal.core.models.repos.virtual import VirtualRepo
10
11
  from dstack._internal.core.services.repos import get_default_branch
@@ -36,51 +37,54 @@ def register_init_repo_args(parser: ArgsParser):
36
37
  )
37
38
 
38
39
 
39
- def init_repo(
40
- api: Client,
41
- repo_path: PathLike,
42
- repo_branch: Optional[str],
43
- repo_hash: Optional[str],
44
- local: bool,
45
- git_identity_file: Optional[PathLike],
46
- oauth_token: Optional[str],
47
- ) -> Repo:
48
- if Path(repo_path).exists():
49
- repo = api.repos.load(
50
- repo_dir=repo_path,
51
- local=local,
52
- init=True,
53
- git_identity_file=git_identity_file,
54
- oauth_token=oauth_token,
55
- )
56
- elif isinstance(repo_path, str):
57
- try:
58
- GitRepoURL.parse(repo_path)
59
- except RepoError as e:
60
- raise CLIError("Invalid repo path") from e
61
- if repo_branch is None and repo_hash is None:
62
- repo_branch = get_default_branch(repo_path)
63
- if repo_branch is None:
64
- raise CLIError(
65
- "Failed to automatically detect remote repo branch."
66
- " Specify --repo-branch or --repo-hash."
67
- )
68
- repo = RemoteRepo.from_url(
69
- repo_url=repo_path,
70
- repo_branch=repo_branch,
71
- repo_hash=repo_hash,
72
- )
73
- api.repos.init(
74
- repo=repo,
75
- git_identity_file=git_identity_file,
76
- oauth_token=oauth_token,
77
- )
78
- else:
79
- raise CLIError("Invalid repo path")
80
- return repo
81
-
82
-
83
40
  def init_default_virtual_repo(api: Client) -> VirtualRepo:
84
41
  repo = VirtualRepo()
85
42
  api.repos.init(repo)
86
43
  return repo
44
+
45
+
46
+ def get_repo_from_url(
47
+ repo_url: str, repo_branch: Optional[str] = None, repo_hash: Optional[str] = None
48
+ ) -> RemoteRepo:
49
+ if repo_branch is None and repo_hash is None:
50
+ repo_branch = get_default_branch(repo_url)
51
+ if repo_branch is None:
52
+ raise CLIError(
53
+ "Failed to automatically detect remote repo branch. Specify branch or hash."
54
+ )
55
+ return RemoteRepo.from_url(
56
+ repo_url=repo_url,
57
+ repo_branch=repo_branch,
58
+ repo_hash=repo_hash,
59
+ )
60
+
61
+
62
+ @overload
63
+ def get_repo_from_dir(repo_dir: PathLike, local: Literal[False] = False) -> RemoteRepo: ...
64
+
65
+
66
+ @overload
67
+ def get_repo_from_dir(repo_dir: PathLike, local: Literal[True]) -> LocalRepo: ...
68
+
69
+
70
+ def get_repo_from_dir(repo_dir: PathLike, local: bool = False) -> Union[RemoteRepo, LocalRepo]:
71
+ if local:
72
+ return LocalRepo.from_dir(repo_dir)
73
+ try:
74
+ return RemoteRepo.from_dir(repo_dir)
75
+ except git.InvalidGitRepositoryError:
76
+ raise CLIError(
77
+ f"Git repo not found: {repo_dir}\n"
78
+ "Use `files` to mount an arbitrary directory:"
79
+ " https://dstack.ai/docs/concepts/tasks/#files"
80
+ )
81
+ except RepoError as e:
82
+ raise CLIError(str(e)) from e
83
+
84
+
85
+ def is_git_repo_url(value: str) -> bool:
86
+ try:
87
+ GitRepoURL.parse(value)
88
+ except RepoError:
89
+ return False
90
+ return True
@@ -7,7 +7,6 @@ from boto3.session import Session
7
7
  from dstack._internal.core.backends.aws import auth, compute, resources
8
8
  from dstack._internal.core.backends.aws.backend import AWSBackend
9
9
  from dstack._internal.core.backends.aws.models import (
10
- AnyAWSBackendConfig,
11
10
  AWSAccessKeyCreds,
12
11
  AWSBackendConfig,
13
12
  AWSBackendConfigWithCreds,
@@ -52,7 +51,12 @@ DEFAULT_REGIONS = REGION_VALUES
52
51
  MAIN_REGION = "us-east-1"
53
52
 
54
53
 
55
- class AWSConfigurator(Configurator):
54
+ class AWSConfigurator(
55
+ Configurator[
56
+ AWSBackendConfig,
57
+ AWSBackendConfigWithCreds,
58
+ ]
59
+ ):
56
60
  TYPE = BackendType.AWS
57
61
  BACKEND_CLASS = AWSBackend
58
62
 
@@ -87,12 +91,12 @@ class AWSConfigurator(Configurator):
87
91
  auth=AWSCreds.parse_obj(config.creds).json(),
88
92
  )
89
93
 
90
- def get_backend_config(
91
- self, record: BackendRecord, include_creds: bool
92
- ) -> AnyAWSBackendConfig:
94
+ def get_backend_config_with_creds(self, record: BackendRecord) -> AWSBackendConfigWithCreds:
95
+ config = self._get_config(record)
96
+ return AWSBackendConfigWithCreds.__response__.parse_obj(config)
97
+
98
+ def get_backend_config_without_creds(self, record: BackendRecord) -> AWSBackendConfig:
93
99
  config = self._get_config(record)
94
- if include_creds:
95
- return AWSBackendConfigWithCreds.__response__.parse_obj(config)
96
100
  return AWSBackendConfig.__response__.parse_obj(config)
97
101
 
98
102
  def get_backend(self, record: BackendRecord) -> AWSBackend:
@@ -24,7 +24,6 @@ from dstack._internal.core.backends.azure import auth, compute, resources
24
24
  from dstack._internal.core.backends.azure import utils as azure_utils
25
25
  from dstack._internal.core.backends.azure.backend import AzureBackend
26
26
  from dstack._internal.core.backends.azure.models import (
27
- AnyAzureBackendConfig,
28
27
  AzureBackendConfig,
29
28
  AzureBackendConfigWithCreds,
30
29
  AzureClientCreds,
@@ -71,7 +70,12 @@ DEFAULT_LOCATIONS = LOCATION_VALUES
71
70
  MAIN_LOCATION = "eastus"
72
71
 
73
72
 
74
- class AzureConfigurator(Configurator):
73
+ class AzureConfigurator(
74
+ Configurator[
75
+ AzureBackendConfig,
76
+ AzureBackendConfigWithCreds,
77
+ ]
78
+ ):
75
79
  TYPE = BackendType.AZURE
76
80
  BACKEND_CLASS = AzureBackend
77
81
 
@@ -130,12 +134,12 @@ class AzureConfigurator(Configurator):
130
134
  auth=AzureCreds.parse_obj(config.creds).__root__.json(),
131
135
  )
132
136
 
133
- def get_backend_config(
134
- self, record: BackendRecord, include_creds: bool
135
- ) -> AnyAzureBackendConfig:
137
+ def get_backend_config_with_creds(self, record: BackendRecord) -> AzureBackendConfigWithCreds:
138
+ config = self._get_config(record)
139
+ return AzureBackendConfigWithCreds.__response__.parse_obj(config)
140
+
141
+ def get_backend_config_without_creds(self, record: BackendRecord) -> AzureBackendConfig:
136
142
  config = self._get_config(record)
137
- if include_creds:
138
- return AzureBackendConfigWithCreds.__response__.parse_obj(config)
139
143
  return AzureBackendConfig.__response__.parse_obj(config)
140
144
 
141
145
  def get_backend(self, record: BackendRecord) -> AzureBackend: