dstack 0.19.25rc1__py3-none-any.whl → 0.19.27__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of dstack might be problematic. Click here for more details.

Files changed (161) hide show
  1. dstack/_internal/cli/commands/__init__.py +2 -2
  2. dstack/_internal/cli/commands/apply.py +3 -61
  3. dstack/_internal/cli/commands/attach.py +1 -1
  4. dstack/_internal/cli/commands/completion.py +1 -1
  5. dstack/_internal/cli/commands/delete.py +2 -2
  6. dstack/_internal/cli/commands/fleet.py +1 -1
  7. dstack/_internal/cli/commands/gateway.py +2 -2
  8. dstack/_internal/cli/commands/init.py +56 -24
  9. dstack/_internal/cli/commands/logs.py +1 -1
  10. dstack/_internal/cli/commands/metrics.py +1 -1
  11. dstack/_internal/cli/commands/offer.py +45 -7
  12. dstack/_internal/cli/commands/project.py +2 -2
  13. dstack/_internal/cli/commands/secrets.py +2 -2
  14. dstack/_internal/cli/commands/server.py +1 -1
  15. dstack/_internal/cli/commands/stop.py +1 -1
  16. dstack/_internal/cli/commands/volume.py +1 -1
  17. dstack/_internal/cli/main.py +2 -2
  18. dstack/_internal/cli/services/completion.py +2 -2
  19. dstack/_internal/cli/services/configurators/__init__.py +6 -2
  20. dstack/_internal/cli/services/configurators/base.py +6 -7
  21. dstack/_internal/cli/services/configurators/fleet.py +1 -3
  22. dstack/_internal/cli/services/configurators/gateway.py +2 -4
  23. dstack/_internal/cli/services/configurators/run.py +293 -58
  24. dstack/_internal/cli/services/configurators/volume.py +2 -4
  25. dstack/_internal/cli/services/profile.py +1 -1
  26. dstack/_internal/cli/services/repos.py +35 -48
  27. dstack/_internal/core/backends/amddevcloud/__init__.py +1 -0
  28. dstack/_internal/core/backends/amddevcloud/backend.py +16 -0
  29. dstack/_internal/core/backends/amddevcloud/compute.py +5 -0
  30. dstack/_internal/core/backends/amddevcloud/configurator.py +29 -0
  31. dstack/_internal/core/backends/aws/compute.py +6 -1
  32. dstack/_internal/core/backends/aws/configurator.py +11 -7
  33. dstack/_internal/core/backends/azure/configurator.py +11 -7
  34. dstack/_internal/core/backends/base/compute.py +33 -5
  35. dstack/_internal/core/backends/base/configurator.py +25 -13
  36. dstack/_internal/core/backends/base/offers.py +2 -0
  37. dstack/_internal/core/backends/cloudrift/configurator.py +13 -7
  38. dstack/_internal/core/backends/configurators.py +15 -0
  39. dstack/_internal/core/backends/cudo/configurator.py +11 -7
  40. dstack/_internal/core/backends/datacrunch/compute.py +5 -1
  41. dstack/_internal/core/backends/datacrunch/configurator.py +13 -7
  42. dstack/_internal/core/backends/digitalocean/__init__.py +1 -0
  43. dstack/_internal/core/backends/digitalocean/backend.py +16 -0
  44. dstack/_internal/core/backends/digitalocean/compute.py +5 -0
  45. dstack/_internal/core/backends/digitalocean/configurator.py +31 -0
  46. dstack/_internal/core/backends/digitalocean_base/__init__.py +1 -0
  47. dstack/_internal/core/backends/digitalocean_base/api_client.py +104 -0
  48. dstack/_internal/core/backends/digitalocean_base/backend.py +5 -0
  49. dstack/_internal/core/backends/digitalocean_base/compute.py +173 -0
  50. dstack/_internal/core/backends/digitalocean_base/configurator.py +57 -0
  51. dstack/_internal/core/backends/digitalocean_base/models.py +43 -0
  52. dstack/_internal/core/backends/gcp/compute.py +32 -8
  53. dstack/_internal/core/backends/gcp/configurator.py +11 -7
  54. dstack/_internal/core/backends/hotaisle/api_client.py +25 -33
  55. dstack/_internal/core/backends/hotaisle/compute.py +1 -6
  56. dstack/_internal/core/backends/hotaisle/configurator.py +13 -7
  57. dstack/_internal/core/backends/kubernetes/configurator.py +13 -7
  58. dstack/_internal/core/backends/lambdalabs/configurator.py +11 -7
  59. dstack/_internal/core/backends/models.py +7 -0
  60. dstack/_internal/core/backends/nebius/compute.py +1 -8
  61. dstack/_internal/core/backends/nebius/configurator.py +11 -7
  62. dstack/_internal/core/backends/nebius/resources.py +21 -11
  63. dstack/_internal/core/backends/oci/compute.py +4 -5
  64. dstack/_internal/core/backends/oci/configurator.py +11 -7
  65. dstack/_internal/core/backends/runpod/configurator.py +11 -7
  66. dstack/_internal/core/backends/template/configurator.py.jinja +11 -7
  67. dstack/_internal/core/backends/tensordock/configurator.py +13 -7
  68. dstack/_internal/core/backends/vastai/configurator.py +11 -7
  69. dstack/_internal/core/backends/vultr/compute.py +1 -5
  70. dstack/_internal/core/backends/vultr/configurator.py +11 -4
  71. dstack/_internal/core/compatibility/fleets.py +5 -0
  72. dstack/_internal/core/compatibility/gpus.py +13 -0
  73. dstack/_internal/core/compatibility/runs.py +9 -1
  74. dstack/_internal/core/models/backends/base.py +5 -1
  75. dstack/_internal/core/models/common.py +3 -3
  76. dstack/_internal/core/models/configurations.py +191 -32
  77. dstack/_internal/core/models/files.py +1 -1
  78. dstack/_internal/core/models/fleets.py +80 -3
  79. dstack/_internal/core/models/profiles.py +41 -11
  80. dstack/_internal/core/models/resources.py +46 -42
  81. dstack/_internal/core/models/runs.py +28 -5
  82. dstack/_internal/core/services/configs/__init__.py +6 -3
  83. dstack/_internal/core/services/profiles.py +2 -2
  84. dstack/_internal/core/services/repos.py +86 -79
  85. dstack/_internal/core/services/ssh/ports.py +1 -1
  86. dstack/_internal/proxy/lib/deps.py +6 -2
  87. dstack/_internal/server/app.py +22 -17
  88. dstack/_internal/server/background/tasks/process_fleets.py +109 -13
  89. dstack/_internal/server/background/tasks/process_gateways.py +4 -1
  90. dstack/_internal/server/background/tasks/process_instances.py +22 -73
  91. dstack/_internal/server/background/tasks/process_probes.py +1 -1
  92. dstack/_internal/server/background/tasks/process_running_jobs.py +12 -4
  93. dstack/_internal/server/background/tasks/process_runs.py +3 -1
  94. dstack/_internal/server/background/tasks/process_submitted_jobs.py +67 -44
  95. dstack/_internal/server/background/tasks/process_terminating_jobs.py +2 -2
  96. dstack/_internal/server/background/tasks/process_volumes.py +1 -1
  97. dstack/_internal/server/db.py +8 -4
  98. dstack/_internal/server/migrations/versions/2498ab323443_add_fleetmodel_consolidation_attempt_.py +44 -0
  99. dstack/_internal/server/models.py +6 -2
  100. dstack/_internal/server/routers/gpus.py +1 -6
  101. dstack/_internal/server/schemas/runner.py +11 -0
  102. dstack/_internal/server/services/backends/__init__.py +14 -8
  103. dstack/_internal/server/services/backends/handlers.py +6 -1
  104. dstack/_internal/server/services/docker.py +5 -5
  105. dstack/_internal/server/services/fleets.py +37 -38
  106. dstack/_internal/server/services/gateways/__init__.py +2 -0
  107. dstack/_internal/server/services/gateways/client.py +5 -2
  108. dstack/_internal/server/services/gateways/connection.py +1 -1
  109. dstack/_internal/server/services/gpus.py +50 -49
  110. dstack/_internal/server/services/instances.py +44 -4
  111. dstack/_internal/server/services/jobs/__init__.py +15 -4
  112. dstack/_internal/server/services/jobs/configurators/base.py +53 -17
  113. dstack/_internal/server/services/jobs/configurators/dev.py +9 -4
  114. dstack/_internal/server/services/jobs/configurators/extensions/cursor.py +6 -8
  115. dstack/_internal/server/services/jobs/configurators/extensions/vscode.py +7 -9
  116. dstack/_internal/server/services/jobs/configurators/service.py +1 -3
  117. dstack/_internal/server/services/jobs/configurators/task.py +3 -3
  118. dstack/_internal/server/services/locking.py +5 -5
  119. dstack/_internal/server/services/logging.py +10 -2
  120. dstack/_internal/server/services/logs/__init__.py +8 -6
  121. dstack/_internal/server/services/logs/aws.py +330 -327
  122. dstack/_internal/server/services/logs/filelog.py +7 -6
  123. dstack/_internal/server/services/logs/gcp.py +141 -139
  124. dstack/_internal/server/services/plugins.py +1 -1
  125. dstack/_internal/server/services/projects.py +2 -5
  126. dstack/_internal/server/services/proxy/repo.py +5 -1
  127. dstack/_internal/server/services/requirements/__init__.py +0 -0
  128. dstack/_internal/server/services/requirements/combine.py +259 -0
  129. dstack/_internal/server/services/runner/client.py +7 -0
  130. dstack/_internal/server/services/runs.py +17 -1
  131. dstack/_internal/server/services/services/__init__.py +8 -2
  132. dstack/_internal/server/services/services/autoscalers.py +2 -0
  133. dstack/_internal/server/services/ssh.py +2 -1
  134. dstack/_internal/server/services/storage/__init__.py +5 -6
  135. dstack/_internal/server/services/storage/gcs.py +49 -49
  136. dstack/_internal/server/services/storage/s3.py +52 -52
  137. dstack/_internal/server/statics/index.html +1 -1
  138. dstack/_internal/server/statics/{main-d151b300fcac3933213d.js → main-4eecc75fbe64067eb1bc.js} +1146 -899
  139. dstack/_internal/server/statics/{main-d151b300fcac3933213d.js.map → main-4eecc75fbe64067eb1bc.js.map} +1 -1
  140. dstack/_internal/server/statics/{main-aec4762350e34d6fbff9.css → main-56191c63d516fd0041c4.css} +1 -1
  141. dstack/_internal/server/testing/common.py +7 -4
  142. dstack/_internal/server/utils/logging.py +3 -3
  143. dstack/_internal/server/utils/provisioning.py +3 -3
  144. dstack/_internal/utils/json_schema.py +3 -1
  145. dstack/_internal/utils/path.py +8 -1
  146. dstack/_internal/utils/ssh.py +7 -0
  147. dstack/_internal/utils/typing.py +14 -0
  148. dstack/api/_public/repos.py +62 -8
  149. dstack/api/_public/runs.py +19 -8
  150. dstack/api/server/__init__.py +17 -19
  151. dstack/api/server/_gpus.py +2 -1
  152. dstack/api/server/_group.py +4 -3
  153. dstack/api/server/_repos.py +20 -3
  154. dstack/plugins/builtin/rest_plugin/_plugin.py +1 -0
  155. dstack/version.py +1 -1
  156. {dstack-0.19.25rc1.dist-info → dstack-0.19.27.dist-info}/METADATA +2 -2
  157. {dstack-0.19.25rc1.dist-info → dstack-0.19.27.dist-info}/RECORD +160 -142
  158. dstack/api/huggingface/__init__.py +0 -73
  159. {dstack-0.19.25rc1.dist-info → dstack-0.19.27.dist-info}/WHEEL +0 -0
  160. {dstack-0.19.25rc1.dist-info → dstack-0.19.27.dist-info}/entry_points.txt +0 -0
  161. {dstack-0.19.25rc1.dist-info → dstack-0.19.27.dist-info}/licenses/LICENSE.md +0 -0
@@ -2,8 +2,8 @@ import argparse
2
2
  import subprocess
3
3
  import sys
4
4
  import time
5
- from pathlib import Path
6
- from typing import Dict, List, Optional, Set
5
+ from pathlib import Path, PurePosixPath
6
+ from typing import Dict, List, Optional, Set, TypeVar
7
7
 
8
8
  import gpuhunt
9
9
  from pydantic import parse_obj_as
@@ -15,12 +15,13 @@ from dstack._internal.cli.services.configurators.base import (
15
15
  BaseApplyConfigurator,
16
16
  )
17
17
  from dstack._internal.cli.services.profile import apply_profile_args, register_profile_args
18
- from dstack._internal.cli.services.repos import init_default_virtual_repo
19
- from dstack._internal.cli.utils.common import (
20
- confirm_ask,
21
- console,
22
- warn,
18
+ from dstack._internal.cli.services.repos import (
19
+ get_repo_from_dir,
20
+ init_default_virtual_repo,
21
+ is_git_repo_url,
22
+ register_init_repo_args,
23
23
  )
24
+ from dstack._internal.cli.utils.common import confirm_ask, console, warn
24
25
  from dstack._internal.cli.utils.rich import MultiItemStatus
25
26
  from dstack._internal.cli.utils.run import get_runs_table, print_run_plan
26
27
  from dstack._internal.core.errors import (
@@ -31,28 +32,37 @@ from dstack._internal.core.errors import (
31
32
  )
32
33
  from dstack._internal.core.models.common import ApplyAction, RegistryAuth
33
34
  from dstack._internal.core.models.configurations import (
35
+ LEGACY_REPO_DIR,
34
36
  AnyRunConfiguration,
35
37
  ApplyConfigurationType,
36
- BaseRunConfiguration,
37
- BaseRunConfigurationWithPorts,
38
+ ConfigurationWithPortsParams,
38
39
  DevEnvironmentConfiguration,
39
40
  PortMapping,
40
41
  RunConfigurationType,
41
42
  ServiceConfiguration,
42
43
  TaskConfiguration,
43
44
  )
45
+ from dstack._internal.core.models.repos import RepoHeadWithCreds
44
46
  from dstack._internal.core.models.repos.base import Repo
45
47
  from dstack._internal.core.models.repos.local import LocalRepo
48
+ from dstack._internal.core.models.repos.remote import RemoteRepo, RemoteRepoCreds
46
49
  from dstack._internal.core.models.resources import CPUSpec
47
50
  from dstack._internal.core.models.runs import JobStatus, JobSubmission, RunSpec, RunStatus
48
51
  from dstack._internal.core.services.configs import ConfigManager
49
52
  from dstack._internal.core.services.diff import diff_models
53
+ from dstack._internal.core.services.repos import (
54
+ InvalidRepoCredentialsError,
55
+ get_repo_creds_and_default_branch,
56
+ load_repo,
57
+ )
50
58
  from dstack._internal.utils.common import local_time
51
59
  from dstack._internal.utils.interpolator import InterpolatorError, VariablesInterpolator
52
60
  from dstack._internal.utils.logging import get_logger
53
61
  from dstack._internal.utils.nested_list import NestedList, NestedListItem
62
+ from dstack._internal.utils.path import is_absolute_posix_path
54
63
  from dstack.api._public.repos import get_ssh_keypair
55
64
  from dstack.api._public.runs import Run
65
+ from dstack.api.server import APIClient
56
66
  from dstack.api.utils import load_profile
57
67
 
58
68
  _KNOWN_AMD_GPUS = {gpu.name.lower() for gpu in gpuhunt.KNOWN_AMD_GPUS}
@@ -63,56 +73,71 @@ _BIND_ADDRESS_ARG = "bind_address"
63
73
 
64
74
  logger = get_logger(__name__)
65
75
 
76
+ RunConfigurationT = TypeVar("RunConfigurationT", bound=AnyRunConfiguration)
77
+
66
78
 
67
- class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
79
+ class BaseRunConfigurator(
80
+ ApplyEnvVarsConfiguratorMixin,
81
+ BaseApplyConfigurator[RunConfigurationT],
82
+ ):
68
83
  TYPE: ApplyConfigurationType
69
84
 
70
85
  def apply_configuration(
71
86
  self,
72
- conf: BaseRunConfiguration,
87
+ conf: RunConfigurationT,
73
88
  configuration_path: str,
74
89
  command_args: argparse.Namespace,
75
90
  configurator_args: argparse.Namespace,
76
91
  unknown_args: List[str],
77
- repo: Optional[Repo] = None,
78
92
  ):
93
+ if configurator_args.repo and configurator_args.no_repo:
94
+ raise CLIError("Either --repo or --no-repo can be specified")
95
+
79
96
  self.apply_args(conf, configurator_args, unknown_args)
80
97
  self.validate_gpu_vendor_and_image(conf)
81
98
  self.validate_cpu_arch_and_image(conf)
82
- config_manager = ConfigManager()
83
- if repo is None:
84
- repo_path = Path.cwd()
85
- repo_config = config_manager.get_repo_config(repo_path)
86
- if repo_config is None:
99
+
100
+ working_dir = conf.working_dir
101
+ if working_dir is None:
102
+ # Use the default working dir for the image for tasks and services if `commands`
103
+ # is not set (emulate pre-0.19.27 JobConfigutor logic), otherwise fall back to
104
+ # `/workflow`.
105
+ if isinstance(conf, DevEnvironmentConfiguration) or conf.commands:
106
+ # relative path for compatibility with pre-0.19.27 servers
107
+ conf.working_dir = "."
87
108
  warn(
88
- "The repo is not initialized. Starting from 0.19.25, repos are optional\n"
89
- "There are three options:\n"
90
- " - Run `dstack init` to initialize the current directory as a repo\n"
91
- " - Specify `--repo`\n"
92
- " - Specify `--no-repo` to not use any repo and supress this warning"
93
- " (this will be the default in the future versions)"
109
+ f'The [code]working_dir[/code] is not set using legacy default [code]"{LEGACY_REPO_DIR}"[/code].'
110
+ " Future versions will default to the [code]image[/code]'s working directory."
94
111
  )
95
- if not command_args.yes and not confirm_ask("Continue without the repo?"):
96
- console.print("\nExiting...")
97
- return
98
- repo = init_default_virtual_repo(self.api)
99
- else:
100
- # Unlikely, but may raise ConfigurationError if the repo does not exist
101
- # on the server side (stale entry in `config.yml`)
102
- repo = self.api.repos.load(repo_path)
103
- if isinstance(repo, LocalRepo):
104
- warn(
105
- f"{repo.repo_dir} is a local repo.\n"
106
- "Local repos are deprecated since 0.19.25"
107
- " and will be removed soon\n"
108
- "There are two options:\n"
109
- " - Migrate to `files`: https://dstack.ai/docs/concepts/tasks/#files\n"
110
- " - Specify `--no-repo` if you don't need the repo at all\n"
111
- "In either case, you can run `dstack init --remove` to remove the repo"
112
- " (only the record about the repo, not its files) and this warning"
113
- )
112
+ elif not is_absolute_posix_path(working_dir):
113
+ legacy_working_dir = PurePosixPath(LEGACY_REPO_DIR) / working_dir
114
+ warn(
115
+ "[code]working_dir[/code] is relative."
116
+ f" Using legacy working directory [code]{legacy_working_dir}[/code]\n\n"
117
+ "Future versions will require absolute path\n"
118
+ f"To keep using legacy working directory, set"
119
+ f" [code]working_dir[/code] to [code]{legacy_working_dir}[/code]\n"
120
+ )
121
+ else:
122
+ # relative path for compatibility with pre-0.19.27 servers
123
+ try:
124
+ conf.working_dir = str(PurePosixPath(working_dir).relative_to(LEGACY_REPO_DIR))
125
+ except ValueError:
126
+ pass
127
+
128
+ if conf.repos and conf.repos[0].path is None:
129
+ warn(
130
+ "[code]repos[0].path[/code] is not set,"
131
+ f" using legacy repo path [code]{LEGACY_REPO_DIR}[/code]\n\n"
132
+ "In a future version the default value will be changed."
133
+ f" To keep using [code]{LEGACY_REPO_DIR}[/code], explicitly set"
134
+ f" [code]repos[0].path[/code] to [code]{LEGACY_REPO_DIR}[/code]\n"
135
+ )
136
+
137
+ config_manager = ConfigManager()
138
+ repo = self.get_repo(conf, configuration_path, configurator_args, config_manager)
114
139
  self.api.ssh_identity_file = get_ssh_keypair(
115
- command_args.ssh_identity_file,
140
+ configurator_args.ssh_identity_file,
116
141
  config_manager.dstack_key_path,
117
142
  )
118
143
  profile = load_profile(Path.cwd(), configurator_args.profile)
@@ -204,6 +229,9 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
204
229
  format_date=local_time,
205
230
  )
206
231
  )
232
+
233
+ _warn_fleet_autocreated(self.api.client, run)
234
+
207
235
  console.print(
208
236
  f"\n[code]{run.name}[/] provisioning completed [secondary]({run.status.value})[/]"
209
237
  )
@@ -270,7 +298,7 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
270
298
 
271
299
  def delete_configuration(
272
300
  self,
273
- conf: AnyRunConfiguration,
301
+ conf: RunConfigurationT,
274
302
  configuration_path: str,
275
303
  command_args: argparse.Namespace,
276
304
  ):
@@ -296,7 +324,14 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
296
324
  console.print(f"Run [code]{conf.name}[/] deleted")
297
325
 
298
326
  @classmethod
299
- def register_args(cls, parser: argparse.ArgumentParser, default_max_offers: int = 3):
327
+ def register_args(cls, parser: argparse.ArgumentParser):
328
+ parser.add_argument(
329
+ "--ssh-identity",
330
+ metavar="SSH_PRIVATE_KEY",
331
+ help="The private SSH key path for SSH tunneling",
332
+ type=Path,
333
+ dest="ssh_identity_file",
334
+ )
300
335
  configuration_group = parser.add_argument_group(f"{cls.TYPE.value} Options")
301
336
  configuration_group.add_argument(
302
337
  "-n",
@@ -308,7 +343,7 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
308
343
  "--max-offers",
309
344
  help="Number of offers to show in the run plan",
310
345
  type=int,
311
- default=default_max_offers,
346
+ default=3,
312
347
  )
313
348
  cls.register_env_args(configuration_group)
314
349
  configuration_group.add_argument(
@@ -335,8 +370,32 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
335
370
  dest="disk_spec",
336
371
  )
337
372
  register_profile_args(parser)
373
+ repo_group = parser.add_argument_group("Repo Options")
374
+ repo_group.add_argument(
375
+ "-P",
376
+ "--repo",
377
+ help=("The repo to use for the run. Can be a local path or a Git repo URL."),
378
+ dest="repo",
379
+ )
380
+ repo_group.add_argument(
381
+ "--repo-branch",
382
+ help="The repo branch to use for the run",
383
+ dest="repo_branch",
384
+ )
385
+ repo_group.add_argument(
386
+ "--repo-hash",
387
+ help="The hash of the repo commit to use for the run",
388
+ dest="repo_hash",
389
+ )
390
+ repo_group.add_argument(
391
+ "--no-repo",
392
+ help="Do not use any repo for the run",
393
+ dest="no_repo",
394
+ action="store_true",
395
+ )
396
+ register_init_repo_args(repo_group)
338
397
 
339
- def apply_args(self, conf: BaseRunConfiguration, args: argparse.Namespace, unknown: List[str]):
398
+ def apply_args(self, conf: RunConfigurationT, args: argparse.Namespace, unknown: List[str]):
340
399
  apply_profile_args(args, conf)
341
400
  if args.run_name:
342
401
  conf.name = args.run_name
@@ -360,7 +419,7 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
360
419
  except InterpolatorError as e:
361
420
  raise ConfigurationError(e.args[0])
362
421
 
363
- def interpolate_env(self, conf: BaseRunConfiguration):
422
+ def interpolate_env(self, conf: RunConfigurationT):
364
423
  env_dict = conf.env.as_dict()
365
424
  interpolator = VariablesInterpolator({"env": env_dict}, skip=["secrets"])
366
425
  try:
@@ -380,7 +439,7 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
380
439
  except InterpolatorError as e:
381
440
  raise ConfigurationError(e.args[0])
382
441
 
383
- def validate_gpu_vendor_and_image(self, conf: BaseRunConfiguration) -> None:
442
+ def validate_gpu_vendor_and_image(self, conf: RunConfigurationT) -> None:
384
443
  """
385
444
  Infers and sets `resources.gpu.vendor` if not set, requires `image` if the vendor is AMD.
386
445
  """
@@ -441,7 +500,7 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
441
500
  "`image` is required if `resources.gpu.vendor` is `tenstorrent`"
442
501
  )
443
502
 
444
- def validate_cpu_arch_and_image(self, conf: BaseRunConfiguration) -> None:
503
+ def validate_cpu_arch_and_image(self, conf: RunConfigurationT) -> None:
445
504
  """
446
505
  Infers `resources.cpu.arch` if not set, requires `image` if the architecture is ARM.
447
506
  """
@@ -464,11 +523,159 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
464
523
  if arch == gpuhunt.CPUArchitecture.ARM and conf.image is None:
465
524
  raise ConfigurationError("`image` is required if `resources.cpu.arch` is `arm`")
466
525
 
526
+ def get_repo(
527
+ self,
528
+ conf: RunConfigurationT,
529
+ configuration_path: str,
530
+ configurator_args: argparse.Namespace,
531
+ config_manager: ConfigManager,
532
+ ) -> Repo:
533
+ if configurator_args.no_repo:
534
+ return init_default_virtual_repo(api=self.api)
535
+
536
+ repo: Optional[Repo] = None
537
+ repo_head: Optional[RepoHeadWithCreds] = None
538
+ repo_branch: Optional[str] = configurator_args.repo_branch
539
+ repo_hash: Optional[str] = configurator_args.repo_hash
540
+ repo_creds: Optional[RemoteRepoCreds] = None
541
+ git_identity_file: Optional[str] = configurator_args.git_identity_file
542
+ git_private_key: Optional[str] = None
543
+ oauth_token: Optional[str] = configurator_args.gh_token
544
+ # Should we (re)initialize the repo?
545
+ # If any Git credentials provided, we reinitialize the repo, as the user may have provided
546
+ # updated credentials.
547
+ init = git_identity_file is not None or oauth_token is not None
548
+
549
+ url: Optional[str] = None
550
+ local_path: Optional[Path] = None
551
+ # dummy value, safe to join with any path
552
+ root_dir = Path(".")
553
+ # True if no repo specified, but we found one in `config.yml`
554
+ legacy_local_path = False
555
+ if repo_arg := configurator_args.repo:
556
+ if is_git_repo_url(repo_arg):
557
+ url = repo_arg
558
+ else:
559
+ local_path = Path(repo_arg)
560
+ # rel paths in `--repo` are resolved relative to the current working dir
561
+ root_dir = Path.cwd()
562
+ elif conf.repos:
563
+ repo_spec = conf.repos[0]
564
+ if repo_spec.url:
565
+ url = repo_spec.url
566
+ elif repo_spec.local_path:
567
+ local_path = Path(repo_spec.local_path)
568
+ # rel paths in the conf are resolved relative to the conf's parent dir
569
+ root_dir = Path(configuration_path).resolve().parent
570
+ else:
571
+ assert False, f"should not reach here: {repo_spec}"
572
+ if repo_branch is None:
573
+ repo_branch = repo_spec.branch
574
+ if repo_hash is None:
575
+ repo_hash = repo_spec.hash
576
+ else:
577
+ local_path = Path.cwd()
578
+ legacy_local_path = True
579
+ if url:
580
+ # "master" is a dummy value, we'll fetch the actual default branch later
581
+ repo = RemoteRepo.from_url(repo_url=url, repo_branch="master")
582
+ repo_head = self.api.repos.get(repo_id=repo.repo_id, with_creds=True)
583
+ elif local_path:
584
+ if legacy_local_path:
585
+ if repo_config := config_manager.get_repo_config(local_path):
586
+ repo = load_repo(repo_config)
587
+ repo_head = self.api.repos.get(repo_id=repo.repo_id, with_creds=True)
588
+ if repo_head is not None:
589
+ warn(
590
+ "The repo is not specified but found and will be used in the run\n"
591
+ "Future versions will not load repos automatically\n"
592
+ "To prepare for future versions and get rid of this warning:\n"
593
+ "- If you need the repo in the run, either specify [code]repos[/code]"
594
+ " in the configuration or use [code]--repo .[/code]\n"
595
+ "- If you don't need the repo in the run, either run"
596
+ " [code]dstack init --remove[/code] once (it removes only the record"
597
+ " about the repo, the repo files will remain intact)"
598
+ " or use [code]--no-repo[/code]"
599
+ )
600
+ else:
601
+ # ignore stale entries in `config.yml`
602
+ repo = None
603
+ init = False
604
+ else:
605
+ original_local_path = local_path
606
+ local_path = local_path.expanduser()
607
+ if not local_path.is_absolute():
608
+ local_path = (root_dir / local_path).resolve()
609
+ if not local_path.exists():
610
+ raise ConfigurationError(
611
+ f"Invalid repo path: {original_local_path} -> {local_path}"
612
+ )
613
+ local: bool = configurator_args.local
614
+ repo = get_repo_from_dir(local_path, local=local)
615
+ repo_head = self.api.repos.get(repo_id=repo.repo_id, with_creds=True)
616
+ if isinstance(repo, RemoteRepo):
617
+ repo_branch = repo.run_repo_data.repo_branch
618
+ repo_hash = repo.run_repo_data.repo_hash
619
+ else:
620
+ assert False, "should not reach here"
621
+
622
+ if repo is None:
623
+ return init_default_virtual_repo(api=self.api)
624
+
625
+ if isinstance(repo, RemoteRepo):
626
+ assert repo.repo_url is not None
627
+
628
+ if repo_head is not None and repo_head.repo_creds is not None:
629
+ if git_identity_file is None and oauth_token is None:
630
+ git_private_key = repo_head.repo_creds.private_key
631
+ oauth_token = repo_head.repo_creds.oauth_token
632
+ else:
633
+ init = True
634
+
635
+ try:
636
+ repo_creds, default_repo_branch = get_repo_creds_and_default_branch(
637
+ repo_url=repo.repo_url,
638
+ identity_file=git_identity_file,
639
+ private_key=git_private_key,
640
+ oauth_token=oauth_token,
641
+ )
642
+ except InvalidRepoCredentialsError as e:
643
+ raise CLIError(*e.args) from e
644
+
645
+ if repo_branch is None and repo_hash is None:
646
+ repo_branch = default_repo_branch
647
+ if repo_branch is None:
648
+ raise CLIError(
649
+ "Failed to automatically detect remote repo branch."
650
+ " Specify branch or hash."
651
+ )
652
+ repo = RemoteRepo.from_url(
653
+ repo_url=repo.repo_url, repo_branch=repo_branch, repo_hash=repo_hash
654
+ )
655
+
656
+ if init:
657
+ self.api.repos.init(
658
+ repo=repo,
659
+ git_identity_file=git_identity_file,
660
+ oauth_token=oauth_token,
661
+ creds=repo_creds,
662
+ )
467
663
 
468
- class RunWithPortsConfigurator(BaseRunConfigurator):
664
+ if isinstance(repo, LocalRepo):
665
+ warn(
666
+ f"{repo.repo_dir} is a local repo\n"
667
+ "Local repos are deprecated since 0.19.25 and will be removed soon\n"
668
+ "There are two options:\n"
669
+ "- Migrate to [code]files[/code]: https://dstack.ai/docs/concepts/tasks/#files\n"
670
+ "- Specify [code]--no-repo[/code] if you don't need the repo at all"
671
+ )
672
+
673
+ return repo
674
+
675
+
676
+ class RunWithPortsConfiguratorMixin:
469
677
  @classmethod
470
- def register_args(cls, parser: argparse.ArgumentParser):
471
- super().register_args(parser)
678
+ def register_ports_args(cls, parser: argparse.ArgumentParser):
472
679
  parser.add_argument(
473
680
  "-p",
474
681
  "--port",
@@ -485,29 +692,42 @@ class RunWithPortsConfigurator(BaseRunConfigurator):
485
692
  metavar="HOST",
486
693
  )
487
694
 
488
- def apply_args(
489
- self, conf: BaseRunConfigurationWithPorts, args: argparse.Namespace, unknown: List[str]
695
+ def apply_ports_args(
696
+ self,
697
+ conf: ConfigurationWithPortsParams,
698
+ args: argparse.Namespace,
490
699
  ):
491
- super().apply_args(conf, args, unknown)
492
700
  if args.ports:
493
701
  conf.ports = list(_merge_ports(conf.ports, args.ports).values())
494
702
 
495
703
 
496
- class TaskConfigurator(RunWithPortsConfigurator):
704
+ class TaskConfigurator(RunWithPortsConfiguratorMixin, BaseRunConfigurator):
497
705
  TYPE = ApplyConfigurationType.TASK
498
706
 
707
+ @classmethod
708
+ def register_args(cls, parser: argparse.ArgumentParser):
709
+ super().register_args(parser)
710
+ cls.register_ports_args(parser)
711
+
499
712
  def apply_args(self, conf: TaskConfiguration, args: argparse.Namespace, unknown: List[str]):
500
713
  super().apply_args(conf, args, unknown)
714
+ self.apply_ports_args(conf, args)
501
715
  self.interpolate_run_args(conf.commands, unknown)
502
716
 
503
717
 
504
- class DevEnvironmentConfigurator(RunWithPortsConfigurator):
718
+ class DevEnvironmentConfigurator(RunWithPortsConfiguratorMixin, BaseRunConfigurator):
505
719
  TYPE = ApplyConfigurationType.DEV_ENVIRONMENT
506
720
 
721
+ @classmethod
722
+ def register_args(cls, parser: argparse.ArgumentParser):
723
+ super().register_args(parser)
724
+ cls.register_ports_args(parser)
725
+
507
726
  def apply_args(
508
727
  self, conf: DevEnvironmentConfiguration, args: argparse.Namespace, unknown: List[str]
509
728
  ):
510
729
  super().apply_args(conf, args, unknown)
730
+ self.apply_ports_args(conf, args)
511
731
  if conf.ide == "vscode" and conf.version is None:
512
732
  conf.version = _detect_vscode_version()
513
733
  if conf.version is None:
@@ -677,6 +897,8 @@ def render_run_spec_diff(old_spec: RunSpec, new_spec: RunSpec) -> Optional[str]:
677
897
  if type(old_spec.profile) is not type(new_spec.profile):
678
898
  item = NestedListItem("Profile")
679
899
  else:
900
+ assert old_spec.profile is not None
901
+ assert new_spec.profile is not None
680
902
  item = NestedListItem(
681
903
  "Profile properties:",
682
904
  children=[
@@ -690,3 +912,16 @@ def render_run_spec_diff(old_spec: RunSpec, new_spec: RunSpec) -> Optional[str]:
690
912
  item = NestedListItem(spec_field.replace("_", " ").capitalize())
691
913
  nested_list.children.append(item)
692
914
  return nested_list.render()
915
+
916
+
917
+ def _warn_fleet_autocreated(api: APIClient, run: Run):
918
+ if run._run.fleet is None:
919
+ return
920
+ fleet = api.fleets.get(project_name=run._project, name=run._run.fleet.name)
921
+ if not fleet.spec.autocreated:
922
+ return
923
+ warn(
924
+ f"\nNo existing fleet matched, so the run created a new fleet [code]{fleet.name}[/code].\n"
925
+ "Future dstack versions won't create fleets automatically.\n"
926
+ "Create a fleet explicitly: https://dstack.ai/docs/concepts/fleets/"
927
+ )
@@ -1,6 +1,6 @@
1
1
  import argparse
2
2
  import time
3
- from typing import List, Optional
3
+ from typing import List
4
4
 
5
5
  from rich.table import Table
6
6
 
@@ -14,7 +14,6 @@ from dstack._internal.cli.utils.rich import MultiItemStatus
14
14
  from dstack._internal.cli.utils.volume import get_volumes_table
15
15
  from dstack._internal.core.errors import ResourceNotExistsError
16
16
  from dstack._internal.core.models.configurations import ApplyConfigurationType
17
- from dstack._internal.core.models.repos.base import Repo
18
17
  from dstack._internal.core.models.volumes import (
19
18
  Volume,
20
19
  VolumeConfiguration,
@@ -26,7 +25,7 @@ from dstack._internal.utils.common import local_time
26
25
  from dstack.api._public import Client
27
26
 
28
27
 
29
- class VolumeConfigurator(BaseApplyConfigurator):
28
+ class VolumeConfigurator(BaseApplyConfigurator[VolumeConfiguration]):
30
29
  TYPE: ApplyConfigurationType = ApplyConfigurationType.VOLUME
31
30
 
32
31
  def apply_configuration(
@@ -36,7 +35,6 @@ class VolumeConfigurator(BaseApplyConfigurator):
36
35
  command_args: argparse.Namespace,
37
36
  configurator_args: argparse.Namespace,
38
37
  unknown_args: List[str],
39
- repo: Optional[Repo] = None,
40
38
  ):
41
39
  self.apply_args(conf, configurator_args, unknown_args)
42
40
  spec = VolumeSpec(
@@ -159,7 +159,7 @@ def apply_profile_args(
159
159
  if args.idle_duration is not None:
160
160
  profile_settings.idle_duration = args.idle_duration
161
161
  elif args.dont_destroy:
162
- profile_settings.idle_duration = "off"
162
+ profile_settings.idle_duration = -1
163
163
  if args.creation_policy_reuse:
164
164
  profile_settings.creation_policy = CreationPolicy.REUSE
165
165
 
@@ -1,13 +1,13 @@
1
1
  import argparse
2
- from pathlib import Path
3
- from typing import Optional
2
+ from typing import Literal, Union, overload
3
+
4
+ import git
4
5
 
5
6
  from dstack._internal.cli.services.configurators.base import ArgsParser
6
7
  from dstack._internal.core.errors import CLIError
7
- from dstack._internal.core.models.repos.base import Repo
8
+ from dstack._internal.core.models.repos.local import LocalRepo
8
9
  from dstack._internal.core.models.repos.remote import GitRepoURL, RemoteRepo, RepoError
9
10
  from dstack._internal.core.models.repos.virtual import VirtualRepo
10
- from dstack._internal.core.services.repos import get_default_branch
11
11
  from dstack._internal.utils.path import PathLike
12
12
  from dstack.api._public import Client
13
13
 
@@ -36,51 +36,38 @@ def register_init_repo_args(parser: ArgsParser):
36
36
  )
37
37
 
38
38
 
39
- def init_repo(
40
- api: Client,
41
- repo_path: PathLike,
42
- repo_branch: Optional[str],
43
- repo_hash: Optional[str],
44
- local: bool,
45
- git_identity_file: Optional[PathLike],
46
- oauth_token: Optional[str],
47
- ) -> Repo:
48
- if Path(repo_path).exists():
49
- repo = api.repos.load(
50
- repo_dir=repo_path,
51
- local=local,
52
- init=True,
53
- git_identity_file=git_identity_file,
54
- oauth_token=oauth_token,
55
- )
56
- elif isinstance(repo_path, str):
57
- try:
58
- GitRepoURL.parse(repo_path)
59
- except RepoError as e:
60
- raise CLIError("Invalid repo path") from e
61
- if repo_branch is None and repo_hash is None:
62
- repo_branch = get_default_branch(repo_path)
63
- if repo_branch is None:
64
- raise CLIError(
65
- "Failed to automatically detect remote repo branch."
66
- " Specify --repo-branch or --repo-hash."
67
- )
68
- repo = RemoteRepo.from_url(
69
- repo_url=repo_path,
70
- repo_branch=repo_branch,
71
- repo_hash=repo_hash,
72
- )
73
- api.repos.init(
74
- repo=repo,
75
- git_identity_file=git_identity_file,
76
- oauth_token=oauth_token,
77
- )
78
- else:
79
- raise CLIError("Invalid repo path")
80
- return repo
81
-
82
-
83
39
  def init_default_virtual_repo(api: Client) -> VirtualRepo:
84
40
  repo = VirtualRepo()
85
41
  api.repos.init(repo)
86
42
  return repo
43
+
44
+
45
+ @overload
46
+ def get_repo_from_dir(repo_dir: PathLike, local: Literal[False] = False) -> RemoteRepo: ...
47
+
48
+
49
+ @overload
50
+ def get_repo_from_dir(repo_dir: PathLike, local: Literal[True]) -> LocalRepo: ...
51
+
52
+
53
+ def get_repo_from_dir(repo_dir: PathLike, local: bool = False) -> Union[RemoteRepo, LocalRepo]:
54
+ if local:
55
+ return LocalRepo.from_dir(repo_dir)
56
+ try:
57
+ return RemoteRepo.from_dir(repo_dir)
58
+ except git.InvalidGitRepositoryError:
59
+ raise CLIError(
60
+ f"Git repo not found: {repo_dir}\n"
61
+ "Use `files` to mount an arbitrary directory:"
62
+ " https://dstack.ai/docs/concepts/tasks/#files"
63
+ )
64
+ except RepoError as e:
65
+ raise CLIError(str(e)) from e
66
+
67
+
68
+ def is_git_repo_url(value: str) -> bool:
69
+ try:
70
+ GitRepoURL.parse(value)
71
+ except RepoError:
72
+ return False
73
+ return True
@@ -0,0 +1 @@
1
+ # This package contains the implementation for the AMDDevCloud backend.
@@ -0,0 +1,16 @@
1
+ from dstack._internal.core.backends.amddevcloud.compute import AMDDevCloudCompute
2
+ from dstack._internal.core.backends.digitalocean_base.backend import BaseDigitalOceanBackend
3
+ from dstack._internal.core.backends.digitalocean_base.models import BaseDigitalOceanConfig
4
+ from dstack._internal.core.models.backends.base import BackendType
5
+
6
+
7
+ class AMDDevCloudBackend(BaseDigitalOceanBackend):
8
+ TYPE = BackendType.AMDDEVCLOUD
9
+ COMPUTE_CLASS = AMDDevCloudCompute
10
+
11
+ def __init__(self, config: BaseDigitalOceanConfig, api_url: str):
12
+ self.config = config
13
+ self._compute = AMDDevCloudCompute(self.config, api_url=api_url, type=self.TYPE)
14
+
15
+ def compute(self) -> AMDDevCloudCompute:
16
+ return self._compute