dstack 0.19.26__py3-none-any.whl → 0.19.28__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dstack might be problematic. Click here for more details.
- dstack/_internal/cli/commands/__init__.py +11 -8
- dstack/_internal/cli/commands/apply.py +6 -3
- dstack/_internal/cli/commands/completion.py +3 -1
- dstack/_internal/cli/commands/config.py +1 -0
- dstack/_internal/cli/commands/init.py +4 -4
- dstack/_internal/cli/commands/offer.py +1 -1
- dstack/_internal/cli/commands/project.py +1 -0
- dstack/_internal/cli/commands/server.py +2 -2
- dstack/_internal/cli/main.py +1 -1
- dstack/_internal/cli/services/configurators/base.py +2 -4
- dstack/_internal/cli/services/configurators/fleet.py +4 -5
- dstack/_internal/cli/services/configurators/gateway.py +3 -5
- dstack/_internal/cli/services/configurators/run.py +165 -43
- dstack/_internal/cli/services/configurators/volume.py +3 -5
- dstack/_internal/cli/services/repos.py +1 -18
- dstack/_internal/core/backends/amddevcloud/__init__.py +1 -0
- dstack/_internal/core/backends/amddevcloud/backend.py +16 -0
- dstack/_internal/core/backends/amddevcloud/compute.py +5 -0
- dstack/_internal/core/backends/amddevcloud/configurator.py +29 -0
- dstack/_internal/core/backends/aws/compute.py +6 -1
- dstack/_internal/core/backends/base/compute.py +33 -5
- dstack/_internal/core/backends/base/offers.py +2 -0
- dstack/_internal/core/backends/configurators.py +15 -0
- dstack/_internal/core/backends/digitalocean/__init__.py +1 -0
- dstack/_internal/core/backends/digitalocean/backend.py +16 -0
- dstack/_internal/core/backends/digitalocean/compute.py +5 -0
- dstack/_internal/core/backends/digitalocean/configurator.py +31 -0
- dstack/_internal/core/backends/digitalocean_base/__init__.py +1 -0
- dstack/_internal/core/backends/digitalocean_base/api_client.py +104 -0
- dstack/_internal/core/backends/digitalocean_base/backend.py +5 -0
- dstack/_internal/core/backends/digitalocean_base/compute.py +173 -0
- dstack/_internal/core/backends/digitalocean_base/configurator.py +57 -0
- dstack/_internal/core/backends/digitalocean_base/models.py +43 -0
- dstack/_internal/core/backends/gcp/compute.py +32 -8
- dstack/_internal/core/backends/hotaisle/api_client.py +25 -33
- dstack/_internal/core/backends/hotaisle/compute.py +1 -6
- dstack/_internal/core/backends/models.py +7 -0
- dstack/_internal/core/backends/nebius/compute.py +0 -7
- dstack/_internal/core/backends/oci/compute.py +4 -5
- dstack/_internal/core/backends/vultr/compute.py +1 -5
- dstack/_internal/core/compatibility/fleets.py +5 -0
- dstack/_internal/core/compatibility/runs.py +10 -1
- dstack/_internal/core/models/backends/base.py +5 -1
- dstack/_internal/core/models/common.py +67 -43
- dstack/_internal/core/models/configurations.py +109 -69
- dstack/_internal/core/models/files.py +1 -1
- dstack/_internal/core/models/fleets.py +115 -25
- dstack/_internal/core/models/instances.py +5 -5
- dstack/_internal/core/models/profiles.py +66 -47
- dstack/_internal/core/models/repos/remote.py +21 -16
- dstack/_internal/core/models/resources.py +69 -65
- dstack/_internal/core/models/runs.py +41 -14
- dstack/_internal/core/services/repos.py +85 -80
- dstack/_internal/server/app.py +5 -0
- dstack/_internal/server/background/tasks/process_fleets.py +117 -13
- dstack/_internal/server/background/tasks/process_instances.py +12 -71
- dstack/_internal/server/background/tasks/process_running_jobs.py +2 -0
- dstack/_internal/server/background/tasks/process_runs.py +2 -0
- dstack/_internal/server/background/tasks/process_submitted_jobs.py +48 -16
- dstack/_internal/server/migrations/versions/2498ab323443_add_fleetmodel_consolidation_attempt_.py +44 -0
- dstack/_internal/server/models.py +11 -7
- dstack/_internal/server/schemas/gateways.py +10 -9
- dstack/_internal/server/schemas/runner.py +1 -0
- dstack/_internal/server/services/backends/handlers.py +2 -0
- dstack/_internal/server/services/docker.py +8 -7
- dstack/_internal/server/services/fleets.py +23 -25
- dstack/_internal/server/services/instances.py +3 -3
- dstack/_internal/server/services/jobs/configurators/base.py +46 -6
- dstack/_internal/server/services/jobs/configurators/dev.py +4 -4
- dstack/_internal/server/services/jobs/configurators/extensions/cursor.py +3 -5
- dstack/_internal/server/services/jobs/configurators/extensions/vscode.py +4 -6
- dstack/_internal/server/services/jobs/configurators/service.py +0 -3
- dstack/_internal/server/services/jobs/configurators/task.py +0 -3
- dstack/_internal/server/services/projects.py +52 -1
- dstack/_internal/server/services/runs.py +16 -0
- dstack/_internal/server/settings.py +46 -0
- dstack/_internal/server/statics/index.html +1 -1
- dstack/_internal/server/statics/{main-aec4762350e34d6fbff9.css → main-5e0d56245c4bd241ec27.css} +1 -1
- dstack/_internal/server/statics/{main-d151b300fcac3933213d.js → main-a2a16772fbf11a14d191.js} +1215 -998
- dstack/_internal/server/statics/{main-d151b300fcac3933213d.js.map → main-a2a16772fbf11a14d191.js.map} +1 -1
- dstack/_internal/server/testing/common.py +6 -3
- dstack/_internal/utils/env.py +85 -11
- dstack/_internal/utils/path.py +8 -1
- dstack/_internal/utils/ssh.py +7 -0
- dstack/api/_public/repos.py +41 -6
- dstack/api/_public/runs.py +14 -1
- dstack/version.py +1 -1
- {dstack-0.19.26.dist-info → dstack-0.19.28.dist-info}/METADATA +2 -2
- {dstack-0.19.26.dist-info → dstack-0.19.28.dist-info}/RECORD +92 -78
- dstack/_internal/server/statics/static/media/github.1f7102513534c83a9d8d735d2b8c12a2.svg +0 -3
- {dstack-0.19.26.dist-info → dstack-0.19.28.dist-info}/WHEEL +0 -0
- {dstack-0.19.26.dist-info → dstack-0.19.28.dist-info}/entry_points.txt +0 -0
- {dstack-0.19.26.dist-info → dstack-0.19.28.dist-info}/licenses/LICENSE.md +0 -0
|
@@ -1,20 +1,22 @@
|
|
|
1
1
|
import argparse
|
|
2
2
|
import os
|
|
3
|
+
import shlex
|
|
3
4
|
from abc import ABC, abstractmethod
|
|
4
|
-
from typing import
|
|
5
|
+
from typing import ClassVar, Optional
|
|
5
6
|
|
|
6
7
|
from rich_argparse import RichHelpFormatter
|
|
7
8
|
|
|
8
9
|
from dstack._internal.cli.services.completion import ProjectNameCompleter
|
|
9
|
-
from dstack._internal.
|
|
10
|
+
from dstack._internal.core.errors import CLIError
|
|
10
11
|
from dstack.api import Client
|
|
11
12
|
|
|
12
13
|
|
|
13
14
|
class BaseCommand(ABC):
|
|
14
|
-
NAME: str = "name the command"
|
|
15
|
-
DESCRIPTION: str = "describe the command"
|
|
16
|
-
DEFAULT_HELP: bool = True
|
|
17
|
-
ALIASES: Optional[
|
|
15
|
+
NAME: ClassVar[str] = "name the command"
|
|
16
|
+
DESCRIPTION: ClassVar[str] = "describe the command"
|
|
17
|
+
DEFAULT_HELP: ClassVar[bool] = True
|
|
18
|
+
ALIASES: ClassVar[Optional[list[str]]] = None
|
|
19
|
+
ACCEPT_EXTRA_ARGS: ClassVar[bool] = False
|
|
18
20
|
|
|
19
21
|
def __init__(self, parser: argparse.ArgumentParser):
|
|
20
22
|
self._parser = parser
|
|
@@ -50,7 +52,8 @@ class BaseCommand(ABC):
|
|
|
50
52
|
|
|
51
53
|
@abstractmethod
|
|
52
54
|
def _command(self, args: argparse.Namespace):
|
|
53
|
-
|
|
55
|
+
if not self.ACCEPT_EXTRA_ARGS and args.extra_args:
|
|
56
|
+
raise CLIError(f"Unrecognized arguments: {shlex.join(args.extra_args)}")
|
|
54
57
|
|
|
55
58
|
|
|
56
59
|
class APIBaseCommand(BaseCommand):
|
|
@@ -65,5 +68,5 @@ class APIBaseCommand(BaseCommand):
|
|
|
65
68
|
).completer = ProjectNameCompleter() # type: ignore[attr-defined]
|
|
66
69
|
|
|
67
70
|
def _command(self, args: argparse.Namespace):
|
|
68
|
-
|
|
71
|
+
super()._command(args)
|
|
69
72
|
self.api = Client.from_config(project_name=args.project)
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import argparse
|
|
2
|
+
import shlex
|
|
2
3
|
|
|
3
4
|
from argcomplete import FilesCompleter # type: ignore[attr-defined]
|
|
4
5
|
|
|
@@ -19,6 +20,7 @@ class ApplyCommand(APIBaseCommand):
|
|
|
19
20
|
NAME = "apply"
|
|
20
21
|
DESCRIPTION = "Apply a configuration"
|
|
21
22
|
DEFAULT_HELP = False
|
|
23
|
+
ACCEPT_EXTRA_ARGS = True
|
|
22
24
|
|
|
23
25
|
def _register(self):
|
|
24
26
|
super()._register()
|
|
@@ -84,13 +86,14 @@ class ApplyCommand(APIBaseCommand):
|
|
|
84
86
|
configurator_class = get_apply_configurator_class(configuration.type)
|
|
85
87
|
configurator = configurator_class(api_client=self.api)
|
|
86
88
|
configurator_parser = configurator.get_parser()
|
|
87
|
-
|
|
89
|
+
configurator_args, unknown_args = configurator_parser.parse_known_args(args.extra_args)
|
|
90
|
+
if unknown_args:
|
|
91
|
+
raise CLIError(f"Unrecognized arguments: {shlex.join(unknown_args)}")
|
|
88
92
|
configurator.apply_configuration(
|
|
89
93
|
conf=configuration,
|
|
90
94
|
configuration_path=configuration_path,
|
|
91
95
|
command_args=args,
|
|
92
|
-
configurator_args=
|
|
93
|
-
unknown_args=unknown,
|
|
96
|
+
configurator_args=configurator_args,
|
|
94
97
|
)
|
|
95
98
|
except KeyboardInterrupt:
|
|
96
99
|
console.print("\nOperation interrupted by user. Exiting...")
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
|
|
1
3
|
import argcomplete
|
|
2
4
|
|
|
3
5
|
from dstack._internal.cli.commands import BaseCommand
|
|
@@ -15,6 +17,6 @@ class CompletionCommand(BaseCommand):
|
|
|
15
17
|
choices=["bash", "zsh"],
|
|
16
18
|
)
|
|
17
19
|
|
|
18
|
-
def _command(self, args):
|
|
20
|
+
def _command(self, args: argparse.Namespace):
|
|
19
21
|
super()._command(args)
|
|
20
22
|
print(argcomplete.shellcode(["dstack"], shell=args.shell)) # type: ignore[attr-defined]
|
|
@@ -6,12 +6,12 @@ from typing import Optional
|
|
|
6
6
|
from dstack._internal.cli.commands import BaseCommand
|
|
7
7
|
from dstack._internal.cli.services.repos import (
|
|
8
8
|
get_repo_from_dir,
|
|
9
|
-
get_repo_from_url,
|
|
10
9
|
is_git_repo_url,
|
|
11
10
|
register_init_repo_args,
|
|
12
11
|
)
|
|
13
|
-
from dstack._internal.cli.utils.common import
|
|
12
|
+
from dstack._internal.cli.utils.common import confirm_ask, console, warn
|
|
14
13
|
from dstack._internal.core.errors import ConfigurationError
|
|
14
|
+
from dstack._internal.core.models.repos.remote import RemoteRepo
|
|
15
15
|
from dstack._internal.core.services.configs import ConfigManager
|
|
16
16
|
from dstack.api import Client
|
|
17
17
|
|
|
@@ -52,7 +52,7 @@ class InitCommand(BaseCommand):
|
|
|
52
52
|
)
|
|
53
53
|
|
|
54
54
|
def _command(self, args: argparse.Namespace):
|
|
55
|
-
|
|
55
|
+
super()._command(args)
|
|
56
56
|
|
|
57
57
|
repo_path: Optional[Path] = None
|
|
58
58
|
repo_url: Optional[str] = None
|
|
@@ -101,7 +101,7 @@ class InitCommand(BaseCommand):
|
|
|
101
101
|
if repo_url is not None:
|
|
102
102
|
# Dummy repo branch to avoid autodetection that fails on private repos.
|
|
103
103
|
# We don't need branch/hash for repo_id anyway.
|
|
104
|
-
repo =
|
|
104
|
+
repo = RemoteRepo.from_url(repo_url, repo_branch="master")
|
|
105
105
|
elif repo_path is not None:
|
|
106
106
|
repo = get_repo_from_dir(repo_path, local=local)
|
|
107
107
|
else:
|
|
@@ -99,7 +99,7 @@ class OfferCommand(APIBaseCommand):
|
|
|
99
99
|
conf = TaskConfiguration(commands=[":"])
|
|
100
100
|
|
|
101
101
|
configurator = OfferConfigurator(api_client=self.api)
|
|
102
|
-
configurator.apply_args(conf, args
|
|
102
|
+
configurator.apply_args(conf, args)
|
|
103
103
|
profile = load_profile(Path.cwd(), profile_name=args.profile)
|
|
104
104
|
|
|
105
105
|
run_spec = RunSpec(
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
+
import argparse
|
|
1
2
|
import os
|
|
2
|
-
from argparse import Namespace
|
|
3
3
|
|
|
4
4
|
from dstack._internal import settings
|
|
5
5
|
from dstack._internal.cli.commands import BaseCommand
|
|
@@ -53,7 +53,7 @@ class ServerCommand(BaseCommand):
|
|
|
53
53
|
)
|
|
54
54
|
self._parser.add_argument("--token", type=str, help="The admin user token")
|
|
55
55
|
|
|
56
|
-
def _command(self, args: Namespace):
|
|
56
|
+
def _command(self, args: argparse.Namespace):
|
|
57
57
|
super()._command(args)
|
|
58
58
|
|
|
59
59
|
if not UVICORN_INSTALLED:
|
dstack/_internal/cli/main.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import argparse
|
|
2
2
|
import os
|
|
3
3
|
from abc import ABC, abstractmethod
|
|
4
|
-
from typing import Generic, List, TypeVar, Union, cast
|
|
4
|
+
from typing import ClassVar, Generic, List, TypeVar, Union, cast
|
|
5
5
|
|
|
6
6
|
from dstack._internal.cli.services.args import env_var
|
|
7
7
|
from dstack._internal.core.errors import ConfigurationError
|
|
@@ -18,7 +18,7 @@ ApplyConfigurationT = TypeVar("ApplyConfigurationT", bound=AnyApplyConfiguration
|
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
class BaseApplyConfigurator(ABC, Generic[ApplyConfigurationT]):
|
|
21
|
-
TYPE: ApplyConfigurationType
|
|
21
|
+
TYPE: ClassVar[ApplyConfigurationType]
|
|
22
22
|
|
|
23
23
|
def __init__(self, api_client: Client):
|
|
24
24
|
self.api = api_client
|
|
@@ -30,7 +30,6 @@ class BaseApplyConfigurator(ABC, Generic[ApplyConfigurationT]):
|
|
|
30
30
|
configuration_path: str,
|
|
31
31
|
command_args: argparse.Namespace,
|
|
32
32
|
configurator_args: argparse.Namespace,
|
|
33
|
-
unknown_args: List[str],
|
|
34
33
|
):
|
|
35
34
|
"""
|
|
36
35
|
Implements `dstack apply` for a given configuration type.
|
|
@@ -40,7 +39,6 @@ class BaseApplyConfigurator(ABC, Generic[ApplyConfigurationT]):
|
|
|
40
39
|
configuration_path: The path to the configuration file.
|
|
41
40
|
command_args: The args parsed by `dstack apply`.
|
|
42
41
|
configurator_args: The known args parsed by `cls.get_parser()`.
|
|
43
|
-
unknown_args: The unknown args after parsing by `cls.get_parser()`.
|
|
44
42
|
"""
|
|
45
43
|
pass
|
|
46
44
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import argparse
|
|
2
2
|
import time
|
|
3
3
|
from pathlib import Path
|
|
4
|
-
from typing import
|
|
4
|
+
from typing import Optional
|
|
5
5
|
|
|
6
6
|
from rich.table import Table
|
|
7
7
|
|
|
@@ -46,7 +46,7 @@ logger = get_logger(__name__)
|
|
|
46
46
|
|
|
47
47
|
|
|
48
48
|
class FleetConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator[FleetConfiguration]):
|
|
49
|
-
TYPE
|
|
49
|
+
TYPE = ApplyConfigurationType.FLEET
|
|
50
50
|
|
|
51
51
|
def apply_configuration(
|
|
52
52
|
self,
|
|
@@ -54,9 +54,8 @@ class FleetConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator[Fle
|
|
|
54
54
|
configuration_path: str,
|
|
55
55
|
command_args: argparse.Namespace,
|
|
56
56
|
configurator_args: argparse.Namespace,
|
|
57
|
-
unknown_args: List[str],
|
|
58
57
|
):
|
|
59
|
-
self.apply_args(conf, configurator_args
|
|
58
|
+
self.apply_args(conf, configurator_args)
|
|
60
59
|
profile = load_profile(Path.cwd(), None)
|
|
61
60
|
spec = FleetSpec(
|
|
62
61
|
configuration=conf,
|
|
@@ -309,7 +308,7 @@ class FleetConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator[Fle
|
|
|
309
308
|
)
|
|
310
309
|
cls.register_env_args(configuration_group)
|
|
311
310
|
|
|
312
|
-
def apply_args(self, conf: FleetConfiguration, args: argparse.Namespace
|
|
311
|
+
def apply_args(self, conf: FleetConfiguration, args: argparse.Namespace):
|
|
313
312
|
if args.name:
|
|
314
313
|
conf.name = args.name
|
|
315
314
|
self.apply_env_vars(conf.env, args)
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import argparse
|
|
2
2
|
import time
|
|
3
|
-
from typing import List
|
|
4
3
|
|
|
5
4
|
from rich.table import Table
|
|
6
5
|
|
|
@@ -27,7 +26,7 @@ from dstack.api._public import Client
|
|
|
27
26
|
|
|
28
27
|
|
|
29
28
|
class GatewayConfigurator(BaseApplyConfigurator[GatewayConfiguration]):
|
|
30
|
-
TYPE
|
|
29
|
+
TYPE = ApplyConfigurationType.GATEWAY
|
|
31
30
|
|
|
32
31
|
def apply_configuration(
|
|
33
32
|
self,
|
|
@@ -35,9 +34,8 @@ class GatewayConfigurator(BaseApplyConfigurator[GatewayConfiguration]):
|
|
|
35
34
|
configuration_path: str,
|
|
36
35
|
command_args: argparse.Namespace,
|
|
37
36
|
configurator_args: argparse.Namespace,
|
|
38
|
-
unknown_args: List[str],
|
|
39
37
|
):
|
|
40
|
-
self.apply_args(conf, configurator_args
|
|
38
|
+
self.apply_args(conf, configurator_args)
|
|
41
39
|
spec = GatewaySpec(
|
|
42
40
|
configuration=conf,
|
|
43
41
|
configuration_path=configuration_path,
|
|
@@ -179,7 +177,7 @@ class GatewayConfigurator(BaseApplyConfigurator[GatewayConfiguration]):
|
|
|
179
177
|
help="The gateway name",
|
|
180
178
|
)
|
|
181
179
|
|
|
182
|
-
def apply_args(self, conf: GatewayConfiguration, args: argparse.Namespace
|
|
180
|
+
def apply_args(self, conf: GatewayConfiguration, args: argparse.Namespace):
|
|
183
181
|
if args.name:
|
|
184
182
|
conf.name = args.name
|
|
185
183
|
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import argparse
|
|
2
|
+
import shlex
|
|
2
3
|
import subprocess
|
|
3
4
|
import sys
|
|
4
5
|
import time
|
|
5
|
-
from pathlib import Path
|
|
6
|
+
from pathlib import Path, PurePosixPath
|
|
6
7
|
from typing import Dict, List, Optional, Set, TypeVar
|
|
7
8
|
|
|
8
9
|
import gpuhunt
|
|
@@ -17,7 +18,6 @@ from dstack._internal.cli.services.configurators.base import (
|
|
|
17
18
|
from dstack._internal.cli.services.profile import apply_profile_args, register_profile_args
|
|
18
19
|
from dstack._internal.cli.services.repos import (
|
|
19
20
|
get_repo_from_dir,
|
|
20
|
-
get_repo_from_url,
|
|
21
21
|
init_default_virtual_repo,
|
|
22
22
|
is_git_repo_url,
|
|
23
23
|
register_init_repo_args,
|
|
@@ -33,8 +33,10 @@ from dstack._internal.core.errors import (
|
|
|
33
33
|
)
|
|
34
34
|
from dstack._internal.core.models.common import ApplyAction, RegistryAuth
|
|
35
35
|
from dstack._internal.core.models.configurations import (
|
|
36
|
+
LEGACY_REPO_DIR,
|
|
36
37
|
AnyRunConfiguration,
|
|
37
38
|
ApplyConfigurationType,
|
|
39
|
+
ConfigurationWithCommandsParams,
|
|
38
40
|
ConfigurationWithPortsParams,
|
|
39
41
|
DevEnvironmentConfiguration,
|
|
40
42
|
PortMapping,
|
|
@@ -42,19 +44,27 @@ from dstack._internal.core.models.configurations import (
|
|
|
42
44
|
ServiceConfiguration,
|
|
43
45
|
TaskConfiguration,
|
|
44
46
|
)
|
|
47
|
+
from dstack._internal.core.models.repos import RepoHeadWithCreds
|
|
45
48
|
from dstack._internal.core.models.repos.base import Repo
|
|
46
49
|
from dstack._internal.core.models.repos.local import LocalRepo
|
|
50
|
+
from dstack._internal.core.models.repos.remote import RemoteRepo, RemoteRepoCreds
|
|
47
51
|
from dstack._internal.core.models.resources import CPUSpec
|
|
48
52
|
from dstack._internal.core.models.runs import JobStatus, JobSubmission, RunSpec, RunStatus
|
|
49
53
|
from dstack._internal.core.services.configs import ConfigManager
|
|
50
54
|
from dstack._internal.core.services.diff import diff_models
|
|
51
|
-
from dstack._internal.core.services.repos import
|
|
55
|
+
from dstack._internal.core.services.repos import (
|
|
56
|
+
InvalidRepoCredentialsError,
|
|
57
|
+
get_repo_creds_and_default_branch,
|
|
58
|
+
load_repo,
|
|
59
|
+
)
|
|
52
60
|
from dstack._internal.utils.common import local_time
|
|
53
61
|
from dstack._internal.utils.interpolator import InterpolatorError, VariablesInterpolator
|
|
54
62
|
from dstack._internal.utils.logging import get_logger
|
|
55
63
|
from dstack._internal.utils.nested_list import NestedList, NestedListItem
|
|
64
|
+
from dstack._internal.utils.path import is_absolute_posix_path
|
|
56
65
|
from dstack.api._public.repos import get_ssh_keypair
|
|
57
66
|
from dstack.api._public.runs import Run
|
|
67
|
+
from dstack.api.server import APIClient
|
|
58
68
|
from dstack.api.utils import load_profile
|
|
59
69
|
|
|
60
70
|
_KNOWN_AMD_GPUS = {gpu.name.lower() for gpu in gpuhunt.KNOWN_AMD_GPUS}
|
|
@@ -72,23 +82,57 @@ class BaseRunConfigurator(
|
|
|
72
82
|
ApplyEnvVarsConfiguratorMixin,
|
|
73
83
|
BaseApplyConfigurator[RunConfigurationT],
|
|
74
84
|
):
|
|
75
|
-
TYPE: ApplyConfigurationType
|
|
76
|
-
|
|
77
85
|
def apply_configuration(
|
|
78
86
|
self,
|
|
79
87
|
conf: RunConfigurationT,
|
|
80
88
|
configuration_path: str,
|
|
81
89
|
command_args: argparse.Namespace,
|
|
82
90
|
configurator_args: argparse.Namespace,
|
|
83
|
-
unknown_args: List[str],
|
|
84
91
|
):
|
|
85
92
|
if configurator_args.repo and configurator_args.no_repo:
|
|
86
93
|
raise CLIError("Either --repo or --no-repo can be specified")
|
|
87
94
|
|
|
88
|
-
self.apply_args(conf, configurator_args
|
|
95
|
+
self.apply_args(conf, configurator_args)
|
|
89
96
|
self.validate_gpu_vendor_and_image(conf)
|
|
90
97
|
self.validate_cpu_arch_and_image(conf)
|
|
91
98
|
|
|
99
|
+
working_dir = conf.working_dir
|
|
100
|
+
if working_dir is None:
|
|
101
|
+
# Use the default working dir for the image for tasks and services if `commands`
|
|
102
|
+
# is not set (emulate pre-0.19.27 JobConfigutor logic), otherwise fall back to
|
|
103
|
+
# `/workflow`.
|
|
104
|
+
if isinstance(conf, DevEnvironmentConfiguration) or conf.commands:
|
|
105
|
+
# relative path for compatibility with pre-0.19.27 servers
|
|
106
|
+
conf.working_dir = "."
|
|
107
|
+
warn(
|
|
108
|
+
f'The [code]working_dir[/code] is not set — using legacy default [code]"{LEGACY_REPO_DIR}"[/code].'
|
|
109
|
+
" Future versions will default to the [code]image[/code]'s working directory."
|
|
110
|
+
)
|
|
111
|
+
elif not is_absolute_posix_path(working_dir):
|
|
112
|
+
legacy_working_dir = PurePosixPath(LEGACY_REPO_DIR) / working_dir
|
|
113
|
+
warn(
|
|
114
|
+
"[code]working_dir[/code] is relative."
|
|
115
|
+
f" Using legacy working directory [code]{legacy_working_dir}[/code]\n\n"
|
|
116
|
+
"Future versions will require absolute path\n"
|
|
117
|
+
f"To keep using legacy working directory, set"
|
|
118
|
+
f" [code]working_dir[/code] to [code]{legacy_working_dir}[/code]\n"
|
|
119
|
+
)
|
|
120
|
+
else:
|
|
121
|
+
# relative path for compatibility with pre-0.19.27 servers
|
|
122
|
+
try:
|
|
123
|
+
conf.working_dir = str(PurePosixPath(working_dir).relative_to(LEGACY_REPO_DIR))
|
|
124
|
+
except ValueError:
|
|
125
|
+
pass
|
|
126
|
+
|
|
127
|
+
if conf.repos and conf.repos[0].path is None:
|
|
128
|
+
warn(
|
|
129
|
+
"[code]repos[0].path[/code] is not set,"
|
|
130
|
+
f" using legacy repo path [code]{LEGACY_REPO_DIR}[/code]\n\n"
|
|
131
|
+
"In a future version the default value will be changed."
|
|
132
|
+
f" To keep using [code]{LEGACY_REPO_DIR}[/code], explicitly set"
|
|
133
|
+
f" [code]repos[0].path[/code] to [code]{LEGACY_REPO_DIR}[/code]\n"
|
|
134
|
+
)
|
|
135
|
+
|
|
92
136
|
config_manager = ConfigManager()
|
|
93
137
|
repo = self.get_repo(conf, configuration_path, configurator_args, config_manager)
|
|
94
138
|
self.api.ssh_identity_file = get_ssh_keypair(
|
|
@@ -184,6 +228,9 @@ class BaseRunConfigurator(
|
|
|
184
228
|
format_date=local_time,
|
|
185
229
|
)
|
|
186
230
|
)
|
|
231
|
+
|
|
232
|
+
_warn_fleet_autocreated(self.api.client, run)
|
|
233
|
+
|
|
187
234
|
console.print(
|
|
188
235
|
f"\n[code]{run.name}[/] provisioning completed [secondary]({run.status.value})[/]"
|
|
189
236
|
)
|
|
@@ -347,7 +394,7 @@ class BaseRunConfigurator(
|
|
|
347
394
|
)
|
|
348
395
|
register_init_repo_args(repo_group)
|
|
349
396
|
|
|
350
|
-
def apply_args(self, conf: RunConfigurationT, args: argparse.Namespace
|
|
397
|
+
def apply_args(self, conf: RunConfigurationT, args: argparse.Namespace):
|
|
351
398
|
apply_profile_args(args, conf)
|
|
352
399
|
if args.run_name:
|
|
353
400
|
conf.name = args.run_name
|
|
@@ -360,16 +407,6 @@ class BaseRunConfigurator(
|
|
|
360
407
|
|
|
361
408
|
self.apply_env_vars(conf.env, args)
|
|
362
409
|
self.interpolate_env(conf)
|
|
363
|
-
self.interpolate_run_args(conf.setup, unknown)
|
|
364
|
-
|
|
365
|
-
def interpolate_run_args(self, value: List[str], unknown):
|
|
366
|
-
run_args = " ".join(unknown)
|
|
367
|
-
interpolator = VariablesInterpolator({"run": {"args": run_args}}, skip=["secrets"])
|
|
368
|
-
try:
|
|
369
|
-
for i in range(len(value)):
|
|
370
|
-
value[i] = interpolator.interpolate_or_error(value[i])
|
|
371
|
-
except InterpolatorError as e:
|
|
372
|
-
raise ConfigurationError(e.args[0])
|
|
373
410
|
|
|
374
411
|
def interpolate_env(self, conf: RunConfigurationT):
|
|
375
412
|
env_dict = conf.env.as_dict()
|
|
@@ -486,15 +523,17 @@ class BaseRunConfigurator(
|
|
|
486
523
|
return init_default_virtual_repo(api=self.api)
|
|
487
524
|
|
|
488
525
|
repo: Optional[Repo] = None
|
|
526
|
+
repo_head: Optional[RepoHeadWithCreds] = None
|
|
489
527
|
repo_branch: Optional[str] = configurator_args.repo_branch
|
|
490
528
|
repo_hash: Optional[str] = configurator_args.repo_hash
|
|
529
|
+
repo_creds: Optional[RemoteRepoCreds] = None
|
|
530
|
+
git_identity_file: Optional[str] = configurator_args.git_identity_file
|
|
531
|
+
git_private_key: Optional[str] = None
|
|
532
|
+
oauth_token: Optional[str] = configurator_args.gh_token
|
|
491
533
|
# Should we (re)initialize the repo?
|
|
492
534
|
# If any Git credentials provided, we reinitialize the repo, as the user may have provided
|
|
493
535
|
# updated credentials.
|
|
494
|
-
init =
|
|
495
|
-
configurator_args.git_identity_file is not None
|
|
496
|
-
or configurator_args.gh_token is not None
|
|
497
|
-
)
|
|
536
|
+
init = git_identity_file is not None or oauth_token is not None
|
|
498
537
|
|
|
499
538
|
url: Optional[str] = None
|
|
500
539
|
local_path: Optional[Path] = None
|
|
@@ -527,15 +566,15 @@ class BaseRunConfigurator(
|
|
|
527
566
|
local_path = Path.cwd()
|
|
528
567
|
legacy_local_path = True
|
|
529
568
|
if url:
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
569
|
+
# "master" is a dummy value, we'll fetch the actual default branch later
|
|
570
|
+
repo = RemoteRepo.from_url(repo_url=url, repo_branch="master")
|
|
571
|
+
repo_head = self.api.repos.get(repo_id=repo.repo_id, with_creds=True)
|
|
533
572
|
elif local_path:
|
|
534
573
|
if legacy_local_path:
|
|
535
574
|
if repo_config := config_manager.get_repo_config(local_path):
|
|
536
575
|
repo = load_repo(repo_config)
|
|
537
|
-
|
|
538
|
-
if
|
|
576
|
+
repo_head = self.api.repos.get(repo_id=repo.repo_id, with_creds=True)
|
|
577
|
+
if repo_head is not None:
|
|
539
578
|
warn(
|
|
540
579
|
"The repo is not specified but found and will be used in the run\n"
|
|
541
580
|
"Future versions will not load repos automatically\n"
|
|
@@ -562,20 +601,55 @@ class BaseRunConfigurator(
|
|
|
562
601
|
)
|
|
563
602
|
local: bool = configurator_args.local
|
|
564
603
|
repo = get_repo_from_dir(local_path, local=local)
|
|
565
|
-
|
|
566
|
-
|
|
604
|
+
repo_head = self.api.repos.get(repo_id=repo.repo_id, with_creds=True)
|
|
605
|
+
if isinstance(repo, RemoteRepo):
|
|
606
|
+
repo_branch = repo.run_repo_data.repo_branch
|
|
607
|
+
repo_hash = repo.run_repo_data.repo_hash
|
|
567
608
|
else:
|
|
568
609
|
assert False, "should not reach here"
|
|
569
610
|
|
|
570
611
|
if repo is None:
|
|
571
612
|
return init_default_virtual_repo(api=self.api)
|
|
572
613
|
|
|
614
|
+
if isinstance(repo, RemoteRepo):
|
|
615
|
+
assert repo.repo_url is not None
|
|
616
|
+
|
|
617
|
+
if repo_head is not None and repo_head.repo_creds is not None:
|
|
618
|
+
if git_identity_file is None and oauth_token is None:
|
|
619
|
+
git_private_key = repo_head.repo_creds.private_key
|
|
620
|
+
oauth_token = repo_head.repo_creds.oauth_token
|
|
621
|
+
else:
|
|
622
|
+
init = True
|
|
623
|
+
|
|
624
|
+
try:
|
|
625
|
+
repo_creds, default_repo_branch = get_repo_creds_and_default_branch(
|
|
626
|
+
repo_url=repo.repo_url,
|
|
627
|
+
identity_file=git_identity_file,
|
|
628
|
+
private_key=git_private_key,
|
|
629
|
+
oauth_token=oauth_token,
|
|
630
|
+
)
|
|
631
|
+
except InvalidRepoCredentialsError as e:
|
|
632
|
+
raise CLIError(*e.args) from e
|
|
633
|
+
|
|
634
|
+
if repo_branch is None and repo_hash is None:
|
|
635
|
+
repo_branch = default_repo_branch
|
|
636
|
+
if repo_branch is None:
|
|
637
|
+
raise CLIError(
|
|
638
|
+
"Failed to automatically detect remote repo branch."
|
|
639
|
+
" Specify branch or hash."
|
|
640
|
+
)
|
|
641
|
+
repo = RemoteRepo.from_url(
|
|
642
|
+
repo_url=repo.repo_url, repo_branch=repo_branch, repo_hash=repo_hash
|
|
643
|
+
)
|
|
644
|
+
|
|
573
645
|
if init:
|
|
574
646
|
self.api.repos.init(
|
|
575
647
|
repo=repo,
|
|
576
|
-
git_identity_file=
|
|
577
|
-
oauth_token=
|
|
648
|
+
git_identity_file=git_identity_file,
|
|
649
|
+
oauth_token=oauth_token,
|
|
650
|
+
creds=repo_creds,
|
|
578
651
|
)
|
|
652
|
+
|
|
579
653
|
if isinstance(repo, LocalRepo):
|
|
580
654
|
warn(
|
|
581
655
|
f"{repo.repo_dir} is a local repo\n"
|
|
@@ -616,18 +690,50 @@ class RunWithPortsConfiguratorMixin:
|
|
|
616
690
|
conf.ports = list(_merge_ports(conf.ports, args.ports).values())
|
|
617
691
|
|
|
618
692
|
|
|
619
|
-
class
|
|
693
|
+
class RunWithCommandsConfiguratorMixin:
|
|
694
|
+
@classmethod
|
|
695
|
+
def register_commands_args(cls, parser: argparse.ArgumentParser):
|
|
696
|
+
parser.add_argument(
|
|
697
|
+
"run_args",
|
|
698
|
+
help=(
|
|
699
|
+
"Run arguments. Available in the configuration [code]commands[/code] as"
|
|
700
|
+
" [code]${{ run.args }}[/code]."
|
|
701
|
+
" Use [code]--[/code] to separate run options from [code]dstack[/code] options"
|
|
702
|
+
),
|
|
703
|
+
nargs="*",
|
|
704
|
+
metavar="RUN_ARGS",
|
|
705
|
+
)
|
|
706
|
+
|
|
707
|
+
def apply_commands_args(
|
|
708
|
+
self,
|
|
709
|
+
conf: ConfigurationWithCommandsParams,
|
|
710
|
+
args: argparse.Namespace,
|
|
711
|
+
):
|
|
712
|
+
commands = conf.commands
|
|
713
|
+
run_args = shlex.join(args.run_args)
|
|
714
|
+
interpolator = VariablesInterpolator({"run": {"args": run_args}}, skip=["secrets"])
|
|
715
|
+
try:
|
|
716
|
+
for i, command in enumerate(commands):
|
|
717
|
+
commands[i] = interpolator.interpolate_or_error(command)
|
|
718
|
+
except InterpolatorError as e:
|
|
719
|
+
raise ConfigurationError(e.args[0])
|
|
720
|
+
|
|
721
|
+
|
|
722
|
+
class TaskConfigurator(
|
|
723
|
+
RunWithPortsConfiguratorMixin, RunWithCommandsConfiguratorMixin, BaseRunConfigurator
|
|
724
|
+
):
|
|
620
725
|
TYPE = ApplyConfigurationType.TASK
|
|
621
726
|
|
|
622
727
|
@classmethod
|
|
623
728
|
def register_args(cls, parser: argparse.ArgumentParser):
|
|
624
729
|
super().register_args(parser)
|
|
625
730
|
cls.register_ports_args(parser)
|
|
731
|
+
cls.register_commands_args(parser)
|
|
626
732
|
|
|
627
|
-
def apply_args(self, conf: TaskConfiguration, args: argparse.Namespace
|
|
628
|
-
super().apply_args(conf, args
|
|
733
|
+
def apply_args(self, conf: TaskConfiguration, args: argparse.Namespace):
|
|
734
|
+
super().apply_args(conf, args)
|
|
629
735
|
self.apply_ports_args(conf, args)
|
|
630
|
-
self.
|
|
736
|
+
self.apply_commands_args(conf, args)
|
|
631
737
|
|
|
632
738
|
|
|
633
739
|
class DevEnvironmentConfigurator(RunWithPortsConfiguratorMixin, BaseRunConfigurator):
|
|
@@ -638,10 +744,8 @@ class DevEnvironmentConfigurator(RunWithPortsConfiguratorMixin, BaseRunConfigura
|
|
|
638
744
|
super().register_args(parser)
|
|
639
745
|
cls.register_ports_args(parser)
|
|
640
746
|
|
|
641
|
-
def apply_args(
|
|
642
|
-
|
|
643
|
-
):
|
|
644
|
-
super().apply_args(conf, args, unknown)
|
|
747
|
+
def apply_args(self, conf: DevEnvironmentConfiguration, args: argparse.Namespace):
|
|
748
|
+
super().apply_args(conf, args)
|
|
645
749
|
self.apply_ports_args(conf, args)
|
|
646
750
|
if conf.ide == "vscode" and conf.version is None:
|
|
647
751
|
conf.version = _detect_vscode_version()
|
|
@@ -661,12 +765,17 @@ class DevEnvironmentConfigurator(RunWithPortsConfiguratorMixin, BaseRunConfigura
|
|
|
661
765
|
)
|
|
662
766
|
|
|
663
767
|
|
|
664
|
-
class ServiceConfigurator(BaseRunConfigurator):
|
|
768
|
+
class ServiceConfigurator(RunWithCommandsConfiguratorMixin, BaseRunConfigurator):
|
|
665
769
|
TYPE = ApplyConfigurationType.SERVICE
|
|
666
770
|
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
771
|
+
@classmethod
|
|
772
|
+
def register_args(cls, parser: argparse.ArgumentParser):
|
|
773
|
+
super().register_args(parser)
|
|
774
|
+
cls.register_commands_args(parser)
|
|
775
|
+
|
|
776
|
+
def apply_args(self, conf: TaskConfiguration, args: argparse.Namespace):
|
|
777
|
+
super().apply_args(conf, args)
|
|
778
|
+
self.apply_commands_args(conf, args)
|
|
670
779
|
|
|
671
780
|
|
|
672
781
|
def _merge_ports(conf: List[PortMapping], args: List[PortMapping]) -> Dict[int, PortMapping]:
|
|
@@ -827,3 +936,16 @@ def render_run_spec_diff(old_spec: RunSpec, new_spec: RunSpec) -> Optional[str]:
|
|
|
827
936
|
item = NestedListItem(spec_field.replace("_", " ").capitalize())
|
|
828
937
|
nested_list.children.append(item)
|
|
829
938
|
return nested_list.render()
|
|
939
|
+
|
|
940
|
+
|
|
941
|
+
def _warn_fleet_autocreated(api: APIClient, run: Run):
|
|
942
|
+
if run._run.fleet is None:
|
|
943
|
+
return
|
|
944
|
+
fleet = api.fleets.get(project_name=run._project, name=run._run.fleet.name)
|
|
945
|
+
if not fleet.spec.autocreated:
|
|
946
|
+
return
|
|
947
|
+
warn(
|
|
948
|
+
f"\nNo existing fleet matched, so the run created a new fleet [code]{fleet.name}[/code].\n"
|
|
949
|
+
"Future dstack versions won't create fleets automatically.\n"
|
|
950
|
+
"Create a fleet explicitly: https://dstack.ai/docs/concepts/fleets/"
|
|
951
|
+
)
|