dstack 0.19.17__py3-none-any.whl → 0.19.19__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dstack might be problematic. Click here for more details.
- dstack/_internal/cli/services/configurators/fleet.py +111 -1
- dstack/_internal/cli/services/profile.py +1 -1
- dstack/_internal/core/backends/aws/compute.py +237 -18
- dstack/_internal/core/backends/base/compute.py +20 -2
- dstack/_internal/core/backends/cudo/compute.py +23 -9
- dstack/_internal/core/backends/gcp/compute.py +13 -7
- dstack/_internal/core/backends/lambdalabs/compute.py +2 -1
- dstack/_internal/core/compatibility/fleets.py +12 -11
- dstack/_internal/core/compatibility/gateways.py +9 -8
- dstack/_internal/core/compatibility/logs.py +4 -3
- dstack/_internal/core/compatibility/runs.py +29 -21
- dstack/_internal/core/compatibility/volumes.py +11 -8
- dstack/_internal/core/errors.py +4 -0
- dstack/_internal/core/models/common.py +45 -2
- dstack/_internal/core/models/configurations.py +9 -1
- dstack/_internal/core/models/fleets.py +2 -1
- dstack/_internal/core/models/profiles.py +8 -5
- dstack/_internal/core/models/resources.py +15 -8
- dstack/_internal/core/models/runs.py +41 -138
- dstack/_internal/core/models/volumes.py +14 -0
- dstack/_internal/core/services/diff.py +56 -3
- dstack/_internal/core/services/ssh/attach.py +2 -0
- dstack/_internal/server/app.py +37 -9
- dstack/_internal/server/background/__init__.py +66 -40
- dstack/_internal/server/background/tasks/process_fleets.py +19 -3
- dstack/_internal/server/background/tasks/process_gateways.py +47 -29
- dstack/_internal/server/background/tasks/process_idle_volumes.py +139 -0
- dstack/_internal/server/background/tasks/process_instances.py +13 -2
- dstack/_internal/server/background/tasks/process_placement_groups.py +4 -2
- dstack/_internal/server/background/tasks/process_running_jobs.py +14 -3
- dstack/_internal/server/background/tasks/process_runs.py +8 -4
- dstack/_internal/server/background/tasks/process_submitted_jobs.py +38 -7
- dstack/_internal/server/background/tasks/process_terminating_jobs.py +5 -3
- dstack/_internal/server/background/tasks/process_volumes.py +2 -2
- dstack/_internal/server/migrations/versions/35e90e1b0d3e_add_rolling_deployment_fields.py +6 -6
- dstack/_internal/server/migrations/versions/d5863798bf41_add_volumemodel_last_job_processed_at.py +40 -0
- dstack/_internal/server/models.py +1 -0
- dstack/_internal/server/routers/backends.py +23 -16
- dstack/_internal/server/routers/files.py +7 -6
- dstack/_internal/server/routers/fleets.py +47 -36
- dstack/_internal/server/routers/gateways.py +27 -18
- dstack/_internal/server/routers/instances.py +18 -13
- dstack/_internal/server/routers/logs.py +7 -3
- dstack/_internal/server/routers/metrics.py +14 -8
- dstack/_internal/server/routers/projects.py +33 -22
- dstack/_internal/server/routers/repos.py +7 -6
- dstack/_internal/server/routers/runs.py +49 -28
- dstack/_internal/server/routers/secrets.py +20 -15
- dstack/_internal/server/routers/server.py +7 -4
- dstack/_internal/server/routers/users.py +22 -19
- dstack/_internal/server/routers/volumes.py +34 -25
- dstack/_internal/server/schemas/logs.py +2 -2
- dstack/_internal/server/schemas/runs.py +17 -5
- dstack/_internal/server/services/fleets.py +358 -75
- dstack/_internal/server/services/gateways/__init__.py +17 -6
- dstack/_internal/server/services/gateways/client.py +5 -3
- dstack/_internal/server/services/instances.py +8 -0
- dstack/_internal/server/services/jobs/__init__.py +45 -0
- dstack/_internal/server/services/jobs/configurators/base.py +12 -1
- dstack/_internal/server/services/locking.py +104 -13
- dstack/_internal/server/services/logging.py +4 -2
- dstack/_internal/server/services/logs/__init__.py +15 -2
- dstack/_internal/server/services/logs/aws.py +2 -4
- dstack/_internal/server/services/logs/filelog.py +33 -27
- dstack/_internal/server/services/logs/gcp.py +3 -5
- dstack/_internal/server/services/proxy/repo.py +4 -1
- dstack/_internal/server/services/runs.py +139 -72
- dstack/_internal/server/services/services/__init__.py +2 -1
- dstack/_internal/server/services/users.py +3 -1
- dstack/_internal/server/services/volumes.py +15 -2
- dstack/_internal/server/settings.py +25 -6
- dstack/_internal/server/statics/index.html +1 -1
- dstack/_internal/server/statics/{main-d151637af20f70b2e796.js → main-64f8273740c4b52c18f5.js} +71 -67
- dstack/_internal/server/statics/{main-d151637af20f70b2e796.js.map → main-64f8273740c4b52c18f5.js.map} +1 -1
- dstack/_internal/server/statics/{main-d48635d8fe670d53961c.css → main-d58fc0460cb0eae7cb5c.css} +1 -1
- dstack/_internal/server/testing/common.py +48 -8
- dstack/_internal/server/utils/routers.py +31 -8
- dstack/_internal/utils/json_utils.py +54 -0
- dstack/api/_public/runs.py +13 -2
- dstack/api/server/_runs.py +12 -2
- dstack/version.py +1 -1
- {dstack-0.19.17.dist-info → dstack-0.19.19.dist-info}/METADATA +17 -14
- {dstack-0.19.17.dist-info → dstack-0.19.19.dist-info}/RECORD +86 -83
- {dstack-0.19.17.dist-info → dstack-0.19.19.dist-info}/WHEEL +0 -0
- {dstack-0.19.17.dist-info → dstack-0.19.19.dist-info}/entry_points.txt +0 -0
- {dstack-0.19.17.dist-info → dstack-0.19.19.dist-info}/licenses/LICENSE.md +0 -0
|
@@ -8,6 +8,7 @@ import google.api_core.exceptions
|
|
|
8
8
|
import google.cloud.compute_v1 as compute_v1
|
|
9
9
|
from cachetools import TTLCache, cachedmethod
|
|
10
10
|
from google.cloud import tpu_v2
|
|
11
|
+
from google.cloud.compute_v1.types.compute import Instance
|
|
11
12
|
from gpuhunt import KNOWN_TPUS
|
|
12
13
|
|
|
13
14
|
import dstack._internal.core.backends.gcp.auth as auth
|
|
@@ -19,6 +20,7 @@ from dstack._internal.core.backends.base.compute import (
|
|
|
19
20
|
ComputeWithGatewaySupport,
|
|
20
21
|
ComputeWithMultinodeSupport,
|
|
21
22
|
ComputeWithPlacementGroupSupport,
|
|
23
|
+
ComputeWithPrivateGatewaySupport,
|
|
22
24
|
ComputeWithVolumeSupport,
|
|
23
25
|
generate_unique_gateway_instance_name,
|
|
24
26
|
generate_unique_instance_name,
|
|
@@ -83,6 +85,7 @@ class GCPCompute(
|
|
|
83
85
|
ComputeWithMultinodeSupport,
|
|
84
86
|
ComputeWithPlacementGroupSupport,
|
|
85
87
|
ComputeWithGatewaySupport,
|
|
88
|
+
ComputeWithPrivateGatewaySupport,
|
|
86
89
|
ComputeWithVolumeSupport,
|
|
87
90
|
Compute,
|
|
88
91
|
):
|
|
@@ -395,11 +398,7 @@ class GCPCompute(
|
|
|
395
398
|
if instance.status in ["PROVISIONING", "STAGING"]:
|
|
396
399
|
return
|
|
397
400
|
if instance.status == "RUNNING":
|
|
398
|
-
|
|
399
|
-
hostname = instance.network_interfaces[0].access_configs[0].nat_i_p
|
|
400
|
-
else:
|
|
401
|
-
hostname = instance.network_interfaces[0].network_i_p
|
|
402
|
-
provisioning_data.hostname = hostname
|
|
401
|
+
provisioning_data.hostname = _get_instance_ip(instance, allocate_public_ip)
|
|
403
402
|
provisioning_data.internal_ip = instance.network_interfaces[0].network_i_p
|
|
404
403
|
return
|
|
405
404
|
raise ProvisioningError(
|
|
@@ -500,7 +499,7 @@ class GCPCompute(
|
|
|
500
499
|
request.instance_resource = gcp_resources.create_instance_struct(
|
|
501
500
|
disk_size=10,
|
|
502
501
|
image_id=_get_gateway_image_id(),
|
|
503
|
-
machine_type="e2-
|
|
502
|
+
machine_type="e2-medium",
|
|
504
503
|
accelerators=[],
|
|
505
504
|
spot=False,
|
|
506
505
|
user_data=get_gateway_user_data(configuration.ssh_key_pub),
|
|
@@ -512,6 +511,7 @@ class GCPCompute(
|
|
|
512
511
|
service_account=self.config.vm_service_account,
|
|
513
512
|
network=self.config.vpc_resource_name,
|
|
514
513
|
subnetwork=subnetwork,
|
|
514
|
+
allocate_public_ip=configuration.public_ip,
|
|
515
515
|
)
|
|
516
516
|
operation = self.instances_client.insert(request=request)
|
|
517
517
|
gcp_resources.wait_for_extended_operation(operation, "instance creation")
|
|
@@ -522,7 +522,7 @@ class GCPCompute(
|
|
|
522
522
|
instance_id=instance_name,
|
|
523
523
|
region=configuration.region, # used for instance termination
|
|
524
524
|
availability_zone=zone,
|
|
525
|
-
ip_address=instance.
|
|
525
|
+
ip_address=_get_instance_ip(instance, configuration.public_ip),
|
|
526
526
|
backend_data=json.dumps({"zone": zone}),
|
|
527
527
|
)
|
|
528
528
|
|
|
@@ -1024,3 +1024,9 @@ def _is_tpu_provisioning_data(provisioning_data: JobProvisioningData) -> bool:
|
|
|
1024
1024
|
backend_data_dict = json.loads(provisioning_data.backend_data)
|
|
1025
1025
|
is_tpu = backend_data_dict.get("is_tpu", False)
|
|
1026
1026
|
return is_tpu
|
|
1027
|
+
|
|
1028
|
+
|
|
1029
|
+
def _get_instance_ip(instance: Instance, public_ip: bool) -> str:
|
|
1030
|
+
if public_ip:
|
|
1031
|
+
return instance.network_interfaces[0].access_configs[0].nat_i_p
|
|
1032
|
+
return instance.network_interfaces[0].network_i_p
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import hashlib
|
|
2
|
+
import shlex
|
|
2
3
|
import subprocess
|
|
3
4
|
import tempfile
|
|
4
5
|
from threading import Thread
|
|
@@ -98,7 +99,7 @@ class LambdaCompute(
|
|
|
98
99
|
arch=provisioning_data.instance_type.resources.cpu_arch,
|
|
99
100
|
)
|
|
100
101
|
# shim is assumed to be run under root
|
|
101
|
-
launch_command = "sudo sh -c
|
|
102
|
+
launch_command = "sudo sh -c " + shlex.quote(" && ".join(commands))
|
|
102
103
|
thread = Thread(
|
|
103
104
|
target=_start_runner,
|
|
104
105
|
kwargs={
|
|
@@ -1,19 +1,20 @@
|
|
|
1
|
-
from typing import
|
|
1
|
+
from typing import Optional
|
|
2
2
|
|
|
3
|
+
from dstack._internal.core.models.common import IncludeExcludeDictType, IncludeExcludeSetType
|
|
3
4
|
from dstack._internal.core.models.fleets import ApplyFleetPlanInput, FleetSpec
|
|
4
5
|
from dstack._internal.core.models.instances import Instance
|
|
5
6
|
|
|
6
7
|
|
|
7
|
-
def get_get_plan_excludes(fleet_spec: FleetSpec) ->
|
|
8
|
-
get_plan_excludes = {}
|
|
8
|
+
def get_get_plan_excludes(fleet_spec: FleetSpec) -> IncludeExcludeDictType:
|
|
9
|
+
get_plan_excludes: IncludeExcludeDictType = {}
|
|
9
10
|
spec_excludes = get_fleet_spec_excludes(fleet_spec)
|
|
10
11
|
if spec_excludes:
|
|
11
12
|
get_plan_excludes["spec"] = spec_excludes
|
|
12
13
|
return get_plan_excludes
|
|
13
14
|
|
|
14
15
|
|
|
15
|
-
def get_apply_plan_excludes(plan_input: ApplyFleetPlanInput) ->
|
|
16
|
-
apply_plan_excludes = {}
|
|
16
|
+
def get_apply_plan_excludes(plan_input: ApplyFleetPlanInput) -> IncludeExcludeDictType:
|
|
17
|
+
apply_plan_excludes: IncludeExcludeDictType = {}
|
|
17
18
|
spec_excludes = get_fleet_spec_excludes(plan_input.spec)
|
|
18
19
|
if spec_excludes:
|
|
19
20
|
apply_plan_excludes["spec"] = spec_excludes
|
|
@@ -28,23 +29,23 @@ def get_apply_plan_excludes(plan_input: ApplyFleetPlanInput) -> Dict:
|
|
|
28
29
|
return {"plan": apply_plan_excludes}
|
|
29
30
|
|
|
30
31
|
|
|
31
|
-
def get_create_fleet_excludes(fleet_spec: FleetSpec) ->
|
|
32
|
-
create_fleet_excludes = {}
|
|
32
|
+
def get_create_fleet_excludes(fleet_spec: FleetSpec) -> IncludeExcludeDictType:
|
|
33
|
+
create_fleet_excludes: IncludeExcludeDictType = {}
|
|
33
34
|
spec_excludes = get_fleet_spec_excludes(fleet_spec)
|
|
34
35
|
if spec_excludes:
|
|
35
36
|
create_fleet_excludes["spec"] = spec_excludes
|
|
36
37
|
return create_fleet_excludes
|
|
37
38
|
|
|
38
39
|
|
|
39
|
-
def get_fleet_spec_excludes(fleet_spec: FleetSpec) -> Optional[
|
|
40
|
+
def get_fleet_spec_excludes(fleet_spec: FleetSpec) -> Optional[IncludeExcludeDictType]:
|
|
40
41
|
"""
|
|
41
42
|
Returns `fleet_spec` exclude mapping to exclude certain fields from the request.
|
|
42
43
|
Use this method to exclude new fields when they are not set to keep
|
|
43
44
|
clients backward-compatibility with older servers.
|
|
44
45
|
"""
|
|
45
|
-
spec_excludes:
|
|
46
|
-
configuration_excludes:
|
|
47
|
-
profile_excludes:
|
|
46
|
+
spec_excludes: IncludeExcludeDictType = {}
|
|
47
|
+
configuration_excludes: IncludeExcludeDictType = {}
|
|
48
|
+
profile_excludes: IncludeExcludeSetType = set()
|
|
48
49
|
profile = fleet_spec.profile
|
|
49
50
|
if profile.fleets is None:
|
|
50
51
|
profile_excludes.add("fleets")
|
|
@@ -1,34 +1,35 @@
|
|
|
1
|
-
from
|
|
2
|
-
|
|
1
|
+
from dstack._internal.core.models.common import IncludeExcludeDictType
|
|
3
2
|
from dstack._internal.core.models.gateways import GatewayConfiguration, GatewaySpec
|
|
4
3
|
|
|
5
4
|
|
|
6
|
-
def get_gateway_spec_excludes(gateway_spec: GatewaySpec) ->
|
|
5
|
+
def get_gateway_spec_excludes(gateway_spec: GatewaySpec) -> IncludeExcludeDictType:
|
|
7
6
|
"""
|
|
8
7
|
Returns `gateway_spec` exclude mapping to exclude certain fields from the request.
|
|
9
8
|
Use this method to exclude new fields when they are not set to keep
|
|
10
9
|
clients backward-compatibility with older servers.
|
|
11
10
|
"""
|
|
12
|
-
spec_excludes = {}
|
|
11
|
+
spec_excludes: IncludeExcludeDictType = {}
|
|
13
12
|
spec_excludes["configuration"] = _get_gateway_configuration_excludes(
|
|
14
13
|
gateway_spec.configuration
|
|
15
14
|
)
|
|
16
15
|
return spec_excludes
|
|
17
16
|
|
|
18
17
|
|
|
19
|
-
def get_create_gateway_excludes(configuration: GatewayConfiguration) ->
|
|
18
|
+
def get_create_gateway_excludes(configuration: GatewayConfiguration) -> IncludeExcludeDictType:
|
|
20
19
|
"""
|
|
21
20
|
Returns an exclude mapping to exclude certain fields from the create gateway request.
|
|
22
21
|
Use this method to exclude new fields when they are not set to keep
|
|
23
22
|
clients backward-compatibility with older servers.
|
|
24
23
|
"""
|
|
25
|
-
create_gateway_excludes = {}
|
|
24
|
+
create_gateway_excludes: IncludeExcludeDictType = {}
|
|
26
25
|
create_gateway_excludes["configuration"] = _get_gateway_configuration_excludes(configuration)
|
|
27
26
|
return create_gateway_excludes
|
|
28
27
|
|
|
29
28
|
|
|
30
|
-
def _get_gateway_configuration_excludes(
|
|
31
|
-
|
|
29
|
+
def _get_gateway_configuration_excludes(
|
|
30
|
+
configuration: GatewayConfiguration,
|
|
31
|
+
) -> IncludeExcludeDictType:
|
|
32
|
+
configuration_excludes: IncludeExcludeDictType = {}
|
|
32
33
|
if configuration.tags is None:
|
|
33
34
|
configuration_excludes["tags"] = True
|
|
34
35
|
return configuration_excludes
|
|
@@ -1,15 +1,16 @@
|
|
|
1
|
-
from typing import
|
|
1
|
+
from typing import Optional
|
|
2
2
|
|
|
3
|
+
from dstack._internal.core.models.common import IncludeExcludeDictType
|
|
3
4
|
from dstack._internal.server.schemas.logs import PollLogsRequest
|
|
4
5
|
|
|
5
6
|
|
|
6
|
-
def get_poll_logs_excludes(request: PollLogsRequest) -> Optional[
|
|
7
|
+
def get_poll_logs_excludes(request: PollLogsRequest) -> Optional[IncludeExcludeDictType]:
|
|
7
8
|
"""
|
|
8
9
|
Returns exclude mapping to exclude certain fields from the request.
|
|
9
10
|
Use this method to exclude new fields when they are not set to keep
|
|
10
11
|
clients backward-compatibility with older servers.
|
|
11
12
|
"""
|
|
12
|
-
excludes = {}
|
|
13
|
+
excludes: IncludeExcludeDictType = {}
|
|
13
14
|
if request.next_token is None:
|
|
14
15
|
excludes["next_token"] = True
|
|
15
16
|
return excludes if excludes else None
|
|
@@ -1,29 +1,39 @@
|
|
|
1
|
-
from typing import
|
|
1
|
+
from typing import Optional
|
|
2
2
|
|
|
3
|
+
from dstack._internal.core.models.common import IncludeExcludeDictType, IncludeExcludeSetType
|
|
3
4
|
from dstack._internal.core.models.configurations import ServiceConfiguration
|
|
4
5
|
from dstack._internal.core.models.runs import ApplyRunPlanInput, JobSpec, JobSubmission, RunSpec
|
|
5
|
-
from dstack._internal.server.schemas.runs import GetRunPlanRequest
|
|
6
|
+
from dstack._internal.server.schemas.runs import GetRunPlanRequest, ListRunsRequest
|
|
6
7
|
|
|
7
8
|
|
|
8
|
-
def
|
|
9
|
+
def get_list_runs_excludes(list_runs_request: ListRunsRequest) -> IncludeExcludeSetType:
|
|
10
|
+
excludes = set()
|
|
11
|
+
if list_runs_request.include_jobs:
|
|
12
|
+
excludes.add("include_jobs")
|
|
13
|
+
if list_runs_request.job_submissions_limit is None:
|
|
14
|
+
excludes.add("job_submissions_limit")
|
|
15
|
+
return excludes
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def get_apply_plan_excludes(plan: ApplyRunPlanInput) -> Optional[IncludeExcludeDictType]:
|
|
9
19
|
"""
|
|
10
20
|
Returns `plan` exclude mapping to exclude certain fields from the request.
|
|
11
21
|
Use this method to exclude new fields when they are not set to keep
|
|
12
22
|
clients backward-compatibility with older servers.
|
|
13
23
|
"""
|
|
14
|
-
apply_plan_excludes = {}
|
|
24
|
+
apply_plan_excludes: IncludeExcludeDictType = {}
|
|
15
25
|
run_spec_excludes = get_run_spec_excludes(plan.run_spec)
|
|
16
26
|
if run_spec_excludes is not None:
|
|
17
27
|
apply_plan_excludes["run_spec"] = run_spec_excludes
|
|
18
28
|
current_resource = plan.current_resource
|
|
19
29
|
if current_resource is not None:
|
|
20
|
-
current_resource_excludes = {}
|
|
30
|
+
current_resource_excludes: IncludeExcludeDictType = {}
|
|
21
31
|
current_resource_excludes["status_message"] = True
|
|
22
32
|
if current_resource.deployment_num == 0:
|
|
23
33
|
current_resource_excludes["deployment_num"] = True
|
|
24
34
|
apply_plan_excludes["current_resource"] = current_resource_excludes
|
|
25
35
|
current_resource_excludes["run_spec"] = get_run_spec_excludes(current_resource.run_spec)
|
|
26
|
-
job_submissions_excludes = {}
|
|
36
|
+
job_submissions_excludes: IncludeExcludeDictType = {}
|
|
27
37
|
current_resource_excludes["jobs"] = {
|
|
28
38
|
"__all__": {
|
|
29
39
|
"job_spec": get_job_spec_excludes([job.job_spec for job in current_resource.jobs]),
|
|
@@ -45,7 +55,7 @@ def get_apply_plan_excludes(plan: ApplyRunPlanInput) -> Optional[Dict]:
|
|
|
45
55
|
job_submissions_excludes["deployment_num"] = True
|
|
46
56
|
latest_job_submission = current_resource.latest_job_submission
|
|
47
57
|
if latest_job_submission is not None:
|
|
48
|
-
latest_job_submission_excludes = {}
|
|
58
|
+
latest_job_submission_excludes: IncludeExcludeDictType = {}
|
|
49
59
|
current_resource_excludes["latest_job_submission"] = latest_job_submission_excludes
|
|
50
60
|
if _should_exclude_job_submission_jpd_cpu_arch(latest_job_submission):
|
|
51
61
|
latest_job_submission_excludes["job_provisioning_data"] = {
|
|
@@ -62,12 +72,12 @@ def get_apply_plan_excludes(plan: ApplyRunPlanInput) -> Optional[Dict]:
|
|
|
62
72
|
return {"plan": apply_plan_excludes}
|
|
63
73
|
|
|
64
74
|
|
|
65
|
-
def get_get_plan_excludes(request: GetRunPlanRequest) -> Optional[
|
|
75
|
+
def get_get_plan_excludes(request: GetRunPlanRequest) -> Optional[IncludeExcludeDictType]:
|
|
66
76
|
"""
|
|
67
77
|
Excludes new fields when they are not set to keep
|
|
68
78
|
clients backward-compatibility with older servers.
|
|
69
79
|
"""
|
|
70
|
-
get_plan_excludes = {}
|
|
80
|
+
get_plan_excludes: IncludeExcludeDictType = {}
|
|
71
81
|
run_spec_excludes = get_run_spec_excludes(request.run_spec)
|
|
72
82
|
if run_spec_excludes is not None:
|
|
73
83
|
get_plan_excludes["run_spec"] = run_spec_excludes
|
|
@@ -76,15 +86,15 @@ def get_get_plan_excludes(request: GetRunPlanRequest) -> Optional[Dict]:
|
|
|
76
86
|
return get_plan_excludes
|
|
77
87
|
|
|
78
88
|
|
|
79
|
-
def get_run_spec_excludes(run_spec: RunSpec) ->
|
|
89
|
+
def get_run_spec_excludes(run_spec: RunSpec) -> IncludeExcludeDictType:
|
|
80
90
|
"""
|
|
81
91
|
Returns `run_spec` exclude mapping to exclude certain fields from the request.
|
|
82
92
|
Use this method to exclude new fields when they are not set to keep
|
|
83
93
|
clients backward-compatibility with older servers.
|
|
84
94
|
"""
|
|
85
|
-
spec_excludes:
|
|
86
|
-
configuration_excludes:
|
|
87
|
-
profile_excludes:
|
|
95
|
+
spec_excludes: IncludeExcludeDictType = {}
|
|
96
|
+
configuration_excludes: IncludeExcludeDictType = {}
|
|
97
|
+
profile_excludes: IncludeExcludeSetType = set()
|
|
88
98
|
configuration = run_spec.configuration
|
|
89
99
|
profile = run_spec.profile
|
|
90
100
|
|
|
@@ -121,18 +131,16 @@ def get_run_spec_excludes(run_spec: RunSpec) -> Optional[Dict]:
|
|
|
121
131
|
spec_excludes["configuration"] = configuration_excludes
|
|
122
132
|
if profile_excludes:
|
|
123
133
|
spec_excludes["profile"] = profile_excludes
|
|
124
|
-
|
|
125
|
-
return spec_excludes
|
|
126
|
-
return None
|
|
134
|
+
return spec_excludes
|
|
127
135
|
|
|
128
136
|
|
|
129
|
-
def get_job_spec_excludes(job_specs: list[JobSpec]) ->
|
|
137
|
+
def get_job_spec_excludes(job_specs: list[JobSpec]) -> IncludeExcludeDictType:
|
|
130
138
|
"""
|
|
131
139
|
Returns `job_spec` exclude mapping to exclude certain fields from the request.
|
|
132
140
|
Use this method to exclude new fields when they are not set to keep
|
|
133
141
|
clients backward-compatibility with older servers.
|
|
134
142
|
"""
|
|
135
|
-
spec_excludes:
|
|
143
|
+
spec_excludes: IncludeExcludeDictType = {}
|
|
136
144
|
|
|
137
145
|
if all(s.repo_code_hash is None for s in job_specs):
|
|
138
146
|
spec_excludes["repo_code_hash"] = True
|
|
@@ -140,10 +148,10 @@ def get_job_spec_excludes(job_specs: list[JobSpec]) -> Optional[dict]:
|
|
|
140
148
|
spec_excludes["repo_data"] = True
|
|
141
149
|
if all(not s.file_archives for s in job_specs):
|
|
142
150
|
spec_excludes["file_archives"] = True
|
|
151
|
+
if all(s.service_port is None for s in job_specs):
|
|
152
|
+
spec_excludes["service_port"] = True
|
|
143
153
|
|
|
144
|
-
|
|
145
|
-
return spec_excludes
|
|
146
|
-
return None
|
|
154
|
+
return spec_excludes
|
|
147
155
|
|
|
148
156
|
|
|
149
157
|
def _should_exclude_job_submission_jpd_cpu_arch(job_submission: JobSubmission) -> bool:
|
|
@@ -1,32 +1,35 @@
|
|
|
1
|
-
from
|
|
2
|
-
|
|
1
|
+
from dstack._internal.core.models.common import IncludeExcludeDictType
|
|
3
2
|
from dstack._internal.core.models.volumes import VolumeConfiguration, VolumeSpec
|
|
4
3
|
|
|
5
4
|
|
|
6
|
-
def get_volume_spec_excludes(volume_spec: VolumeSpec) ->
|
|
5
|
+
def get_volume_spec_excludes(volume_spec: VolumeSpec) -> IncludeExcludeDictType:
|
|
7
6
|
"""
|
|
8
7
|
Returns `volume_spec` exclude mapping to exclude certain fields from the request.
|
|
9
8
|
Use this method to exclude new fields when they are not set to keep
|
|
10
9
|
clients backward-compatibility with older servers.
|
|
11
10
|
"""
|
|
12
|
-
spec_excludes = {}
|
|
11
|
+
spec_excludes: IncludeExcludeDictType = {}
|
|
13
12
|
spec_excludes["configuration"] = _get_volume_configuration_excludes(volume_spec.configuration)
|
|
14
13
|
return spec_excludes
|
|
15
14
|
|
|
16
15
|
|
|
17
|
-
def get_create_volume_excludes(configuration: VolumeConfiguration) ->
|
|
16
|
+
def get_create_volume_excludes(configuration: VolumeConfiguration) -> IncludeExcludeDictType:
|
|
18
17
|
"""
|
|
19
18
|
Returns an exclude mapping to exclude certain fields from the create volume request.
|
|
20
19
|
Use this method to exclude new fields when they are not set to keep
|
|
21
20
|
clients backward-compatibility with older servers.
|
|
22
21
|
"""
|
|
23
|
-
create_volume_excludes = {}
|
|
22
|
+
create_volume_excludes: IncludeExcludeDictType = {}
|
|
24
23
|
create_volume_excludes["configuration"] = _get_volume_configuration_excludes(configuration)
|
|
25
24
|
return create_volume_excludes
|
|
26
25
|
|
|
27
26
|
|
|
28
|
-
def _get_volume_configuration_excludes(
|
|
29
|
-
|
|
27
|
+
def _get_volume_configuration_excludes(
|
|
28
|
+
configuration: VolumeConfiguration,
|
|
29
|
+
) -> IncludeExcludeDictType:
|
|
30
|
+
configuration_excludes: IncludeExcludeDictType = {}
|
|
30
31
|
if configuration.tags is None:
|
|
31
32
|
configuration_excludes["tags"] = True
|
|
33
|
+
if configuration.auto_cleanup_duration is None:
|
|
34
|
+
configuration_excludes["auto_cleanup_duration"] = True
|
|
32
35
|
return configuration_excludes
|
dstack/_internal/core/errors.py
CHANGED
|
@@ -110,6 +110,10 @@ class PlacementGroupInUseError(ComputeError):
|
|
|
110
110
|
pass
|
|
111
111
|
|
|
112
112
|
|
|
113
|
+
class PlacementGroupNotSupportedError(ComputeError):
|
|
114
|
+
pass
|
|
115
|
+
|
|
116
|
+
|
|
113
117
|
class NotYetTerminated(ComputeError):
|
|
114
118
|
"""
|
|
115
119
|
Used by Compute.terminate_instance to signal that instance termination is not complete
|
|
@@ -1,11 +1,21 @@
|
|
|
1
1
|
import re
|
|
2
2
|
from enum import Enum
|
|
3
|
-
from typing import Union
|
|
3
|
+
from typing import Any, Callable, Optional, Union
|
|
4
4
|
|
|
5
|
+
import orjson
|
|
5
6
|
from pydantic import Field
|
|
6
7
|
from pydantic_duality import DualBaseModel
|
|
7
8
|
from typing_extensions import Annotated
|
|
8
9
|
|
|
10
|
+
from dstack._internal.utils.json_utils import pydantic_orjson_dumps
|
|
11
|
+
|
|
12
|
+
IncludeExcludeFieldType = Union[int, str]
|
|
13
|
+
IncludeExcludeSetType = set[IncludeExcludeFieldType]
|
|
14
|
+
IncludeExcludeDictType = dict[
|
|
15
|
+
IncludeExcludeFieldType, Union[bool, IncludeExcludeSetType, "IncludeExcludeDictType"]
|
|
16
|
+
]
|
|
17
|
+
IncludeExcludeType = Union[IncludeExcludeSetType, IncludeExcludeDictType]
|
|
18
|
+
|
|
9
19
|
|
|
10
20
|
# DualBaseModel creates two classes for the model:
|
|
11
21
|
# one with extra = "forbid" (CoreModel/CoreModel.__request__),
|
|
@@ -13,7 +23,40 @@ from typing_extensions import Annotated
|
|
|
13
23
|
# This allows to use the same model both for a strict parsing of the user input and
|
|
14
24
|
# for a permissive parsing of the server responses.
|
|
15
25
|
class CoreModel(DualBaseModel):
|
|
16
|
-
|
|
26
|
+
class Config:
|
|
27
|
+
json_loads = orjson.loads
|
|
28
|
+
json_dumps = pydantic_orjson_dumps
|
|
29
|
+
|
|
30
|
+
def json(
|
|
31
|
+
self,
|
|
32
|
+
*,
|
|
33
|
+
include: Optional[IncludeExcludeType] = None,
|
|
34
|
+
exclude: Optional[IncludeExcludeType] = None,
|
|
35
|
+
by_alias: bool = False,
|
|
36
|
+
skip_defaults: Optional[bool] = None, # ignore as it's deprecated
|
|
37
|
+
exclude_unset: bool = False,
|
|
38
|
+
exclude_defaults: bool = False,
|
|
39
|
+
exclude_none: bool = False,
|
|
40
|
+
encoder: Optional[Callable[[Any], Any]] = None,
|
|
41
|
+
models_as_dict: bool = True, # does not seems to be needed by dstack or dependencies
|
|
42
|
+
**dumps_kwargs: Any,
|
|
43
|
+
) -> str:
|
|
44
|
+
"""
|
|
45
|
+
Override `json()` method so that it calls `dict()`.
|
|
46
|
+
Allows changing how models are serialized by overriding `dict()` only.
|
|
47
|
+
By default, `json()` won't call `dict()`, so changes applied in `dict()` won't take place.
|
|
48
|
+
"""
|
|
49
|
+
data = self.dict(
|
|
50
|
+
by_alias=by_alias,
|
|
51
|
+
include=include,
|
|
52
|
+
exclude=exclude,
|
|
53
|
+
exclude_unset=exclude_unset,
|
|
54
|
+
exclude_defaults=exclude_defaults,
|
|
55
|
+
exclude_none=exclude_none,
|
|
56
|
+
)
|
|
57
|
+
if self.__custom_root_type__:
|
|
58
|
+
data = data["__root__"]
|
|
59
|
+
return self.__config__.json_dumps(data, default=encoder, **dumps_kwargs)
|
|
17
60
|
|
|
18
61
|
|
|
19
62
|
class Duration(int):
|
|
@@ -4,6 +4,7 @@ from enum import Enum
|
|
|
4
4
|
from pathlib import PurePosixPath
|
|
5
5
|
from typing import Any, Dict, List, Optional, Union
|
|
6
6
|
|
|
7
|
+
import orjson
|
|
7
8
|
from pydantic import Field, ValidationError, conint, constr, root_validator, validator
|
|
8
9
|
from typing_extensions import Annotated, Literal
|
|
9
10
|
|
|
@@ -18,6 +19,9 @@ from dstack._internal.core.models.resources import Range, ResourcesSpec
|
|
|
18
19
|
from dstack._internal.core.models.services import AnyModel, OpenAIChatModel
|
|
19
20
|
from dstack._internal.core.models.unix import UnixUser
|
|
20
21
|
from dstack._internal.core.models.volumes import MountPoint, VolumeConfiguration, parse_mount_point
|
|
22
|
+
from dstack._internal.utils.json_utils import (
|
|
23
|
+
pydantic_orjson_dumps_with_indent,
|
|
24
|
+
)
|
|
21
25
|
|
|
22
26
|
CommandsList = List[str]
|
|
23
27
|
ValidPort = conint(gt=0, le=65536)
|
|
@@ -394,8 +398,9 @@ class TaskConfiguration(
|
|
|
394
398
|
|
|
395
399
|
class ServiceConfigurationParams(CoreModel):
|
|
396
400
|
port: Annotated[
|
|
401
|
+
# NOTE: it's a PortMapping for historical reasons. Only `port.container_port` is used.
|
|
397
402
|
Union[ValidPort, constr(regex=r"^[0-9]+:[0-9]+$"), PortMapping],
|
|
398
|
-
Field(description="The port
|
|
403
|
+
Field(description="The port the application listens on"),
|
|
399
404
|
]
|
|
400
405
|
gateway: Annotated[
|
|
401
406
|
Optional[Union[bool, str]],
|
|
@@ -573,6 +578,9 @@ class DstackConfiguration(CoreModel):
|
|
|
573
578
|
]
|
|
574
579
|
|
|
575
580
|
class Config:
|
|
581
|
+
json_loads = orjson.loads
|
|
582
|
+
json_dumps = pydantic_orjson_dumps_with_indent
|
|
583
|
+
|
|
576
584
|
@staticmethod
|
|
577
585
|
def schema_extra(schema: Dict[str, Any]):
|
|
578
586
|
schema["$schema"] = "http://json-schema.org/draft-07/schema#"
|
|
@@ -8,7 +8,7 @@ from pydantic import Field, root_validator, validator
|
|
|
8
8
|
from typing_extensions import Annotated, Literal
|
|
9
9
|
|
|
10
10
|
from dstack._internal.core.models.backends.base import BackendType
|
|
11
|
-
from dstack._internal.core.models.common import CoreModel
|
|
11
|
+
from dstack._internal.core.models.common import ApplyAction, CoreModel
|
|
12
12
|
from dstack._internal.core.models.envs import Env
|
|
13
13
|
from dstack._internal.core.models.instances import Instance, InstanceOfferWithAvailability, SSHKey
|
|
14
14
|
from dstack._internal.core.models.profiles import (
|
|
@@ -324,6 +324,7 @@ class FleetPlan(CoreModel):
|
|
|
324
324
|
offers: List[InstanceOfferWithAvailability]
|
|
325
325
|
total_offers: int
|
|
326
326
|
max_offer_price: Optional[float] = None
|
|
327
|
+
action: Optional[ApplyAction] = None # default value for backward compatibility
|
|
327
328
|
|
|
328
329
|
def get_effective_spec(self) -> FleetSpec:
|
|
329
330
|
if self.effective_spec is not None:
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
from enum import Enum
|
|
2
2
|
from typing import Any, Dict, List, Optional, Union, overload
|
|
3
3
|
|
|
4
|
+
import orjson
|
|
4
5
|
from pydantic import Field, root_validator, validator
|
|
5
6
|
from typing_extensions import Annotated, Literal
|
|
6
7
|
|
|
7
8
|
from dstack._internal.core.models.backends.base import BackendType
|
|
8
9
|
from dstack._internal.core.models.common import CoreModel, Duration
|
|
9
10
|
from dstack._internal.utils.common import list_enum_values_for_annotation
|
|
11
|
+
from dstack._internal.utils.json_utils import pydantic_orjson_dumps_with_indent
|
|
10
12
|
from dstack._internal.utils.tags import tags_validator
|
|
11
13
|
|
|
12
14
|
DEFAULT_RETRY_DURATION = 3600
|
|
@@ -74,11 +76,9 @@ def parse_off_duration(v: Optional[Union[int, str, bool]]) -> Optional[Union[str
|
|
|
74
76
|
return parse_duration(v)
|
|
75
77
|
|
|
76
78
|
|
|
77
|
-
def parse_idle_duration(v: Optional[Union[int, str
|
|
78
|
-
if v
|
|
79
|
+
def parse_idle_duration(v: Optional[Union[int, str]]) -> Optional[Union[str, int]]:
|
|
80
|
+
if v == "off" or v == -1:
|
|
79
81
|
return -1
|
|
80
|
-
if v is True:
|
|
81
|
-
return None
|
|
82
82
|
return parse_duration(v)
|
|
83
83
|
|
|
84
84
|
|
|
@@ -249,7 +249,7 @@ class ProfileParams(CoreModel):
|
|
|
249
249
|
),
|
|
250
250
|
] = None
|
|
251
251
|
idle_duration: Annotated[
|
|
252
|
-
Optional[Union[Literal["off"], str, int
|
|
252
|
+
Optional[Union[Literal["off"], str, int]],
|
|
253
253
|
Field(
|
|
254
254
|
description=(
|
|
255
255
|
"Time to wait before terminating idle instances."
|
|
@@ -343,6 +343,9 @@ class ProfilesConfig(CoreModel):
|
|
|
343
343
|
profiles: List[Profile]
|
|
344
344
|
|
|
345
345
|
class Config:
|
|
346
|
+
json_loads = orjson.loads
|
|
347
|
+
json_dumps = pydantic_orjson_dumps_with_indent
|
|
348
|
+
|
|
346
349
|
schema_extra = {"$schema": "http://json-schema.org/draft-07/schema#"}
|
|
347
350
|
|
|
348
351
|
def default(self) -> Optional[Profile]:
|
|
@@ -382,14 +382,6 @@ class ResourcesSpec(CoreModel):
|
|
|
382
382
|
gpu: Annotated[Optional[GPUSpec], Field(description="The GPU requirements")] = None
|
|
383
383
|
disk: Annotated[Optional[DiskSpec], Field(description="The disk resources")] = DEFAULT_DISK
|
|
384
384
|
|
|
385
|
-
# TODO: Remove in 0.20. Added for backward compatibility.
|
|
386
|
-
@root_validator
|
|
387
|
-
def _post_validate(cls, values):
|
|
388
|
-
cpu = values.get("cpu")
|
|
389
|
-
if isinstance(cpu, CPUSpec) and cpu.arch in [None, gpuhunt.CPUArchitecture.X86]:
|
|
390
|
-
values["cpu"] = cpu.count
|
|
391
|
-
return values
|
|
392
|
-
|
|
393
385
|
def pretty_format(self) -> str:
|
|
394
386
|
# TODO: Remove in 0.20. Use self.cpu directly
|
|
395
387
|
cpu = parse_obj_as(CPUSpec, self.cpu)
|
|
@@ -407,3 +399,18 @@ class ResourcesSpec(CoreModel):
|
|
|
407
399
|
resources.update(disk_size=self.disk.size)
|
|
408
400
|
res = pretty_resources(**resources)
|
|
409
401
|
return res
|
|
402
|
+
|
|
403
|
+
def dict(self, *args, **kwargs) -> Dict:
|
|
404
|
+
# super() does not work with pydantic-duality
|
|
405
|
+
res = CoreModel.dict(self, *args, **kwargs)
|
|
406
|
+
self._update_serialized_cpu(res)
|
|
407
|
+
return res
|
|
408
|
+
|
|
409
|
+
# TODO: Remove in 0.20. Added for backward compatibility.
|
|
410
|
+
def _update_serialized_cpu(self, values: Dict):
|
|
411
|
+
cpu = values["cpu"]
|
|
412
|
+
if cpu:
|
|
413
|
+
arch = cpu.get("arch")
|
|
414
|
+
count = cpu.get("count")
|
|
415
|
+
if count and arch in [None, gpuhunt.CPUArchitecture.X86.value]:
|
|
416
|
+
values["cpu"] = count
|