dstack 0.19.25rc1__py3-none-any.whl → 0.19.27__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dstack might be problematic. Click here for more details.
- dstack/_internal/cli/commands/__init__.py +2 -2
- dstack/_internal/cli/commands/apply.py +3 -61
- dstack/_internal/cli/commands/attach.py +1 -1
- dstack/_internal/cli/commands/completion.py +1 -1
- dstack/_internal/cli/commands/delete.py +2 -2
- dstack/_internal/cli/commands/fleet.py +1 -1
- dstack/_internal/cli/commands/gateway.py +2 -2
- dstack/_internal/cli/commands/init.py +56 -24
- dstack/_internal/cli/commands/logs.py +1 -1
- dstack/_internal/cli/commands/metrics.py +1 -1
- dstack/_internal/cli/commands/offer.py +45 -7
- dstack/_internal/cli/commands/project.py +2 -2
- dstack/_internal/cli/commands/secrets.py +2 -2
- dstack/_internal/cli/commands/server.py +1 -1
- dstack/_internal/cli/commands/stop.py +1 -1
- dstack/_internal/cli/commands/volume.py +1 -1
- dstack/_internal/cli/main.py +2 -2
- dstack/_internal/cli/services/completion.py +2 -2
- dstack/_internal/cli/services/configurators/__init__.py +6 -2
- dstack/_internal/cli/services/configurators/base.py +6 -7
- dstack/_internal/cli/services/configurators/fleet.py +1 -3
- dstack/_internal/cli/services/configurators/gateway.py +2 -4
- dstack/_internal/cli/services/configurators/run.py +293 -58
- dstack/_internal/cli/services/configurators/volume.py +2 -4
- dstack/_internal/cli/services/profile.py +1 -1
- dstack/_internal/cli/services/repos.py +35 -48
- dstack/_internal/core/backends/amddevcloud/__init__.py +1 -0
- dstack/_internal/core/backends/amddevcloud/backend.py +16 -0
- dstack/_internal/core/backends/amddevcloud/compute.py +5 -0
- dstack/_internal/core/backends/amddevcloud/configurator.py +29 -0
- dstack/_internal/core/backends/aws/compute.py +6 -1
- dstack/_internal/core/backends/aws/configurator.py +11 -7
- dstack/_internal/core/backends/azure/configurator.py +11 -7
- dstack/_internal/core/backends/base/compute.py +33 -5
- dstack/_internal/core/backends/base/configurator.py +25 -13
- dstack/_internal/core/backends/base/offers.py +2 -0
- dstack/_internal/core/backends/cloudrift/configurator.py +13 -7
- dstack/_internal/core/backends/configurators.py +15 -0
- dstack/_internal/core/backends/cudo/configurator.py +11 -7
- dstack/_internal/core/backends/datacrunch/compute.py +5 -1
- dstack/_internal/core/backends/datacrunch/configurator.py +13 -7
- dstack/_internal/core/backends/digitalocean/__init__.py +1 -0
- dstack/_internal/core/backends/digitalocean/backend.py +16 -0
- dstack/_internal/core/backends/digitalocean/compute.py +5 -0
- dstack/_internal/core/backends/digitalocean/configurator.py +31 -0
- dstack/_internal/core/backends/digitalocean_base/__init__.py +1 -0
- dstack/_internal/core/backends/digitalocean_base/api_client.py +104 -0
- dstack/_internal/core/backends/digitalocean_base/backend.py +5 -0
- dstack/_internal/core/backends/digitalocean_base/compute.py +173 -0
- dstack/_internal/core/backends/digitalocean_base/configurator.py +57 -0
- dstack/_internal/core/backends/digitalocean_base/models.py +43 -0
- dstack/_internal/core/backends/gcp/compute.py +32 -8
- dstack/_internal/core/backends/gcp/configurator.py +11 -7
- dstack/_internal/core/backends/hotaisle/api_client.py +25 -33
- dstack/_internal/core/backends/hotaisle/compute.py +1 -6
- dstack/_internal/core/backends/hotaisle/configurator.py +13 -7
- dstack/_internal/core/backends/kubernetes/configurator.py +13 -7
- dstack/_internal/core/backends/lambdalabs/configurator.py +11 -7
- dstack/_internal/core/backends/models.py +7 -0
- dstack/_internal/core/backends/nebius/compute.py +1 -8
- dstack/_internal/core/backends/nebius/configurator.py +11 -7
- dstack/_internal/core/backends/nebius/resources.py +21 -11
- dstack/_internal/core/backends/oci/compute.py +4 -5
- dstack/_internal/core/backends/oci/configurator.py +11 -7
- dstack/_internal/core/backends/runpod/configurator.py +11 -7
- dstack/_internal/core/backends/template/configurator.py.jinja +11 -7
- dstack/_internal/core/backends/tensordock/configurator.py +13 -7
- dstack/_internal/core/backends/vastai/configurator.py +11 -7
- dstack/_internal/core/backends/vultr/compute.py +1 -5
- dstack/_internal/core/backends/vultr/configurator.py +11 -4
- dstack/_internal/core/compatibility/fleets.py +5 -0
- dstack/_internal/core/compatibility/gpus.py +13 -0
- dstack/_internal/core/compatibility/runs.py +9 -1
- dstack/_internal/core/models/backends/base.py +5 -1
- dstack/_internal/core/models/common.py +3 -3
- dstack/_internal/core/models/configurations.py +191 -32
- dstack/_internal/core/models/files.py +1 -1
- dstack/_internal/core/models/fleets.py +80 -3
- dstack/_internal/core/models/profiles.py +41 -11
- dstack/_internal/core/models/resources.py +46 -42
- dstack/_internal/core/models/runs.py +28 -5
- dstack/_internal/core/services/configs/__init__.py +6 -3
- dstack/_internal/core/services/profiles.py +2 -2
- dstack/_internal/core/services/repos.py +86 -79
- dstack/_internal/core/services/ssh/ports.py +1 -1
- dstack/_internal/proxy/lib/deps.py +6 -2
- dstack/_internal/server/app.py +22 -17
- dstack/_internal/server/background/tasks/process_fleets.py +109 -13
- dstack/_internal/server/background/tasks/process_gateways.py +4 -1
- dstack/_internal/server/background/tasks/process_instances.py +22 -73
- dstack/_internal/server/background/tasks/process_probes.py +1 -1
- dstack/_internal/server/background/tasks/process_running_jobs.py +12 -4
- dstack/_internal/server/background/tasks/process_runs.py +3 -1
- dstack/_internal/server/background/tasks/process_submitted_jobs.py +67 -44
- dstack/_internal/server/background/tasks/process_terminating_jobs.py +2 -2
- dstack/_internal/server/background/tasks/process_volumes.py +1 -1
- dstack/_internal/server/db.py +8 -4
- dstack/_internal/server/migrations/versions/2498ab323443_add_fleetmodel_consolidation_attempt_.py +44 -0
- dstack/_internal/server/models.py +6 -2
- dstack/_internal/server/routers/gpus.py +1 -6
- dstack/_internal/server/schemas/runner.py +11 -0
- dstack/_internal/server/services/backends/__init__.py +14 -8
- dstack/_internal/server/services/backends/handlers.py +6 -1
- dstack/_internal/server/services/docker.py +5 -5
- dstack/_internal/server/services/fleets.py +37 -38
- dstack/_internal/server/services/gateways/__init__.py +2 -0
- dstack/_internal/server/services/gateways/client.py +5 -2
- dstack/_internal/server/services/gateways/connection.py +1 -1
- dstack/_internal/server/services/gpus.py +50 -49
- dstack/_internal/server/services/instances.py +44 -4
- dstack/_internal/server/services/jobs/__init__.py +15 -4
- dstack/_internal/server/services/jobs/configurators/base.py +53 -17
- dstack/_internal/server/services/jobs/configurators/dev.py +9 -4
- dstack/_internal/server/services/jobs/configurators/extensions/cursor.py +6 -8
- dstack/_internal/server/services/jobs/configurators/extensions/vscode.py +7 -9
- dstack/_internal/server/services/jobs/configurators/service.py +1 -3
- dstack/_internal/server/services/jobs/configurators/task.py +3 -3
- dstack/_internal/server/services/locking.py +5 -5
- dstack/_internal/server/services/logging.py +10 -2
- dstack/_internal/server/services/logs/__init__.py +8 -6
- dstack/_internal/server/services/logs/aws.py +330 -327
- dstack/_internal/server/services/logs/filelog.py +7 -6
- dstack/_internal/server/services/logs/gcp.py +141 -139
- dstack/_internal/server/services/plugins.py +1 -1
- dstack/_internal/server/services/projects.py +2 -5
- dstack/_internal/server/services/proxy/repo.py +5 -1
- dstack/_internal/server/services/requirements/__init__.py +0 -0
- dstack/_internal/server/services/requirements/combine.py +259 -0
- dstack/_internal/server/services/runner/client.py +7 -0
- dstack/_internal/server/services/runs.py +17 -1
- dstack/_internal/server/services/services/__init__.py +8 -2
- dstack/_internal/server/services/services/autoscalers.py +2 -0
- dstack/_internal/server/services/ssh.py +2 -1
- dstack/_internal/server/services/storage/__init__.py +5 -6
- dstack/_internal/server/services/storage/gcs.py +49 -49
- dstack/_internal/server/services/storage/s3.py +52 -52
- dstack/_internal/server/statics/index.html +1 -1
- dstack/_internal/server/statics/{main-d151b300fcac3933213d.js → main-4eecc75fbe64067eb1bc.js} +1146 -899
- dstack/_internal/server/statics/{main-d151b300fcac3933213d.js.map → main-4eecc75fbe64067eb1bc.js.map} +1 -1
- dstack/_internal/server/statics/{main-aec4762350e34d6fbff9.css → main-56191c63d516fd0041c4.css} +1 -1
- dstack/_internal/server/testing/common.py +7 -4
- dstack/_internal/server/utils/logging.py +3 -3
- dstack/_internal/server/utils/provisioning.py +3 -3
- dstack/_internal/utils/json_schema.py +3 -1
- dstack/_internal/utils/path.py +8 -1
- dstack/_internal/utils/ssh.py +7 -0
- dstack/_internal/utils/typing.py +14 -0
- dstack/api/_public/repos.py +62 -8
- dstack/api/_public/runs.py +19 -8
- dstack/api/server/__init__.py +17 -19
- dstack/api/server/_gpus.py +2 -1
- dstack/api/server/_group.py +4 -3
- dstack/api/server/_repos.py +20 -3
- dstack/plugins/builtin/rest_plugin/_plugin.py +1 -0
- dstack/version.py +1 -1
- {dstack-0.19.25rc1.dist-info → dstack-0.19.27.dist-info}/METADATA +2 -2
- {dstack-0.19.25rc1.dist-info → dstack-0.19.27.dist-info}/RECORD +160 -142
- dstack/api/huggingface/__init__.py +0 -73
- {dstack-0.19.25rc1.dist-info → dstack-0.19.27.dist-info}/WHEEL +0 -0
- {dstack-0.19.25rc1.dist-info → dstack-0.19.27.dist-info}/entry_points.txt +0 -0
- {dstack-0.19.25rc1.dist-info → dstack-0.19.27.dist-info}/licenses/LICENSE.md +0 -0
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from typing import Optional
|
|
2
|
+
|
|
3
|
+
from dstack._internal.core.backends.amddevcloud.backend import AMDDevCloudBackend
|
|
4
|
+
from dstack._internal.core.backends.base.configurator import BackendRecord
|
|
5
|
+
from dstack._internal.core.backends.digitalocean_base.api_client import DigitalOceanAPIClient
|
|
6
|
+
from dstack._internal.core.backends.digitalocean_base.backend import BaseDigitalOceanBackend
|
|
7
|
+
from dstack._internal.core.backends.digitalocean_base.configurator import (
|
|
8
|
+
BaseDigitalOceanConfigurator,
|
|
9
|
+
)
|
|
10
|
+
from dstack._internal.core.backends.digitalocean_base.models import AnyBaseDigitalOceanCreds
|
|
11
|
+
from dstack._internal.core.models.backends.base import (
|
|
12
|
+
BackendType,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class AMDDevCloudConfigurator(BaseDigitalOceanConfigurator):
|
|
17
|
+
TYPE = BackendType.AMDDEVCLOUD
|
|
18
|
+
BACKEND_CLASS = AMDDevCloudBackend
|
|
19
|
+
API_URL = "https://api-amd.digitalocean.com"
|
|
20
|
+
|
|
21
|
+
def get_backend(self, record: BackendRecord) -> BaseDigitalOceanBackend:
|
|
22
|
+
config = self._get_config(record)
|
|
23
|
+
return AMDDevCloudBackend(config=config, api_url=self.API_URL)
|
|
24
|
+
|
|
25
|
+
def _validate_creds(self, creds: AnyBaseDigitalOceanCreds, project_name: Optional[str] = None):
|
|
26
|
+
api_client = DigitalOceanAPIClient(creds.api_key, self.API_URL)
|
|
27
|
+
api_client.validate_api_key()
|
|
28
|
+
if project_name:
|
|
29
|
+
api_client.validate_project_name(project_name)
|
|
@@ -292,7 +292,12 @@ class AWSCompute(
|
|
|
292
292
|
image_id=image_id,
|
|
293
293
|
instance_type=instance_offer.instance.name,
|
|
294
294
|
iam_instance_profile=self.config.iam_instance_profile,
|
|
295
|
-
user_data=get_user_data(
|
|
295
|
+
user_data=get_user_data(
|
|
296
|
+
authorized_keys=instance_config.get_public_keys(),
|
|
297
|
+
# Custom OS images may lack ufw, so don't attempt to set up the firewall.
|
|
298
|
+
# Rely on security groups and the image's built-in firewall rules instead.
|
|
299
|
+
skip_firewall_setup=self.config.os_images is not None,
|
|
300
|
+
),
|
|
296
301
|
tags=aws_resources.make_tags(tags),
|
|
297
302
|
security_group_id=security_group_id,
|
|
298
303
|
spot=instance_offer.instance.resources.spot,
|
|
@@ -7,7 +7,6 @@ from boto3.session import Session
|
|
|
7
7
|
from dstack._internal.core.backends.aws import auth, compute, resources
|
|
8
8
|
from dstack._internal.core.backends.aws.backend import AWSBackend
|
|
9
9
|
from dstack._internal.core.backends.aws.models import (
|
|
10
|
-
AnyAWSBackendConfig,
|
|
11
10
|
AWSAccessKeyCreds,
|
|
12
11
|
AWSBackendConfig,
|
|
13
12
|
AWSBackendConfigWithCreds,
|
|
@@ -52,7 +51,12 @@ DEFAULT_REGIONS = REGION_VALUES
|
|
|
52
51
|
MAIN_REGION = "us-east-1"
|
|
53
52
|
|
|
54
53
|
|
|
55
|
-
class AWSConfigurator(
|
|
54
|
+
class AWSConfigurator(
|
|
55
|
+
Configurator[
|
|
56
|
+
AWSBackendConfig,
|
|
57
|
+
AWSBackendConfigWithCreds,
|
|
58
|
+
]
|
|
59
|
+
):
|
|
56
60
|
TYPE = BackendType.AWS
|
|
57
61
|
BACKEND_CLASS = AWSBackend
|
|
58
62
|
|
|
@@ -87,12 +91,12 @@ class AWSConfigurator(Configurator):
|
|
|
87
91
|
auth=AWSCreds.parse_obj(config.creds).json(),
|
|
88
92
|
)
|
|
89
93
|
|
|
90
|
-
def
|
|
91
|
-
self
|
|
92
|
-
|
|
94
|
+
def get_backend_config_with_creds(self, record: BackendRecord) -> AWSBackendConfigWithCreds:
|
|
95
|
+
config = self._get_config(record)
|
|
96
|
+
return AWSBackendConfigWithCreds.__response__.parse_obj(config)
|
|
97
|
+
|
|
98
|
+
def get_backend_config_without_creds(self, record: BackendRecord) -> AWSBackendConfig:
|
|
93
99
|
config = self._get_config(record)
|
|
94
|
-
if include_creds:
|
|
95
|
-
return AWSBackendConfigWithCreds.__response__.parse_obj(config)
|
|
96
100
|
return AWSBackendConfig.__response__.parse_obj(config)
|
|
97
101
|
|
|
98
102
|
def get_backend(self, record: BackendRecord) -> AWSBackend:
|
|
@@ -24,7 +24,6 @@ from dstack._internal.core.backends.azure import auth, compute, resources
|
|
|
24
24
|
from dstack._internal.core.backends.azure import utils as azure_utils
|
|
25
25
|
from dstack._internal.core.backends.azure.backend import AzureBackend
|
|
26
26
|
from dstack._internal.core.backends.azure.models import (
|
|
27
|
-
AnyAzureBackendConfig,
|
|
28
27
|
AzureBackendConfig,
|
|
29
28
|
AzureBackendConfigWithCreds,
|
|
30
29
|
AzureClientCreds,
|
|
@@ -71,7 +70,12 @@ DEFAULT_LOCATIONS = LOCATION_VALUES
|
|
|
71
70
|
MAIN_LOCATION = "eastus"
|
|
72
71
|
|
|
73
72
|
|
|
74
|
-
class AzureConfigurator(
|
|
73
|
+
class AzureConfigurator(
|
|
74
|
+
Configurator[
|
|
75
|
+
AzureBackendConfig,
|
|
76
|
+
AzureBackendConfigWithCreds,
|
|
77
|
+
]
|
|
78
|
+
):
|
|
75
79
|
TYPE = BackendType.AZURE
|
|
76
80
|
BACKEND_CLASS = AzureBackend
|
|
77
81
|
|
|
@@ -130,12 +134,12 @@ class AzureConfigurator(Configurator):
|
|
|
130
134
|
auth=AzureCreds.parse_obj(config.creds).__root__.json(),
|
|
131
135
|
)
|
|
132
136
|
|
|
133
|
-
def
|
|
134
|
-
self
|
|
135
|
-
|
|
137
|
+
def get_backend_config_with_creds(self, record: BackendRecord) -> AzureBackendConfigWithCreds:
|
|
138
|
+
config = self._get_config(record)
|
|
139
|
+
return AzureBackendConfigWithCreds.__response__.parse_obj(config)
|
|
140
|
+
|
|
141
|
+
def get_backend_config_without_creds(self, record: BackendRecord) -> AzureBackendConfig:
|
|
136
142
|
config = self._get_config(record)
|
|
137
|
-
if include_creds:
|
|
138
|
-
return AzureBackendConfigWithCreds.__response__.parse_obj(config)
|
|
139
143
|
return AzureBackendConfig.__response__.parse_obj(config)
|
|
140
144
|
|
|
141
145
|
def get_backend(self, record: BackendRecord) -> AzureBackend:
|
|
@@ -4,6 +4,7 @@ import re
|
|
|
4
4
|
import string
|
|
5
5
|
import threading
|
|
6
6
|
from abc import ABC, abstractmethod
|
|
7
|
+
from collections.abc import Iterable
|
|
7
8
|
from functools import lru_cache
|
|
8
9
|
from pathlib import Path
|
|
9
10
|
from typing import Dict, List, Literal, Optional
|
|
@@ -19,7 +20,7 @@ from dstack._internal.core.consts import (
|
|
|
19
20
|
DSTACK_RUNNER_SSH_PORT,
|
|
20
21
|
DSTACK_SHIM_HTTP_PORT,
|
|
21
22
|
)
|
|
22
|
-
from dstack._internal.core.models.configurations import
|
|
23
|
+
from dstack._internal.core.models.configurations import LEGACY_REPO_DIR
|
|
23
24
|
from dstack._internal.core.models.gateways import (
|
|
24
25
|
GatewayComputeConfiguration,
|
|
25
26
|
GatewayProvisioningData,
|
|
@@ -45,6 +46,7 @@ logger = get_logger(__name__)
|
|
|
45
46
|
|
|
46
47
|
DSTACK_SHIM_BINARY_NAME = "dstack-shim"
|
|
47
48
|
DSTACK_RUNNER_BINARY_NAME = "dstack-runner"
|
|
49
|
+
DEFAULT_PRIVATE_SUBNETS = ("10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16")
|
|
48
50
|
|
|
49
51
|
GoArchType = Literal["amd64", "arm64"]
|
|
50
52
|
|
|
@@ -507,12 +509,16 @@ def get_user_data(
|
|
|
507
509
|
base_path: Optional[PathLike] = None,
|
|
508
510
|
bin_path: Optional[PathLike] = None,
|
|
509
511
|
backend_shim_env: Optional[Dict[str, str]] = None,
|
|
512
|
+
skip_firewall_setup: bool = False,
|
|
513
|
+
firewall_allow_from_subnets: Iterable[str] = DEFAULT_PRIVATE_SUBNETS,
|
|
510
514
|
) -> str:
|
|
511
515
|
shim_commands = get_shim_commands(
|
|
512
516
|
authorized_keys=authorized_keys,
|
|
513
517
|
base_path=base_path,
|
|
514
518
|
bin_path=bin_path,
|
|
515
519
|
backend_shim_env=backend_shim_env,
|
|
520
|
+
skip_firewall_setup=skip_firewall_setup,
|
|
521
|
+
firewall_allow_from_subnets=firewall_allow_from_subnets,
|
|
516
522
|
)
|
|
517
523
|
commands = (backend_specific_commands or []) + shim_commands
|
|
518
524
|
return get_cloud_config(
|
|
@@ -554,8 +560,13 @@ def get_shim_commands(
|
|
|
554
560
|
bin_path: Optional[PathLike] = None,
|
|
555
561
|
backend_shim_env: Optional[Dict[str, str]] = None,
|
|
556
562
|
arch: Optional[str] = None,
|
|
563
|
+
skip_firewall_setup: bool = False,
|
|
564
|
+
firewall_allow_from_subnets: Iterable[str] = DEFAULT_PRIVATE_SUBNETS,
|
|
557
565
|
) -> List[str]:
|
|
558
|
-
commands = get_setup_cloud_instance_commands(
|
|
566
|
+
commands = get_setup_cloud_instance_commands(
|
|
567
|
+
skip_firewall_setup=skip_firewall_setup,
|
|
568
|
+
firewall_allow_from_subnets=firewall_allow_from_subnets,
|
|
569
|
+
)
|
|
559
570
|
commands += get_shim_pre_start_commands(
|
|
560
571
|
base_path=base_path,
|
|
561
572
|
bin_path=bin_path,
|
|
@@ -638,8 +649,11 @@ def get_dstack_shim_download_url(arch: Optional[str] = None) -> str:
|
|
|
638
649
|
return url_template.format(version=version, arch=arch)
|
|
639
650
|
|
|
640
651
|
|
|
641
|
-
def get_setup_cloud_instance_commands(
|
|
642
|
-
|
|
652
|
+
def get_setup_cloud_instance_commands(
|
|
653
|
+
skip_firewall_setup: bool,
|
|
654
|
+
firewall_allow_from_subnets: Iterable[str],
|
|
655
|
+
) -> list[str]:
|
|
656
|
+
commands = [
|
|
643
657
|
# Workaround for https://github.com/NVIDIA/nvidia-container-toolkit/issues/48
|
|
644
658
|
# Attempts to patch /etc/docker/daemon.json while keeping any custom settings it may have.
|
|
645
659
|
(
|
|
@@ -653,6 +667,19 @@ def get_setup_cloud_instance_commands() -> list[str]:
|
|
|
653
667
|
"'"
|
|
654
668
|
),
|
|
655
669
|
]
|
|
670
|
+
if not skip_firewall_setup:
|
|
671
|
+
commands += [
|
|
672
|
+
"ufw --force reset", # Some OS images have default rules like `allow 80`. Delete them
|
|
673
|
+
"ufw default deny incoming",
|
|
674
|
+
"ufw default allow outgoing",
|
|
675
|
+
"ufw allow ssh",
|
|
676
|
+
]
|
|
677
|
+
for subnet in firewall_allow_from_subnets:
|
|
678
|
+
commands.append(f"ufw allow from {subnet}")
|
|
679
|
+
commands += [
|
|
680
|
+
"ufw --force enable",
|
|
681
|
+
]
|
|
682
|
+
return commands
|
|
656
683
|
|
|
657
684
|
|
|
658
685
|
def get_shim_pre_start_commands(
|
|
@@ -773,7 +800,8 @@ def get_docker_commands(
|
|
|
773
800
|
f" --ssh-port {DSTACK_RUNNER_SSH_PORT}"
|
|
774
801
|
" --temp-dir /tmp/runner"
|
|
775
802
|
" --home-dir /root"
|
|
776
|
-
|
|
803
|
+
# TODO: Not used, left for compatibility with old runners. Remove eventually.
|
|
804
|
+
f" --working-dir {LEGACY_REPO_DIR}"
|
|
777
805
|
),
|
|
778
806
|
]
|
|
779
807
|
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
from abc import ABC, abstractmethod
|
|
2
|
-
from typing import Any, ClassVar, List, Optional
|
|
2
|
+
from typing import Any, ClassVar, Generic, List, Optional, TypeVar
|
|
3
3
|
from uuid import UUID
|
|
4
4
|
|
|
5
5
|
from dstack._internal.core.backends.base.backend import Backend
|
|
6
6
|
from dstack._internal.core.backends.models import (
|
|
7
|
-
AnyBackendConfig,
|
|
8
7
|
AnyBackendConfigWithCreds,
|
|
8
|
+
AnyBackendConfigWithoutCreds,
|
|
9
9
|
)
|
|
10
10
|
from dstack._internal.core.errors import BackendInvalidCredentialsError
|
|
11
11
|
from dstack._internal.core.models.backends.base import BackendType
|
|
@@ -15,6 +15,11 @@ from dstack._internal.core.models.common import CoreModel
|
|
|
15
15
|
# We'll introduce our own base limit that can be customized per backend if required.
|
|
16
16
|
TAGS_MAX_NUM = 25
|
|
17
17
|
|
|
18
|
+
BackendConfigWithoutCredsT = TypeVar(
|
|
19
|
+
"BackendConfigWithoutCredsT", bound=AnyBackendConfigWithoutCreds
|
|
20
|
+
)
|
|
21
|
+
BackendConfigWithCredsT = TypeVar("BackendConfigWithCredsT", bound=AnyBackendConfigWithCreds)
|
|
22
|
+
|
|
18
23
|
|
|
19
24
|
class BackendRecord(CoreModel):
|
|
20
25
|
"""
|
|
@@ -39,7 +44,7 @@ class StoredBackendRecord(BackendRecord):
|
|
|
39
44
|
backend_id: UUID
|
|
40
45
|
|
|
41
46
|
|
|
42
|
-
class Configurator(ABC):
|
|
47
|
+
class Configurator(ABC, Generic[BackendConfigWithoutCredsT, BackendConfigWithCredsT]):
|
|
43
48
|
"""
|
|
44
49
|
`Configurator` is responsible for configuring backends
|
|
45
50
|
and initializing `Backend` instances from backend configs.
|
|
@@ -52,7 +57,7 @@ class Configurator(ABC):
|
|
|
52
57
|
BACKEND_CLASS: ClassVar[type[Backend]]
|
|
53
58
|
|
|
54
59
|
@abstractmethod
|
|
55
|
-
def validate_config(self, config:
|
|
60
|
+
def validate_config(self, config: BackendConfigWithCredsT, default_creds_enabled: bool):
|
|
56
61
|
"""
|
|
57
62
|
Validates backend config including backend creds and other parameters.
|
|
58
63
|
Raises `ServerClientError` or its subclass if config is invalid.
|
|
@@ -61,9 +66,7 @@ class Configurator(ABC):
|
|
|
61
66
|
pass
|
|
62
67
|
|
|
63
68
|
@abstractmethod
|
|
64
|
-
def create_backend(
|
|
65
|
-
self, project_name: str, config: AnyBackendConfigWithCreds
|
|
66
|
-
) -> BackendRecord:
|
|
69
|
+
def create_backend(self, project_name: str, config: BackendConfigWithCredsT) -> BackendRecord:
|
|
67
70
|
"""
|
|
68
71
|
Sets up backend given backend config and returns
|
|
69
72
|
text-encoded config and creds to be stored in the DB.
|
|
@@ -78,13 +81,22 @@ class Configurator(ABC):
|
|
|
78
81
|
pass
|
|
79
82
|
|
|
80
83
|
@abstractmethod
|
|
81
|
-
def
|
|
82
|
-
self, record: StoredBackendRecord
|
|
83
|
-
) ->
|
|
84
|
+
def get_backend_config_with_creds(
|
|
85
|
+
self, record: StoredBackendRecord
|
|
86
|
+
) -> BackendConfigWithCredsT:
|
|
87
|
+
"""
|
|
88
|
+
Constructs `BackendConfig` with credentials included.
|
|
89
|
+
Used internally and when project admins need to see backend's creds.
|
|
90
|
+
"""
|
|
91
|
+
pass
|
|
92
|
+
|
|
93
|
+
@abstractmethod
|
|
94
|
+
def get_backend_config_without_creds(
|
|
95
|
+
self, record: StoredBackendRecord
|
|
96
|
+
) -> BackendConfigWithoutCredsT:
|
|
84
97
|
"""
|
|
85
|
-
Constructs `BackendConfig`
|
|
86
|
-
|
|
87
|
-
Otherwise, no sensitive information should be included.
|
|
98
|
+
Constructs `BackendConfig` without sensitive information.
|
|
99
|
+
Used for API responses where creds should not be exposed.
|
|
88
100
|
"""
|
|
89
101
|
pass
|
|
90
102
|
|
|
@@ -34,6 +34,8 @@ def get_catalog_offers(
|
|
|
34
34
|
provider = backend.value
|
|
35
35
|
if backend == BackendType.LAMBDA:
|
|
36
36
|
provider = "lambdalabs"
|
|
37
|
+
if backend == BackendType.AMDDEVCLOUD:
|
|
38
|
+
provider = "digitalocean"
|
|
37
39
|
q = requirements_to_query_filter(requirements)
|
|
38
40
|
q.provider = [provider]
|
|
39
41
|
offers = []
|
|
@@ -8,7 +8,6 @@ from dstack._internal.core.backends.base.configurator import (
|
|
|
8
8
|
from dstack._internal.core.backends.cloudrift.api_client import RiftClient
|
|
9
9
|
from dstack._internal.core.backends.cloudrift.backend import CloudRiftBackend
|
|
10
10
|
from dstack._internal.core.backends.cloudrift.models import (
|
|
11
|
-
AnyCloudRiftBackendConfig,
|
|
12
11
|
AnyCloudRiftCreds,
|
|
13
12
|
CloudRiftBackendConfig,
|
|
14
13
|
CloudRiftBackendConfigWithCreds,
|
|
@@ -21,7 +20,12 @@ from dstack._internal.core.models.backends.base import (
|
|
|
21
20
|
)
|
|
22
21
|
|
|
23
22
|
|
|
24
|
-
class CloudRiftConfigurator(
|
|
23
|
+
class CloudRiftConfigurator(
|
|
24
|
+
Configurator[
|
|
25
|
+
CloudRiftBackendConfig,
|
|
26
|
+
CloudRiftBackendConfigWithCreds,
|
|
27
|
+
]
|
|
28
|
+
):
|
|
25
29
|
TYPE = BackendType.CLOUDRIFT
|
|
26
30
|
BACKEND_CLASS = CloudRiftBackend
|
|
27
31
|
|
|
@@ -40,12 +44,14 @@ class CloudRiftConfigurator(Configurator):
|
|
|
40
44
|
auth=CloudRiftCreds.parse_obj(config.creds).json(),
|
|
41
45
|
)
|
|
42
46
|
|
|
43
|
-
def
|
|
44
|
-
self, record: BackendRecord
|
|
45
|
-
) ->
|
|
47
|
+
def get_backend_config_with_creds(
|
|
48
|
+
self, record: BackendRecord
|
|
49
|
+
) -> CloudRiftBackendConfigWithCreds:
|
|
50
|
+
config = self._get_config(record)
|
|
51
|
+
return CloudRiftBackendConfigWithCreds.__response__.parse_obj(config)
|
|
52
|
+
|
|
53
|
+
def get_backend_config_without_creds(self, record: BackendRecord) -> CloudRiftBackendConfig:
|
|
46
54
|
config = self._get_config(record)
|
|
47
|
-
if include_creds:
|
|
48
|
-
return CloudRiftBackendConfigWithCreds.__response__.parse_obj(config)
|
|
49
55
|
return CloudRiftBackendConfig.__response__.parse_obj(config)
|
|
50
56
|
|
|
51
57
|
def get_backend(self, record: BackendRecord) -> CloudRiftBackend:
|
|
@@ -5,6 +5,12 @@ from dstack._internal.core.models.backends.base import BackendType
|
|
|
5
5
|
|
|
6
6
|
_CONFIGURATOR_CLASSES: List[Type[Configurator]] = []
|
|
7
7
|
|
|
8
|
+
try:
|
|
9
|
+
from dstack._internal.core.backends.amddevcloud.configurator import AMDDevCloudConfigurator
|
|
10
|
+
|
|
11
|
+
_CONFIGURATOR_CLASSES.append(AMDDevCloudConfigurator)
|
|
12
|
+
except ImportError:
|
|
13
|
+
pass
|
|
8
14
|
|
|
9
15
|
try:
|
|
10
16
|
from dstack._internal.core.backends.aws.configurator import AWSConfigurator
|
|
@@ -47,6 +53,15 @@ try:
|
|
|
47
53
|
except ImportError:
|
|
48
54
|
pass
|
|
49
55
|
|
|
56
|
+
try:
|
|
57
|
+
from dstack._internal.core.backends.digitalocean.configurator import (
|
|
58
|
+
DigitalOceanConfigurator,
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
_CONFIGURATOR_CLASSES.append(DigitalOceanConfigurator)
|
|
62
|
+
except ImportError:
|
|
63
|
+
pass
|
|
64
|
+
|
|
50
65
|
try:
|
|
51
66
|
from dstack._internal.core.backends.gcp.configurator import GCPConfigurator
|
|
52
67
|
|
|
@@ -8,7 +8,6 @@ from dstack._internal.core.backends.base.configurator import (
|
|
|
8
8
|
from dstack._internal.core.backends.cudo import api_client
|
|
9
9
|
from dstack._internal.core.backends.cudo.backend import CudoBackend
|
|
10
10
|
from dstack._internal.core.backends.cudo.models import (
|
|
11
|
-
AnyCudoBackendConfig,
|
|
12
11
|
CudoBackendConfig,
|
|
13
12
|
CudoBackendConfigWithCreds,
|
|
14
13
|
CudoConfig,
|
|
@@ -18,7 +17,12 @@ from dstack._internal.core.backends.cudo.models import (
|
|
|
18
17
|
from dstack._internal.core.models.backends.base import BackendType
|
|
19
18
|
|
|
20
19
|
|
|
21
|
-
class CudoConfigurator(
|
|
20
|
+
class CudoConfigurator(
|
|
21
|
+
Configurator[
|
|
22
|
+
CudoBackendConfig,
|
|
23
|
+
CudoBackendConfigWithCreds,
|
|
24
|
+
]
|
|
25
|
+
):
|
|
22
26
|
TYPE = BackendType.CUDO
|
|
23
27
|
BACKEND_CLASS = CudoBackend
|
|
24
28
|
|
|
@@ -35,12 +39,12 @@ class CudoConfigurator(Configurator):
|
|
|
35
39
|
auth=CudoCreds.parse_obj(config.creds).json(),
|
|
36
40
|
)
|
|
37
41
|
|
|
38
|
-
def
|
|
39
|
-
self
|
|
40
|
-
|
|
42
|
+
def get_backend_config_with_creds(self, record: BackendRecord) -> CudoBackendConfigWithCreds:
|
|
43
|
+
config = self._get_config(record)
|
|
44
|
+
return CudoBackendConfigWithCreds.__response__.parse_obj(config)
|
|
45
|
+
|
|
46
|
+
def get_backend_config_without_creds(self, record: BackendRecord) -> CudoBackendConfig:
|
|
41
47
|
config = self._get_config(record)
|
|
42
|
-
if include_creds:
|
|
43
|
-
return CudoBackendConfigWithCreds.__response__.parse_obj(config)
|
|
44
48
|
return CudoBackendConfig.__response__.parse_obj(config)
|
|
45
49
|
|
|
46
50
|
def get_backend(self, record: BackendRecord) -> CudoBackend:
|
|
@@ -161,7 +161,10 @@ class DataCrunchCompute(
|
|
|
161
161
|
try:
|
|
162
162
|
self.client.instances.action(id_list=[instance_id], action="delete")
|
|
163
163
|
except APIException as e:
|
|
164
|
-
if e.message
|
|
164
|
+
if e.message in [
|
|
165
|
+
"Invalid instance id",
|
|
166
|
+
"Can't discontinue a discontinued instance",
|
|
167
|
+
]:
|
|
165
168
|
logger.debug("Skipping instance %s termination. Instance not found.", instance_id)
|
|
166
169
|
return
|
|
167
170
|
raise
|
|
@@ -243,6 +246,7 @@ def _deploy_instance(
|
|
|
243
246
|
hostname=hostname,
|
|
244
247
|
description=description,
|
|
245
248
|
startup_script_id=startup_script_id,
|
|
249
|
+
pricing="FIXED_PRICE",
|
|
246
250
|
is_spot=is_spot,
|
|
247
251
|
location=location,
|
|
248
252
|
os_volume={"name": "OS volume", "size": disk_size},
|
|
@@ -10,7 +10,6 @@ from dstack._internal.core.backends.base.configurator import (
|
|
|
10
10
|
)
|
|
11
11
|
from dstack._internal.core.backends.datacrunch.backend import DataCrunchBackend
|
|
12
12
|
from dstack._internal.core.backends.datacrunch.models import (
|
|
13
|
-
AnyDataCrunchBackendConfig,
|
|
14
13
|
DataCrunchBackendConfig,
|
|
15
14
|
DataCrunchBackendConfigWithCreds,
|
|
16
15
|
DataCrunchConfig,
|
|
@@ -22,7 +21,12 @@ from dstack._internal.core.models.backends.base import (
|
|
|
22
21
|
)
|
|
23
22
|
|
|
24
23
|
|
|
25
|
-
class DataCrunchConfigurator(
|
|
24
|
+
class DataCrunchConfigurator(
|
|
25
|
+
Configurator[
|
|
26
|
+
DataCrunchBackendConfig,
|
|
27
|
+
DataCrunchBackendConfigWithCreds,
|
|
28
|
+
]
|
|
29
|
+
):
|
|
26
30
|
TYPE = BackendType.DATACRUNCH
|
|
27
31
|
BACKEND_CLASS = DataCrunchBackend
|
|
28
32
|
|
|
@@ -41,12 +45,14 @@ class DataCrunchConfigurator(Configurator):
|
|
|
41
45
|
auth=DataCrunchCreds.parse_obj(config.creds).json(),
|
|
42
46
|
)
|
|
43
47
|
|
|
44
|
-
def
|
|
45
|
-
self, record: BackendRecord
|
|
46
|
-
) ->
|
|
48
|
+
def get_backend_config_with_creds(
|
|
49
|
+
self, record: BackendRecord
|
|
50
|
+
) -> DataCrunchBackendConfigWithCreds:
|
|
51
|
+
config = self._get_config(record)
|
|
52
|
+
return DataCrunchBackendConfigWithCreds.__response__.parse_obj(config)
|
|
53
|
+
|
|
54
|
+
def get_backend_config_without_creds(self, record: BackendRecord) -> DataCrunchBackendConfig:
|
|
47
55
|
config = self._get_config(record)
|
|
48
|
-
if include_creds:
|
|
49
|
-
return DataCrunchBackendConfigWithCreds.__response__.parse_obj(config)
|
|
50
56
|
return DataCrunchBackendConfig.__response__.parse_obj(config)
|
|
51
57
|
|
|
52
58
|
def get_backend(self, record: BackendRecord) -> DataCrunchBackend:
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# DigitalOcean backend for dstack
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
from dstack._internal.core.backends.digitalocean.compute import DigitalOceanCompute
|
|
2
|
+
from dstack._internal.core.backends.digitalocean_base.backend import BaseDigitalOceanBackend
|
|
3
|
+
from dstack._internal.core.backends.digitalocean_base.models import BaseDigitalOceanConfig
|
|
4
|
+
from dstack._internal.core.models.backends.base import BackendType
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class DigitalOceanBackend(BaseDigitalOceanBackend):
|
|
8
|
+
TYPE = BackendType.DIGITALOCEAN
|
|
9
|
+
COMPUTE_CLASS = DigitalOceanCompute
|
|
10
|
+
|
|
11
|
+
def __init__(self, config: BaseDigitalOceanConfig, api_url: str):
|
|
12
|
+
self.config = config
|
|
13
|
+
self._compute = DigitalOceanCompute(self.config, api_url=api_url, type=self.TYPE)
|
|
14
|
+
|
|
15
|
+
def compute(self) -> DigitalOceanCompute:
|
|
16
|
+
return self._compute
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
from typing import Optional
|
|
2
|
+
|
|
3
|
+
from dstack._internal.core.backends.base.configurator import BackendRecord
|
|
4
|
+
from dstack._internal.core.backends.digitalocean.backend import DigitalOceanBackend
|
|
5
|
+
from dstack._internal.core.backends.digitalocean_base.api_client import DigitalOceanAPIClient
|
|
6
|
+
from dstack._internal.core.backends.digitalocean_base.backend import BaseDigitalOceanBackend
|
|
7
|
+
from dstack._internal.core.backends.digitalocean_base.configurator import (
|
|
8
|
+
BaseDigitalOceanConfigurator,
|
|
9
|
+
)
|
|
10
|
+
from dstack._internal.core.backends.digitalocean_base.models import (
|
|
11
|
+
AnyBaseDigitalOceanCreds,
|
|
12
|
+
)
|
|
13
|
+
from dstack._internal.core.models.backends.base import (
|
|
14
|
+
BackendType,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class DigitalOceanConfigurator(BaseDigitalOceanConfigurator):
|
|
19
|
+
TYPE = BackendType.DIGITALOCEAN
|
|
20
|
+
BACKEND_CLASS = DigitalOceanBackend
|
|
21
|
+
API_URL = "https://api.digitalocean.com"
|
|
22
|
+
|
|
23
|
+
def get_backend(self, record: BackendRecord) -> BaseDigitalOceanBackend:
|
|
24
|
+
config = self._get_config(record)
|
|
25
|
+
return DigitalOceanBackend(config=config, api_url=self.API_URL)
|
|
26
|
+
|
|
27
|
+
def _validate_creds(self, creds: AnyBaseDigitalOceanCreds, project_name: Optional[str] = None):
|
|
28
|
+
api_client = DigitalOceanAPIClient(creds.api_key, self.API_URL)
|
|
29
|
+
api_client.validate_api_key()
|
|
30
|
+
if project_name:
|
|
31
|
+
api_client.validate_project_name(project_name)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# This package contains the base classes for DigitalOcean and AMDDevCloud backends.
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
from typing import Any, Dict, List, Optional
|
|
2
|
+
|
|
3
|
+
import requests
|
|
4
|
+
|
|
5
|
+
from dstack._internal.core.backends.base.configurator import raise_invalid_credentials_error
|
|
6
|
+
from dstack._internal.core.errors import NoCapacityError
|
|
7
|
+
from dstack._internal.utils.logging import get_logger
|
|
8
|
+
|
|
9
|
+
logger = get_logger(__name__)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class DigitalOceanAPIClient:
|
|
13
|
+
def __init__(self, api_key: str, api_url: str):
|
|
14
|
+
self.api_key = api_key
|
|
15
|
+
self.base_url = api_url
|
|
16
|
+
|
|
17
|
+
def validate_api_key(self) -> bool:
|
|
18
|
+
try:
|
|
19
|
+
response = self._make_request("GET", "/v2/account")
|
|
20
|
+
response.raise_for_status()
|
|
21
|
+
return True
|
|
22
|
+
except requests.HTTPError as e:
|
|
23
|
+
status = e.response.status_code
|
|
24
|
+
if status == 401:
|
|
25
|
+
raise_invalid_credentials_error(
|
|
26
|
+
fields=[["creds", "api_key"]], details="Invaild API key"
|
|
27
|
+
)
|
|
28
|
+
raise e
|
|
29
|
+
|
|
30
|
+
def validate_project_name(self, project_name: str) -> bool:
|
|
31
|
+
if self.get_project_id(project_name) is None:
|
|
32
|
+
raise_invalid_credentials_error(
|
|
33
|
+
fields=[["project_name"]],
|
|
34
|
+
details=f"Project with name '{project_name}' does not exist",
|
|
35
|
+
)
|
|
36
|
+
return True
|
|
37
|
+
|
|
38
|
+
def list_ssh_keys(self) -> List[Dict[str, Any]]:
|
|
39
|
+
response = self._make_request("GET", "/v2/account/keys")
|
|
40
|
+
response.raise_for_status()
|
|
41
|
+
return response.json()["ssh_keys"]
|
|
42
|
+
|
|
43
|
+
def list_projects(self) -> List[Dict[str, Any]]:
|
|
44
|
+
response = self._make_request("GET", "/v2/projects")
|
|
45
|
+
response.raise_for_status()
|
|
46
|
+
return response.json()["projects"]
|
|
47
|
+
|
|
48
|
+
def get_project_id(self, project_name: str) -> Optional[str]:
|
|
49
|
+
projects = self.list_projects()
|
|
50
|
+
for project in projects:
|
|
51
|
+
if project["name"] == project_name:
|
|
52
|
+
return project["id"]
|
|
53
|
+
return None
|
|
54
|
+
|
|
55
|
+
def create_ssh_key(self, name: str, public_key: str) -> Dict[str, Any]:
|
|
56
|
+
payload = {"name": name, "public_key": public_key}
|
|
57
|
+
response = self._make_request("POST", "/v2/account/keys", json=payload)
|
|
58
|
+
response.raise_for_status()
|
|
59
|
+
return response.json()["ssh_key"]
|
|
60
|
+
|
|
61
|
+
def get_or_create_ssh_key(self, name: str, public_key: str) -> int:
|
|
62
|
+
ssh_keys = self.list_ssh_keys()
|
|
63
|
+
for ssh_key in ssh_keys:
|
|
64
|
+
if ssh_key["public_key"].strip() == public_key.strip():
|
|
65
|
+
return ssh_key["id"]
|
|
66
|
+
|
|
67
|
+
ssh_key = self.create_ssh_key(name, public_key)
|
|
68
|
+
return ssh_key["id"]
|
|
69
|
+
|
|
70
|
+
def create_droplet(self, droplet_config: Dict[str, Any]) -> Dict[str, Any]:
|
|
71
|
+
response = self._make_request("POST", "/v2/droplets", json=droplet_config)
|
|
72
|
+
if response.status_code == 422:
|
|
73
|
+
raise NoCapacityError(response.json()["message"])
|
|
74
|
+
response.raise_for_status()
|
|
75
|
+
return response.json()["droplet"]
|
|
76
|
+
|
|
77
|
+
def get_droplet(self, droplet_id: str) -> Dict[str, Any]:
|
|
78
|
+
response = self._make_request("GET", f"/v2/droplets/{droplet_id}")
|
|
79
|
+
response.raise_for_status()
|
|
80
|
+
return response.json()["droplet"]
|
|
81
|
+
|
|
82
|
+
def delete_droplet(self, droplet_id: str) -> None:
|
|
83
|
+
response = self._make_request("DELETE", f"/v2/droplets/{droplet_id}")
|
|
84
|
+
if response.status_code == 404:
|
|
85
|
+
logger.debug("DigitalOcean droplet %s not found", droplet_id)
|
|
86
|
+
return
|
|
87
|
+
response.raise_for_status()
|
|
88
|
+
|
|
89
|
+
def _make_request(
|
|
90
|
+
self, method: str, endpoint: str, json: Optional[Dict[str, Any]] = None, timeout: int = 30
|
|
91
|
+
) -> requests.Response:
|
|
92
|
+
url = f"{self.base_url}{endpoint}"
|
|
93
|
+
headers = {
|
|
94
|
+
"Authorization": f"Bearer {self.api_key}",
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
response = requests.request(
|
|
98
|
+
method=method,
|
|
99
|
+
url=url,
|
|
100
|
+
headers=headers,
|
|
101
|
+
json=json,
|
|
102
|
+
timeout=timeout,
|
|
103
|
+
)
|
|
104
|
+
return response
|