dstack 0.18.44__py3-none-any.whl → 0.19.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (267) hide show
  1. dstack/_internal/cli/commands/gateway.py +15 -3
  2. dstack/_internal/cli/commands/logs.py +0 -22
  3. dstack/_internal/cli/commands/stats.py +8 -17
  4. dstack/_internal/cli/main.py +1 -5
  5. dstack/_internal/cli/services/configurators/fleet.py +4 -39
  6. dstack/_internal/cli/services/configurators/run.py +22 -21
  7. dstack/_internal/cli/services/profile.py +34 -83
  8. dstack/_internal/cli/utils/gateway.py +1 -1
  9. dstack/_internal/core/backends/__init__.py +56 -39
  10. dstack/_internal/core/backends/aws/__init__.py +0 -25
  11. dstack/_internal/core/backends/aws/auth.py +1 -10
  12. dstack/_internal/core/backends/aws/backend.py +26 -0
  13. dstack/_internal/core/backends/aws/compute.py +20 -45
  14. dstack/_internal/{server/services/backends/configurators/aws.py → core/backends/aws/configurator.py} +46 -85
  15. dstack/_internal/core/backends/aws/models.py +135 -0
  16. dstack/_internal/core/backends/aws/resources.py +1 -1
  17. dstack/_internal/core/backends/azure/__init__.py +0 -20
  18. dstack/_internal/core/backends/azure/auth.py +2 -11
  19. dstack/_internal/core/backends/azure/backend.py +21 -0
  20. dstack/_internal/core/backends/azure/compute.py +13 -27
  21. dstack/_internal/{server/services/backends/configurators/azure.py → core/backends/azure/configurator.py} +141 -210
  22. dstack/_internal/core/backends/azure/models.py +89 -0
  23. dstack/_internal/core/backends/base/__init__.py +0 -12
  24. dstack/_internal/core/backends/base/backend.py +18 -0
  25. dstack/_internal/core/backends/base/compute.py +153 -33
  26. dstack/_internal/core/backends/base/configurator.py +105 -0
  27. dstack/_internal/core/backends/base/models.py +14 -0
  28. dstack/_internal/core/backends/configurators.py +138 -0
  29. dstack/_internal/core/backends/cudo/__init__.py +0 -15
  30. dstack/_internal/core/backends/cudo/backend.py +16 -0
  31. dstack/_internal/core/backends/cudo/compute.py +8 -26
  32. dstack/_internal/core/backends/cudo/configurator.py +72 -0
  33. dstack/_internal/core/backends/cudo/models.py +37 -0
  34. dstack/_internal/core/backends/datacrunch/__init__.py +0 -15
  35. dstack/_internal/core/backends/datacrunch/backend.py +16 -0
  36. dstack/_internal/core/backends/datacrunch/compute.py +8 -25
  37. dstack/_internal/core/backends/datacrunch/configurator.py +66 -0
  38. dstack/_internal/core/backends/datacrunch/models.py +38 -0
  39. dstack/_internal/core/{models/backends/dstack.py → backends/dstack/models.py} +7 -7
  40. dstack/_internal/core/backends/gcp/__init__.py +0 -16
  41. dstack/_internal/core/backends/gcp/auth.py +2 -11
  42. dstack/_internal/core/backends/gcp/backend.py +17 -0
  43. dstack/_internal/core/backends/gcp/compute.py +13 -43
  44. dstack/_internal/{server/services/backends/configurators/gcp.py → core/backends/gcp/configurator.py} +46 -103
  45. dstack/_internal/core/backends/gcp/models.py +125 -0
  46. dstack/_internal/core/backends/kubernetes/__init__.py +0 -15
  47. dstack/_internal/core/backends/kubernetes/backend.py +16 -0
  48. dstack/_internal/core/backends/kubernetes/compute.py +16 -5
  49. dstack/_internal/core/backends/kubernetes/configurator.py +55 -0
  50. dstack/_internal/core/backends/kubernetes/models.py +72 -0
  51. dstack/_internal/core/backends/lambdalabs/__init__.py +0 -16
  52. dstack/_internal/core/backends/lambdalabs/backend.py +17 -0
  53. dstack/_internal/core/backends/lambdalabs/compute.py +7 -28
  54. dstack/_internal/core/backends/lambdalabs/configurator.py +82 -0
  55. dstack/_internal/core/backends/lambdalabs/models.py +37 -0
  56. dstack/_internal/core/backends/local/__init__.py +0 -13
  57. dstack/_internal/core/backends/local/backend.py +14 -0
  58. dstack/_internal/core/backends/local/compute.py +16 -2
  59. dstack/_internal/core/backends/models.py +128 -0
  60. dstack/_internal/core/backends/oci/__init__.py +0 -15
  61. dstack/_internal/core/backends/oci/auth.py +1 -5
  62. dstack/_internal/core/backends/oci/backend.py +16 -0
  63. dstack/_internal/core/backends/oci/compute.py +9 -23
  64. dstack/_internal/{server/services/backends/configurators/oci.py → core/backends/oci/configurator.py} +40 -85
  65. dstack/_internal/core/{models/backends/oci.py → backends/oci/models.py} +24 -25
  66. dstack/_internal/core/backends/oci/region.py +1 -1
  67. dstack/_internal/core/backends/runpod/__init__.py +0 -15
  68. dstack/_internal/core/backends/runpod/backend.py +16 -0
  69. dstack/_internal/core/backends/runpod/compute.py +7 -3
  70. dstack/_internal/core/backends/runpod/configurator.py +59 -0
  71. dstack/_internal/core/backends/runpod/models.py +54 -0
  72. dstack/_internal/core/backends/template/__init__.py +0 -0
  73. dstack/_internal/core/backends/tensordock/__init__.py +0 -15
  74. dstack/_internal/core/backends/tensordock/backend.py +16 -0
  75. dstack/_internal/core/backends/tensordock/compute.py +8 -27
  76. dstack/_internal/core/backends/tensordock/configurator.py +68 -0
  77. dstack/_internal/core/backends/tensordock/models.py +38 -0
  78. dstack/_internal/core/backends/vastai/__init__.py +0 -15
  79. dstack/_internal/core/backends/vastai/backend.py +16 -0
  80. dstack/_internal/core/backends/vastai/compute.py +2 -2
  81. dstack/_internal/core/backends/vastai/configurator.py +66 -0
  82. dstack/_internal/core/backends/vastai/models.py +37 -0
  83. dstack/_internal/core/backends/vultr/__init__.py +0 -15
  84. dstack/_internal/core/backends/vultr/backend.py +16 -0
  85. dstack/_internal/core/backends/vultr/compute.py +10 -24
  86. dstack/_internal/core/backends/vultr/configurator.py +64 -0
  87. dstack/_internal/core/backends/vultr/models.py +34 -0
  88. dstack/_internal/core/models/backends/__init__.py +0 -184
  89. dstack/_internal/core/models/backends/base.py +0 -19
  90. dstack/_internal/core/models/configurations.py +20 -15
  91. dstack/_internal/core/models/envs.py +4 -3
  92. dstack/_internal/core/models/fleets.py +17 -22
  93. dstack/_internal/core/models/gateways.py +3 -3
  94. dstack/_internal/core/models/instances.py +24 -0
  95. dstack/_internal/core/models/profiles.py +41 -46
  96. dstack/_internal/core/models/projects.py +1 -1
  97. dstack/_internal/core/models/repos/base.py +0 -5
  98. dstack/_internal/core/models/repos/local.py +3 -3
  99. dstack/_internal/core/models/repos/remote.py +26 -12
  100. dstack/_internal/core/models/repos/virtual.py +1 -1
  101. dstack/_internal/core/models/resources.py +45 -76
  102. dstack/_internal/core/models/runs.py +17 -19
  103. dstack/_internal/core/models/volumes.py +1 -3
  104. dstack/_internal/core/services/profiles.py +7 -16
  105. dstack/_internal/core/services/repos.py +0 -4
  106. dstack/_internal/server/app.py +0 -3
  107. dstack/_internal/server/background/tasks/process_gateways.py +4 -8
  108. dstack/_internal/server/background/tasks/process_instances.py +14 -9
  109. dstack/_internal/server/background/tasks/process_metrics.py +1 -1
  110. dstack/_internal/server/background/tasks/process_placement_groups.py +4 -1
  111. dstack/_internal/server/background/tasks/process_prometheus_metrics.py +1 -1
  112. dstack/_internal/server/background/tasks/process_running_jobs.py +14 -5
  113. dstack/_internal/server/background/tasks/process_submitted_jobs.py +16 -37
  114. dstack/_internal/server/background/tasks/process_volumes.py +5 -2
  115. dstack/_internal/server/migrations/versions/7bc2586e8b9e_make_instancemodel_pool_id_optional.py +36 -0
  116. dstack/_internal/server/migrations/versions/bc8ca4a505c6_store_backendtype_as_string.py +171 -0
  117. dstack/_internal/server/models.py +48 -9
  118. dstack/_internal/server/routers/backends.py +14 -23
  119. dstack/_internal/server/routers/instances.py +3 -4
  120. dstack/_internal/server/routers/metrics.py +10 -8
  121. dstack/_internal/server/routers/prometheus.py +1 -1
  122. dstack/_internal/server/routers/repos.py +1 -2
  123. dstack/_internal/server/routers/runs.py +13 -59
  124. dstack/_internal/server/schemas/gateways.py +14 -23
  125. dstack/_internal/server/schemas/projects.py +7 -2
  126. dstack/_internal/server/schemas/repos.py +2 -38
  127. dstack/_internal/server/schemas/runner.py +1 -0
  128. dstack/_internal/server/schemas/runs.py +1 -24
  129. dstack/_internal/server/services/backends/__init__.py +85 -158
  130. dstack/_internal/server/services/config.py +52 -576
  131. dstack/_internal/server/services/fleets.py +8 -103
  132. dstack/_internal/server/services/gateways/__init__.py +12 -4
  133. dstack/_internal/server/services/{pools.py → instances.py} +22 -329
  134. dstack/_internal/server/services/jobs/__init__.py +9 -6
  135. dstack/_internal/server/services/jobs/configurators/base.py +16 -0
  136. dstack/_internal/server/services/jobs/configurators/dev.py +9 -1
  137. dstack/_internal/server/services/jobs/configurators/extensions/cursor.py +42 -0
  138. dstack/_internal/server/services/metrics.py +39 -13
  139. dstack/_internal/server/services/offers.py +1 -1
  140. dstack/_internal/server/services/projects.py +23 -14
  141. dstack/_internal/server/services/prometheus.py +176 -18
  142. dstack/_internal/server/services/runs.py +24 -16
  143. dstack/_internal/server/services/volumes.py +8 -4
  144. dstack/_internal/server/statics/index.html +1 -1
  145. dstack/_internal/server/statics/{main-4eb116b97819badd1e2c.js → main-4fd5a4770eff59325ee3.js} +7 -7
  146. dstack/_internal/server/statics/{main-4eb116b97819badd1e2c.js.map → main-4fd5a4770eff59325ee3.js.map} +1 -1
  147. dstack/_internal/server/testing/common.py +58 -32
  148. dstack/_internal/utils/json_schema.py +6 -0
  149. dstack/_internal/utils/ssh.py +2 -1
  150. dstack/api/__init__.py +4 -0
  151. dstack/api/_public/__init__.py +16 -20
  152. dstack/api/_public/backends.py +1 -1
  153. dstack/api/_public/repos.py +36 -36
  154. dstack/api/_public/runs.py +167 -83
  155. dstack/api/server/__init__.py +11 -13
  156. dstack/api/server/_backends.py +12 -16
  157. dstack/api/server/_fleets.py +15 -57
  158. dstack/api/server/_gateways.py +3 -14
  159. dstack/api/server/_repos.py +1 -4
  160. dstack/api/server/_runs.py +21 -100
  161. dstack/api/server/_volumes.py +10 -5
  162. dstack/version.py +1 -1
  163. {dstack-0.18.44.dist-info → dstack-0.19.0rc1.dist-info}/METADATA +1 -1
  164. {dstack-0.18.44.dist-info → dstack-0.19.0rc1.dist-info}/RECORD +218 -204
  165. tests/_internal/cli/services/configurators/test_profile.py +6 -6
  166. tests/_internal/core/backends/aws/test_configurator.py +35 -0
  167. tests/_internal/core/backends/aws/test_resources.py +1 -1
  168. tests/_internal/core/backends/azure/test_configurator.py +61 -0
  169. tests/_internal/core/backends/cudo/__init__.py +0 -0
  170. tests/_internal/core/backends/cudo/test_configurator.py +37 -0
  171. tests/_internal/core/backends/datacrunch/__init__.py +0 -0
  172. tests/_internal/core/backends/datacrunch/test_configurator.py +17 -0
  173. tests/_internal/core/backends/gcp/test_configurator.py +42 -0
  174. tests/_internal/core/backends/kubernetes/test_configurator.py +43 -0
  175. tests/_internal/core/backends/lambdalabs/__init__.py +0 -0
  176. tests/_internal/core/backends/lambdalabs/test_configurator.py +38 -0
  177. tests/_internal/core/backends/oci/test_configurator.py +55 -0
  178. tests/_internal/core/backends/runpod/__init__.py +0 -0
  179. tests/_internal/core/backends/runpod/test_configurator.py +33 -0
  180. tests/_internal/core/backends/tensordock/__init__.py +0 -0
  181. tests/_internal/core/backends/tensordock/test_configurator.py +38 -0
  182. tests/_internal/core/backends/vastai/__init__.py +0 -0
  183. tests/_internal/core/backends/vastai/test_configurator.py +33 -0
  184. tests/_internal/core/backends/vultr/__init__.py +0 -0
  185. tests/_internal/core/backends/vultr/test_configurator.py +33 -0
  186. tests/_internal/server/background/tasks/test_process_gateways.py +4 -0
  187. tests/_internal/server/background/tasks/test_process_instances.py +49 -48
  188. tests/_internal/server/background/tasks/test_process_metrics.py +0 -3
  189. tests/_internal/server/background/tasks/test_process_placement_groups.py +2 -0
  190. tests/_internal/server/background/tasks/test_process_prometheus_metrics.py +0 -3
  191. tests/_internal/server/background/tasks/test_process_running_jobs.py +0 -21
  192. tests/_internal/server/background/tasks/test_process_runs.py +8 -22
  193. tests/_internal/server/background/tasks/test_process_submitted_jobs.py +3 -40
  194. tests/_internal/server/background/tasks/test_process_submitted_volumes.py +2 -0
  195. tests/_internal/server/background/tasks/test_process_terminating_jobs.py +10 -15
  196. tests/_internal/server/routers/test_backends.py +6 -764
  197. tests/_internal/server/routers/test_fleets.py +0 -26
  198. tests/_internal/server/routers/test_gateways.py +27 -3
  199. tests/_internal/server/routers/test_instances.py +0 -10
  200. tests/_internal/server/routers/test_metrics.py +27 -0
  201. tests/_internal/server/routers/test_projects.py +56 -0
  202. tests/_internal/server/routers/test_prometheus.py +116 -27
  203. tests/_internal/server/routers/test_repos.py +0 -15
  204. tests/_internal/server/routers/test_runs.py +4 -219
  205. tests/_internal/server/routers/test_volumes.py +2 -3
  206. tests/_internal/server/services/backends/__init__.py +0 -0
  207. tests/_internal/server/services/jobs/configurators/test_task.py +35 -0
  208. tests/_internal/server/services/test_config.py +7 -4
  209. tests/_internal/server/services/test_fleets.py +1 -4
  210. tests/_internal/server/services/{test_pools.py → test_instances.py} +11 -49
  211. tests/_internal/server/services/test_metrics.py +9 -5
  212. tests/_internal/server/services/test_repos.py +1 -14
  213. tests/_internal/server/services/test_runs.py +0 -4
  214. dstack/_internal/cli/commands/pool.py +0 -581
  215. dstack/_internal/cli/commands/run.py +0 -75
  216. dstack/_internal/core/backends/aws/config.py +0 -18
  217. dstack/_internal/core/backends/azure/config.py +0 -12
  218. dstack/_internal/core/backends/base/config.py +0 -5
  219. dstack/_internal/core/backends/cudo/config.py +0 -9
  220. dstack/_internal/core/backends/datacrunch/config.py +0 -9
  221. dstack/_internal/core/backends/gcp/config.py +0 -22
  222. dstack/_internal/core/backends/kubernetes/config.py +0 -6
  223. dstack/_internal/core/backends/lambdalabs/config.py +0 -9
  224. dstack/_internal/core/backends/nebius/__init__.py +0 -15
  225. dstack/_internal/core/backends/nebius/api_client.py +0 -319
  226. dstack/_internal/core/backends/nebius/compute.py +0 -220
  227. dstack/_internal/core/backends/nebius/config.py +0 -6
  228. dstack/_internal/core/backends/nebius/types.py +0 -37
  229. dstack/_internal/core/backends/oci/config.py +0 -6
  230. dstack/_internal/core/backends/runpod/config.py +0 -17
  231. dstack/_internal/core/backends/tensordock/config.py +0 -9
  232. dstack/_internal/core/backends/vastai/config.py +0 -6
  233. dstack/_internal/core/backends/vultr/config.py +0 -9
  234. dstack/_internal/core/models/backends/aws.py +0 -86
  235. dstack/_internal/core/models/backends/azure.py +0 -68
  236. dstack/_internal/core/models/backends/cudo.py +0 -43
  237. dstack/_internal/core/models/backends/datacrunch.py +0 -44
  238. dstack/_internal/core/models/backends/gcp.py +0 -67
  239. dstack/_internal/core/models/backends/kubernetes.py +0 -40
  240. dstack/_internal/core/models/backends/lambdalabs.py +0 -43
  241. dstack/_internal/core/models/backends/nebius.py +0 -54
  242. dstack/_internal/core/models/backends/runpod.py +0 -42
  243. dstack/_internal/core/models/backends/tensordock.py +0 -44
  244. dstack/_internal/core/models/backends/vastai.py +0 -43
  245. dstack/_internal/core/models/backends/vultr.py +0 -40
  246. dstack/_internal/core/models/pools.py +0 -43
  247. dstack/_internal/server/routers/pools.py +0 -142
  248. dstack/_internal/server/schemas/pools.py +0 -38
  249. dstack/_internal/server/services/backends/configurators/base.py +0 -72
  250. dstack/_internal/server/services/backends/configurators/cudo.py +0 -87
  251. dstack/_internal/server/services/backends/configurators/datacrunch.py +0 -79
  252. dstack/_internal/server/services/backends/configurators/kubernetes.py +0 -63
  253. dstack/_internal/server/services/backends/configurators/lambdalabs.py +0 -98
  254. dstack/_internal/server/services/backends/configurators/nebius.py +0 -85
  255. dstack/_internal/server/services/backends/configurators/runpod.py +0 -67
  256. dstack/_internal/server/services/backends/configurators/tensordock.py +0 -82
  257. dstack/_internal/server/services/backends/configurators/vastai.py +0 -80
  258. dstack/_internal/server/services/backends/configurators/vultr.py +0 -80
  259. dstack/api/_public/pools.py +0 -41
  260. dstack/api/_public/resources.py +0 -105
  261. dstack/api/server/_pools.py +0 -63
  262. tests/_internal/server/routers/test_pools.py +0 -612
  263. /dstack/_internal/{server/services/backends/configurators → core/backends/dstack}/__init__.py +0 -0
  264. {dstack-0.18.44.dist-info → dstack-0.19.0rc1.dist-info}/LICENSE.md +0 -0
  265. {dstack-0.18.44.dist-info → dstack-0.19.0rc1.dist-info}/WHEEL +0 -0
  266. {dstack-0.18.44.dist-info → dstack-0.19.0rc1.dist-info}/entry_points.txt +0 -0
  267. {dstack-0.18.44.dist-info → dstack-0.19.0rc1.dist-info}/top_level.txt +0 -0
@@ -13,6 +13,10 @@ from dstack._internal.cli.utils.common import (
13
13
  )
14
14
  from dstack._internal.cli.utils.gateway import get_gateways_table, print_gateways_table
15
15
  from dstack._internal.core.models.backends.base import BackendType
16
+ from dstack._internal.core.models.gateways import GatewayConfiguration
17
+ from dstack._internal.utils.logging import get_logger
18
+
19
+ logger = get_logger(__name__)
16
20
 
17
21
 
18
22
  class GatewayCommand(APIBaseCommand):
@@ -41,7 +45,9 @@ class GatewayCommand(APIBaseCommand):
41
45
  )
42
46
 
43
47
  create_parser = subparsers.add_parser(
44
- "create", help="Add a gateway", formatter_class=self._parser.formatter_class
48
+ "create",
49
+ help="Add a gateway. Deprecated in favor of `dstack apply` with gateway configuration.",
50
+ formatter_class=self._parser.formatter_class,
45
51
  )
46
52
  create_parser.set_defaults(subfunc=self._create)
47
53
  create_parser.add_argument(
@@ -100,10 +106,16 @@ class GatewayCommand(APIBaseCommand):
100
106
  pass
101
107
 
102
108
  def _create(self, args: argparse.Namespace):
109
+ logger.warning(
110
+ "`dstack gateway create` is deperecated in favor of `dstack apply` with gateway configurations."
111
+ )
103
112
  with console.status("Creating gateway..."):
104
- gateway = self.api.client.gateways.create(
105
- self.api.project, args.name, BackendType(args.backend), args.region
113
+ configuration = GatewayConfiguration(
114
+ name=args.name,
115
+ backend=BackendType(args.backend),
116
+ region=args.region,
106
117
  )
118
+ gateway = self.api.client.gateways.create(self.api.project, configuration)
107
119
  if args.set_default:
108
120
  self.api.client.gateways.set_default(self.api.project, gateway.name)
109
121
  if args.domain:
@@ -1,6 +1,5 @@
1
1
  import argparse
2
2
  import sys
3
- from pathlib import Path
4
3
 
5
4
  from dstack._internal.cli.commands import APIBaseCommand
6
5
  from dstack._internal.cli.services.completion import RunNameCompleter
@@ -19,19 +18,6 @@ class LogsCommand(APIBaseCommand):
19
18
  self._parser.add_argument(
20
19
  "-d", "--diagnose", action="store_true", help="Show run diagnostic logs"
21
20
  )
22
- self._parser.add_argument(
23
- "-a",
24
- "--attach",
25
- action="store_true",
26
- help="Set up an SSH tunnel and print logs as they follow",
27
- )
28
- self._parser.add_argument(
29
- "--ssh-identity",
30
- metavar="SSH_PRIVATE_KEY",
31
- help="The private SSH key path for SSH tunneling",
32
- type=Path,
33
- dest="ssh_identity_file",
34
- )
35
21
  self._parser.add_argument(
36
22
  "--replica",
37
23
  help="The replica number. Defaults to 0.",
@@ -51,14 +37,6 @@ class LogsCommand(APIBaseCommand):
51
37
  run = self.api.runs.get(args.run_name)
52
38
  if run is None:
53
39
  raise CLIError(f"Run {args.run_name} not found")
54
- if not args.diagnose and args.attach:
55
- if run.status.is_finished():
56
- raise CLIError(f"Run {args.run_name} is finished")
57
- else:
58
- logger.warning(
59
- "`dstack logs --attach` is deprecated in favor of `dstack attach --logs`"
60
- )
61
- run.attach(args.ssh_identity_file)
62
40
  logs = run.logs(
63
41
  diagnose=args.diagnose,
64
42
  replica_num=args.replica,
@@ -2,7 +2,6 @@ import argparse
2
2
  import time
3
3
  from typing import Any, Dict, List, Optional, Union
4
4
 
5
- import requests
6
5
  from rich.live import Live
7
6
  from rich.table import Table
8
7
 
@@ -64,22 +63,14 @@ class StatsCommand(APIBaseCommand):
64
63
 
65
64
  def _get_run_jobs_metrics(api: Client, run: Run) -> List[JobMetrics]:
66
65
  metrics = []
67
- try:
68
- for job in run._run.jobs:
69
- job_metrics = api.client.metrics.get_job_metrics(
70
- project_name=api.project,
71
- run_name=run.name,
72
- replica_num=job.job_spec.replica_num,
73
- job_num=job.job_spec.job_num,
74
- )
75
- metrics.append(job_metrics)
76
- except requests.exceptions.HTTPError as e:
77
- if e.response.status_code == 404:
78
- raise CLIError(
79
- "Metrics API is not supported for server versions before 0.18.18. "
80
- "Update the server to use `dstack stats`."
81
- )
82
- raise
66
+ for job in run._run.jobs:
67
+ job_metrics = api.client.metrics.get_job_metrics(
68
+ project_name=api.project,
69
+ run_name=run.name,
70
+ replica_num=job.job_spec.replica_num,
71
+ job_num=job.job_spec.job_num,
72
+ )
73
+ metrics.append(job_metrics)
83
74
  return metrics
84
75
 
85
76
 
@@ -13,9 +13,7 @@ from dstack._internal.cli.commands.fleet import FleetCommand
13
13
  from dstack._internal.cli.commands.gateway import GatewayCommand
14
14
  from dstack._internal.cli.commands.init import InitCommand
15
15
  from dstack._internal.cli.commands.logs import LogsCommand
16
- from dstack._internal.cli.commands.pool import PoolCommand
17
16
  from dstack._internal.cli.commands.ps import PsCommand
18
- from dstack._internal.cli.commands.run import RunCommand
19
17
  from dstack._internal.cli.commands.server import ServerCommand
20
18
  from dstack._internal.cli.commands.stats import StatsCommand
21
19
  from dstack._internal.cli.commands.stop import StopCommand
@@ -40,7 +38,7 @@ def main():
40
38
  parser = argparse.ArgumentParser(
41
39
  description=(
42
40
  "Not sure where to start? Call [code]dstack init[/].\n"
43
- "Define a [code].dstack.yml[/] configuration file and run it via [code]dstack run[/]\n"
41
+ "Define a [code].dstack.yml[/] configuration file and run it via [code]dstack apply[/]\n"
44
42
  ),
45
43
  formatter_class=RichHelpFormatter,
46
44
  epilog=(
@@ -65,11 +63,9 @@ def main():
65
63
  DeleteCommand.register(subparsers)
66
64
  FleetCommand.register(subparsers)
67
65
  GatewayCommand.register(subparsers)
68
- PoolCommand.register(subparsers)
69
66
  InitCommand.register(subparsers)
70
67
  LogsCommand.register(subparsers)
71
68
  PsCommand.register(subparsers)
72
- RunCommand.register(subparsers)
73
69
  ServerCommand.register(subparsers)
74
70
  StatsCommand.register(subparsers)
75
71
  StopCommand.register(subparsers)
@@ -3,7 +3,6 @@ import time
3
3
  from pathlib import Path
4
4
  from typing import List, Optional
5
5
 
6
- import requests
7
6
  from rich.table import Table
8
7
 
9
8
  from dstack._internal.cli.services.configurators.base import (
@@ -32,7 +31,6 @@ from dstack._internal.core.models.repos.base import Repo
32
31
  from dstack._internal.utils.common import local_time
33
32
  from dstack._internal.utils.logging import get_logger
34
33
  from dstack._internal.utils.ssh import convert_ssh_key_to_pem, generate_public_key, pkey_from_str
35
- from dstack.api._public import Client
36
34
  from dstack.api.utils import load_profile
37
35
 
38
36
  logger = get_logger(__name__)
@@ -60,7 +58,10 @@ class FleetConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
60
58
  _preprocess_spec(spec)
61
59
 
62
60
  with console.status("Getting apply plan..."):
63
- plan = _get_plan(api=self.api, spec=spec)
61
+ plan = self.api.client.fleets.get_plan(
62
+ project_name=self.api.project,
63
+ spec=spec,
64
+ )
64
65
  _print_plan_header(plan)
65
66
 
66
67
  action_message = ""
@@ -234,42 +235,6 @@ def _resolve_ssh_key(ssh_key_path: Optional[str]) -> Optional[SSHKey]:
234
235
  exit()
235
236
 
236
237
 
237
- def _get_plan(api: Client, spec: FleetSpec) -> FleetPlan:
238
- try:
239
- return api.client.fleets.get_plan(
240
- project_name=api.project,
241
- spec=spec,
242
- )
243
- except requests.exceptions.HTTPError as e:
244
- # Handle older server versions that do not have /get_plan for fleets
245
- # TODO: Can be removed in 0.19
246
- if e.response.status_code == 405:
247
- logger.warning(
248
- "Fleet apply plan is not fully supported before 0.18.17. "
249
- "Update the server to view full-featured apply plan."
250
- )
251
- user = api.client.users.get_my_user()
252
- spec.configuration_path = None
253
- current_resource = None
254
- if spec.configuration.name is not None:
255
- try:
256
- current_resource = api.client.fleets.get(
257
- project_name=api.project, name=spec.configuration.name
258
- )
259
- except ResourceNotExistsError:
260
- pass
261
- return FleetPlan(
262
- project_name=api.project,
263
- user=user.username,
264
- spec=spec,
265
- current_resource=current_resource,
266
- offers=[],
267
- total_offers=0,
268
- max_offer_price=0,
269
- )
270
- raise e
271
-
272
-
273
238
  def _print_plan_header(plan: FleetPlan):
274
239
  def th(s: str) -> str:
275
240
  return f"[bold]{s}[/bold]"
@@ -85,28 +85,11 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
85
85
  )
86
86
  profile = load_profile(Path.cwd(), configurator_args.profile)
87
87
  with console.status("Getting apply plan..."):
88
- run_plan = self.api.runs.get_plan(
88
+ run_plan = self.api.runs.get_run_plan(
89
89
  configuration=conf,
90
90
  repo=repo,
91
91
  configuration_path=configuration_path,
92
- backends=profile.backends,
93
- regions=profile.regions,
94
- instance_types=profile.instance_types,
95
- reservation=profile.reservation,
96
- spot_policy=profile.spot_policy,
97
- retry_policy=profile.retry_policy,
98
- utilization_policy=profile.utilization_policy,
99
- max_duration=profile.max_duration,
100
- stop_duration=profile.stop_duration,
101
- max_price=profile.max_price,
102
- working_dir=conf.working_dir,
103
- run_name=conf.name,
104
- pool_name=profile.pool_name,
105
- instance_name=profile.instance_name,
106
- creation_policy=profile.creation_policy,
107
- termination_policy=profile.termination_policy,
108
- termination_policy_idle=profile.termination_idle_time,
109
- idle_duration=profile.idle_duration,
92
+ profile=profile,
110
93
  )
111
94
 
112
95
  print_run_plan(run_plan, offers_limit=configurator_args.max_offers)
@@ -165,8 +148,8 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
165
148
 
166
149
  try:
167
150
  with console.status("Applying plan..."):
168
- run = self.api.runs.exec_plan(
169
- run_plan, repo, reserve_ports=not command_args.detach
151
+ run = self.api.runs.apply_plan(
152
+ run_plan=run_plan, repo=repo, reserve_ports=not command_args.detach
170
153
  )
171
154
  except ServerClientError as e:
172
155
  raise CLIError(e.msg)
@@ -459,6 +442,14 @@ class DevEnvironmentConfigurator(RunWithPortsConfigurator):
459
442
  "Fix by opening [code]Command Palette[/code], executing [code]Shell Command: "
460
443
  "Install 'code' command in PATH[/code], and restarting terminal.[/]\n"
461
444
  )
445
+ if conf.ide == "cursor" and conf.version is None:
446
+ conf.version = _detect_cursor_version()
447
+ if conf.version is None:
448
+ console.print(
449
+ "[secondary]Unable to detect the Cursor version and pre-install extensions. "
450
+ "Fix by opening [code]Command Palette[/code], executing [code]Shell Command: "
451
+ "Install 'cursor' command in PATH[/code], and restarting terminal.[/]\n"
452
+ )
462
453
 
463
454
 
464
455
  class ServiceConfigurator(BaseRunConfigurator):
@@ -497,6 +488,16 @@ def _detect_vscode_version(exe: str = "code") -> Optional[str]:
497
488
  return None
498
489
 
499
490
 
491
+ def _detect_cursor_version(exe: str = "cursor") -> Optional[str]:
492
+ try:
493
+ run = subprocess.run([exe, "--version"], capture_output=True)
494
+ except FileNotFoundError:
495
+ return None
496
+ if run.returncode == 0:
497
+ return run.stdout.decode().split("\n")[1].strip()
498
+ return None
499
+
500
+
500
501
  def _print_service_urls(run: Run) -> None:
501
502
  if run._run.run_spec.configuration.type != RunConfigurationType.SERVICE.value:
502
503
  return
@@ -4,21 +4,18 @@ from typing import Union
4
4
 
5
5
  from dstack._internal.core.models.configurations import AnyRunConfiguration
6
6
  from dstack._internal.core.models.profiles import (
7
- DEFAULT_INSTANCE_RETRY_DURATION,
8
- DEFAULT_POOL_TERMINATION_IDLE_TIME,
9
7
  CreationPolicy,
10
8
  Profile,
11
- ProfileRetryPolicy,
9
+ ProfileRetry,
12
10
  SpotPolicy,
13
- TerminationPolicy,
14
11
  parse_duration,
15
12
  parse_max_duration,
16
13
  )
17
14
 
18
15
 
19
- def register_profile_args(parser: argparse.ArgumentParser, pool_add: bool = False):
16
+ def register_profile_args(parser: argparse.ArgumentParser):
20
17
  """
21
- Registers `parser` with `dstack run` and `dstack pool add`
18
+ Registers `parser` with `dstack apply` run configuration
22
19
  CLI arguments that override `profiles.yml` settings.
23
20
  """
24
21
  profile_group = parser.add_argument_group("Profile")
@@ -36,14 +33,13 @@ def register_profile_args(parser: argparse.ArgumentParser, pool_add: bool = Fals
36
33
  help="The maximum price per hour, in dollars",
37
34
  dest="max_price",
38
35
  )
39
- if not pool_add:
40
- profile_group.add_argument(
41
- "--max-duration",
42
- type=max_duration,
43
- dest="max_duration",
44
- help="The maximum duration of the run",
45
- metavar="DURATION",
46
- )
36
+ profile_group.add_argument(
37
+ "--max-duration",
38
+ type=max_duration,
39
+ dest="max_duration",
40
+ help="The maximum duration of the run",
41
+ metavar="DURATION",
42
+ )
47
43
  profile_group.add_argument(
48
44
  "-b",
49
45
  "--backend",
@@ -67,42 +63,28 @@ def register_profile_args(parser: argparse.ArgumentParser, pool_add: bool = Fals
67
63
  dest="instance_types",
68
64
  help="The cloud-specific instance types that will be tried for provisioning",
69
65
  )
70
- if pool_add:
71
- pools_group_exc = parser
72
- else:
73
- pools_group = parser.add_argument_group("Pools")
74
- pools_group_exc = pools_group.add_mutually_exclusive_group()
75
- pools_group_exc.add_argument(
76
- "--pool",
77
- dest="pool_name",
78
- help="The name of the pool. If not set, the default pool will be used",
79
- )
80
- pools_group_exc.add_argument(
66
+
67
+ fleets_group = parser.add_argument_group("Fleets")
68
+ fleets_group_exc = fleets_group.add_mutually_exclusive_group()
69
+ fleets_group_exc.add_argument(
81
70
  "-R",
82
71
  "--reuse",
83
72
  dest="creation_policy_reuse",
84
73
  action="store_true",
85
- help="Reuse instance from pool",
74
+ help="Reuse an existing instance from fleet (do not provision a new one)",
86
75
  )
87
- pools_group_exc.add_argument(
76
+ fleets_group_exc.add_argument(
88
77
  "--dont-destroy",
89
78
  dest="dont_destroy",
90
79
  action="store_true",
91
- help="Do not destroy instance after the run is finished",
80
+ help="Do not destroy instance after the run is finished (if the run provisions a new instance)",
92
81
  )
93
- pools_group_exc.add_argument(
82
+ fleets_group_exc.add_argument(
94
83
  "--idle-duration",
95
84
  dest="idle_duration",
96
85
  type=str,
97
- help="Time to wait before destroying the idle instance",
86
+ help="Time to wait before destroying the idle instance (if the run provisions a new instance)",
98
87
  )
99
- if not pool_add:
100
- pools_group_exc.add_argument(
101
- "--instance",
102
- dest="instance_name",
103
- metavar="NAME",
104
- help="Reuse instance from pool with name [code]NAME[/]",
105
- )
106
88
 
107
89
  spot_group = parser.add_argument_group("Spot policy")
108
90
  spot_group_exc = spot_group.add_mutually_exclusive_group()
@@ -137,10 +119,8 @@ def register_profile_args(parser: argparse.ArgumentParser, pool_add: bool = Fals
137
119
 
138
120
  retry_group = parser.add_argument_group("Retry policy")
139
121
  retry_group_exc = retry_group.add_mutually_exclusive_group()
140
- retry_group_exc.add_argument("--retry", action="store_const", dest="retry_policy", const=True)
141
- retry_group_exc.add_argument(
142
- "--no-retry", action="store_const", dest="retry_policy", const=False
143
- )
122
+ retry_group_exc.add_argument("--retry", action="store_const", dest="retry", const=True)
123
+ retry_group_exc.add_argument("--no-retry", action="store_const", dest="retry", const=False)
144
124
  retry_group_exc.add_argument(
145
125
  "--retry-duration", type=retry_duration, dest="retry_duration", metavar="DURATION"
146
126
  )
@@ -149,7 +129,6 @@ def register_profile_args(parser: argparse.ArgumentParser, pool_add: bool = Fals
149
129
  def apply_profile_args(
150
130
  args: argparse.Namespace,
151
131
  profile_settings: Union[Profile, AnyRunConfiguration],
152
- pool_add: bool = False,
153
132
  ):
154
133
  """
155
134
  Overrides `profile_settings` settings with arguments registered by `register_profile_args()`.
@@ -165,53 +144,25 @@ def apply_profile_args(
165
144
  profile_settings.instance_types = args.instance_types
166
145
  if args.max_price is not None:
167
146
  profile_settings.max_price = args.max_price
168
- if not pool_add:
169
- if args.max_duration is not None:
170
- profile_settings.max_duration = args.max_duration
171
-
172
- if args.pool_name:
173
- profile_settings.pool_name = args.pool_name
147
+ if args.max_duration is not None:
148
+ profile_settings.max_duration = args.max_duration
174
149
 
175
150
  if args.idle_duration is not None:
176
- profile_settings.termination_idle_time = args.idle_duration
177
- if pool_add and args.idle_duration is None:
178
- profile_settings.termination_idle_time = DEFAULT_POOL_TERMINATION_IDLE_TIME
179
-
180
- if args.dont_destroy:
181
- profile_settings.termination_policy = TerminationPolicy.DONT_DESTROY
182
- if not pool_add:
183
- if args.instance_name:
184
- profile_settings.instance_name = args.instance_name
185
- if args.creation_policy_reuse:
186
- profile_settings.creation_policy = CreationPolicy.REUSE
151
+ profile_settings.idle_duration = args.idle_duration
152
+ elif args.dont_destroy:
153
+ profile_settings.idle_duration = False
154
+ if args.creation_policy_reuse:
155
+ profile_settings.creation_policy = CreationPolicy.REUSE
187
156
 
188
157
  if args.spot_policy is not None:
189
158
  profile_settings.spot_policy = args.spot_policy
190
- if pool_add and args.spot_policy is None: # ONDEMAND by default for `dstack pool add`
191
- profile_settings.spot_policy = SpotPolicy.ONDEMAND
192
159
 
193
- if not pool_add:
194
- if args.retry_policy is not None:
195
- if not profile_settings.retry_policy:
196
- profile_settings.retry_policy = ProfileRetryPolicy()
197
- profile_settings.retry_policy.retry = args.retry_policy
198
- elif args.retry_duration is not None:
199
- if not profile_settings.retry_policy:
200
- profile_settings.retry_policy = ProfileRetryPolicy()
201
- profile_settings.retry_policy.retry = True
202
- profile_settings.retry_policy.duration = args.retry_duration
203
- else:
204
- if args.retry_policy is not None:
205
- if not profile_settings.retry_policy:
206
- profile_settings.retry_policy = ProfileRetryPolicy()
207
- profile_settings.retry_policy.retry = args.retry_policy
208
- if profile_settings.retry_policy.retry:
209
- profile_settings.retry_policy.duration = DEFAULT_INSTANCE_RETRY_DURATION
210
- elif args.retry_duration is not None:
211
- if not profile_settings.retry_policy:
212
- profile_settings.retry_policy = ProfileRetryPolicy()
213
- profile_settings.retry_policy.retry = True
214
- profile_settings.retry_policy.duration = args.retry_duration # --retry-duration
160
+ if args.retry is not None:
161
+ profile_settings.retry = args.retry
162
+ elif args.retry_duration is not None:
163
+ profile_settings.retry = ProfileRetry(
164
+ duration=args.retry_duration,
165
+ )
215
166
 
216
167
 
217
168
  def max_duration(v: str) -> int:
@@ -34,7 +34,7 @@ def get_gateways_table(
34
34
  for gateway in gateways:
35
35
  row = {
36
36
  "NAME": gateway.name,
37
- "BACKEND": f"{gateway.backend.value} ({gateway.region})",
37
+ "BACKEND": f"{gateway.configuration.backend.value} ({gateway.configuration.region})",
38
38
  "HOSTNAME": gateway.hostname,
39
39
  "DOMAIN": gateway.wildcard_domain,
40
40
  "DEFAULT": "✓" if gateway.default else "",
@@ -1,42 +1,59 @@
1
+ from dstack._internal.core.backends.base.compute import (
2
+ ComputeWithCreateInstanceSupport,
3
+ ComputeWithGatewaySupport,
4
+ ComputeWithMultinodeSupport,
5
+ ComputeWithPlacementGroupSupport,
6
+ ComputeWithPrivateGatewaySupport,
7
+ ComputeWithReservationSupport,
8
+ ComputeWithVolumeSupport,
9
+ )
10
+ from dstack._internal.core.backends.base.configurator import Configurator
11
+ from dstack._internal.core.backends.configurators import list_available_configurator_classes
1
12
  from dstack._internal.core.models.backends.base import BackendType
2
13
 
3
- BACKENDS_WITH_MULTINODE_SUPPORT = [
4
- BackendType.AWS,
5
- BackendType.AZURE,
6
- BackendType.GCP,
7
- BackendType.REMOTE,
8
- BackendType.OCI,
9
- BackendType.VULTR,
10
- ]
11
- BACKENDS_WITH_CREATE_INSTANCE_SUPPORT = [
12
- BackendType.AWS,
13
- BackendType.DSTACK,
14
- BackendType.AZURE,
15
- BackendType.CUDO,
16
- BackendType.DATACRUNCH,
17
- BackendType.GCP,
18
- BackendType.LAMBDA,
19
- BackendType.OCI,
20
- BackendType.TENSORDOCK,
21
- BackendType.VULTR,
22
- ]
23
- BACKENDS_WITH_PLACEMENT_GROUPS_SUPPORT = [
24
- BackendType.AWS,
25
- ]
26
- BACKENDS_WITH_RESERVATION_SUPPORT = [
27
- BackendType.AWS,
28
- ]
29
14
 
30
- BACKENDS_WITH_GATEWAY_SUPPORT = [
31
- BackendType.AWS,
32
- BackendType.AZURE,
33
- BackendType.GCP,
34
- BackendType.KUBERNETES,
35
- ]
36
- BACKENDS_WITH_PRIVATE_GATEWAY_SUPPORT = [BackendType.AWS]
37
- BACKENDS_WITH_VOLUMES_SUPPORT = [
38
- BackendType.AWS,
39
- BackendType.GCP,
40
- BackendType.LOCAL,
41
- BackendType.RUNPOD,
42
- ]
15
+ def _get_backends_with_compute_feature(
16
+ configurator_classes: list[type[Configurator]],
17
+ compute_feature_class: type,
18
+ ) -> list[BackendType]:
19
+ backend_types = []
20
+ for configurator_class in configurator_classes:
21
+ compute_class = configurator_class.BACKEND_CLASS.COMPUTE_CLASS
22
+ if issubclass(compute_class, compute_feature_class):
23
+ backend_types.append(configurator_class.TYPE)
24
+ return backend_types
25
+
26
+
27
+ _configurator_classes = list_available_configurator_classes()
28
+
29
+
30
+ # The following backend lists do not include unavailable backends (i.e. backends missing deps).
31
+ # TODO: Add LocalBackend to lists if it's enabled
32
+ BACKENDS_WITH_CREATE_INSTANCE_SUPPORT = _get_backends_with_compute_feature(
33
+ configurator_classes=_configurator_classes,
34
+ compute_feature_class=ComputeWithCreateInstanceSupport,
35
+ )
36
+ BACKENDS_WITH_MULTINODE_SUPPORT = [BackendType.REMOTE] + _get_backends_with_compute_feature(
37
+ configurator_classes=_configurator_classes,
38
+ compute_feature_class=ComputeWithMultinodeSupport,
39
+ )
40
+ BACKENDS_WITH_PLACEMENT_GROUPS_SUPPORT = _get_backends_with_compute_feature(
41
+ configurator_classes=_configurator_classes,
42
+ compute_feature_class=ComputeWithPlacementGroupSupport,
43
+ )
44
+ BACKENDS_WITH_RESERVATION_SUPPORT = _get_backends_with_compute_feature(
45
+ configurator_classes=_configurator_classes,
46
+ compute_feature_class=ComputeWithReservationSupport,
47
+ )
48
+ BACKENDS_WITH_GATEWAY_SUPPORT = _get_backends_with_compute_feature(
49
+ configurator_classes=_configurator_classes,
50
+ compute_feature_class=ComputeWithGatewaySupport,
51
+ )
52
+ BACKENDS_WITH_PRIVATE_GATEWAY_SUPPORT = _get_backends_with_compute_feature(
53
+ configurator_classes=_configurator_classes,
54
+ compute_feature_class=ComputeWithPrivateGatewaySupport,
55
+ )
56
+ BACKENDS_WITH_VOLUMES_SUPPORT = _get_backends_with_compute_feature(
57
+ configurator_classes=_configurator_classes,
58
+ compute_feature_class=ComputeWithVolumeSupport,
59
+ )
@@ -1,25 +0,0 @@
1
- import botocore.exceptions
2
-
3
- from dstack._internal.core.backends.aws.compute import AWSCompute
4
- from dstack._internal.core.backends.aws.config import AWSConfig
5
- from dstack._internal.core.backends.base import Backend
6
- from dstack._internal.core.errors import BackendInvalidCredentialsError
7
- from dstack._internal.core.models.backends.base import BackendType
8
-
9
-
10
- class AWSBackend(Backend):
11
- TYPE: BackendType = BackendType.AWS
12
-
13
- def __init__(self, config: AWSConfig):
14
- self.config = config
15
- self._compute = AWSCompute(self.config)
16
- self._check_credentials()
17
-
18
- def compute(self) -> AWSCompute:
19
- return self._compute
20
-
21
- def _check_credentials(self):
22
- try:
23
- pass
24
- except (botocore.exceptions.ClientError, botocore.exceptions.NoCredentialsError):
25
- raise BackendInvalidCredentialsError()
@@ -2,8 +2,8 @@ import boto3.session
2
2
  import botocore.exceptions
3
3
  from boto3.session import Session
4
4
 
5
+ from dstack._internal.core.backends.aws.models import AnyAWSCreds, AWSAccessKeyCreds
5
6
  from dstack._internal.core.errors import BackendAuthError
6
- from dstack._internal.core.models.backends.aws import AnyAWSCreds, AWSAccessKeyCreds
7
7
  from dstack._internal.core.models.common import is_core_model_instance
8
8
 
9
9
 
@@ -29,12 +29,3 @@ def validate_credentials(session: Session):
29
29
  sts.get_caller_identity()
30
30
  except (botocore.exceptions.ClientError, botocore.exceptions.NoCredentialsError):
31
31
  raise BackendAuthError()
32
-
33
-
34
- def default_creds_available() -> bool:
35
- session = boto3.session.Session()
36
- try:
37
- validate_credentials(session)
38
- except BackendAuthError:
39
- return False
40
- return True
@@ -0,0 +1,26 @@
1
+ import botocore.exceptions
2
+
3
+ from dstack._internal.core.backends.aws.compute import AWSCompute
4
+ from dstack._internal.core.backends.aws.models import AWSConfig
5
+ from dstack._internal.core.backends.base.backend import Backend
6
+ from dstack._internal.core.errors import BackendInvalidCredentialsError
7
+ from dstack._internal.core.models.backends.base import BackendType
8
+
9
+
10
+ class AWSBackend(Backend):
11
+ TYPE = BackendType.AWS
12
+ COMPUTE_CLASS = AWSCompute
13
+
14
+ def __init__(self, config: AWSConfig):
15
+ self.config = config
16
+ self._compute = AWSCompute(self.config)
17
+ self._check_credentials()
18
+
19
+ def compute(self) -> AWSCompute:
20
+ return self._compute
21
+
22
+ def _check_credentials(self):
23
+ try:
24
+ pass
25
+ except (botocore.exceptions.ClientError, botocore.exceptions.NoCredentialsError):
26
+ raise BackendInvalidCredentialsError()