dstack 0.19.0rc1__py3-none-any.whl → 0.19.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of dstack might be problematic. Click here for more details.

Files changed (37) hide show
  1. dstack/_internal/cli/commands/metrics.py +138 -0
  2. dstack/_internal/cli/commands/stats.py +5 -119
  3. dstack/_internal/cli/main.py +2 -0
  4. dstack/_internal/core/backends/base/compute.py +3 -0
  5. dstack/_internal/core/backends/base/models.py +7 -7
  6. dstack/_internal/core/backends/configurators.py +9 -0
  7. dstack/_internal/core/backends/models.py +8 -0
  8. dstack/_internal/core/backends/nebius/__init__.py +0 -0
  9. dstack/_internal/core/backends/nebius/backend.py +16 -0
  10. dstack/_internal/core/backends/nebius/compute.py +270 -0
  11. dstack/_internal/core/backends/nebius/configurator.py +74 -0
  12. dstack/_internal/core/backends/nebius/models.py +108 -0
  13. dstack/_internal/core/backends/nebius/resources.py +222 -0
  14. dstack/_internal/core/errors.py +14 -0
  15. dstack/_internal/core/models/backends/base.py +2 -0
  16. dstack/_internal/proxy/lib/schemas/model_proxy.py +3 -3
  17. dstack/_internal/server/background/tasks/process_instances.py +26 -12
  18. dstack/_internal/server/routers/prometheus.py +5 -12
  19. dstack/_internal/server/security/permissions.py +19 -1
  20. dstack/_internal/server/services/jobs/configurators/extensions/cursor.py +1 -1
  21. dstack/_internal/server/services/prometheus.py +175 -112
  22. dstack/_internal/server/statics/index.html +1 -1
  23. dstack/_internal/server/statics/{main-4fd5a4770eff59325ee3.js → main-bcb3228138bc8483cc0b.js} +7278 -131
  24. dstack/_internal/server/statics/{main-4fd5a4770eff59325ee3.js.map → main-bcb3228138bc8483cc0b.js.map} +1 -1
  25. dstack/_internal/server/statics/{main-da9f8c06a69c20dac23e.css → main-c0bdaac8f1ea67d499eb.css} +1 -1
  26. dstack/_internal/utils/event_loop.py +30 -0
  27. dstack/version.py +1 -1
  28. {dstack-0.19.0rc1.dist-info → dstack-0.19.2.dist-info}/METADATA +27 -11
  29. {dstack-0.19.0rc1.dist-info → dstack-0.19.2.dist-info}/RECORD +37 -28
  30. tests/_internal/server/background/tasks/test_process_instances.py +68 -2
  31. tests/_internal/server/routers/test_backends.py +116 -0
  32. tests/_internal/server/routers/test_prometheus.py +158 -120
  33. tests/_internal/utils/test_event_loop.py +18 -0
  34. {dstack-0.19.0rc1.dist-info → dstack-0.19.2.dist-info}/LICENSE.md +0 -0
  35. {dstack-0.19.0rc1.dist-info → dstack-0.19.2.dist-info}/WHEEL +0 -0
  36. {dstack-0.19.0rc1.dist-info → dstack-0.19.2.dist-info}/entry_points.txt +0 -0
  37. {dstack-0.19.0rc1.dist-info → dstack-0.19.2.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,30 @@
1
+ import asyncio
2
+ import threading
3
+ from collections.abc import Awaitable
4
+ from typing import TypeVar
5
+
6
+ T = TypeVar("T")
7
+
8
+
9
+ class DaemonEventLoop:
10
+ """
11
+ A wrapper around asyncio.EventLoop that runs the loop in a daemon thread.
12
+ The thread is started with the first `await_` call.
13
+ """
14
+
15
+ def __init__(self) -> None:
16
+ self._loop = asyncio.new_event_loop()
17
+ self._start_lock = threading.Lock()
18
+ self._started = False
19
+
20
+ def await_(self, awaitable: Awaitable[T]) -> T:
21
+ with self._start_lock:
22
+ if not self._started:
23
+ threading.Thread(target=self._loop.run_forever, daemon=True).start()
24
+ self._started = True
25
+ future = asyncio.run_coroutine_threadsafe(_coroutine(awaitable), self._loop)
26
+ return future.result()
27
+
28
+
29
+ async def _coroutine(awaitable: Awaitable[T]) -> T:
30
+ return await awaitable
dstack/version.py CHANGED
@@ -1,3 +1,3 @@
1
- __version__ = "0.19.0rc1"
1
+ __version__ = "0.19.2"
2
2
  __is_release__ = True
3
3
  base_image = "0.7"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dstack
3
- Version: 0.19.0rc1
3
+ Version: 0.19.2
4
4
  Summary: dstack is an open-source orchestration engine for running AI workloads on any cloud or on-premises.
5
5
  Home-page: https://dstack.ai
6
6
  Author: Andrey Cheptsov
@@ -23,6 +23,7 @@ Requires-Dist: typing-extensions>=4.0.0
23
23
  Requires-Dist: cryptography
24
24
  Requires-Dist: packaging
25
25
  Requires-Dist: python-dateutil
26
+ Requires-Dist: cachetools
26
27
  Requires-Dist: gitpython
27
28
  Requires-Dist: jsonschema
28
29
  Requires-Dist: paramiko>=3.2.0
@@ -37,7 +38,7 @@ Requires-Dist: websocket-client
37
38
  Requires-Dist: python-multipart>=0.0.16
38
39
  Requires-Dist: filelock
39
40
  Requires-Dist: psutil
40
- Requires-Dist: gpuhunt<0.2.0,>=0.1.0
41
+ Requires-Dist: gpuhunt<0.2.0,>=0.1.1
41
42
  Requires-Dist: argcomplete>=3.5.0
42
43
  Provides-Extra: all
43
44
  Requires-Dist: fastapi; extra == "all"
@@ -58,7 +59,6 @@ Requires-Dist: python-dxf==12.1.0; extra == "all"
58
59
  Requires-Dist: sentry-sdk[fastapi]; extra == "all"
59
60
  Requires-Dist: alembic-postgresql-enum; extra == "all"
60
61
  Requires-Dist: asyncpg; extra == "all"
61
- Requires-Dist: cachetools; extra == "all"
62
62
  Requires-Dist: python-json-logger>=3.1.0; extra == "all"
63
63
  Requires-Dist: prometheus-client; extra == "all"
64
64
  Requires-Dist: grpcio>=1.50; extra == "all"
@@ -80,6 +80,7 @@ Requires-Dist: google-cloud-tpu>=1.18.3; extra == "all"
80
80
  Requires-Dist: datacrunch; extra == "all"
81
81
  Requires-Dist: kubernetes; extra == "all"
82
82
  Requires-Dist: oci; extra == "all"
83
+ Requires-Dist: nebius<0.3,>=0.2.19; python_version >= "3.10" and extra == "all"
83
84
  Provides-Extra: aws
84
85
  Requires-Dist: fastapi; extra == "aws"
85
86
  Requires-Dist: starlette>=0.26.0; extra == "aws"
@@ -99,7 +100,6 @@ Requires-Dist: python-dxf==12.1.0; extra == "aws"
99
100
  Requires-Dist: sentry-sdk[fastapi]; extra == "aws"
100
101
  Requires-Dist: alembic-postgresql-enum; extra == "aws"
101
102
  Requires-Dist: asyncpg; extra == "aws"
102
- Requires-Dist: cachetools; extra == "aws"
103
103
  Requires-Dist: python-json-logger>=3.1.0; extra == "aws"
104
104
  Requires-Dist: prometheus-client; extra == "aws"
105
105
  Requires-Dist: grpcio>=1.50; extra == "aws"
@@ -124,7 +124,6 @@ Requires-Dist: python-dxf==12.1.0; extra == "azure"
124
124
  Requires-Dist: sentry-sdk[fastapi]; extra == "azure"
125
125
  Requires-Dist: alembic-postgresql-enum; extra == "azure"
126
126
  Requires-Dist: asyncpg; extra == "azure"
127
- Requires-Dist: cachetools; extra == "azure"
128
127
  Requires-Dist: python-json-logger>=3.1.0; extra == "azure"
129
128
  Requires-Dist: prometheus-client; extra == "azure"
130
129
  Requires-Dist: grpcio>=1.50; extra == "azure"
@@ -153,7 +152,6 @@ Requires-Dist: python-dxf==12.1.0; extra == "datacrunch"
153
152
  Requires-Dist: sentry-sdk[fastapi]; extra == "datacrunch"
154
153
  Requires-Dist: alembic-postgresql-enum; extra == "datacrunch"
155
154
  Requires-Dist: asyncpg; extra == "datacrunch"
156
- Requires-Dist: cachetools; extra == "datacrunch"
157
155
  Requires-Dist: python-json-logger>=3.1.0; extra == "datacrunch"
158
156
  Requires-Dist: prometheus-client; extra == "datacrunch"
159
157
  Requires-Dist: grpcio>=1.50; extra == "datacrunch"
@@ -185,7 +183,6 @@ Requires-Dist: python-dxf==12.1.0; extra == "gcp"
185
183
  Requires-Dist: sentry-sdk[fastapi]; extra == "gcp"
186
184
  Requires-Dist: alembic-postgresql-enum; extra == "gcp"
187
185
  Requires-Dist: asyncpg; extra == "gcp"
188
- Requires-Dist: cachetools; extra == "gcp"
189
186
  Requires-Dist: python-json-logger>=3.1.0; extra == "gcp"
190
187
  Requires-Dist: prometheus-client; extra == "gcp"
191
188
  Requires-Dist: grpcio>=1.50; extra == "gcp"
@@ -215,7 +212,6 @@ Requires-Dist: python-dxf==12.1.0; extra == "kubernetes"
215
212
  Requires-Dist: sentry-sdk[fastapi]; extra == "kubernetes"
216
213
  Requires-Dist: alembic-postgresql-enum; extra == "kubernetes"
217
214
  Requires-Dist: asyncpg; extra == "kubernetes"
218
- Requires-Dist: cachetools; extra == "kubernetes"
219
215
  Requires-Dist: python-json-logger>=3.1.0; extra == "kubernetes"
220
216
  Requires-Dist: prometheus-client; extra == "kubernetes"
221
217
  Requires-Dist: grpcio>=1.50; extra == "kubernetes"
@@ -239,12 +235,34 @@ Requires-Dist: python-dxf==12.1.0; extra == "lambda"
239
235
  Requires-Dist: sentry-sdk[fastapi]; extra == "lambda"
240
236
  Requires-Dist: alembic-postgresql-enum; extra == "lambda"
241
237
  Requires-Dist: asyncpg; extra == "lambda"
242
- Requires-Dist: cachetools; extra == "lambda"
243
238
  Requires-Dist: python-json-logger>=3.1.0; extra == "lambda"
244
239
  Requires-Dist: prometheus-client; extra == "lambda"
245
240
  Requires-Dist: grpcio>=1.50; extra == "lambda"
246
241
  Requires-Dist: boto3; extra == "lambda"
247
242
  Requires-Dist: botocore; extra == "lambda"
243
+ Provides-Extra: nebius
244
+ Requires-Dist: fastapi; extra == "nebius"
245
+ Requires-Dist: starlette>=0.26.0; extra == "nebius"
246
+ Requires-Dist: uvicorn; extra == "nebius"
247
+ Requires-Dist: aiorwlock; extra == "nebius"
248
+ Requires-Dist: aiocache; extra == "nebius"
249
+ Requires-Dist: httpx; extra == "nebius"
250
+ Requires-Dist: jinja2; extra == "nebius"
251
+ Requires-Dist: watchfiles; extra == "nebius"
252
+ Requires-Dist: sqlalchemy[asyncio]>=2.0.0; extra == "nebius"
253
+ Requires-Dist: sqlalchemy-utils>=0.40.0; extra == "nebius"
254
+ Requires-Dist: alembic>=1.10.2; extra == "nebius"
255
+ Requires-Dist: apscheduler<4; extra == "nebius"
256
+ Requires-Dist: aiosqlite; extra == "nebius"
257
+ Requires-Dist: docker>=6.0.0; extra == "nebius"
258
+ Requires-Dist: python-dxf==12.1.0; extra == "nebius"
259
+ Requires-Dist: sentry-sdk[fastapi]; extra == "nebius"
260
+ Requires-Dist: alembic-postgresql-enum; extra == "nebius"
261
+ Requires-Dist: asyncpg; extra == "nebius"
262
+ Requires-Dist: python-json-logger>=3.1.0; extra == "nebius"
263
+ Requires-Dist: prometheus-client; extra == "nebius"
264
+ Requires-Dist: grpcio>=1.50; extra == "nebius"
265
+ Requires-Dist: nebius<0.3,>=0.2.19; extra == "nebius"
248
266
  Provides-Extra: oci
249
267
  Requires-Dist: fastapi; extra == "oci"
250
268
  Requires-Dist: starlette>=0.26.0; extra == "oci"
@@ -264,7 +282,6 @@ Requires-Dist: python-dxf==12.1.0; extra == "oci"
264
282
  Requires-Dist: sentry-sdk[fastapi]; extra == "oci"
265
283
  Requires-Dist: alembic-postgresql-enum; extra == "oci"
266
284
  Requires-Dist: asyncpg; extra == "oci"
267
- Requires-Dist: cachetools; extra == "oci"
268
285
  Requires-Dist: python-json-logger>=3.1.0; extra == "oci"
269
286
  Requires-Dist: prometheus-client; extra == "oci"
270
287
  Requires-Dist: grpcio>=1.50; extra == "oci"
@@ -288,7 +305,6 @@ Requires-Dist: python-dxf==12.1.0; extra == "server"
288
305
  Requires-Dist: sentry-sdk[fastapi]; extra == "server"
289
306
  Requires-Dist: alembic-postgresql-enum; extra == "server"
290
307
  Requires-Dist: asyncpg; extra == "server"
291
- Requires-Dist: cachetools; extra == "server"
292
308
  Requires-Dist: python-json-logger>=3.1.0; extra == "server"
293
309
  Requires-Dist: prometheus-client; extra == "server"
294
310
  Requires-Dist: grpcio>=1.50; extra == "server"
@@ -1,10 +1,10 @@
1
1
  dstack/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
2
- dstack/version.py,sha256=-Q9COxDViZow52OSN2VafeyzPtL8-5rmrIp9kMGNcuc,67
2
+ dstack/version.py,sha256=6z601r_k8B66H4FWmrWo34Dj29lhhWy4y-3_XVO3_V8,64
3
3
  dstack/_internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
4
4
  dstack/_internal/compat.py,sha256=bF9U9fTMfL8UVhCouedoUSTYFl7UAOiU0WXrnRoByxw,40
5
5
  dstack/_internal/settings.py,sha256=8XODoSW2joaEndvZxuHUPSFK85sGgJ7fVL976isYeJM,557
6
6
  dstack/_internal/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
7
- dstack/_internal/cli/main.py,sha256=yaEEdjzonW18IULYqu9-oAEbusRSgjmFiHP6a-FgVHA,3489
7
+ dstack/_internal/cli/main.py,sha256=tVSDJxBI4qJhsDePgPkkJCEB-e98j7HSWzvcSfA64G8,3594
8
8
  dstack/_internal/cli/commands/__init__.py,sha256=e8EjD8Ej7Q3_APlxAN7NBwY7dGnW-DR91nH_RgcjBXg,2023
9
9
  dstack/_internal/cli/commands/apply.py,sha256=dZH2B3Yr-lB7hs-QeUnvX0FV0Yk2x19dEO5uSFo4098,4909
10
10
  dstack/_internal/cli/commands/attach.py,sha256=resmqoqkIOptp9Vn4BMs8m42Txl4mPzFDhHcv_lkWJk,5033
@@ -15,9 +15,10 @@ dstack/_internal/cli/commands/fleet.py,sha256=J0Yi5CAYvIQF_NC4ZBsDYGc3grTOtvaF7s
15
15
  dstack/_internal/cli/commands/gateway.py,sha256=DcD6P_MvXbSL9aXkLX9hgGYSAzARjgY6RSbrCMzdNcg,6075
16
16
  dstack/_internal/cli/commands/init.py,sha256=bLhSlViNWtjflB6xNq_PuCR2o2A06h222luh1NeUgVA,1169
17
17
  dstack/_internal/cli/commands/logs.py,sha256=o8ehPAKM12Xn9thg2jjnYdr7_wKqF-00ziVry8IVVwE,1528
18
+ dstack/_internal/cli/commands/metrics.py,sha256=nxdTcyCvHNev7Mn19zGQ7vTHUBGY6hyvPtY6Z731SOU,5373
18
19
  dstack/_internal/cli/commands/ps.py,sha256=WUQKL-1HcoM-oFF_WX4zplO8yKGMWfVbsivg0L7LP2w,1666
19
20
  dstack/_internal/cli/commands/server.py,sha256=dCEdmVp1OlwFnSGw9k8mBb26wZy1D_ctgn5pmK8ZrCQ,2802
20
- dstack/_internal/cli/commands/stats.py,sha256=9EyGw60Oc_KkPLHz-IN1sIFZ4AJUadfsU0PNXQ7Kvqg,5031
21
+ dstack/_internal/cli/commands/stats.py,sha256=_CiJPQZuKHUzA1x7CHJP-leth3MmqiQ2-jwojq6G-e8,390
21
22
  dstack/_internal/cli/commands/stop.py,sha256=i7TreejSemrQcL7C37La22KkwM98O09IDMRNt8cw1NI,1000
22
23
  dstack/_internal/cli/commands/volume.py,sha256=h0bQQm6zx1Bq0kdBX5ksqPZk55wz-md_Kitm0RTDReg,3214
23
24
  dstack/_internal/cli/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -41,10 +42,10 @@ dstack/_internal/cli/utils/updates.py,sha256=9KQcm_TEE_PiU0yc5WH4xRse0WIeHOubi9E
41
42
  dstack/_internal/cli/utils/volume.py,sha256=mU9I06dVMFbpjfkefxrZNoSWadKLoib3U14rHudNQN4,1975
42
43
  dstack/_internal/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
43
44
  dstack/_internal/core/consts.py,sha256=c1Yd5UY6Qx7KeuYgloXWncWhMsYj6TqwlElda7NtB98,254
44
- dstack/_internal/core/errors.py,sha256=UvqMIBVHG1NIUP0w0g5dWjnxOz37sfFi4lh4yuYK0vw,2793
45
+ dstack/_internal/core/errors.py,sha256=elI7vcTXJvCpg_TOcpar5nBTKiQ71vZMEly6aM_N0jg,3212
45
46
  dstack/_internal/core/backends/__init__.py,sha256=fwgV8CN8Ap6MZmWklMGHHf0roliBtqne-ijjVOpWcgc,2467
46
- dstack/_internal/core/backends/configurators.py,sha256=EFzzlP6R4J82WkKpltGWG4HPmg5Iqw8Nvg3wuCxskxY,3583
47
- dstack/_internal/core/backends/models.py,sha256=YoVHk-dcFMcrbOE9dxtUTYO3ZJlpRy3MBND2KB6u2aA,3833
47
+ dstack/_internal/core/backends/configurators.py,sha256=JxGfZwcmL90akMFkAzzZ_fzPvU2No0pBaBqU_g0D-y0,3775
48
+ dstack/_internal/core/backends/models.py,sha256=aKQOrDEStouuwY4MacSen7SkoyAa6HR6a6PFq5-cbNk,4088
48
49
  dstack/_internal/core/backends/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
49
50
  dstack/_internal/core/backends/aws/auth.py,sha256=TzjRq01exWToWUf9BjT_58zB7L_wnNZduTdVlPmUfxg,1061
50
51
  dstack/_internal/core/backends/aws/backend.py,sha256=pjROH-S9pgrSMm-Eox_ocL7cTU6mIMRxvURq7Vi-2J8,876
@@ -62,9 +63,9 @@ dstack/_internal/core/backends/azure/resources.py,sha256=keE3ruSSNWhSYMCkG7832TY
62
63
  dstack/_internal/core/backends/azure/utils.py,sha256=taHMJq6UHRzUXLUcO2P5VCKy3wJaye2bG-6QdkEPNdY,1741
63
64
  dstack/_internal/core/backends/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
64
65
  dstack/_internal/core/backends/base/backend.py,sha256=hdFMHED1RMV9GVfLSU0yGhGE-cXlbLvt1coDA885PMM,505
65
- dstack/_internal/core/backends/base/compute.py,sha256=pWO-28QT-soRATGUcdWoGbNff6VuMJLyJ5_gmJ_SG1U,23696
66
+ dstack/_internal/core/backends/base/compute.py,sha256=d34JDZDOGoHKDFpjtJsMpB9O0Akq_CwfHaQsPnqJIB4,23866
66
67
  dstack/_internal/core/backends/base/configurator.py,sha256=OCv8N2oxcxy3In2zS1PKiCJ0a-COZwxGjBz2FYkQnfg,3807
67
- dstack/_internal/core/backends/base/models.py,sha256=B8CscQIEttg9xySzaQ5h3VIczjEFxfh4tKTlL0JVwUo,388
68
+ dstack/_internal/core/backends/base/models.py,sha256=Ij0osOl-T-ABsKLoVg2eY81DMkwdWkevAnjXj2QnLXI,532
68
69
  dstack/_internal/core/backends/base/offers.py,sha256=89H6g6Dw5WvXDEBnKoZUF-fGIE3Clc8T4lEjZ_jr71A,5909
69
70
  dstack/_internal/core/backends/cudo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
70
71
  dstack/_internal/core/backends/cudo/api_client.py,sha256=ygq1Gx7ZvwKaifdXtvzDSw4xR4ZH6UWd5J47BjuaGh0,3685
@@ -102,6 +103,12 @@ dstack/_internal/core/backends/lambdalabs/models.py,sha256=c9dAYd0wjun2dvMLUZO4D
102
103
  dstack/_internal/core/backends/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
103
104
  dstack/_internal/core/backends/local/backend.py,sha256=KJuNXUXrg60NhLywnExD1EXH2gK0TL-kzNWUPRNZ_Zg,427
104
105
  dstack/_internal/core/backends/local/compute.py,sha256=OQub3qFQdWGm8Laceh-TU4KPRzJwQxRgf_tMwmXFIQk,3613
106
+ dstack/_internal/core/backends/nebius/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
107
+ dstack/_internal/core/backends/nebius/backend.py,sha256=2XqZIbSR8VzlfOnuVklXlDxNmwAkQj7txQN8VXF1j2E,566
108
+ dstack/_internal/core/backends/nebius/compute.py,sha256=ffmEsx10too4ZT2WvoBJMteHRieuOJOXxggvlzo7fpc,10999
109
+ dstack/_internal/core/backends/nebius/configurator.py,sha256=qG7n2ZpCcy7E0F9yqJlGo6yCu29Y8vT-y61fc1EGbKo,2747
110
+ dstack/_internal/core/backends/nebius/models.py,sha256=5-JZDOv3k0BZ3do6F5MBoRuBnpOq54Ulh_am1KfxxJA,3535
111
+ dstack/_internal/core/backends/nebius/resources.py,sha256=SB4-BFWGGlPaPLS70pmJK2faHdTc8kRnWpqcKCVxPSk,7209
105
112
  dstack/_internal/core/backends/oci/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
106
113
  dstack/_internal/core/backends/oci/auth.py,sha256=ErDfuAvmrKlFg5OCYz5BhtlGHQmSiupl9Uaxx3W-wTE,800
107
114
  dstack/_internal/core/backends/oci/backend.py,sha256=yXjVCt7n6BVLH0byYFbNFf-P9J0FwlNfxsYbKGMdoI4,536
@@ -160,7 +167,7 @@ dstack/_internal/core/models/unix.py,sha256=KxnSQELnkAjjuUgYcQKVkf-UAbYREBD8WCWD
160
167
  dstack/_internal/core/models/users.py,sha256=o_rd0GAmd6jufypVUs9P12NRri3rgAPDt-KxnqNNsGw,703
161
168
  dstack/_internal/core/models/volumes.py,sha256=EQmfTnB3oN6zbLwMqwda0X1T7oKLRRt_h5lTXvQ-siQ,6096
162
169
  dstack/_internal/core/models/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
163
- dstack/_internal/core/models/backends/base.py,sha256=nayexPe9kU0FytYI6LdY8aKaTHRG07der7Ia0nlKc9c,1023
170
+ dstack/_internal/core/models/backends/base.py,sha256=Fbhs90NBMceXhjORwj7c2_pVriapCGSUMBFwuR-wRIg,1091
164
171
  dstack/_internal/core/models/repos/__init__.py,sha256=7Qo1QgJ852LklUuM-mlCNFodp_XrQ4iqV7uRPiX_qm0,885
165
172
  dstack/_internal/core/models/repos/base.py,sha256=nErlSR3AbG9fwDE5vuJK5DIrP8JB3fG8a7mXqosJ9gY,846
166
173
  dstack/_internal/core/models/repos/local.py,sha256=DmW_e6qOWctZwuEpCqME-JpxYHTgH5I1gcK7zjSMfao,2394
@@ -218,7 +225,7 @@ dstack/_internal/proxy/lib/repo.py,sha256=zkWZ9XZzQHfCa-eifec7H7UYnJZLgeRuiQls7R
218
225
  dstack/_internal/proxy/lib/routers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
219
226
  dstack/_internal/proxy/lib/routers/model_proxy.py,sha256=57GFRpVRXcVY-347HnUSUr4w4RsxsjLuuZiJs8DwDpM,3895
220
227
  dstack/_internal/proxy/lib/schemas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
221
- dstack/_internal/proxy/lib/schemas/model_proxy.py,sha256=a7QbgPE6QwFPlB890OQO7hRu5I6dAArsVJ4k5euFaC4,1851
228
+ dstack/_internal/proxy/lib/schemas/model_proxy.py,sha256=a-LT18-SrVVriF3YsYPUvil8e_9rLDiOTxfZy8hGsqU,1895
222
229
  dstack/_internal/proxy/lib/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
223
230
  dstack/_internal/proxy/lib/services/service_connection.py,sha256=Gf_ucng9dSKEfpi2kvxZekDSWisY2by6O7_PNZcQi1Q,5924
224
231
  dstack/_internal/proxy/lib/services/model_proxy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -242,7 +249,7 @@ dstack/_internal/server/background/tasks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5
242
249
  dstack/_internal/server/background/tasks/common.py,sha256=N7xSXbf2MoBWgbJ1e3AEzYBTf1Gn-pDXYND8Zr_YCJQ,970
243
250
  dstack/_internal/server/background/tasks/process_fleets.py,sha256=lKXUvN_b7DNjD3psHzyCt_JYsTxPFuQ86iXi8fj8GkM,3202
244
251
  dstack/_internal/server/background/tasks/process_gateways.py,sha256=hoUI1CSqbHt_uMwnzTRAEDl-LBw0wUk_W4xobIbdvRc,7017
245
- dstack/_internal/server/background/tasks/process_instances.py,sha256=TvpMnK211tXcOPAMF3qngGH9DycwKJeEchoYOtKICrg,37284
252
+ dstack/_internal/server/background/tasks/process_instances.py,sha256=De-nbzvvwiO_xtI16tRsosvQXAC3uzjynHAeWpcX51M,37970
246
253
  dstack/_internal/server/background/tasks/process_metrics.py,sha256=acySfsacpYbTPV9Yivs-oU37z1S2sUdWhRHdJkfBcCA,5332
247
254
  dstack/_internal/server/background/tasks/process_placement_groups.py,sha256=FqGfbzvfILdnPUfxjFPAM1ij2xd2mCDi8qufiBcUMI8,4107
248
255
  dstack/_internal/server/background/tasks/process_prometheus_metrics.py,sha256=u8hCXjOOek7VLEsmLy2VnDXFmIwTNjrJwcpWG7a1zW0,5093
@@ -325,7 +332,7 @@ dstack/_internal/server/routers/instances.py,sha256=XOogTC9My2Zv0ck37_PbHKoZI-j4
325
332
  dstack/_internal/server/routers/logs.py,sha256=_Euk283LbhlwHibJTKM-7YcpbeQFtWBqMfbOry3PSkU,1159
326
333
  dstack/_internal/server/routers/metrics.py,sha256=VFgWhkOvxVFDLlRM_kXHYFylLcfCD6UjXInvcd7H4dY,2314
327
334
  dstack/_internal/server/routers/projects.py,sha256=0R-w_6WXUbNo6fREAexFUQ3RoOJF2D_Iz35elKjym14,2717
328
- dstack/_internal/server/routers/prometheus.py,sha256=CIz1GoCVnSAevWhTPnbIJKUO-ntz8tvK6Q_2vicDLoo,1246
335
+ dstack/_internal/server/routers/prometheus.py,sha256=UAcOE8dpGZe4Wd0EOIlFPMbjaNjzX8A7iHlooeRvsfo,944
329
336
  dstack/_internal/server/routers/repos.py,sha256=P_zLoEQderxhCeHQJwRkrIhVcc0-cpabfyde22bWVRk,3362
330
337
  dstack/_internal/server/routers/runs.py,sha256=oPqyIRPwkMjj12M1IdMF2UitatqvljISAXnJAjfEJyQ,5352
331
338
  dstack/_internal/server/routers/secrets.py,sha256=50_qJCTYRpnGSlLyS93gqoV17wWewOVmM65PcG1bT_Y,856
@@ -347,7 +354,7 @@ dstack/_internal/server/schemas/secrets.py,sha256=mfqLSM7PqxVQ-GIWB6RfPRUOvSvvaR
347
354
  dstack/_internal/server/schemas/users.py,sha256=FuDqwRVe3mOmv497vOZKjI0a_d4Wt2g4ZiCJcyfHEKA,495
348
355
  dstack/_internal/server/schemas/volumes.py,sha256=9iwaQLMhA6aj9XmtdU_9jWVhpzNOtFbDByAe-WYfYh0,673
349
356
  dstack/_internal/server/security/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
350
- dstack/_internal/server/security/permissions.py,sha256=ZIUIP5HGk1KG7vGfpZCxP4LsMuKHAz64iOUsAC7FebY,4425
357
+ dstack/_internal/server/security/permissions.py,sha256=FJ_8YPhjmebA4jQjtQoAGEaj1Hahb_po0tYRCQ18aaE,4940
351
358
  dstack/_internal/server/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
352
359
  dstack/_internal/server/services/config.py,sha256=X61ypyuBQqAuWi-mVFJ7Uk1DyGfHqk8_d2C6xIG0yxE,10434
353
360
  dstack/_internal/server/services/docker.py,sha256=3EcYPiVsrNBGDQYOb60QJ241mzTT6lJROYQXIwt-8dk,5351
@@ -360,7 +367,7 @@ dstack/_internal/server/services/offers.py,sha256=tTld2ZcYdbhzShtMIf1YfTyIADtpN3
360
367
  dstack/_internal/server/services/permissions.py,sha256=l7Ngdelmn65vjw13NcOdaC6lBYMRuSw6FbHzYwdK3nE,1005
361
368
  dstack/_internal/server/services/placement.py,sha256=DWZ8-iAE3o0J0xaHikuJYZzpuBiq7lj41LiAP1PfoEs,1773
362
369
  dstack/_internal/server/services/projects.py,sha256=Y4LEkSvOVUHHP-F2qlrwBR7rFu0CFFhbHmDTKrrNuXE,15071
363
- dstack/_internal/server/services/prometheus.py,sha256=1hXSsML-xIiQHv2DKVuJTfK0FRAlseF3GESUQUvy3NE,9625
370
+ dstack/_internal/server/services/prometheus.py,sha256=xq5G-Q2BJup9lS2F6__0wUVTs-k1Gr3dYclGzo2WoWo,12474
364
371
  dstack/_internal/server/services/repos.py,sha256=f9ztN7jz_2gvD9hXF5sJwWDVyG2-NHRfjIdSukowPh8,9342
365
372
  dstack/_internal/server/services/runs.py,sha256=B2jZtTOxavUHr6WqKMXqgLzB3xWsHTkWKykcvcT2lXI,37245
366
373
  dstack/_internal/server/services/storage.py,sha256=6I0xI_3_RpJNbKZwHjDnjrEwXGdHfiaeb5li15T-M1I,1884
@@ -385,7 +392,7 @@ dstack/_internal/server/services/jobs/configurators/service.py,sha256=FOWrLE-6YF
385
392
  dstack/_internal/server/services/jobs/configurators/task.py,sha256=0-B3oO-61Eq4-mmlLmqJPliFKHhvvIV0tqc12slcQuA,1436
386
393
  dstack/_internal/server/services/jobs/configurators/extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
387
394
  dstack/_internal/server/services/jobs/configurators/extensions/base.py,sha256=xJbHxaaSJ1zjn8zuuApP1Xt2uBaedPhhc-IY0NtDDJQ,418
388
- dstack/_internal/server/services/jobs/configurators/extensions/cursor.py,sha256=r2siUu__sz86xDR_oK2xODmmzef1FzOS3veekGTS2-s,1645
395
+ dstack/_internal/server/services/jobs/configurators/extensions/cursor.py,sha256=00HB1kC_eMlioEW0nZI7Ly78b-RSehySVNhC9pajBP8,1636
389
396
  dstack/_internal/server/services/jobs/configurators/extensions/vscode.py,sha256=DAj8OEVLyL1x8Jko2EXKhnAkcSnlO1sJk6o6eiiVkDI,1611
390
397
  dstack/_internal/server/services/logs/__init__.py,sha256=NAjO1KeYvuDznN2EkfAaJt9S6Y00fo_dl3ob3WmsdGQ,3088
391
398
  dstack/_internal/server/services/logs/aws.py,sha256=949k8t9H9v_-aedDjDWkw8yPVyhZemmsszcDDEL5Tb4,13711
@@ -431,10 +438,10 @@ dstack/_internal/server/statics/e467d7d60aae81ab198b.svg,sha256=_XHc9mfQZgGkcy4h
431
438
  dstack/_internal/server/statics/eb9b344b73818fe2b71a.png,sha256=2H14eOCQRyZhFGJ1Kn2LH1j70kTF1Qop4vH-tiKqyPI,85
432
439
  dstack/_internal/server/statics/f517dd626eb964120de0.png,sha256=4QQuNa8SqmcZ67HK6739OHCyjnAJseU1bkcn454KRQs,159
433
440
  dstack/_internal/server/statics/f958aecddee5d8e3222c.png,sha256=8CoZkVNgRfOAe62X1dU-AZDvwh_nESKaQblEmaX2Xrs,87
434
- dstack/_internal/server/statics/index.html,sha256=JPOmBF44MDRAYRPj8eT7Dm4ZnwVLXrRyq32Qku5-fPo,10468
435
- dstack/_internal/server/statics/main-4fd5a4770eff59325ee3.js,sha256=kt1vdr7aEynUtIrrzu9gVEde2kxBMmv4LsTaZDIUNV8,6201881
436
- dstack/_internal/server/statics/main-4fd5a4770eff59325ee3.js.map,sha256=BAYKYPDgLlBT5KAU8L8SS8LYOuU7rDPIL3PpigxAgNo,8139109
437
- dstack/_internal/server/statics/main-da9f8c06a69c20dac23e.css,sha256=2ObS4Rg6yWkbk7nQqw-lnLVdYC3L8X5N7xZMKcavMWg,1296312
441
+ dstack/_internal/server/statics/index.html,sha256=XenYZSciBY6Ysgsf-stlogOrfpZ4YCoVOLLvTNqokI4,10468
442
+ dstack/_internal/server/statics/main-bcb3228138bc8483cc0b.js,sha256=klLuC7jhEGhx8bi67rpdwGxmazLND-Fy18QOFaJxj1I,6517024
443
+ dstack/_internal/server/statics/main-bcb3228138bc8483cc0b.js.map,sha256=pE7tRB7ifhSCy27H9D5pjP2pw_uQfPpFXkbrIGsV0tw,8553628
444
+ dstack/_internal/server/statics/main-c0bdaac8f1ea67d499eb.css,sha256=3tz-eO3jCGYZ1qRY3f8x8lre_NMHNWSFxp4w1g1py3E,1336523
438
445
  dstack/_internal/server/statics/manifest.json,sha256=430w2BoWVmYYVr14lDvUxx-ROPt3VjigzeMqfLeiSCM,340
439
446
  dstack/_internal/server/statics/robots.txt,sha256=kNJLw79pisHhc3OVAimMzKcq3x9WT6sF9IS4xI0crdI,67
440
447
  dstack/_internal/server/statics/assets/android-chrome-144x144.png,sha256=tB3V-95O-VVEoawN5V1XFoMQRSK0I6gthraV8bATGaw,23414
@@ -523,6 +530,7 @@ dstack/_internal/utils/common.py,sha256=c4qqqBqwhh44EVK002bq3PuvwtQUS77b30Ka8GZn
523
530
  dstack/_internal/utils/crypto.py,sha256=2RTSyzePuwwqc1X2HO6lwcSFyZ2kujnqluoICQ2DLJQ,1462
524
531
  dstack/_internal/utils/dxf.py,sha256=wguK9s6-69kqSDZkxd1kFEr6VlH5ixvFRJxizyOuJ8I,3229
525
532
  dstack/_internal/utils/env.py,sha256=HRbIspHpKHh05fMZeV23-hrZoV6vVMuniefD08u6ey0,357
533
+ dstack/_internal/utils/event_loop.py,sha256=DO2ADtWfH2z8F2hBbg_EADSWzleQYGVZ9D1XYDpH-tk,880
526
534
  dstack/_internal/utils/gpu.py,sha256=ZeWpy1nRLVh-FwBZdxbMoVjjCF0DWJlWfNoVgFhGx2w,1776
527
535
  dstack/_internal/utils/hash.py,sha256=mCERRtj9QwbpoP3vveBqbniSJiNMHG0vPSzp4fxmKv0,920
528
536
  dstack/_internal/utils/ignore.py,sha256=kC2dI1aKudFGJWPLnJX34eLBl4u9TOqw0mSElY-bIRM,3053
@@ -639,7 +647,7 @@ tests/_internal/server/background/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeR
639
647
  tests/_internal/server/background/tasks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
640
648
  tests/_internal/server/background/tasks/test_process_fleets.py,sha256=Dl31_TwxoCzYqkVNPWGLsYxmGL2sZfEK3rQXLFyPIz8,2701
641
649
  tests/_internal/server/background/tasks/test_process_gateways.py,sha256=lOP4jPXDtadAgYp0aFND_fp5R_X19M58CaOlgnDAEck,5085
642
- tests/_internal/server/background/tasks/test_process_instances.py,sha256=b7Shke7p3FLPT-qTVDMnwuCK4KqR4kLw9UFfsr0TSVg,24549
650
+ tests/_internal/server/background/tasks/test_process_instances.py,sha256=zRUlcD04l_wYpM5r3PnODYxE95ngu1_RD5KJRTTr-io,27998
643
651
  tests/_internal/server/background/tasks/test_process_metrics.py,sha256=z-u4HXJE5EMVH9kwU_POHmvp55ldAvuLpEMkaebBtsg,4976
644
652
  tests/_internal/server/background/tasks/test_process_placement_groups.py,sha256=19LYbIMZIIeKAN0b9KOMyS-cHUx0FoOojqQuM8Oeiq4,1620
645
653
  tests/_internal/server/background/tasks/test_process_prometheus_metrics.py,sha256=I9DgIJXVGS7UvbFgm4HFnzWiCICBpy72NjDPKU_7WII,7178
@@ -649,14 +657,14 @@ tests/_internal/server/background/tasks/test_process_submitted_jobs.py,sha256=DC
649
657
  tests/_internal/server/background/tasks/test_process_submitted_volumes.py,sha256=pM7PwApOEDP0ogan91_a5ceF0foHQGCNKE9wlTJy9BA,2263
650
658
  tests/_internal/server/background/tasks/test_process_terminating_jobs.py,sha256=RWXc2MXp2n8Mte1URlkJfgxUZKDnTQntuWtJ5JdyTBQ,14053
651
659
  tests/_internal/server/routers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
652
- tests/_internal/server/routers/test_backends.py,sha256=xYZEBeISlS5mctGnzcq57QBSu08zFjcQGBxW8rnmcXw,33661
660
+ tests/_internal/server/routers/test_backends.py,sha256=LhD2PNnz4q2HcBzMkvwtyh9DcXuxOaoUd5pwKTNoAWU,39238
653
661
  tests/_internal/server/routers/test_fleets.py,sha256=1VIX-52lWbGnQ0m22KzHYOhLBioS7YO-H_pK7IkhVlA,34079
654
662
  tests/_internal/server/routers/test_gateways.py,sha256=59r93kdVIuyMHGPNTMCKUdIDpGJADjNs0vEV3xprxTw,24205
655
663
  tests/_internal/server/routers/test_instances.py,sha256=78HFMU9Xel8BNZL3TqnuvrKEAhqULOlnSpd1Ja3oXkU,9188
656
664
  tests/_internal/server/routers/test_logs.py,sha256=NZwyJlgjMOGq4XEx7-VDjTpniYPhZpsbZvB0dTawaog,3989
657
665
  tests/_internal/server/routers/test_metrics.py,sha256=xMdDFZW73Zl06QfggjatfwTut37s0soeliJivkCgBks,7620
658
666
  tests/_internal/server/routers/test_projects.py,sha256=Z3Ok7onAjUYS4ADvKvN-SwSxYKvlvf4MG5Y8baqQU14,25964
659
- tests/_internal/server/routers/test_prometheus.py,sha256=L3qxaJQzir2nJbFtRHZMC7Pw6xWBFcv4-0buQUSdek8,18933
667
+ tests/_internal/server/routers/test_prometheus.py,sha256=iYpgSVkV2yTSKV306rhAffuWGyQV3l2i88crwHIQWYo,23696
660
668
  tests/_internal/server/routers/test_repos.py,sha256=G4dKuFGd_UrxAHwh_XLl1xCHK_DCsiJcXBsHODw3yJk,16682
661
669
  tests/_internal/server/routers/test_runs.py,sha256=q02oBrUcp4JoJOL68jbxlfFxH9B8JO9Bkb7v_Qg-Aug,62984
662
670
  tests/_internal/server/routers/test_server.py,sha256=ROkuRNNJEkMQuK8guZ3Qy3iRRfiWvPIJJJDc09BI0D4,489
@@ -694,6 +702,7 @@ tests/_internal/server/utils/test_routers.py,sha256=vhaTGaZ8r3gbwM3AQlgvy4tFgzjZ
694
702
  tests/_internal/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
695
703
  tests/_internal/utils/test_common.py,sha256=Wyw2QzZuaznXarLAyq3QyEsXvXDQrVW1uXIZms1XI0U,6098
696
704
  tests/_internal/utils/test_env.py,sha256=MKg6gHw9JGROqg8Lovslr_BPfNol_eIVVrJSBcLCRJY,1109
705
+ tests/_internal/utils/test_event_loop.py,sha256=MpkFb8_rqS25ulfzZDw3RyR4LIMGrqPGzXHxeBFA1kQ,426
697
706
  tests/_internal/utils/test_gpu.py,sha256=vTyyWXRgZzil5RhzzA2tWW5owH_jgXXJBtW1Dx9rswo,2051
698
707
  tests/_internal/utils/test_interpolator.py,sha256=hjovKQhOkSdBFQyBXfnYG3kVJDG6x_jSiWizAxFfifs,1844
699
708
  tests/_internal/utils/test_network.py,sha256=4nxej28JgoDWfZ-uwqh5HtUKtCQCmEspUGQ8O5g6ixU,968
@@ -701,9 +710,9 @@ tests/_internal/utils/test_path.py,sha256=rzS-1YCxsFUocBe42dghLOMFNymPruGrA7bqFZ
701
710
  tests/_internal/utils/test_ssh.py,sha256=V-cBFPhD--9eM9d1uQQgpj2gnYLA3c43f4cX9uJ6E-U,1743
702
711
  tests/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
703
712
  tests/api/test_utils.py,sha256=SSSqHcNE5cZVqDq4n2sKZthRoXaZ_Bx7z1AAN5xTM9s,391
704
- dstack-0.19.0rc1.dist-info/LICENSE.md,sha256=qDABaRGjSKVOib1U8viw2P_96sIK7Puo426784oD9f8,15976
705
- dstack-0.19.0rc1.dist-info/METADATA,sha256=TB4oTDpqjbz5UruEwcMWcwY_0cQZl4kNZuhwZlS30Ww,18234
706
- dstack-0.19.0rc1.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
707
- dstack-0.19.0rc1.dist-info/entry_points.txt,sha256=GnLrMS8hx3rWAySQjA7tPNhtixV6a-brRkmal1PKoHc,58
708
- dstack-0.19.0rc1.dist-info/top_level.txt,sha256=3BrIO1zrqxT9P20ymhRM6k15meZXzbPL6ykBlDZG2_k,13
709
- dstack-0.19.0rc1.dist-info/RECORD,,
713
+ dstack-0.19.2.dist-info/LICENSE.md,sha256=qDABaRGjSKVOib1U8viw2P_96sIK7Puo426784oD9f8,15976
714
+ dstack-0.19.2.dist-info/METADATA,sha256=y1cv3qjZHTcWjObJuh0n5gqemdq0_d5AZXZ63t-G7kI,19039
715
+ dstack-0.19.2.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
716
+ dstack-0.19.2.dist-info/entry_points.txt,sha256=GnLrMS8hx3rWAySQjA7tPNhtixV6a-brRkmal1PKoHc,58
717
+ dstack-0.19.2.dist-info/top_level.txt,sha256=3BrIO1zrqxT9P20ymhRM6k15meZXzbPL6ykBlDZG2_k,13
718
+ dstack-0.19.2.dist-info/RECORD,,
@@ -8,7 +8,7 @@ import pytest
8
8
  from freezegun import freeze_time
9
9
  from sqlalchemy.ext.asyncio import AsyncSession
10
10
 
11
- from dstack._internal.core.errors import BackendError
11
+ from dstack._internal.core.errors import BackendError, NotYetTerminated, ProvisioningError
12
12
  from dstack._internal.core.models.backends.base import BackendType
13
13
  from dstack._internal.core.models.instances import (
14
14
  Gpu,
@@ -35,6 +35,8 @@ from dstack._internal.server.testing.common import (
35
35
  create_repo,
36
36
  create_run,
37
37
  create_user,
38
+ get_instance_offer_with_availability,
39
+ get_job_provisioning_data,
38
40
  get_remote_connection_info,
39
41
  )
40
42
  from dstack._internal.utils.common import get_current_datetime
@@ -382,7 +384,9 @@ class TestTerminate:
382
384
 
383
385
  @pytest.mark.asyncio
384
386
  @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
385
- @pytest.mark.parametrize("error", [BackendError("err"), RuntimeError("err")])
387
+ @pytest.mark.parametrize(
388
+ "error", [BackendError("err"), RuntimeError("err"), NotYetTerminated("")]
389
+ )
386
390
  async def test_terminate_retry(self, test_db, session: AsyncSession, error: Exception):
387
391
  project = await create_project(session=session)
388
392
  instance = await create_instance(
@@ -557,6 +561,68 @@ class TestCreateInstance:
557
561
  assert instance.total_blocks == expected_blocks
558
562
  assert instance.busy_blocks == 0
559
563
 
564
+ @pytest.mark.parametrize("err", [RuntimeError("Unexpected"), ProvisioningError("Expected")])
565
+ async def test_tries_second_offer_if_first_fails(self, session: AsyncSession, err: Exception):
566
+ project = await create_project(session=session)
567
+ instance = await create_instance(
568
+ session=session, project=project, status=InstanceStatus.PENDING
569
+ )
570
+ aws_mock = Mock()
571
+ aws_mock.TYPE = BackendType.AWS
572
+ offer = get_instance_offer_with_availability(backend=BackendType.AWS, price=1.0)
573
+ aws_mock.compute.return_value = Mock(spec=ComputeMockSpec)
574
+ aws_mock.compute.return_value.get_offers_cached.return_value = [offer]
575
+ aws_mock.compute.return_value.create_instance.side_effect = err
576
+ gcp_mock = Mock()
577
+ gcp_mock.TYPE = BackendType.GCP
578
+ offer = get_instance_offer_with_availability(backend=BackendType.GCP, price=2.0)
579
+ gcp_mock.compute.return_value = Mock(spec=ComputeMockSpec)
580
+ gcp_mock.compute.return_value.get_offers_cached.return_value = [offer]
581
+ gcp_mock.compute.return_value.create_instance.return_value = get_job_provisioning_data(
582
+ backend=offer.backend, region=offer.region, price=offer.price
583
+ )
584
+ with patch("dstack._internal.server.services.backends.get_project_backends") as m:
585
+ m.return_value = [aws_mock, gcp_mock]
586
+ await process_instances()
587
+
588
+ await session.refresh(instance)
589
+ assert instance.status == InstanceStatus.PROVISIONING
590
+ aws_mock.compute.return_value.create_instance.assert_called_once()
591
+ assert instance.backend == BackendType.GCP
592
+
593
+ @pytest.mark.parametrize("err", [RuntimeError("Unexpected"), ProvisioningError("Expected")])
594
+ async def test_fails_if_all_offers_fail(self, session: AsyncSession, err: Exception):
595
+ project = await create_project(session=session)
596
+ instance = await create_instance(
597
+ session=session, project=project, status=InstanceStatus.PENDING
598
+ )
599
+ aws_mock = Mock()
600
+ aws_mock.TYPE = BackendType.AWS
601
+ offer = get_instance_offer_with_availability(backend=BackendType.AWS, price=1.0)
602
+ aws_mock.compute.return_value = Mock(spec=ComputeMockSpec)
603
+ aws_mock.compute.return_value.get_offers_cached.return_value = [offer]
604
+ aws_mock.compute.return_value.create_instance.side_effect = err
605
+ with patch("dstack._internal.server.services.backends.get_project_backends") as m:
606
+ m.return_value = [aws_mock]
607
+ await process_instances()
608
+
609
+ await session.refresh(instance)
610
+ assert instance.status == InstanceStatus.TERMINATED
611
+ assert instance.termination_reason == "All offers failed"
612
+
613
+ async def test_fails_if_no_offers(self, session: AsyncSession):
614
+ project = await create_project(session=session)
615
+ instance = await create_instance(
616
+ session=session, project=project, status=InstanceStatus.PENDING
617
+ )
618
+ with patch("dstack._internal.server.services.backends.get_project_backends") as m:
619
+ m.return_value = []
620
+ await process_instances()
621
+
622
+ await session.refresh(instance)
623
+ assert instance.status == InstanceStatus.TERMINATED
624
+ assert instance.termination_reason == "No offers found"
625
+
560
626
 
561
627
  @pytest.mark.asyncio
562
628
  @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
@@ -1,4 +1,5 @@
1
1
  import json
2
+ import sys
2
3
  from datetime import datetime, timezone
3
4
  from unittest.mock import Mock, patch
4
5
 
@@ -25,7 +26,14 @@ from dstack._internal.server.testing.common import (
25
26
  get_auth_headers,
26
27
  get_volume_provisioning_data,
27
28
  )
29
+ from dstack._internal.utils.crypto import generate_rsa_key_pair_bytes
28
30
 
31
+ FAKE_NEBIUS_SERVICE_ACCOUNT_CREDS = {
32
+ "type": "service_account",
33
+ "service_account_id": "serviceaccount-e00test",
34
+ "public_key_id": "publickey-e00test",
35
+ "private_key_content": generate_rsa_key_pair_bytes()[0].decode(),
36
+ }
29
37
  FAKE_OCI_CLIENT_CREDS = {
30
38
  "type": "client",
31
39
  "user": "ocid1.user.oc1..aaaaaaaa",
@@ -62,6 +70,7 @@ class TestListBackendTypes:
62
70
  "gcp",
63
71
  "kubernetes",
64
72
  "lambda",
73
+ *(["nebius"] if sys.version_info >= (3, 10) else []),
65
74
  "oci",
66
75
  "runpod",
67
76
  "tensordock",
@@ -182,6 +191,113 @@ class TestCreateBackend:
182
191
  res = await session.execute(select(BackendModel))
183
192
  assert len(res.scalars().all()) == 1
184
193
 
194
+ @pytest.mark.asyncio
195
+ @pytest.mark.skipif(sys.version_info < (3, 10), reason="Nebius requires Python 3.10")
196
+ @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
197
+ class TestNebius:
198
+ async def test_creates(self, test_db, session: AsyncSession, client: AsyncClient):
199
+ user = await create_user(session=session, global_role=GlobalRole.USER)
200
+ project = await create_project(session=session, owner=user)
201
+ await add_project_member(
202
+ session=session, project=project, user=user, project_role=ProjectRole.ADMIN
203
+ )
204
+ body = {
205
+ "type": "nebius",
206
+ "creds": FAKE_NEBIUS_SERVICE_ACCOUNT_CREDS,
207
+ }
208
+ with patch(
209
+ "dstack._internal.core.backends.nebius.resources.get_region_to_project_id_map"
210
+ ) as get_region_to_project_id_map:
211
+ get_region_to_project_id_map.return_value = {"eu-north1": "project-e00test"}
212
+ response = await client.post(
213
+ f"/api/project/{project.name}/backends/create",
214
+ headers=get_auth_headers(user.token),
215
+ json=body,
216
+ )
217
+ assert response.status_code == 200, response.json()
218
+ res = await session.execute(select(BackendModel))
219
+ assert len(res.scalars().all()) == 1
220
+
221
+ async def test_not_creates_with_invalid_creds(
222
+ self, test_db, session: AsyncSession, client: AsyncClient
223
+ ):
224
+ user = await create_user(session=session, global_role=GlobalRole.USER)
225
+ project = await create_project(session=session, owner=user)
226
+ await add_project_member(
227
+ session=session, project=project, user=user, project_role=ProjectRole.ADMIN
228
+ )
229
+ body = {
230
+ "type": "nebius",
231
+ "creds": FAKE_NEBIUS_SERVICE_ACCOUNT_CREDS,
232
+ }
233
+ with patch(
234
+ "dstack._internal.core.backends.nebius.resources.get_region_to_project_id_map"
235
+ ) as get_region_to_project_id_map:
236
+ get_region_to_project_id_map.side_effect = ValueError()
237
+ response = await client.post(
238
+ f"/api/project/{project.name}/backends/create",
239
+ headers=get_auth_headers(user.token),
240
+ json=body,
241
+ )
242
+ assert response.status_code == 400, response.json()
243
+ res = await session.execute(select(BackendModel))
244
+ assert len(res.scalars().all()) == 0
245
+
246
+ async def test_creates_with_regions(
247
+ self, test_db, session: AsyncSession, client: AsyncClient
248
+ ):
249
+ user = await create_user(session=session, global_role=GlobalRole.USER)
250
+ project = await create_project(session=session, owner=user)
251
+ await add_project_member(
252
+ session=session, project=project, user=user, project_role=ProjectRole.ADMIN
253
+ )
254
+ body = {
255
+ "type": "nebius",
256
+ "creds": FAKE_NEBIUS_SERVICE_ACCOUNT_CREDS,
257
+ "regions": ["eu-north1"],
258
+ }
259
+ with patch(
260
+ "dstack._internal.core.backends.nebius.resources.get_region_to_project_id_map"
261
+ ) as get_region_to_project_id_map:
262
+ get_region_to_project_id_map.return_value = {
263
+ "eu-north1": "project-e00test",
264
+ "eu-west1": "project-e01test",
265
+ }
266
+ response = await client.post(
267
+ f"/api/project/{project.name}/backends/create",
268
+ headers=get_auth_headers(user.token),
269
+ json=body,
270
+ )
271
+ assert response.status_code == 200, response.json()
272
+ res = await session.execute(select(BackendModel))
273
+ assert len(res.scalars().all()) == 1
274
+
275
+ async def test_not_creates_with_invalid_regions(
276
+ self, test_db, session: AsyncSession, client: AsyncClient
277
+ ):
278
+ user = await create_user(session=session, global_role=GlobalRole.USER)
279
+ project = await create_project(session=session, owner=user)
280
+ await add_project_member(
281
+ session=session, project=project, user=user, project_role=ProjectRole.ADMIN
282
+ )
283
+ body = {
284
+ "type": "nebius",
285
+ "creds": FAKE_NEBIUS_SERVICE_ACCOUNT_CREDS,
286
+ "regions": ["xx-xxxx1"],
287
+ }
288
+ with patch(
289
+ "dstack._internal.core.backends.nebius.resources.get_region_to_project_id_map"
290
+ ) as get_region_to_project_id_map:
291
+ get_region_to_project_id_map.return_value = {"eu-north1": "project-e00test"}
292
+ response = await client.post(
293
+ f"/api/project/{project.name}/backends/create",
294
+ headers=get_auth_headers(user.token),
295
+ json=body,
296
+ )
297
+ assert response.status_code == 400, response.json()
298
+ res = await session.execute(select(BackendModel))
299
+ assert len(res.scalars().all()) == 0
300
+
185
301
  @pytest.mark.asyncio
186
302
  @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
187
303
  async def test_creates_oci_backend(self, test_db, session: AsyncSession, client: AsyncClient):