dstack 0.19.1__py3-none-any.whl → 0.19.2__py3-none-any.whl

dstack/_internal/utils/event_loop.py

@@ -0,0 +1,30 @@
+import asyncio
+import threading
+from collections.abc import Awaitable
+from typing import TypeVar
+
+T = TypeVar("T")
+
+
+class DaemonEventLoop:
+    """
+    A wrapper around asyncio.EventLoop that runs the loop in a daemon thread.
+    The thread is started with the first `await_` call.
+    """
+
+    def __init__(self) -> None:
+        self._loop = asyncio.new_event_loop()
+        self._start_lock = threading.Lock()
+        self._started = False
+
+    def await_(self, awaitable: Awaitable[T]) -> T:
+        with self._start_lock:
+            if not self._started:
+                threading.Thread(target=self._loop.run_forever, daemon=True).start()
+                self._started = True
+        future = asyncio.run_coroutine_threadsafe(_coroutine(awaitable), self._loop)
+        return future.result()
+
+
+async def _coroutine(awaitable: Awaitable[T]) -> T:
+    return await awaitable

dstack/version.py

@@ -1,3 +1,3 @@
-__version__ = "0.19.1"
+__version__ = "0.19.2"
 __is_release__ = True
 base_image = "0.7"

{dstack-0.19.1.dist-info → dstack-0.19.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dstack
-Version: 0.19.1
+Version: 0.19.2
 Summary: dstack is an open-source orchestration engine for running AI workloads on any cloud or on-premises.
 Home-page: https://dstack.ai
 Author: Andrey Cheptsov
@@ -23,6 +23,7 @@ Requires-Dist: typing-extensions>=4.0.0
 Requires-Dist: cryptography
 Requires-Dist: packaging
 Requires-Dist: python-dateutil
+Requires-Dist: cachetools
 Requires-Dist: gitpython
 Requires-Dist: jsonschema
 Requires-Dist: paramiko>=3.2.0
@@ -37,7 +38,7 @@ Requires-Dist: websocket-client
 Requires-Dist: python-multipart>=0.0.16
 Requires-Dist: filelock
 Requires-Dist: psutil
-Requires-Dist: gpuhunt<0.2.0,>=0.1.0
+Requires-Dist: gpuhunt<0.2.0,>=0.1.1
 Requires-Dist: argcomplete>=3.5.0
 Provides-Extra: all
 Requires-Dist: fastapi; extra == "all"
@@ -58,7 +59,6 @@ Requires-Dist: python-dxf==12.1.0; extra == "all"
 Requires-Dist: sentry-sdk[fastapi]; extra == "all"
 Requires-Dist: alembic-postgresql-enum; extra == "all"
 Requires-Dist: asyncpg; extra == "all"
-Requires-Dist: cachetools; extra == "all"
 Requires-Dist: python-json-logger>=3.1.0; extra == "all"
 Requires-Dist: prometheus-client; extra == "all"
 Requires-Dist: grpcio>=1.50; extra == "all"
@@ -80,6 +80,7 @@ Requires-Dist: google-cloud-tpu>=1.18.3; extra == "all"
 Requires-Dist: datacrunch; extra == "all"
 Requires-Dist: kubernetes; extra == "all"
 Requires-Dist: oci; extra == "all"
+Requires-Dist: nebius<0.3,>=0.2.19; python_version >= "3.10" and extra == "all"
 Provides-Extra: aws
 Requires-Dist: fastapi; extra == "aws"
 Requires-Dist: starlette>=0.26.0; extra == "aws"
@@ -99,7 +100,6 @@ Requires-Dist: python-dxf==12.1.0; extra == "aws"
 Requires-Dist: sentry-sdk[fastapi]; extra == "aws"
 Requires-Dist: alembic-postgresql-enum; extra == "aws"
 Requires-Dist: asyncpg; extra == "aws"
-Requires-Dist: cachetools; extra == "aws"
 Requires-Dist: python-json-logger>=3.1.0; extra == "aws"
 Requires-Dist: prometheus-client; extra == "aws"
 Requires-Dist: grpcio>=1.50; extra == "aws"
@@ -124,7 +124,6 @@ Requires-Dist: python-dxf==12.1.0; extra == "azure"
 Requires-Dist: sentry-sdk[fastapi]; extra == "azure"
 Requires-Dist: alembic-postgresql-enum; extra == "azure"
 Requires-Dist: asyncpg; extra == "azure"
-Requires-Dist: cachetools; extra == "azure"
 Requires-Dist: python-json-logger>=3.1.0; extra == "azure"
 Requires-Dist: prometheus-client; extra == "azure"
 Requires-Dist: grpcio>=1.50; extra == "azure"
@@ -153,7 +152,6 @@ Requires-Dist: python-dxf==12.1.0; extra == "datacrunch"
 Requires-Dist: sentry-sdk[fastapi]; extra == "datacrunch"
 Requires-Dist: alembic-postgresql-enum; extra == "datacrunch"
 Requires-Dist: asyncpg; extra == "datacrunch"
-Requires-Dist: cachetools; extra == "datacrunch"
 Requires-Dist: python-json-logger>=3.1.0; extra == "datacrunch"
 Requires-Dist: prometheus-client; extra == "datacrunch"
 Requires-Dist: grpcio>=1.50; extra == "datacrunch"
@@ -185,7 +183,6 @@ Requires-Dist: python-dxf==12.1.0; extra == "gcp"
 Requires-Dist: sentry-sdk[fastapi]; extra == "gcp"
 Requires-Dist: alembic-postgresql-enum; extra == "gcp"
 Requires-Dist: asyncpg; extra == "gcp"
-Requires-Dist: cachetools; extra == "gcp"
 Requires-Dist: python-json-logger>=3.1.0; extra == "gcp"
 Requires-Dist: prometheus-client; extra == "gcp"
 Requires-Dist: grpcio>=1.50; extra == "gcp"
@@ -215,7 +212,6 @@ Requires-Dist: python-dxf==12.1.0; extra == "kubernetes"
 Requires-Dist: sentry-sdk[fastapi]; extra == "kubernetes"
 Requires-Dist: alembic-postgresql-enum; extra == "kubernetes"
 Requires-Dist: asyncpg; extra == "kubernetes"
-Requires-Dist: cachetools; extra == "kubernetes"
 Requires-Dist: python-json-logger>=3.1.0; extra == "kubernetes"
 Requires-Dist: prometheus-client; extra == "kubernetes"
 Requires-Dist: grpcio>=1.50; extra == "kubernetes"
@@ -239,12 +235,34 @@ Requires-Dist: python-dxf==12.1.0; extra == "lambda"
 Requires-Dist: sentry-sdk[fastapi]; extra == "lambda"
 Requires-Dist: alembic-postgresql-enum; extra == "lambda"
 Requires-Dist: asyncpg; extra == "lambda"
-Requires-Dist: cachetools; extra == "lambda"
 Requires-Dist: python-json-logger>=3.1.0; extra == "lambda"
 Requires-Dist: prometheus-client; extra == "lambda"
 Requires-Dist: grpcio>=1.50; extra == "lambda"
 Requires-Dist: boto3; extra == "lambda"
 Requires-Dist: botocore; extra == "lambda"
+Provides-Extra: nebius
+Requires-Dist: fastapi; extra == "nebius"
+Requires-Dist: starlette>=0.26.0; extra == "nebius"
+Requires-Dist: uvicorn; extra == "nebius"
+Requires-Dist: aiorwlock; extra == "nebius"
+Requires-Dist: aiocache; extra == "nebius"
+Requires-Dist: httpx; extra == "nebius"
+Requires-Dist: jinja2; extra == "nebius"
+Requires-Dist: watchfiles; extra == "nebius"
+Requires-Dist: sqlalchemy[asyncio]>=2.0.0; extra == "nebius"
+Requires-Dist: sqlalchemy-utils>=0.40.0; extra == "nebius"
+Requires-Dist: alembic>=1.10.2; extra == "nebius"
+Requires-Dist: apscheduler<4; extra == "nebius"
+Requires-Dist: aiosqlite; extra == "nebius"
+Requires-Dist: docker>=6.0.0; extra == "nebius"
+Requires-Dist: python-dxf==12.1.0; extra == "nebius"
+Requires-Dist: sentry-sdk[fastapi]; extra == "nebius"
+Requires-Dist: alembic-postgresql-enum; extra == "nebius"
+Requires-Dist: asyncpg; extra == "nebius"
+Requires-Dist: python-json-logger>=3.1.0; extra == "nebius"
+Requires-Dist: prometheus-client; extra == "nebius"
+Requires-Dist: grpcio>=1.50; extra == "nebius"
+Requires-Dist: nebius<0.3,>=0.2.19; extra == "nebius"
 Provides-Extra: oci
 Requires-Dist: fastapi; extra == "oci"
 Requires-Dist: starlette>=0.26.0; extra == "oci"
@@ -264,7 +282,6 @@ Requires-Dist: python-dxf==12.1.0; extra == "oci"
 Requires-Dist: sentry-sdk[fastapi]; extra == "oci"
 Requires-Dist: alembic-postgresql-enum; extra == "oci"
 Requires-Dist: asyncpg; extra == "oci"
-Requires-Dist: cachetools; extra == "oci"
 Requires-Dist: python-json-logger>=3.1.0; extra == "oci"
 Requires-Dist: prometheus-client; extra == "oci"
 Requires-Dist: grpcio>=1.50; extra == "oci"
@@ -288,7 +305,6 @@ Requires-Dist: python-dxf==12.1.0; extra == "server"
 Requires-Dist: sentry-sdk[fastapi]; extra == "server"
 Requires-Dist: alembic-postgresql-enum; extra == "server"
 Requires-Dist: asyncpg; extra == "server"
-Requires-Dist: cachetools; extra == "server"
 Requires-Dist: python-json-logger>=3.1.0; extra == "server"
 Requires-Dist: prometheus-client; extra == "server"
 Requires-Dist: grpcio>=1.50; extra == "server"

{dstack-0.19.1.dist-info → dstack-0.19.2.dist-info}/RECORD

@@ -1,10 +1,10 @@
 dstack/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/version.py,sha256=ja4J6HzBpBX3wxm5CKLUUgzAwmmr8naAhq3SBch6VIw,64
+dstack/version.py,sha256=6z601r_k8B66H4FWmrWo34Dj29lhhWy4y-3_XVO3_V8,64
 dstack/_internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/compat.py,sha256=bF9U9fTMfL8UVhCouedoUSTYFl7UAOiU0WXrnRoByxw,40
 dstack/_internal/settings.py,sha256=8XODoSW2joaEndvZxuHUPSFK85sGgJ7fVL976isYeJM,557
 dstack/_internal/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/_internal/cli/main.py,sha256=yaEEdjzonW18IULYqu9-oAEbusRSgjmFiHP6a-FgVHA,3489
+dstack/_internal/cli/main.py,sha256=tVSDJxBI4qJhsDePgPkkJCEB-e98j7HSWzvcSfA64G8,3594
 dstack/_internal/cli/commands/__init__.py,sha256=e8EjD8Ej7Q3_APlxAN7NBwY7dGnW-DR91nH_RgcjBXg,2023
 dstack/_internal/cli/commands/apply.py,sha256=dZH2B3Yr-lB7hs-QeUnvX0FV0Yk2x19dEO5uSFo4098,4909
 dstack/_internal/cli/commands/attach.py,sha256=resmqoqkIOptp9Vn4BMs8m42Txl4mPzFDhHcv_lkWJk,5033
@@ -15,9 +15,10 @@ dstack/_internal/cli/commands/fleet.py,sha256=J0Yi5CAYvIQF_NC4ZBsDYGc3grTOtvaF7s
 dstack/_internal/cli/commands/gateway.py,sha256=DcD6P_MvXbSL9aXkLX9hgGYSAzARjgY6RSbrCMzdNcg,6075
 dstack/_internal/cli/commands/init.py,sha256=bLhSlViNWtjflB6xNq_PuCR2o2A06h222luh1NeUgVA,1169
 dstack/_internal/cli/commands/logs.py,sha256=o8ehPAKM12Xn9thg2jjnYdr7_wKqF-00ziVry8IVVwE,1528
+dstack/_internal/cli/commands/metrics.py,sha256=nxdTcyCvHNev7Mn19zGQ7vTHUBGY6hyvPtY6Z731SOU,5373
 dstack/_internal/cli/commands/ps.py,sha256=WUQKL-1HcoM-oFF_WX4zplO8yKGMWfVbsivg0L7LP2w,1666
 dstack/_internal/cli/commands/server.py,sha256=dCEdmVp1OlwFnSGw9k8mBb26wZy1D_ctgn5pmK8ZrCQ,2802
-dstack/_internal/cli/commands/stats.py,sha256=9EyGw60Oc_KkPLHz-IN1sIFZ4AJUadfsU0PNXQ7Kvqg,5031
+dstack/_internal/cli/commands/stats.py,sha256=_CiJPQZuKHUzA1x7CHJP-leth3MmqiQ2-jwojq6G-e8,390
 dstack/_internal/cli/commands/stop.py,sha256=i7TreejSemrQcL7C37La22KkwM98O09IDMRNt8cw1NI,1000
 dstack/_internal/cli/commands/volume.py,sha256=h0bQQm6zx1Bq0kdBX5ksqPZk55wz-md_Kitm0RTDReg,3214
 dstack/_internal/cli/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -41,10 +42,10 @@ dstack/_internal/cli/utils/updates.py,sha256=9KQcm_TEE_PiU0yc5WH4xRse0WIeHOubi9E
 dstack/_internal/cli/utils/volume.py,sha256=mU9I06dVMFbpjfkefxrZNoSWadKLoib3U14rHudNQN4,1975
 dstack/_internal/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/core/consts.py,sha256=c1Yd5UY6Qx7KeuYgloXWncWhMsYj6TqwlElda7NtB98,254
-dstack/_internal/core/errors.py,sha256=UvqMIBVHG1NIUP0w0g5dWjnxOz37sfFi4lh4yuYK0vw,2793
+dstack/_internal/core/errors.py,sha256=elI7vcTXJvCpg_TOcpar5nBTKiQ71vZMEly6aM_N0jg,3212
 dstack/_internal/core/backends/__init__.py,sha256=fwgV8CN8Ap6MZmWklMGHHf0roliBtqne-ijjVOpWcgc,2467
-dstack/_internal/core/backends/configurators.py,sha256=EFzzlP6R4J82WkKpltGWG4HPmg5Iqw8Nvg3wuCxskxY,3583
-dstack/_internal/core/backends/models.py,sha256=YoVHk-dcFMcrbOE9dxtUTYO3ZJlpRy3MBND2KB6u2aA,3833
+dstack/_internal/core/backends/configurators.py,sha256=JxGfZwcmL90akMFkAzzZ_fzPvU2No0pBaBqU_g0D-y0,3775
+dstack/_internal/core/backends/models.py,sha256=aKQOrDEStouuwY4MacSen7SkoyAa6HR6a6PFq5-cbNk,4088
 dstack/_internal/core/backends/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/core/backends/aws/auth.py,sha256=TzjRq01exWToWUf9BjT_58zB7L_wnNZduTdVlPmUfxg,1061
 dstack/_internal/core/backends/aws/backend.py,sha256=pjROH-S9pgrSMm-Eox_ocL7cTU6mIMRxvURq7Vi-2J8,876
@@ -62,9 +63,9 @@ dstack/_internal/core/backends/azure/resources.py,sha256=keE3ruSSNWhSYMCkG7832TY
 dstack/_internal/core/backends/azure/utils.py,sha256=taHMJq6UHRzUXLUcO2P5VCKy3wJaye2bG-6QdkEPNdY,1741
 dstack/_internal/core/backends/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/core/backends/base/backend.py,sha256=hdFMHED1RMV9GVfLSU0yGhGE-cXlbLvt1coDA885PMM,505
-dstack/_internal/core/backends/base/compute.py,sha256=pWO-28QT-soRATGUcdWoGbNff6VuMJLyJ5_gmJ_SG1U,23696
+dstack/_internal/core/backends/base/compute.py,sha256=d34JDZDOGoHKDFpjtJsMpB9O0Akq_CwfHaQsPnqJIB4,23866
 dstack/_internal/core/backends/base/configurator.py,sha256=OCv8N2oxcxy3In2zS1PKiCJ0a-COZwxGjBz2FYkQnfg,3807
-dstack/_internal/core/backends/base/models.py,sha256=B8CscQIEttg9xySzaQ5h3VIczjEFxfh4tKTlL0JVwUo,388
+dstack/_internal/core/backends/base/models.py,sha256=Ij0osOl-T-ABsKLoVg2eY81DMkwdWkevAnjXj2QnLXI,532
 dstack/_internal/core/backends/base/offers.py,sha256=89H6g6Dw5WvXDEBnKoZUF-fGIE3Clc8T4lEjZ_jr71A,5909
 dstack/_internal/core/backends/cudo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/core/backends/cudo/api_client.py,sha256=ygq1Gx7ZvwKaifdXtvzDSw4xR4ZH6UWd5J47BjuaGh0,3685
@@ -102,6 +103,12 @@ dstack/_internal/core/backends/lambdalabs/models.py,sha256=c9dAYd0wjun2dvMLUZO4D
 dstack/_internal/core/backends/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/core/backends/local/backend.py,sha256=KJuNXUXrg60NhLywnExD1EXH2gK0TL-kzNWUPRNZ_Zg,427
 dstack/_internal/core/backends/local/compute.py,sha256=OQub3qFQdWGm8Laceh-TU4KPRzJwQxRgf_tMwmXFIQk,3613
+dstack/_internal/core/backends/nebius/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dstack/_internal/core/backends/nebius/backend.py,sha256=2XqZIbSR8VzlfOnuVklXlDxNmwAkQj7txQN8VXF1j2E,566
+dstack/_internal/core/backends/nebius/compute.py,sha256=ffmEsx10too4ZT2WvoBJMteHRieuOJOXxggvlzo7fpc,10999
+dstack/_internal/core/backends/nebius/configurator.py,sha256=qG7n2ZpCcy7E0F9yqJlGo6yCu29Y8vT-y61fc1EGbKo,2747
+dstack/_internal/core/backends/nebius/models.py,sha256=5-JZDOv3k0BZ3do6F5MBoRuBnpOq54Ulh_am1KfxxJA,3535
+dstack/_internal/core/backends/nebius/resources.py,sha256=SB4-BFWGGlPaPLS70pmJK2faHdTc8kRnWpqcKCVxPSk,7209
 dstack/_internal/core/backends/oci/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/core/backends/oci/auth.py,sha256=ErDfuAvmrKlFg5OCYz5BhtlGHQmSiupl9Uaxx3W-wTE,800
 dstack/_internal/core/backends/oci/backend.py,sha256=yXjVCt7n6BVLH0byYFbNFf-P9J0FwlNfxsYbKGMdoI4,536
@@ -160,7 +167,7 @@ dstack/_internal/core/models/unix.py,sha256=KxnSQELnkAjjuUgYcQKVkf-UAbYREBD8WCWD
 dstack/_internal/core/models/users.py,sha256=o_rd0GAmd6jufypVUs9P12NRri3rgAPDt-KxnqNNsGw,703
 dstack/_internal/core/models/volumes.py,sha256=EQmfTnB3oN6zbLwMqwda0X1T7oKLRRt_h5lTXvQ-siQ,6096
 dstack/_internal/core/models/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/_internal/core/models/backends/base.py,sha256=nayexPe9kU0FytYI6LdY8aKaTHRG07der7Ia0nlKc9c,1023
+dstack/_internal/core/models/backends/base.py,sha256=Fbhs90NBMceXhjORwj7c2_pVriapCGSUMBFwuR-wRIg,1091
 dstack/_internal/core/models/repos/__init__.py,sha256=7Qo1QgJ852LklUuM-mlCNFodp_XrQ4iqV7uRPiX_qm0,885
 dstack/_internal/core/models/repos/base.py,sha256=nErlSR3AbG9fwDE5vuJK5DIrP8JB3fG8a7mXqosJ9gY,846
 dstack/_internal/core/models/repos/local.py,sha256=DmW_e6qOWctZwuEpCqME-JpxYHTgH5I1gcK7zjSMfao,2394
@@ -218,7 +225,7 @@ dstack/_internal/proxy/lib/repo.py,sha256=zkWZ9XZzQHfCa-eifec7H7UYnJZLgeRuiQls7R
 dstack/_internal/proxy/lib/routers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/proxy/lib/routers/model_proxy.py,sha256=57GFRpVRXcVY-347HnUSUr4w4RsxsjLuuZiJs8DwDpM,3895
 dstack/_internal/proxy/lib/schemas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/_internal/proxy/lib/schemas/model_proxy.py,sha256=a7QbgPE6QwFPlB890OQO7hRu5I6dAArsVJ4k5euFaC4,1851
+dstack/_internal/proxy/lib/schemas/model_proxy.py,sha256=a-LT18-SrVVriF3YsYPUvil8e_9rLDiOTxfZy8hGsqU,1895
 dstack/_internal/proxy/lib/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/proxy/lib/services/service_connection.py,sha256=Gf_ucng9dSKEfpi2kvxZekDSWisY2by6O7_PNZcQi1Q,5924
 dstack/_internal/proxy/lib/services/model_proxy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -242,7 +249,7 @@ dstack/_internal/server/background/tasks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5
 dstack/_internal/server/background/tasks/common.py,sha256=N7xSXbf2MoBWgbJ1e3AEzYBTf1Gn-pDXYND8Zr_YCJQ,970
 dstack/_internal/server/background/tasks/process_fleets.py,sha256=lKXUvN_b7DNjD3psHzyCt_JYsTxPFuQ86iXi8fj8GkM,3202
 dstack/_internal/server/background/tasks/process_gateways.py,sha256=hoUI1CSqbHt_uMwnzTRAEDl-LBw0wUk_W4xobIbdvRc,7017
-dstack/_internal/server/background/tasks/process_instances.py,sha256=Kc7CbWK4mFOsKwOqp-Pt0ewTsB5OZ5gkPyv9T6TNbpM,37674
+dstack/_internal/server/background/tasks/process_instances.py,sha256=De-nbzvvwiO_xtI16tRsosvQXAC3uzjynHAeWpcX51M,37970
 dstack/_internal/server/background/tasks/process_metrics.py,sha256=acySfsacpYbTPV9Yivs-oU37z1S2sUdWhRHdJkfBcCA,5332
 dstack/_internal/server/background/tasks/process_placement_groups.py,sha256=FqGfbzvfILdnPUfxjFPAM1ij2xd2mCDi8qufiBcUMI8,4107
 dstack/_internal/server/background/tasks/process_prometheus_metrics.py,sha256=u8hCXjOOek7VLEsmLy2VnDXFmIwTNjrJwcpWG7a1zW0,5093
@@ -325,7 +332,7 @@ dstack/_internal/server/routers/instances.py,sha256=XOogTC9My2Zv0ck37_PbHKoZI-j4
 dstack/_internal/server/routers/logs.py,sha256=_Euk283LbhlwHibJTKM-7YcpbeQFtWBqMfbOry3PSkU,1159
 dstack/_internal/server/routers/metrics.py,sha256=VFgWhkOvxVFDLlRM_kXHYFylLcfCD6UjXInvcd7H4dY,2314
 dstack/_internal/server/routers/projects.py,sha256=0R-w_6WXUbNo6fREAexFUQ3RoOJF2D_Iz35elKjym14,2717
-dstack/_internal/server/routers/prometheus.py,sha256=OuC17kgKkb2ErxDD5QZ_ZdZft5A8dMIAFlIzQ_04NEo,744
+dstack/_internal/server/routers/prometheus.py,sha256=UAcOE8dpGZe4Wd0EOIlFPMbjaNjzX8A7iHlooeRvsfo,944
 dstack/_internal/server/routers/repos.py,sha256=P_zLoEQderxhCeHQJwRkrIhVcc0-cpabfyde22bWVRk,3362
 dstack/_internal/server/routers/runs.py,sha256=oPqyIRPwkMjj12M1IdMF2UitatqvljISAXnJAjfEJyQ,5352
 dstack/_internal/server/routers/secrets.py,sha256=50_qJCTYRpnGSlLyS93gqoV17wWewOVmM65PcG1bT_Y,856
@@ -347,7 +354,7 @@ dstack/_internal/server/schemas/secrets.py,sha256=mfqLSM7PqxVQ-GIWB6RfPRUOvSvvaR
 dstack/_internal/server/schemas/users.py,sha256=FuDqwRVe3mOmv497vOZKjI0a_d4Wt2g4ZiCJcyfHEKA,495
 dstack/_internal/server/schemas/volumes.py,sha256=9iwaQLMhA6aj9XmtdU_9jWVhpzNOtFbDByAe-WYfYh0,673
 dstack/_internal/server/security/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dstack/_internal/server/security/permissions.py,sha256=ZIUIP5HGk1KG7vGfpZCxP4LsMuKHAz64iOUsAC7FebY,4425
+dstack/_internal/server/security/permissions.py,sha256=FJ_8YPhjmebA4jQjtQoAGEaj1Hahb_po0tYRCQ18aaE,4940
 dstack/_internal/server/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dstack/_internal/server/services/config.py,sha256=X61ypyuBQqAuWi-mVFJ7Uk1DyGfHqk8_d2C6xIG0yxE,10434
 dstack/_internal/server/services/docker.py,sha256=3EcYPiVsrNBGDQYOb60QJ241mzTT6lJROYQXIwt-8dk,5351
@@ -431,10 +438,10 @@ dstack/_internal/server/statics/e467d7d60aae81ab198b.svg,sha256=_XHc9mfQZgGkcy4h
 dstack/_internal/server/statics/eb9b344b73818fe2b71a.png,sha256=2H14eOCQRyZhFGJ1Kn2LH1j70kTF1Qop4vH-tiKqyPI,85
 dstack/_internal/server/statics/f517dd626eb964120de0.png,sha256=4QQuNa8SqmcZ67HK6739OHCyjnAJseU1bkcn454KRQs,159
 dstack/_internal/server/statics/f958aecddee5d8e3222c.png,sha256=8CoZkVNgRfOAe62X1dU-AZDvwh_nESKaQblEmaX2Xrs,87
-dstack/_internal/server/statics/index.html,sha256=1RngYCyoktp3XhwHvC2jeCdolq0T-w-LDfNJHInNeI0,10468
-dstack/_internal/server/statics/main-4a0fe83e84574654e397.js,sha256=lNEeGSQNL6XYc_nXkZwtL35FOJkqbAGUexPC38feqsQ,6204421
-dstack/_internal/server/statics/main-4a0fe83e84574654e397.js.map,sha256=WoJHj0c5pkjUN-aGI-v8oyylaxU3y4KgJ_G8QiwtvA8,8143363
-dstack/_internal/server/statics/main-da9f8c06a69c20dac23e.css,sha256=2ObS4Rg6yWkbk7nQqw-lnLVdYC3L8X5N7xZMKcavMWg,1296312
+dstack/_internal/server/statics/index.html,sha256=XenYZSciBY6Ysgsf-stlogOrfpZ4YCoVOLLvTNqokI4,10468
+dstack/_internal/server/statics/main-bcb3228138bc8483cc0b.js,sha256=klLuC7jhEGhx8bi67rpdwGxmazLND-Fy18QOFaJxj1I,6517024
+dstack/_internal/server/statics/main-bcb3228138bc8483cc0b.js.map,sha256=pE7tRB7ifhSCy27H9D5pjP2pw_uQfPpFXkbrIGsV0tw,8553628
+dstack/_internal/server/statics/main-c0bdaac8f1ea67d499eb.css,sha256=3tz-eO3jCGYZ1qRY3f8x8lre_NMHNWSFxp4w1g1py3E,1336523
 dstack/_internal/server/statics/manifest.json,sha256=430w2BoWVmYYVr14lDvUxx-ROPt3VjigzeMqfLeiSCM,340
 dstack/_internal/server/statics/robots.txt,sha256=kNJLw79pisHhc3OVAimMzKcq3x9WT6sF9IS4xI0crdI,67
 dstack/_internal/server/statics/assets/android-chrome-144x144.png,sha256=tB3V-95O-VVEoawN5V1XFoMQRSK0I6gthraV8bATGaw,23414
@@ -523,6 +530,7 @@ dstack/_internal/utils/common.py,sha256=c4qqqBqwhh44EVK002bq3PuvwtQUS77b30Ka8GZn
 dstack/_internal/utils/crypto.py,sha256=2RTSyzePuwwqc1X2HO6lwcSFyZ2kujnqluoICQ2DLJQ,1462
 dstack/_internal/utils/dxf.py,sha256=wguK9s6-69kqSDZkxd1kFEr6VlH5ixvFRJxizyOuJ8I,3229
 dstack/_internal/utils/env.py,sha256=HRbIspHpKHh05fMZeV23-hrZoV6vVMuniefD08u6ey0,357
+dstack/_internal/utils/event_loop.py,sha256=DO2ADtWfH2z8F2hBbg_EADSWzleQYGVZ9D1XYDpH-tk,880
 dstack/_internal/utils/gpu.py,sha256=ZeWpy1nRLVh-FwBZdxbMoVjjCF0DWJlWfNoVgFhGx2w,1776
 dstack/_internal/utils/hash.py,sha256=mCERRtj9QwbpoP3vveBqbniSJiNMHG0vPSzp4fxmKv0,920
 dstack/_internal/utils/ignore.py,sha256=kC2dI1aKudFGJWPLnJX34eLBl4u9TOqw0mSElY-bIRM,3053
@@ -639,7 +647,7 @@ tests/_internal/server/background/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeR
 tests/_internal/server/background/tasks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/_internal/server/background/tasks/test_process_fleets.py,sha256=Dl31_TwxoCzYqkVNPWGLsYxmGL2sZfEK3rQXLFyPIz8,2701
 tests/_internal/server/background/tasks/test_process_gateways.py,sha256=lOP4jPXDtadAgYp0aFND_fp5R_X19M58CaOlgnDAEck,5085
-tests/_internal/server/background/tasks/test_process_instances.py,sha256=WC32HvynBuSxwFtAyMTHS4eVzqCnyGufcrIUTEVoozI,27944
+tests/_internal/server/background/tasks/test_process_instances.py,sha256=zRUlcD04l_wYpM5r3PnODYxE95ngu1_RD5KJRTTr-io,27998
 tests/_internal/server/background/tasks/test_process_metrics.py,sha256=z-u4HXJE5EMVH9kwU_POHmvp55ldAvuLpEMkaebBtsg,4976
 tests/_internal/server/background/tasks/test_process_placement_groups.py,sha256=19LYbIMZIIeKAN0b9KOMyS-cHUx0FoOojqQuM8Oeiq4,1620
 tests/_internal/server/background/tasks/test_process_prometheus_metrics.py,sha256=I9DgIJXVGS7UvbFgm4HFnzWiCICBpy72NjDPKU_7WII,7178
@@ -649,14 +657,14 @@ tests/_internal/server/background/tasks/test_process_submitted_jobs.py,sha256=DC
 tests/_internal/server/background/tasks/test_process_submitted_volumes.py,sha256=pM7PwApOEDP0ogan91_a5ceF0foHQGCNKE9wlTJy9BA,2263
 tests/_internal/server/background/tasks/test_process_terminating_jobs.py,sha256=RWXc2MXp2n8Mte1URlkJfgxUZKDnTQntuWtJ5JdyTBQ,14053
 tests/_internal/server/routers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tests/_internal/server/routers/test_backends.py,sha256=xYZEBeISlS5mctGnzcq57QBSu08zFjcQGBxW8rnmcXw,33661
+tests/_internal/server/routers/test_backends.py,sha256=LhD2PNnz4q2HcBzMkvwtyh9DcXuxOaoUd5pwKTNoAWU,39238
 tests/_internal/server/routers/test_fleets.py,sha256=1VIX-52lWbGnQ0m22KzHYOhLBioS7YO-H_pK7IkhVlA,34079
 tests/_internal/server/routers/test_gateways.py,sha256=59r93kdVIuyMHGPNTMCKUdIDpGJADjNs0vEV3xprxTw,24205
 tests/_internal/server/routers/test_instances.py,sha256=78HFMU9Xel8BNZL3TqnuvrKEAhqULOlnSpd1Ja3oXkU,9188
 tests/_internal/server/routers/test_logs.py,sha256=NZwyJlgjMOGq4XEx7-VDjTpniYPhZpsbZvB0dTawaog,3989
 tests/_internal/server/routers/test_metrics.py,sha256=xMdDFZW73Zl06QfggjatfwTut37s0soeliJivkCgBks,7620
 tests/_internal/server/routers/test_projects.py,sha256=Z3Ok7onAjUYS4ADvKvN-SwSxYKvlvf4MG5Y8baqQU14,25964
-tests/_internal/server/routers/test_prometheus.py,sha256=LqJwWn5ztSLIGnvZgj-sD7BFW-JuePFt6k__ymF5Btw,22711
+tests/_internal/server/routers/test_prometheus.py,sha256=iYpgSVkV2yTSKV306rhAffuWGyQV3l2i88crwHIQWYo,23696
 tests/_internal/server/routers/test_repos.py,sha256=G4dKuFGd_UrxAHwh_XLl1xCHK_DCsiJcXBsHODw3yJk,16682
 tests/_internal/server/routers/test_runs.py,sha256=q02oBrUcp4JoJOL68jbxlfFxH9B8JO9Bkb7v_Qg-Aug,62984
 tests/_internal/server/routers/test_server.py,sha256=ROkuRNNJEkMQuK8guZ3Qy3iRRfiWvPIJJJDc09BI0D4,489
@@ -694,6 +702,7 @@ tests/_internal/server/utils/test_routers.py,sha256=vhaTGaZ8r3gbwM3AQlgvy4tFgzjZ
 tests/_internal/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/_internal/utils/test_common.py,sha256=Wyw2QzZuaznXarLAyq3QyEsXvXDQrVW1uXIZms1XI0U,6098
 tests/_internal/utils/test_env.py,sha256=MKg6gHw9JGROqg8Lovslr_BPfNol_eIVVrJSBcLCRJY,1109
+tests/_internal/utils/test_event_loop.py,sha256=MpkFb8_rqS25ulfzZDw3RyR4LIMGrqPGzXHxeBFA1kQ,426
 tests/_internal/utils/test_gpu.py,sha256=vTyyWXRgZzil5RhzzA2tWW5owH_jgXXJBtW1Dx9rswo,2051
 tests/_internal/utils/test_interpolator.py,sha256=hjovKQhOkSdBFQyBXfnYG3kVJDG6x_jSiWizAxFfifs,1844
 tests/_internal/utils/test_network.py,sha256=4nxej28JgoDWfZ-uwqh5HtUKtCQCmEspUGQ8O5g6ixU,968
@@ -701,9 +710,9 @@ tests/_internal/utils/test_path.py,sha256=rzS-1YCxsFUocBe42dghLOMFNymPruGrA7bqFZ
 tests/_internal/utils/test_ssh.py,sha256=V-cBFPhD--9eM9d1uQQgpj2gnYLA3c43f4cX9uJ6E-U,1743
 tests/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/api/test_utils.py,sha256=SSSqHcNE5cZVqDq4n2sKZthRoXaZ_Bx7z1AAN5xTM9s,391
-dstack-0.19.1.dist-info/LICENSE.md,sha256=qDABaRGjSKVOib1U8viw2P_96sIK7Puo426784oD9f8,15976
-dstack-0.19.1.dist-info/METADATA,sha256=sj_wcanWBaGU9ecMn6I32zvXuYFniyN_6K6ehbcO3tA,18231
-dstack-0.19.1.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-dstack-0.19.1.dist-info/entry_points.txt,sha256=GnLrMS8hx3rWAySQjA7tPNhtixV6a-brRkmal1PKoHc,58
-dstack-0.19.1.dist-info/top_level.txt,sha256=3BrIO1zrqxT9P20ymhRM6k15meZXzbPL6ykBlDZG2_k,13
-dstack-0.19.1.dist-info/RECORD,,
+dstack-0.19.2.dist-info/LICENSE.md,sha256=qDABaRGjSKVOib1U8viw2P_96sIK7Puo426784oD9f8,15976
+dstack-0.19.2.dist-info/METADATA,sha256=y1cv3qjZHTcWjObJuh0n5gqemdq0_d5AZXZ63t-G7kI,19039
+dstack-0.19.2.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+dstack-0.19.2.dist-info/entry_points.txt,sha256=GnLrMS8hx3rWAySQjA7tPNhtixV6a-brRkmal1PKoHc,58
+dstack-0.19.2.dist-info/top_level.txt,sha256=3BrIO1zrqxT9P20ymhRM6k15meZXzbPL6ykBlDZG2_k,13
+dstack-0.19.2.dist-info/RECORD,,

tests/_internal/server/background/tasks/test_process_instances.py

@@ -8,7 +8,7 @@ import pytest
 from freezegun import freeze_time
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from dstack._internal.core.errors import BackendError, ProvisioningError
+from dstack._internal.core.errors import BackendError, NotYetTerminated, ProvisioningError
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.instances import (
     Gpu,
@@ -384,7 +384,9 @@ class TestTerminate:
 
     @pytest.mark.asyncio
     @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
-    @pytest.mark.parametrize("error", [BackendError("err"), RuntimeError("err")])
+    @pytest.mark.parametrize(
+        "error", [BackendError("err"), RuntimeError("err"), NotYetTerminated("")]
+    )
     async def test_terminate_retry(self, test_db, session: AsyncSession, error: Exception):
         project = await create_project(session=session)
         instance = await create_instance(

tests/_internal/server/routers/test_backends.py

@@ -1,4 +1,5 @@
 import json
+import sys
 from datetime import datetime, timezone
 from unittest.mock import Mock, patch
 
@@ -25,7 +26,14 @@ from dstack._internal.server.testing.common import (
     get_auth_headers,
     get_volume_provisioning_data,
 )
+from dstack._internal.utils.crypto import generate_rsa_key_pair_bytes
 
+FAKE_NEBIUS_SERVICE_ACCOUNT_CREDS = {
+    "type": "service_account",
+    "service_account_id": "serviceaccount-e00test",
+    "public_key_id": "publickey-e00test",
+    "private_key_content": generate_rsa_key_pair_bytes()[0].decode(),
+}
 FAKE_OCI_CLIENT_CREDS = {
     "type": "client",
     "user": "ocid1.user.oc1..aaaaaaaa",
@@ -62,6 +70,7 @@ class TestListBackendTypes:
             "gcp",
             "kubernetes",
             "lambda",
+            *(["nebius"] if sys.version_info >= (3, 10) else []),
             "oci",
             "runpod",
             "tensordock",
@@ -182,6 +191,113 @@ class TestCreateBackend:
         res = await session.execute(select(BackendModel))
         assert len(res.scalars().all()) == 1
 
+    @pytest.mark.asyncio
+    @pytest.mark.skipif(sys.version_info < (3, 10), reason="Nebius requires Python 3.10")
+    @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
+    class TestNebius:
+        async def test_creates(self, test_db, session: AsyncSession, client: AsyncClient):
+            user = await create_user(session=session, global_role=GlobalRole.USER)
+            project = await create_project(session=session, owner=user)
+            await add_project_member(
+                session=session, project=project, user=user, project_role=ProjectRole.ADMIN
+            )
+            body = {
+                "type": "nebius",
+                "creds": FAKE_NEBIUS_SERVICE_ACCOUNT_CREDS,
+            }
+            with patch(
+                "dstack._internal.core.backends.nebius.resources.get_region_to_project_id_map"
+            ) as get_region_to_project_id_map:
+                get_region_to_project_id_map.return_value = {"eu-north1": "project-e00test"}
+                response = await client.post(
+                    f"/api/project/{project.name}/backends/create",
+                    headers=get_auth_headers(user.token),
+                    json=body,
+                )
+            assert response.status_code == 200, response.json()
+            res = await session.execute(select(BackendModel))
+            assert len(res.scalars().all()) == 1
+
+        async def test_not_creates_with_invalid_creds(
+            self, test_db, session: AsyncSession, client: AsyncClient
+        ):
+            user = await create_user(session=session, global_role=GlobalRole.USER)
+            project = await create_project(session=session, owner=user)
+            await add_project_member(
+                session=session, project=project, user=user, project_role=ProjectRole.ADMIN
+            )
+            body = {
+                "type": "nebius",
+                "creds": FAKE_NEBIUS_SERVICE_ACCOUNT_CREDS,
+            }
+            with patch(
+                "dstack._internal.core.backends.nebius.resources.get_region_to_project_id_map"
+            ) as get_region_to_project_id_map:
+                get_region_to_project_id_map.side_effect = ValueError()
+                response = await client.post(
+                    f"/api/project/{project.name}/backends/create",
+                    headers=get_auth_headers(user.token),
+                    json=body,
+                )
+            assert response.status_code == 400, response.json()
+            res = await session.execute(select(BackendModel))
+            assert len(res.scalars().all()) == 0
+
+        async def test_creates_with_regions(
+            self, test_db, session: AsyncSession, client: AsyncClient
+        ):
+            user = await create_user(session=session, global_role=GlobalRole.USER)
+            project = await create_project(session=session, owner=user)
+            await add_project_member(
+                session=session, project=project, user=user, project_role=ProjectRole.ADMIN
+            )
+            body = {
+                "type": "nebius",
+                "creds": FAKE_NEBIUS_SERVICE_ACCOUNT_CREDS,
+                "regions": ["eu-north1"],
+            }
+            with patch(
+                "dstack._internal.core.backends.nebius.resources.get_region_to_project_id_map"
+            ) as get_region_to_project_id_map:
+                get_region_to_project_id_map.return_value = {
+                    "eu-north1": "project-e00test",
+                    "eu-west1": "project-e01test",
+                }
+                response = await client.post(
+                    f"/api/project/{project.name}/backends/create",
+                    headers=get_auth_headers(user.token),
+                    json=body,
+                )
+            assert response.status_code == 200, response.json()
+            res = await session.execute(select(BackendModel))
+            assert len(res.scalars().all()) == 1
+
+        async def test_not_creates_with_invalid_regions(
+            self, test_db, session: AsyncSession, client: AsyncClient
+        ):
+            user = await create_user(session=session, global_role=GlobalRole.USER)
+            project = await create_project(session=session, owner=user)
+            await add_project_member(
+                session=session, project=project, user=user, project_role=ProjectRole.ADMIN
+            )
+            body = {
+                "type": "nebius",
+                "creds": FAKE_NEBIUS_SERVICE_ACCOUNT_CREDS,
+                "regions": ["xx-xxxx1"],
+            }
+            with patch(
+                "dstack._internal.core.backends.nebius.resources.get_region_to_project_id_map"
+            ) as get_region_to_project_id_map:
+                get_region_to_project_id_map.return_value = {"eu-north1": "project-e00test"}
+                response = await client.post(
+                    f"/api/project/{project.name}/backends/create",
+                    headers=get_auth_headers(user.token),
+                    json=body,
+                )
+            assert response.status_code == 400, response.json()
+            res = await session.execute(select(BackendModel))
+            assert len(res.scalars().all()) == 0
+
     @pytest.mark.asyncio
     @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
     async def test_creates_oci_backend(self, test_db, session: AsyncSession, client: AsyncClient):

tests/_internal/server/routers/test_prometheus.py

@@ -28,6 +28,7 @@ from dstack._internal.server.testing.common import (
     create_repo,
     create_run,
     create_user,
+    get_auth_headers,
     get_instance_offer_with_availability,
     get_job_provisioning_data,
     get_job_runtime_data,
@@ -38,6 +39,7 @@ from dstack._internal.server.testing.common import (
 @pytest.fixture
 def enable_metrics(monkeypatch: pytest.MonkeyPatch):
     monkeypatch.setattr("dstack._internal.server.settings.ENABLE_PROMETHEUS_METRICS", True)
+    monkeypatch.setattr("dstack._internal.server.routers.prometheus._auth._token", None)
 
 
 FAKE_NOW = datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc)
@@ -289,6 +291,25 @@ class TestGetPrometheusMetrics:
         response = await client.get("/metrics")
         assert response.status_code == 404
 
+    @pytest.mark.parametrize("token", [None, "foo"])
+    async def test_returns_403_if_not_authenticated(
+        self, monkeypatch: pytest.MonkeyPatch, client: AsyncClient, token: Optional[str]
+    ):
+        monkeypatch.setattr("dstack._internal.server.routers.prometheus._auth._token", "secret")
+        if token is not None:
+            headers = get_auth_headers(token)
+        else:
+            headers = None
+        response = await client.get("/metrics", headers=headers)
+        assert response.status_code == 403
+
+    async def test_returns_200_if_token_is_valid(
+        self, monkeypatch: pytest.MonkeyPatch, client: AsyncClient
+    ):
+        monkeypatch.setattr("dstack._internal.server.routers.prometheus._auth._token", "secret")
+        response = await client.get("/metrics", headers=get_auth_headers("secret"))
+        assert response.status_code == 200
+
 
 async def _create_project(session: AsyncSession, name: str, user: UserModel) -> ProjectModel:
     project = await create_project(session=session, owner=user, name=name)

tests/_internal/utils/test_event_loop.py

@@ -0,0 +1,18 @@
+import asyncio
+
+from dstack._internal.utils.event_loop import DaemonEventLoop
+
+
+def test_daemon_event_loop():
+    q = asyncio.Queue()
+
+    async def worker(i):
+        await q.put(i)
+
+    async def all_workers():
+        await asyncio.gather(*[worker(i) for i in range(3)])
+
+    loop = DaemonEventLoop()
+    loop.await_(all_workers())
+    assert q.qsize() == 3
+    assert {loop.await_(q.get()) for _ in range(3)} == {0, 1, 2}