dstack 0.19.21__py3-none-any.whl → 0.19.23rc1__py3-none-any.whl

dstack/_internal/server/testing/common.py

@@ -35,6 +35,7 @@ from dstack._internal.core.models.fleets import (
     SSHParams,
 )
 from dstack._internal.core.models.gateways import GatewayComputeConfiguration, GatewayStatus
+from dstack._internal.core.models.health import HealthStatus
 from dstack._internal.core.models.instances import (
     Disk,
     Gpu,
@@ -85,11 +86,13 @@ from dstack._internal.server.models import (
     FleetModel,
     GatewayComputeModel,
     GatewayModel,
+    InstanceHealthCheckModel,
     InstanceModel,
     JobMetricsPoint,
     JobModel,
     JobPrometheusMetrics,
     PlacementGroupModel,
+    ProbeModel,
     ProjectModel,
     RepoCredsModel,
     RepoModel,
@@ -368,6 +371,7 @@ async def create_job(
         instance_assigned=instance_assigned,
         used_instance_id=instance.id if instance is not None else None,
         disconnected_at=disconnected_at,
+        probes=[],
     )
     session.add(job)
     await session.commit()
@@ -439,6 +443,26 @@ def get_job_runtime_data(
     )
 
 
+async def create_probe(
+    session: AsyncSession,
+    job: JobModel,
+    probe_num: int = 0,
+    due: datetime = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc),
+    success_streak: int = 0,
+) -> ProbeModel:
+    probe = ProbeModel(
+        name=f"{job.job_name}-{probe_num}",
+        job=job,
+        probe_num=probe_num,
+        due=due,
+        success_streak=success_streak,
+        active=True,
+    )
+    session.add(probe)
+    await session.commit()
+    return probe
+
+
 async def create_gateway(
     session: AsyncSession,
     project_id: UUID,
@@ -592,6 +616,7 @@ async def create_instance(
     fleet: Optional[FleetModel] = None,
     status: InstanceStatus = InstanceStatus.IDLE,
     unreachable: bool = False,
+    health_status: HealthStatus = HealthStatus.HEALTHY,
     created_at: datetime = datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc),
     finished_at: Optional[datetime] = None,
     spot: bool = False,
@@ -654,6 +679,7 @@ async def create_instance(
         status=status,
         last_processed_at=last_processed_at,
         unreachable=unreachable,
+        health=health_status,
         created_at=created_at,
         started_at=created_at,
         finished_at=finished_at,
@@ -774,6 +800,24 @@ def get_ssh_key() -> SSHKey:
     )
 
 
+async def create_instance_health_check(
+    session: AsyncSession,
+    instance: InstanceModel,
+    collected_at: datetime = datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc),
+    status: HealthStatus = HealthStatus.HEALTHY,
+    response: str = "{}",
+) -> InstanceHealthCheckModel:
+    health_check = InstanceHealthCheckModel(
+        instance_id=instance.id,
+        collected_at=collected_at,
+        status=status,
+        response=response,
+    )
+    session.add(health_check)
+    await session.commit()
+    return health_check
+
+
 async def create_volume(
     session: AsyncSession,
     project: ProjectModel,

dstack/_internal/{core/backends/remote → server/utils}/provisioning.py

@@ -224,23 +224,28 @@ def get_shim_healthcheck(client: paramiko.SSHClient) -> str:
     retries = 20
     iter_delay = 3
     for _ in range(retries):
-        try:
-            _, stdout, stderr = client.exec_command(
-                f"curl -s http://localhost:{DSTACK_SHIM_HTTP_PORT}/api/healthcheck", timeout=15
-            )
-            out = stdout.read().strip().decode()
-            err = stderr.read().strip().decode()
-            if err:
-                raise ProvisioningError(
-                    f"The command 'get_shim_healthcheck' didn't work. stdout: {out}, stderr: {err}"
-                )
-            if not out:
-                logger.debug("healthcheck is empty. retry")
-                time.sleep(iter_delay)
-                continue
-            return out
-        except (paramiko.SSHException, OSError) as e:
-            raise ProvisioningError(f"get_shim_healthcheck failed: {e}") from e
+        healthcheck = _get_shim_healthcheck(client)
+        if healthcheck is not None:
+            return healthcheck
+        logger.debug("healthcheck is empty. retry")
+        time.sleep(iter_delay)
+    raise ProvisioningError("Cannot get HealthcheckResponse")
+
+
+def _get_shim_healthcheck(client: paramiko.SSHClient) -> Optional[str]:
+    try:
+        _, stdout, stderr = client.exec_command(
+            f"curl -s http://localhost:{DSTACK_SHIM_HTTP_PORT}/api/healthcheck", timeout=15
+        )
+        out = stdout.read().strip().decode()
+        err = stderr.read().strip().decode()
+    except (paramiko.SSHException, OSError) as e:
+        raise ProvisioningError(f"get_shim_healthcheck failed: {e}") from e
+    if err:
+        raise ProvisioningError(f"get_shim_healthcheck didn't work. stdout: {out}, stderr: {err}")
+    if not out:
+        return None
+    return out
 
 
 def host_info_to_instance_type(host_info: Dict[str, Any], cpu_arch: GoArchType) -> InstanceType:

dstack/_internal/settings.py

@@ -19,6 +19,9 @@ DSTACK_BASE_IMAGE_UBUNTU_VERSION = os.getenv(
 )
 DSTACK_DIND_IMAGE = os.getenv("DSTACK_DIND_IMAGE", "dstackai/dind")
 
+CLI_LOG_LEVEL = os.getenv("DSTACK_CLI_LOG_LEVEL", "INFO").upper()
+CLI_FILE_LOG_LEVEL = os.getenv("DSTACK_CLI_FILE_LOG_LEVEL", "DEBUG").upper()
+
 # Development settings
 
 LOCAL_BACKEND_ENABLED = os.getenv("DSTACK_LOCAL_BACKEND_ENABLED") is not None

dstack/_internal/utils/common.py

@@ -222,6 +222,21 @@ def remove_prefix(text: str, prefix: str) -> str:
     return text
 
 
+def has_duplicates(iterable: Iterable[Any]) -> bool:
+    """
+    Checks if there are any duplicate items in the given iterable.
+
+    O(n^2) implementation, but works with iterables with unhashable items.
+    For iterables with hashable items, prefer len(set(iterable)) != len(iterable).
+    """
+    seen = []
+    for item in iterable:
+        if item in seen:
+            return True
+        seen.append(item)
+    return False
+
+
 T = TypeVar("T")
 
 

dstack/api/server/__init__.py

@@ -173,7 +173,7 @@ class APIClient:
                 raise ClientError(
                     f"Unexpected error: status code {resp.status_code}"
                     f" when requesting {resp.request.url}."
-                    " Check server logs or run with DSTACK_CLI_LOG_LEVEL=DEBUG to see more details"
+                    " Check the server logs for backend issues, and the CLI logs at (~/.dstack/logs/cli/latest.log) local CLI output"
                 )
         return resp
 

dstack/version.py

@@ -1,4 +1,4 @@
-__version__ = "0.19.21"
+__version__ = "0.19.23rc1"
 __is_release__ = True
 base_image = "0.10"
 base_image_ubuntu_version = "22.04"

{dstack-0.19.21.dist-info → dstack-0.19.23rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dstack
-Version: 0.19.21
+Version: 0.19.23rc1
 Summary: dstack is an open-source orchestration engine for running AI workloads on any cloud or on-premises.
 Project-URL: Homepage, https://dstack.ai
 Project-URL: Source, https://github.com/dstackai/dstack
@@ -22,7 +22,7 @@ Requires-Dist: cryptography
 Requires-Dist: cursor
 Requires-Dist: filelock
 Requires-Dist: gitpython
-Requires-Dist: gpuhunt==0.1.6
+Requires-Dist: gpuhunt==0.1.7
 Requires-Dist: ignore-python>=0.2.0
 Requires-Dist: jsonschema
 Requires-Dist: orjson
@@ -73,13 +73,13 @@ Requires-Dist: grpcio>=1.50; extra == 'all'
 Requires-Dist: httpx; extra == 'all'
 Requires-Dist: jinja2; extra == 'all'
 Requires-Dist: kubernetes; extra == 'all'
-Requires-Dist: nebius<0.3,>=0.2.19; (python_version >= '3.10') and extra == 'all'
+Requires-Dist: nebius<0.3,>=0.2.40; (python_version >= '3.10') and extra == 'all'
 Requires-Dist: oci>=2.150.0; extra == 'all'
 Requires-Dist: prometheus-client; extra == 'all'
 Requires-Dist: pyopenssl>=23.2.0; extra == 'all'
 Requires-Dist: python-dxf==12.1.0; extra == 'all'
 Requires-Dist: python-json-logger>=3.1.0; extra == 'all'
-Requires-Dist: sentry-sdk[fastapi]; extra == 'all'
+Requires-Dist: sentry-sdk[fastapi]>=2.27.0; extra == 'all'
 Requires-Dist: sqlalchemy-utils>=0.40.0; extra == 'all'
 Requires-Dist: sqlalchemy[asyncio]>=2.0.0; extra == 'all'
 Requires-Dist: starlette>=0.26.0; extra == 'all'
@@ -104,7 +104,7 @@ Requires-Dist: jinja2; extra == 'aws'
 Requires-Dist: prometheus-client; extra == 'aws'
 Requires-Dist: python-dxf==12.1.0; extra == 'aws'
 Requires-Dist: python-json-logger>=3.1.0; extra == 'aws'
-Requires-Dist: sentry-sdk[fastapi]; extra == 'aws'
+Requires-Dist: sentry-sdk[fastapi]>=2.27.0; extra == 'aws'
 Requires-Dist: sqlalchemy-utils>=0.40.0; extra == 'aws'
 Requires-Dist: sqlalchemy[asyncio]>=2.0.0; extra == 'aws'
 Requires-Dist: starlette>=0.26.0; extra == 'aws'
@@ -133,7 +133,7 @@ Requires-Dist: jinja2; extra == 'azure'
 Requires-Dist: prometheus-client; extra == 'azure'
 Requires-Dist: python-dxf==12.1.0; extra == 'azure'
 Requires-Dist: python-json-logger>=3.1.0; extra == 'azure'
-Requires-Dist: sentry-sdk[fastapi]; extra == 'azure'
+Requires-Dist: sentry-sdk[fastapi]>=2.27.0; extra == 'azure'
 Requires-Dist: sqlalchemy-utils>=0.40.0; extra == 'azure'
 Requires-Dist: sqlalchemy[asyncio]>=2.0.0; extra == 'azure'
 Requires-Dist: starlette>=0.26.0; extra == 'azure'
@@ -156,7 +156,7 @@ Requires-Dist: jinja2; extra == 'datacrunch'
 Requires-Dist: prometheus-client; extra == 'datacrunch'
 Requires-Dist: python-dxf==12.1.0; extra == 'datacrunch'
 Requires-Dist: python-json-logger>=3.1.0; extra == 'datacrunch'
-Requires-Dist: sentry-sdk[fastapi]; extra == 'datacrunch'
+Requires-Dist: sentry-sdk[fastapi]>=2.27.0; extra == 'datacrunch'
 Requires-Dist: sqlalchemy-utils>=0.40.0; extra == 'datacrunch'
 Requires-Dist: sqlalchemy[asyncio]>=2.0.0; extra == 'datacrunch'
 Requires-Dist: starlette>=0.26.0; extra == 'datacrunch'
@@ -193,7 +193,7 @@ Requires-Dist: jinja2; extra == 'gcp'
 Requires-Dist: prometheus-client; extra == 'gcp'
 Requires-Dist: python-dxf==12.1.0; extra == 'gcp'
 Requires-Dist: python-json-logger>=3.1.0; extra == 'gcp'
-Requires-Dist: sentry-sdk[fastapi]; extra == 'gcp'
+Requires-Dist: sentry-sdk[fastapi]>=2.27.0; extra == 'gcp'
 Requires-Dist: sqlalchemy-utils>=0.40.0; extra == 'gcp'
 Requires-Dist: sqlalchemy[asyncio]>=2.0.0; extra == 'gcp'
 Requires-Dist: starlette>=0.26.0; extra == 'gcp'
@@ -216,7 +216,7 @@ Requires-Dist: kubernetes; extra == 'kubernetes'
 Requires-Dist: prometheus-client; extra == 'kubernetes'
 Requires-Dist: python-dxf==12.1.0; extra == 'kubernetes'
 Requires-Dist: python-json-logger>=3.1.0; extra == 'kubernetes'
-Requires-Dist: sentry-sdk[fastapi]; extra == 'kubernetes'
+Requires-Dist: sentry-sdk[fastapi]>=2.27.0; extra == 'kubernetes'
 Requires-Dist: sqlalchemy-utils>=0.40.0; extra == 'kubernetes'
 Requires-Dist: sqlalchemy[asyncio]>=2.0.0; extra == 'kubernetes'
 Requires-Dist: starlette>=0.26.0; extra == 'kubernetes'
@@ -240,7 +240,7 @@ Requires-Dist: jinja2; extra == 'lambda'
 Requires-Dist: prometheus-client; extra == 'lambda'
 Requires-Dist: python-dxf==12.1.0; extra == 'lambda'
 Requires-Dist: python-json-logger>=3.1.0; extra == 'lambda'
-Requires-Dist: sentry-sdk[fastapi]; extra == 'lambda'
+Requires-Dist: sentry-sdk[fastapi]>=2.27.0; extra == 'lambda'
 Requires-Dist: sqlalchemy-utils>=0.40.0; extra == 'lambda'
 Requires-Dist: sqlalchemy[asyncio]>=2.0.0; extra == 'lambda'
 Requires-Dist: starlette>=0.26.0; extra == 'lambda'
@@ -259,11 +259,11 @@ Requires-Dist: fastapi; extra == 'nebius'
 Requires-Dist: grpcio>=1.50; extra == 'nebius'
 Requires-Dist: httpx; extra == 'nebius'
 Requires-Dist: jinja2; extra == 'nebius'
-Requires-Dist: nebius<0.3,>=0.2.19; (python_version >= '3.10') and extra == 'nebius'
+Requires-Dist: nebius<0.3,>=0.2.40; (python_version >= '3.10') and extra == 'nebius'
 Requires-Dist: prometheus-client; extra == 'nebius'
 Requires-Dist: python-dxf==12.1.0; extra == 'nebius'
 Requires-Dist: python-json-logger>=3.1.0; extra == 'nebius'
-Requires-Dist: sentry-sdk[fastapi]; extra == 'nebius'
+Requires-Dist: sentry-sdk[fastapi]>=2.27.0; extra == 'nebius'
 Requires-Dist: sqlalchemy-utils>=0.40.0; extra == 'nebius'
 Requires-Dist: sqlalchemy[asyncio]>=2.0.0; extra == 'nebius'
 Requires-Dist: starlette>=0.26.0; extra == 'nebius'
@@ -288,7 +288,7 @@ Requires-Dist: prometheus-client; extra == 'oci'
 Requires-Dist: pyopenssl>=23.2.0; extra == 'oci'
 Requires-Dist: python-dxf==12.1.0; extra == 'oci'
 Requires-Dist: python-json-logger>=3.1.0; extra == 'oci'
-Requires-Dist: sentry-sdk[fastapi]; extra == 'oci'
+Requires-Dist: sentry-sdk[fastapi]>=2.27.0; extra == 'oci'
 Requires-Dist: sqlalchemy-utils>=0.40.0; extra == 'oci'
 Requires-Dist: sqlalchemy[asyncio]>=2.0.0; extra == 'oci'
 Requires-Dist: starlette>=0.26.0; extra == 'oci'
@@ -310,7 +310,7 @@ Requires-Dist: jinja2; extra == 'server'
 Requires-Dist: prometheus-client; extra == 'server'
 Requires-Dist: python-dxf==12.1.0; extra == 'server'
 Requires-Dist: python-json-logger>=3.1.0; extra == 'server'
-Requires-Dist: sentry-sdk[fastapi]; extra == 'server'
+Requires-Dist: sentry-sdk[fastapi]>=2.27.0; extra == 'server'
 Requires-Dist: sqlalchemy-utils>=0.40.0; extra == 'server'
 Requires-Dist: sqlalchemy[asyncio]>=2.0.0; extra == 'server'
 Requires-Dist: starlette>=0.26.0; extra == 'server'