dstack 0.19.20__py3-none-any.whl → 0.19.22__py3-none-any.whl

dstack/_internal/server/testing/common.py

@@ -35,6 +35,7 @@ from dstack._internal.core.models.fleets import (
     SSHParams,
 )
 from dstack._internal.core.models.gateways import GatewayComputeConfiguration, GatewayStatus
+from dstack._internal.core.models.health import HealthStatus
 from dstack._internal.core.models.instances import (
     Disk,
     Gpu,
@@ -68,6 +69,7 @@ from dstack._internal.core.models.runs import (
     Requirements,
     RunSpec,
     RunStatus,
+    RunTerminationReason,
 )
 from dstack._internal.core.models.users import GlobalRole
 from dstack._internal.core.models.volumes import (
@@ -84,11 +86,13 @@ from dstack._internal.server.models import (
     FleetModel,
     GatewayComputeModel,
     GatewayModel,
+    InstanceHealthCheckModel,
     InstanceModel,
     JobMetricsPoint,
     JobModel,
     JobPrometheusMetrics,
     PlacementGroupModel,
+    ProbeModel,
     ProjectModel,
     RepoCredsModel,
     RepoModel,
@@ -282,12 +286,15 @@ async def create_run(
     user: UserModel,
     run_name: str = "test-run",
     status: RunStatus = RunStatus.SUBMITTED,
+    termination_reason: Optional[RunTerminationReason] = None,
     submitted_at: datetime = datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc),
     run_spec: Optional[RunSpec] = None,
     run_id: Optional[UUID] = None,
     deleted: bool = False,
     priority: int = 0,
     deployment_num: int = 0,
+    resubmission_attempt: int = 0,
+    next_triggered_at: Optional[datetime] = None,
 ) -> RunModel:
     if run_spec is None:
         run_spec = get_run_spec(
@@ -305,12 +312,15 @@ async def create_run(
         submitted_at=submitted_at,
         run_name=run_name,
         status=status,
+        termination_reason=termination_reason,
         run_spec=run_spec.json(),
         last_processed_at=submitted_at,
         jobs=[],
         priority=priority,
         deployment_num=deployment_num,
         desired_replica_count=1,
+        resubmission_attempt=resubmission_attempt,
+        next_triggered_at=next_triggered_at,
     )
     session.add(run)
     await session.commit()
@@ -361,6 +371,7 @@ async def create_job(
         instance_assigned=instance_assigned,
         used_instance_id=instance.id if instance is not None else None,
         disconnected_at=disconnected_at,
+        probes=[],
     )
     session.add(job)
     await session.commit()
@@ -432,6 +443,26 @@ def get_job_runtime_data(
     )
 
 
+async def create_probe(
+    session: AsyncSession,
+    job: JobModel,
+    probe_num: int = 0,
+    due: datetime = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc),
+    success_streak: int = 0,
+) -> ProbeModel:
+    probe = ProbeModel(
+        name=f"{job.job_name}-{probe_num}",
+        job=job,
+        probe_num=probe_num,
+        due=due,
+        success_streak=success_streak,
+        active=True,
+    )
+    session.add(probe)
+    await session.commit()
+    return probe
+
+
 async def create_gateway(
     session: AsyncSession,
     project_id: UUID,
@@ -585,6 +616,7 @@ async def create_instance(
     fleet: Optional[FleetModel] = None,
     status: InstanceStatus = InstanceStatus.IDLE,
     unreachable: bool = False,
+    health_status: HealthStatus = HealthStatus.HEALTHY,
     created_at: datetime = datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc),
     finished_at: Optional[datetime] = None,
     spot: bool = False,
@@ -647,6 +679,7 @@ async def create_instance(
         status=status,
         last_processed_at=last_processed_at,
         unreachable=unreachable,
+        health=health_status,
         created_at=created_at,
         started_at=created_at,
         finished_at=finished_at,
@@ -767,6 +800,24 @@ def get_ssh_key() -> SSHKey:
     )
 
 
+async def create_instance_health_check(
+    session: AsyncSession,
+    instance: InstanceModel,
+    collected_at: datetime = datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc),
+    status: HealthStatus = HealthStatus.HEALTHY,
+    response: str = "{}",
+) -> InstanceHealthCheckModel:
+    health_check = InstanceHealthCheckModel(
+        instance_id=instance.id,
+        collected_at=collected_at,
+        status=status,
+        response=response,
+    )
+    session.add(health_check)
+    await session.commit()
+    return health_check
+
+
 async def create_volume(
     session: AsyncSession,
     project: ProjectModel,

dstack/_internal/{core/backends/remote → server/utils}/provisioning.py

@@ -224,23 +224,28 @@ def get_shim_healthcheck(client: paramiko.SSHClient) -> str:
     retries = 20
     iter_delay = 3
     for _ in range(retries):
-        try:
-            _, stdout, stderr = client.exec_command(
-                f"curl -s http://localhost:{DSTACK_SHIM_HTTP_PORT}/api/healthcheck", timeout=15
-            )
-            out = stdout.read().strip().decode()
-            err = stderr.read().strip().decode()
-            if err:
-                raise ProvisioningError(
-                    f"The command 'get_shim_healthcheck' didn't work. stdout: {out}, stderr: {err}"
-                )
-            if not out:
-                logger.debug("healthcheck is empty. retry")
-                time.sleep(iter_delay)
-                continue
-            return out
-        except (paramiko.SSHException, OSError) as e:
-            raise ProvisioningError(f"get_shim_healthcheck failed: {e}") from e
+        healthcheck = _get_shim_healthcheck(client)
+        if healthcheck is not None:
+            return healthcheck
+        logger.debug("healthcheck is empty. retry")
+        time.sleep(iter_delay)
+    raise ProvisioningError("Cannot get HealthcheckResponse")
+
+
+def _get_shim_healthcheck(client: paramiko.SSHClient) -> Optional[str]:
+    try:
+        _, stdout, stderr = client.exec_command(
+            f"curl -s http://localhost:{DSTACK_SHIM_HTTP_PORT}/api/healthcheck", timeout=15
+        )
+        out = stdout.read().strip().decode()
+        err = stderr.read().strip().decode()
+    except (paramiko.SSHException, OSError) as e:
+        raise ProvisioningError(f"get_shim_healthcheck failed: {e}") from e
+    if err:
+        raise ProvisioningError(f"get_shim_healthcheck didn't work. stdout: {out}, stderr: {err}")
+    if not out:
+        return None
+    return out
 
 
 def host_info_to_instance_type(host_info: Dict[str, Any], cpu_arch: GoArchType) -> InstanceType:

dstack/_internal/server/utils/sentry_utils.py

@@ -0,0 +1,12 @@
+import functools
+
+import sentry_sdk
+
+
+def instrument_background_task(f):
+    @functools.wraps(f)
+    async def wrapper(*args, **kwargs):
+        with sentry_sdk.start_transaction(name=f"background.{f.__name__}"):
+            return await f(*args, **kwargs)
+
+    return wrapper

dstack/_internal/settings.py

@@ -19,6 +19,9 @@ DSTACK_BASE_IMAGE_UBUNTU_VERSION = os.getenv(
 )
 DSTACK_DIND_IMAGE = os.getenv("DSTACK_DIND_IMAGE", "dstackai/dind")
 
+CLI_LOG_LEVEL = os.getenv("DSTACK_CLI_LOG_LEVEL", "INFO").upper()
+CLI_FILE_LOG_LEVEL = os.getenv("DSTACK_CLI_FILE_LOG_LEVEL", "DEBUG").upper()
+
 # Development settings
 
 LOCAL_BACKEND_ENABLED = os.getenv("DSTACK_LOCAL_BACKEND_ENABLED") is not None

dstack/_internal/utils/common.py

@@ -222,6 +222,21 @@ def remove_prefix(text: str, prefix: str) -> str:
     return text
 
 
+def has_duplicates(iterable: Iterable[Any]) -> bool:
+    """
+    Checks if there are any duplicate items in the given iterable.
+
+    O(n^2) implementation, but works with iterables with unhashable items.
+    For iterables with hashable items, prefer len(set(iterable)) != len(iterable).
+    """
+    seen = []
+    for item in iterable:
+        if item in seen:
+            return True
+        seen.append(item)
+    return False
+
+
 T = TypeVar("T")
 
 

dstack/_internal/utils/cron.py

@@ -0,0 +1,5 @@
+from apscheduler.triggers.cron import CronTrigger
+
+
+def validate_cron(cron_expr: str):
+    CronTrigger.from_crontab(cron_expr)

dstack/api/server/__init__.py

@@ -173,7 +173,7 @@ class APIClient:
                 raise ClientError(
                     f"Unexpected error: status code {resp.status_code}"
                     f" when requesting {resp.request.url}."
-                    " Check server logs or run with DSTACK_CLI_LOG_LEVEL=DEBUG to see more details"
+                    " Check the server logs for backend issues, and the CLI logs at (~/.dstack/logs/cli/latest.log) local CLI output"
                 )
         return resp
 

dstack/version.py

@@ -1,4 +1,4 @@
-__version__ = "0.19.20"
+__version__ = "0.19.22"
 __is_release__ = True
 base_image = "0.10"
 base_image_ubuntu_version = "22.04"

{dstack-0.19.20.dist-info → dstack-0.19.22.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dstack
-Version: 0.19.20
+Version: 0.19.22
 Summary: dstack is an open-source orchestration engine for running AI workloads on any cloud or on-premises.
 Project-URL: Homepage, https://dstack.ai
 Project-URL: Source, https://github.com/dstackai/dstack
@@ -15,13 +15,14 @@ Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9
+Requires-Dist: apscheduler<4
 Requires-Dist: argcomplete>=3.5.0
 Requires-Dist: cachetools
 Requires-Dist: cryptography
 Requires-Dist: cursor
 Requires-Dist: filelock
 Requires-Dist: gitpython
-Requires-Dist: gpuhunt==0.1.6
+Requires-Dist: gpuhunt==0.1.7
 Requires-Dist: ignore-python>=0.2.0
 Requires-Dist: jsonschema
 Requires-Dist: orjson
@@ -46,7 +47,6 @@ Requires-Dist: aiorwlock; extra == 'all'
 Requires-Dist: aiosqlite; extra == 'all'
 Requires-Dist: alembic-postgresql-enum; extra == 'all'
 Requires-Dist: alembic>=1.10.2; extra == 'all'
-Requires-Dist: apscheduler<4; extra == 'all'
 Requires-Dist: asyncpg; extra == 'all'
 Requires-Dist: azure-identity>=1.12.0; extra == 'all'
 Requires-Dist: azure-mgmt-authorization>=3.0.0; extra == 'all'
@@ -79,7 +79,7 @@ Requires-Dist: prometheus-client; extra == 'all'
 Requires-Dist: pyopenssl>=23.2.0; extra == 'all'
 Requires-Dist: python-dxf==12.1.0; extra == 'all'
 Requires-Dist: python-json-logger>=3.1.0; extra == 'all'
-Requires-Dist: sentry-sdk[fastapi]; extra == 'all'
+Requires-Dist: sentry-sdk[fastapi]>=2.27.0; extra == 'all'
 Requires-Dist: sqlalchemy-utils>=0.40.0; extra == 'all'
 Requires-Dist: sqlalchemy[asyncio]>=2.0.0; extra == 'all'
 Requires-Dist: starlette>=0.26.0; extra == 'all'
@@ -92,7 +92,6 @@ Requires-Dist: aiorwlock; extra == 'aws'
 Requires-Dist: aiosqlite; extra == 'aws'
 Requires-Dist: alembic-postgresql-enum; extra == 'aws'
 Requires-Dist: alembic>=1.10.2; extra == 'aws'
-Requires-Dist: apscheduler<4; extra == 'aws'
 Requires-Dist: asyncpg; extra == 'aws'
 Requires-Dist: backports-entry-points-selectable; extra == 'aws'
 Requires-Dist: boto3>=1.38.13; extra == 'aws'
@@ -105,7 +104,7 @@ Requires-Dist: jinja2; extra == 'aws'
 Requires-Dist: prometheus-client; extra == 'aws'
 Requires-Dist: python-dxf==12.1.0; extra == 'aws'
 Requires-Dist: python-json-logger>=3.1.0; extra == 'aws'
-Requires-Dist: sentry-sdk[fastapi]; extra == 'aws'
+Requires-Dist: sentry-sdk[fastapi]>=2.27.0; extra == 'aws'
 Requires-Dist: sqlalchemy-utils>=0.40.0; extra == 'aws'
 Requires-Dist: sqlalchemy[asyncio]>=2.0.0; extra == 'aws'
 Requires-Dist: starlette>=0.26.0; extra == 'aws'
@@ -117,7 +116,6 @@ Requires-Dist: aiorwlock; extra == 'azure'
 Requires-Dist: aiosqlite; extra == 'azure'
 Requires-Dist: alembic-postgresql-enum; extra == 'azure'
 Requires-Dist: alembic>=1.10.2; extra == 'azure'
-Requires-Dist: apscheduler<4; extra == 'azure'
 Requires-Dist: asyncpg; extra == 'azure'
 Requires-Dist: azure-identity>=1.12.0; extra == 'azure'
 Requires-Dist: azure-mgmt-authorization>=3.0.0; extra == 'azure'
@@ -135,7 +133,7 @@ Requires-Dist: jinja2; extra == 'azure'
 Requires-Dist: prometheus-client; extra == 'azure'
 Requires-Dist: python-dxf==12.1.0; extra == 'azure'
 Requires-Dist: python-json-logger>=3.1.0; extra == 'azure'
-Requires-Dist: sentry-sdk[fastapi]; extra == 'azure'
+Requires-Dist: sentry-sdk[fastapi]>=2.27.0; extra == 'azure'
 Requires-Dist: sqlalchemy-utils>=0.40.0; extra == 'azure'
 Requires-Dist: sqlalchemy[asyncio]>=2.0.0; extra == 'azure'
 Requires-Dist: starlette>=0.26.0; extra == 'azure'
@@ -147,7 +145,6 @@ Requires-Dist: aiorwlock; extra == 'datacrunch'
 Requires-Dist: aiosqlite; extra == 'datacrunch'
 Requires-Dist: alembic-postgresql-enum; extra == 'datacrunch'
 Requires-Dist: alembic>=1.10.2; extra == 'datacrunch'
-Requires-Dist: apscheduler<4; extra == 'datacrunch'
 Requires-Dist: asyncpg; extra == 'datacrunch'
 Requires-Dist: backports-entry-points-selectable; extra == 'datacrunch'
 Requires-Dist: datacrunch; extra == 'datacrunch'
@@ -159,7 +156,7 @@ Requires-Dist: jinja2; extra == 'datacrunch'
 Requires-Dist: prometheus-client; extra == 'datacrunch'
 Requires-Dist: python-dxf==12.1.0; extra == 'datacrunch'
 Requires-Dist: python-json-logger>=3.1.0; extra == 'datacrunch'
-Requires-Dist: sentry-sdk[fastapi]; extra == 'datacrunch'
+Requires-Dist: sentry-sdk[fastapi]>=2.27.0; extra == 'datacrunch'
 Requires-Dist: sqlalchemy-utils>=0.40.0; extra == 'datacrunch'
 Requires-Dist: sqlalchemy[asyncio]>=2.0.0; extra == 'datacrunch'
 Requires-Dist: starlette>=0.26.0; extra == 'datacrunch'
@@ -179,7 +176,6 @@ Requires-Dist: aiorwlock; extra == 'gcp'
 Requires-Dist: aiosqlite; extra == 'gcp'
 Requires-Dist: alembic-postgresql-enum; extra == 'gcp'
 Requires-Dist: alembic>=1.10.2; extra == 'gcp'
-Requires-Dist: apscheduler<4; extra == 'gcp'
 Requires-Dist: asyncpg; extra == 'gcp'
 Requires-Dist: backports-entry-points-selectable; extra == 'gcp'
 Requires-Dist: docker>=6.0.0; extra == 'gcp'
@@ -197,7 +193,7 @@ Requires-Dist: jinja2; extra == 'gcp'
 Requires-Dist: prometheus-client; extra == 'gcp'
 Requires-Dist: python-dxf==12.1.0; extra == 'gcp'
 Requires-Dist: python-json-logger>=3.1.0; extra == 'gcp'
-Requires-Dist: sentry-sdk[fastapi]; extra == 'gcp'
+Requires-Dist: sentry-sdk[fastapi]>=2.27.0; extra == 'gcp'
 Requires-Dist: sqlalchemy-utils>=0.40.0; extra == 'gcp'
 Requires-Dist: sqlalchemy[asyncio]>=2.0.0; extra == 'gcp'
 Requires-Dist: starlette>=0.26.0; extra == 'gcp'
@@ -209,7 +205,6 @@ Requires-Dist: aiorwlock; extra == 'kubernetes'
 Requires-Dist: aiosqlite; extra == 'kubernetes'
 Requires-Dist: alembic-postgresql-enum; extra == 'kubernetes'
 Requires-Dist: alembic>=1.10.2; extra == 'kubernetes'
-Requires-Dist: apscheduler<4; extra == 'kubernetes'
 Requires-Dist: asyncpg; extra == 'kubernetes'
 Requires-Dist: backports-entry-points-selectable; extra == 'kubernetes'
 Requires-Dist: docker>=6.0.0; extra == 'kubernetes'
@@ -221,7 +216,7 @@ Requires-Dist: kubernetes; extra == 'kubernetes'
 Requires-Dist: prometheus-client; extra == 'kubernetes'
 Requires-Dist: python-dxf==12.1.0; extra == 'kubernetes'
 Requires-Dist: python-json-logger>=3.1.0; extra == 'kubernetes'
-Requires-Dist: sentry-sdk[fastapi]; extra == 'kubernetes'
+Requires-Dist: sentry-sdk[fastapi]>=2.27.0; extra == 'kubernetes'
 Requires-Dist: sqlalchemy-utils>=0.40.0; extra == 'kubernetes'
 Requires-Dist: sqlalchemy[asyncio]>=2.0.0; extra == 'kubernetes'
 Requires-Dist: starlette>=0.26.0; extra == 'kubernetes'
@@ -233,7 +228,6 @@ Requires-Dist: aiorwlock; extra == 'lambda'
 Requires-Dist: aiosqlite; extra == 'lambda'
 Requires-Dist: alembic-postgresql-enum; extra == 'lambda'
 Requires-Dist: alembic>=1.10.2; extra == 'lambda'
-Requires-Dist: apscheduler<4; extra == 'lambda'
 Requires-Dist: asyncpg; extra == 'lambda'
 Requires-Dist: backports-entry-points-selectable; extra == 'lambda'
 Requires-Dist: boto3>=1.38.13; extra == 'lambda'
@@ -246,7 +240,7 @@ Requires-Dist: jinja2; extra == 'lambda'
 Requires-Dist: prometheus-client; extra == 'lambda'
 Requires-Dist: python-dxf==12.1.0; extra == 'lambda'
 Requires-Dist: python-json-logger>=3.1.0; extra == 'lambda'
-Requires-Dist: sentry-sdk[fastapi]; extra == 'lambda'
+Requires-Dist: sentry-sdk[fastapi]>=2.27.0; extra == 'lambda'
 Requires-Dist: sqlalchemy-utils>=0.40.0; extra == 'lambda'
 Requires-Dist: sqlalchemy[asyncio]>=2.0.0; extra == 'lambda'
 Requires-Dist: starlette>=0.26.0; extra == 'lambda'
@@ -258,7 +252,6 @@ Requires-Dist: aiorwlock; extra == 'nebius'
 Requires-Dist: aiosqlite; extra == 'nebius'
 Requires-Dist: alembic-postgresql-enum; extra == 'nebius'
 Requires-Dist: alembic>=1.10.2; extra == 'nebius'
-Requires-Dist: apscheduler<4; extra == 'nebius'
 Requires-Dist: asyncpg; extra == 'nebius'
 Requires-Dist: backports-entry-points-selectable; extra == 'nebius'
 Requires-Dist: docker>=6.0.0; extra == 'nebius'
@@ -270,7 +263,7 @@ Requires-Dist: nebius<0.3,>=0.2.19; (python_version >= '3.10') and extra == 'neb
 Requires-Dist: prometheus-client; extra == 'nebius'
 Requires-Dist: python-dxf==12.1.0; extra == 'nebius'
 Requires-Dist: python-json-logger>=3.1.0; extra == 'nebius'
-Requires-Dist: sentry-sdk[fastapi]; extra == 'nebius'
+Requires-Dist: sentry-sdk[fastapi]>=2.27.0; extra == 'nebius'
 Requires-Dist: sqlalchemy-utils>=0.40.0; extra == 'nebius'
 Requires-Dist: sqlalchemy[asyncio]>=2.0.0; extra == 'nebius'
 Requires-Dist: starlette>=0.26.0; extra == 'nebius'
@@ -282,7 +275,6 @@ Requires-Dist: aiorwlock; extra == 'oci'
 Requires-Dist: aiosqlite; extra == 'oci'
 Requires-Dist: alembic-postgresql-enum; extra == 'oci'
 Requires-Dist: alembic>=1.10.2; extra == 'oci'
-Requires-Dist: apscheduler<4; extra == 'oci'
 Requires-Dist: asyncpg; extra == 'oci'
 Requires-Dist: backports-entry-points-selectable; extra == 'oci'
 Requires-Dist: cryptography>=44.0.3; extra == 'oci'
@@ -296,7 +288,7 @@ Requires-Dist: prometheus-client; extra == 'oci'
 Requires-Dist: pyopenssl>=23.2.0; extra == 'oci'
 Requires-Dist: python-dxf==12.1.0; extra == 'oci'
 Requires-Dist: python-json-logger>=3.1.0; extra == 'oci'
-Requires-Dist: sentry-sdk[fastapi]; extra == 'oci'
+Requires-Dist: sentry-sdk[fastapi]>=2.27.0; extra == 'oci'
 Requires-Dist: sqlalchemy-utils>=0.40.0; extra == 'oci'
 Requires-Dist: sqlalchemy[asyncio]>=2.0.0; extra == 'oci'
 Requires-Dist: starlette>=0.26.0; extra == 'oci'
@@ -308,7 +300,6 @@ Requires-Dist: aiorwlock; extra == 'server'
 Requires-Dist: aiosqlite; extra == 'server'
 Requires-Dist: alembic-postgresql-enum; extra == 'server'
 Requires-Dist: alembic>=1.10.2; extra == 'server'
-Requires-Dist: apscheduler<4; extra == 'server'
 Requires-Dist: asyncpg; extra == 'server'
 Requires-Dist: backports-entry-points-selectable; extra == 'server'
 Requires-Dist: docker>=6.0.0; extra == 'server'
@@ -319,7 +310,7 @@ Requires-Dist: jinja2; extra == 'server'
 Requires-Dist: prometheus-client; extra == 'server'
 Requires-Dist: python-dxf==12.1.0; extra == 'server'
 Requires-Dist: python-json-logger>=3.1.0; extra == 'server'
-Requires-Dist: sentry-sdk[fastapi]; extra == 'server'
+Requires-Dist: sentry-sdk[fastapi]>=2.27.0; extra == 'server'
 Requires-Dist: sqlalchemy-utils>=0.40.0; extra == 'server'
 Requires-Dist: sqlalchemy[asyncio]>=2.0.0; extra == 'server'
 Requires-Dist: starlette>=0.26.0; extra == 'server'