dstack 0.19.34__py3-none-any.whl → 0.19.35__py3-none-any.whl

dstack/_internal/server/services/compute_groups.py

@@ -0,0 +1,22 @@
+from dstack._internal.core.models.compute_groups import ComputeGroup, ComputeGroupProvisioningData
+from dstack._internal.server.models import ComputeGroupModel
+
+
+def compute_group_model_to_compute_group(compute_group_model: ComputeGroupModel) -> ComputeGroup:
+    provisioning_data = get_compute_group_provisioning_data(compute_group_model)
+    return ComputeGroup(
+        id=compute_group_model.id,
+        project_name=compute_group_model.project.name,
+        status=compute_group_model.status,
+        name=provisioning_data.compute_group_name,
+        created_at=compute_group_model.created_at,
+        provisioning_data=provisioning_data,
+    )
+
+
+def get_compute_group_provisioning_data(
+    compute_group_model: ComputeGroupModel,
+) -> ComputeGroupProvisioningData:
+    return ComputeGroupProvisioningData.__response__.parse_raw(
+        compute_group_model.provisioning_data
+    )

dstack/_internal/server/services/fleets.py

@@ -650,6 +650,7 @@ def get_fleet_requirements(fleet_spec: FleetSpec) -> Requirements:
         max_price=profile.max_price,
         spot=get_policy_map(profile.spot_policy, default=SpotPolicy.ONDEMAND),
         reservation=fleet_spec.configuration.reservation,
+        multinode=fleet_spec.configuration.placement == InstanceGroupPlacement.CLUSTER,
     )
     return requirements
 

dstack/_internal/server/services/jobs/__init__.py

@@ -96,6 +96,19 @@ def find_job(jobs: List[Job], replica_num: int, job_num: int) -> Job:
     )
 
 
+def find_jobs(
+    jobs: List[Job],
+    replica_num: Optional[int] = None,
+    job_num: Optional[int] = None,
+) -> list[Job]:
+    res = jobs
+    if replica_num is not None:
+        res = [j for j in res if j.job_spec.replica_num == replica_num]
+    if job_num is not None:
+        res = [j for j in res if j.job_spec.job_num == job_num]
+    return res
+
+
 async def get_run_job_model(
     session: AsyncSession,
     project: ProjectModel,

dstack/_internal/server/services/jobs/configurators/base.py

@@ -161,7 +161,7 @@ class JobConfigurator(ABC):
             stop_duration=self._stop_duration(),
             utilization_policy=self._utilization_policy(),
             registry_auth=self._registry_auth(),
-            requirements=self._requirements(),
+            requirements=self._requirements(jobs_per_replica),
             retry=self._retry(),
             working_dir=self._working_dir(),
             volumes=self._volumes(job_num),
@@ -295,13 +295,14 @@ class JobConfigurator(ABC):
     def _registry_auth(self) -> Optional[RegistryAuth]:
         return self.run_spec.configuration.registry_auth
 
-    def _requirements(self) -> Requirements:
+    def _requirements(self, jobs_per_replica: int) -> Requirements:
         spot_policy = self._spot_policy()
         return Requirements(
             resources=self.run_spec.configuration.resources,
             max_price=self.run_spec.merged_profile.max_price,
             spot=None if spot_policy == SpotPolicy.AUTO else (spot_policy == SpotPolicy.SPOT),
             reservation=self.run_spec.merged_profile.reservation,
+            multinode=jobs_per_replica > 1,
         )
 
     def _retry(self) -> Optional[Retry]:

dstack/_internal/server/services/requirements/combine.py

@@ -63,6 +63,7 @@ def combine_fleet_and_run_requirements(
             reservation=_get_single_value_optional(
                 fleet_requirements.reservation, run_requirements.reservation
             ),
+            multinode=fleet_requirements.multinode or run_requirements.multinode,
         )
     except CombineError:
         return None

dstack/_internal/server/services/runs.py

@@ -34,6 +34,7 @@ from dstack._internal.core.models.profiles import (
 )
 from dstack._internal.core.models.repos.virtual import DEFAULT_VIRTUAL_REPO_ID, VirtualRunRepoData
 from dstack._internal.core.models.runs import (
+    LEGACY_REPO_DIR,
     ApplyRunPlanInput,
     Job,
     JobPlan,
@@ -308,6 +309,7 @@ async def get_plan(
     user: UserModel,
     run_spec: RunSpec,
     max_offers: Optional[int],
+    legacy_default_working_dir: bool = False,
 ) -> RunPlan:
     # Spec must be copied by parsing to calculate merged_profile
     effective_run_spec = RunSpec.parse_obj(run_spec.dict())
@@ -317,7 +319,11 @@ async def get_plan(
         spec=effective_run_spec,
     )
     effective_run_spec = RunSpec.parse_obj(effective_run_spec.dict())
-    _validate_run_spec_and_set_defaults(user, effective_run_spec)
+    _validate_run_spec_and_set_defaults(
+        user=user,
+        run_spec=effective_run_spec,
+        legacy_default_working_dir=legacy_default_working_dir,
+    )
 
     profile = effective_run_spec.merged_profile
     creation_policy = profile.creation_policy
@@ -413,6 +419,7 @@ async def apply_plan(
     project: ProjectModel,
     plan: ApplyRunPlanInput,
     force: bool,
+    legacy_default_working_dir: bool = False,
 ) -> Run:
     run_spec = plan.run_spec
     run_spec = await apply_plugin_policies(
@@ -422,7 +429,9 @@ async def apply_plan(
     )
     # Spec must be copied by parsing to calculate merged_profile
     run_spec = RunSpec.parse_obj(run_spec.dict())
-    _validate_run_spec_and_set_defaults(user, run_spec)
+    _validate_run_spec_and_set_defaults(
+        user=user, run_spec=run_spec, legacy_default_working_dir=legacy_default_working_dir
+    )
     if run_spec.run_name is None:
         return await submit_run(
             session=session,
@@ -600,6 +609,7 @@ def create_job_model_for_new_submission(
         job_spec_data=job.job_spec.json(),
         job_provisioning_data=None,
         probes=[],
+        waiting_master_job=job.job_spec.job_num != 0,
     )
 
 
@@ -985,7 +995,9 @@ def _get_job_submission_cost(job_submission: JobSubmission) -> float:
     return job_submission.job_provisioning_data.price * duration_hours
 
 
-def _validate_run_spec_and_set_defaults(user: UserModel, run_spec: RunSpec):
+def _validate_run_spec_and_set_defaults(
+    user: UserModel, run_spec: RunSpec, legacy_default_working_dir: bool = False
+):
     # This function may set defaults for null run_spec values,
     # although most defaults are resolved when building job_spec
     # so that we can keep both the original user-supplied value (null in run_spec)
@@ -1040,6 +1052,8 @@ def _validate_run_spec_and_set_defaults(user: UserModel, run_spec: RunSpec):
             run_spec.ssh_key_pub = user.ssh_public_key
         else:
             raise ServerClientError("ssh_key_pub must be set if the user has no ssh_public_key")
+    if run_spec.configuration.working_dir is None and legacy_default_working_dir:
+        run_spec.configuration.working_dir = LEGACY_REPO_DIR
 
 
 _UPDATABLE_SPEC_FIELDS = ["configuration_path", "configuration"]

dstack/_internal/server/testing/common.py

@@ -13,6 +13,7 @@ from dstack._internal.core.backends.base.compute import (
     Compute,
     ComputeWithCreateInstanceSupport,
     ComputeWithGatewaySupport,
+    ComputeWithGroupProvisioningSupport,
     ComputeWithMultinodeSupport,
     ComputeWithPlacementGroupSupport,
     ComputeWithPrivateGatewaySupport,
@@ -22,6 +23,10 @@ from dstack._internal.core.backends.base.compute import (
 )
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.common import NetworkMode
+from dstack._internal.core.models.compute_groups import (
+    ComputeGroupProvisioningData,
+    ComputeGroupStatus,
+)
 from dstack._internal.core.models.configurations import (
     AnyRunConfiguration,
     DevEnvironmentConfiguration,
@@ -83,6 +88,7 @@ from dstack._internal.core.models.volumes import (
 )
 from dstack._internal.server.models import (
     BackendModel,
+    ComputeGroupModel,
     DecryptedString,
     FileArchiveModel,
     FleetModel,
@@ -353,6 +359,7 @@ async def create_job(
     instance_assigned: bool = False,
     disconnected_at: Optional[datetime] = None,
     registered: bool = False,
+    waiting_master_job: Optional[bool] = None,
 ) -> JobModel:
     if deployment_num is None:
         deployment_num = run.deployment_num
@@ -384,6 +391,7 @@ async def create_job(
         disconnected_at=disconnected_at,
         probes=[],
         registered=registered,
+        waiting_master_job=waiting_master_job,
     )
     session.add(job)
     await session.commit()
@@ -455,6 +463,48 @@ def get_job_runtime_data(
     )
 
 
+def get_compute_group_provisioning_data(
+    compute_group_id: str = "test_compute_group",
+    compute_group_name: str = "test_compute_group",
+    backend: BackendType = BackendType.RUNPOD,
+    region: str = "US",
+    job_provisioning_datas: Optional[list[JobProvisioningData]] = None,
+    backend_data: Optional[str] = None,
+) -> ComputeGroupProvisioningData:
+    if job_provisioning_datas is None:
+        job_provisioning_datas = []
+    return ComputeGroupProvisioningData(
+        compute_group_id=compute_group_id,
+        compute_group_name=compute_group_name,
+        backend=backend,
+        region=region,
+        job_provisioning_datas=job_provisioning_datas,
+        backend_data=backend_data,
+    )
+
+
+async def create_compute_group(
+    session: AsyncSession,
+    project: ProjectModel,
+    fleet: FleetModel,
+    status: ComputeGroupStatus = ComputeGroupStatus.RUNNING,
+    provisioning_data: Optional[ComputeGroupProvisioningData] = None,
+    last_processed_at: datetime = datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc),
+):
+    if provisioning_data is None:
+        provisioning_data = get_compute_group_provisioning_data()
+    compute_group = ComputeGroupModel(
+        project=project,
+        fleet=fleet,
+        status=status,
+        provisioning_data=provisioning_data.json(),
+        last_processed_at=last_processed_at,
+    )
+    session.add(compute_group)
+    await session.commit()
+    return compute_group
+
+
 async def create_probe(
     session: AsyncSession,
     job: JobModel,
@@ -1136,6 +1186,7 @@ class AsyncContextManager:
 class ComputeMockSpec(
     Compute,
     ComputeWithCreateInstanceSupport,
+    ComputeWithGroupProvisioningSupport,
     ComputeWithPrivilegedSupport,
     ComputeWithMultinodeSupport,
     ComputeWithReservationSupport,

dstack/_internal/server/utils/routers.py

@@ -1,12 +1,13 @@
 from typing import Any, Dict, List, Optional
 
 import orjson
+import packaging.version
 from fastapi import HTTPException, Request, Response, status
-from packaging import version
 
 from dstack._internal.core.errors import ServerClientError, ServerClientErrorCode
 from dstack._internal.core.models.common import CoreModel
 from dstack._internal.utils.json_utils import get_orjson_default_options, orjson_default
+from dstack._internal.utils.version import parse_version
 
 
 class CustomORJSONResponse(Response):
@@ -122,8 +123,15 @@ def get_request_size(request: Request) -> int:
     return int(request.headers["content-length"])
 
 
+def get_client_version(request: Request) -> Optional[packaging.version.Version]:
+    version = request.headers.get("x-api-version")
+    if version is None:
+        return None
+    return parse_version(version)
+
+
 def check_client_server_compatibility(
-    client_version: Optional[str],
+    client_version: Optional[packaging.version.Version],
     server_version: Optional[str],
 ) -> Optional[CustomORJSONResponse]:
     """
@@ -132,28 +140,18 @@ def check_client_server_compatibility(
     """
     if client_version is None or server_version is None:
         return None
-    parsed_server_version = version.parse(server_version)
-    # latest allows client to bypass compatibility check (e.g. frontend)
-    if client_version == "latest":
+    parsed_server_version = parse_version(server_version)
+    if parsed_server_version is None:
         return None
-    try:
-        parsed_client_version = version.parse(client_version)
-    except version.InvalidVersion:
-        return CustomORJSONResponse(
-            status_code=status.HTTP_400_BAD_REQUEST,
-            content={
-                "detail": get_server_client_error_details(
-                    ServerClientError("Bad API version specified")
-                )
-            },
-        )
     # We preserve full client backward compatibility across patch releases.
     # Server is always partially backward-compatible (so no check).
-    if parsed_client_version > parsed_server_version and (
-        parsed_client_version.major > parsed_server_version.major
-        or parsed_client_version.minor > parsed_server_version.minor
+    if client_version > parsed_server_version and (
+        client_version.major > parsed_server_version.major
+        or client_version.minor > parsed_server_version.minor
     ):
-        return error_incompatible_versions(client_version, server_version, ask_cli_update=False)
+        return error_incompatible_versions(
+            str(client_version), server_version, ask_cli_update=False
+        )
     return None
 
 

dstack/_internal/settings.py

@@ -1,9 +1,10 @@
 import os
 
 from dstack import version
+from dstack._internal.utils.version import parse_version
 
 DSTACK_VERSION = os.getenv("DSTACK_VERSION", version.__version__)
-if DSTACK_VERSION == "0.0.0":
+if parse_version(DSTACK_VERSION) is None:
     # The build backend (hatching) requires not None for versions,
     # but the code currently treats None as dev version.
     # TODO: update the code to treat 0.0.0 as dev version.
@@ -33,3 +34,5 @@ class FeatureFlags:
     large features. This class may be empty if there are no such features in
     development. Feature flags are environment variables of the form DSTACK_FF_*
     """
+
+    pass

dstack/_internal/utils/version.py

@@ -0,0 +1,22 @@
+from typing import Optional
+
+import packaging.version
+
+
+def parse_version(version_string: str) -> Optional[packaging.version.Version]:
+    """
+    Returns a `packaging.version.Version` instance or `None` if the version is dev/latest.
+
+    Values parsed as the dev/latest version:
+    * the "latest" literal
+    * any "0.0.0" release, e.g., "0.0.0", "0.0.0a1", "0.0.0.dev0"
+    """
+    if version_string == "latest":
+        return None
+    try:
+        version = packaging.version.parse(version_string)
+    except packaging.version.InvalidVersion as e:
+        raise ValueError(f"Invalid version: {version_string}") from e
+    if version.release == (0, 0, 0):
+        return None
+    return version

dstack/version.py

@@ -1,4 +1,4 @@
-__version__ = "0.19.34"
+__version__ = "0.19.35"
 __is_release__ = True
 base_image = "0.11"
 base_image_ubuntu_version = "22.04"

{dstack-0.19.34.dist-info → dstack-0.19.35.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dstack
-Version: 0.19.34
+Version: 0.19.35
 Summary: dstack is an open-source orchestration engine for running AI workloads on any cloud or on-premises.
 Project-URL: Homepage, https://dstack.ai
 Project-URL: Source, https://github.com/dstackai/dstack
@@ -73,7 +73,7 @@ Requires-Dist: grpcio>=1.50; extra == 'all'
 Requires-Dist: httpx; extra == 'all'
 Requires-Dist: jinja2; extra == 'all'
 Requires-Dist: kubernetes; extra == 'all'
-Requires-Dist: nebius<=0.2.72,>=0.2.40; (python_version >= '3.10') and extra == 'all'
+Requires-Dist: nebius<0.4,>=0.3.4; (python_version >= '3.10') and extra == 'all'
 Requires-Dist: oci>=2.150.0; extra == 'all'
 Requires-Dist: prometheus-client; extra == 'all'
 Requires-Dist: pyopenssl>=23.2.0; extra == 'all'
@@ -259,7 +259,7 @@ Requires-Dist: fastapi; extra == 'nebius'
 Requires-Dist: grpcio>=1.50; extra == 'nebius'
 Requires-Dist: httpx; extra == 'nebius'
 Requires-Dist: jinja2; extra == 'nebius'
-Requires-Dist: nebius<=0.2.72,>=0.2.40; (python_version >= '3.10') and extra == 'nebius'
+Requires-Dist: nebius<0.4,>=0.3.4; (python_version >= '3.10') and extra == 'nebius'
 Requires-Dist: prometheus-client; extra == 'nebius'
 Requires-Dist: python-dxf==12.1.0; extra == 'nebius'
 Requires-Dist: python-json-logger>=3.1.0; extra == 'nebius'