dstack 0.19.27__py3-none-any.whl → 0.19.29__py3-none-any.whl

dstack/_internal/core/models/fleets.py

@@ -2,13 +2,18 @@ import ipaddress
 import uuid
 from datetime import datetime
 from enum import Enum
-from typing import Any, Dict, List, Optional, Type, Union
+from typing import Any, Dict, List, Optional, Union
 
 from pydantic import Field, root_validator, validator
 from typing_extensions import Annotated, Literal
 
 from dstack._internal.core.models.backends.base import BackendType
-from dstack._internal.core.models.common import ApplyAction, CoreModel
+from dstack._internal.core.models.common import (
+    ApplyAction,
+    CoreConfig,
+    CoreModel,
+    generate_dual_core_model,
+)
 from dstack._internal.core.models.envs import Env
 from dstack._internal.core.models.instances import Instance, InstanceOfferWithAvailability, SSHKey
 from dstack._internal.core.models.profiles import (
@@ -202,6 +207,21 @@ class FleetNodesSpec(CoreModel):
         return values
 
 
+class InstanceGroupParamsConfig(CoreConfig):
+    @staticmethod
+    def schema_extra(schema: Dict[str, Any]):
+        del schema["properties"]["termination_policy"]
+        del schema["properties"]["termination_idle_time"]
+        add_extra_schema_types(
+            schema["properties"]["nodes"],
+            extra_types=[{"type": "integer"}, {"type": "string"}],
+        )
+        add_extra_schema_types(
+            schema["properties"]["idle_duration"],
+            extra_types=[{"type": "string"}],
+        )
+
+
 class InstanceGroupParams(CoreModel):
     env: Annotated[
         Env,
@@ -297,20 +317,6 @@ class InstanceGroupParams(CoreModel):
     termination_policy: Annotated[Optional[TerminationPolicy], Field(exclude=True)] = None
     termination_idle_time: Annotated[Optional[Union[str, int]], Field(exclude=True)] = None
 
-    class Config(CoreModel.Config):
-        @staticmethod
-        def schema_extra(schema: Dict[str, Any], model: Type):
-            del schema["properties"]["termination_policy"]
-            del schema["properties"]["termination_idle_time"]
-            add_extra_schema_types(
-                schema["properties"]["nodes"],
-                extra_types=[{"type": "integer"}, {"type": "string"}],
-            )
-            add_extra_schema_types(
-                schema["properties"]["idle_duration"],
-                extra_types=[{"type": "string"}],
-            )
-
     @validator("nodes", pre=True)
     def parse_nodes(cls, v: Optional[Union[dict, str]]) -> Optional[dict]:
         if isinstance(v, str) and ".." in v:
@@ -331,7 +337,17 @@ class FleetProps(CoreModel):
     name: Annotated[Optional[str], Field(description="The fleet name")] = None
 
 
-class FleetConfiguration(InstanceGroupParams, FleetProps):
+class FleetConfigurationConfig(InstanceGroupParamsConfig):
+    @staticmethod
+    def schema_extra(schema: Dict[str, Any]):
+        InstanceGroupParamsConfig.schema_extra(schema)
+
+
+class FleetConfiguration(
+    InstanceGroupParams,
+    FleetProps,
+    generate_dual_core_model(FleetConfigurationConfig),
+):
     tags: Annotated[
         Optional[Dict[str, str]],
         Field(
@@ -346,7 +362,14 @@ class FleetConfiguration(InstanceGroupParams, FleetProps):
     _validate_tags = validator("tags", pre=True, allow_reuse=True)(tags_validator)
 
 
-class FleetSpec(CoreModel):
+class FleetSpecConfig(CoreConfig):
+    @staticmethod
+    def schema_extra(schema: Dict[str, Any]):
+        prop = schema.get("properties", {})
+        prop.pop("merged_profile", None)
+
+
+class FleetSpec(generate_dual_core_model(FleetSpecConfig)):
     configuration: FleetConfiguration
     configuration_path: Optional[str] = None
     profile: Profile
@@ -356,12 +379,6 @@ class FleetSpec(CoreModel):
     # TODO: make merged_profile a computed field after migrating to pydanticV2
     merged_profile: Annotated[Profile, Field(exclude=True)] = None
 
-    class Config(CoreModel.Config):
-        @staticmethod
-        def schema_extra(schema: Dict[str, Any], model: Type) -> None:
-            prop = schema.get("properties", {})
-            prop.pop("merged_profile", None)
-
     @root_validator
     def _merged_profile(cls, values) -> Dict:
         try:

dstack/_internal/core/models/instances.py

@@ -7,7 +7,10 @@ import gpuhunt
 from pydantic import root_validator
 
 from dstack._internal.core.models.backends.base import BackendType
-from dstack._internal.core.models.common import CoreModel
+from dstack._internal.core.models.common import (
+    CoreModel,
+    FrozenCoreModel,
+)
 from dstack._internal.core.models.envs import Env
 from dstack._internal.core.models.health import HealthStatus
 from dstack._internal.core.models.volumes import Volume
@@ -117,14 +120,11 @@ class InstanceType(CoreModel):
     resources: Resources
 
 
-class SSHConnectionParams(CoreModel):
+class SSHConnectionParams(FrozenCoreModel):
     hostname: str
     username: str
     port: int
 
-    class Config(CoreModel.Config):
-        frozen = True
-
 
 class SSHKey(CoreModel):
     public: str

dstack/_internal/core/models/profiles.py

@@ -6,7 +6,12 @@ from pydantic import Field, root_validator, validator
 from typing_extensions import Annotated, Literal
 
 from dstack._internal.core.models.backends.base import BackendType
-from dstack._internal.core.models.common import CoreModel, Duration
+from dstack._internal.core.models.common import (
+    CoreConfig,
+    CoreModel,
+    Duration,
+    generate_dual_core_model,
+)
 from dstack._internal.utils.common import list_enum_values_for_annotation
 from dstack._internal.utils.cron import validate_cron
 from dstack._internal.utils.json_schema import add_extra_schema_types
@@ -112,7 +117,16 @@ class RetryEvent(str, Enum):
     ERROR = "error"
 
 
-class ProfileRetry(CoreModel):
+class ProfileRetryConfig(CoreConfig):
+    @staticmethod
+    def schema_extra(schema: Dict[str, Any]):
+        add_extra_schema_types(
+            schema["properties"]["duration"],
+            extra_types=[{"type": "string"}],
+        )
+
+
+class ProfileRetry(generate_dual_core_model(ProfileRetryConfig)):
     on_events: Annotated[
         Optional[List[RetryEvent]],
         Field(
@@ -128,14 +142,6 @@ class ProfileRetry(CoreModel):
         Field(description="The maximum period of retrying the run, e.g., `4h` or `1d`"),
     ] = None
 
-    class Config(CoreModel.Config):
-        @staticmethod
-        def schema_extra(schema: Dict[str, Any]):
-            add_extra_schema_types(
-                schema["properties"]["duration"],
-                extra_types=[{"type": "string"}],
-            )
-
     _validate_duration = validator("duration", pre=True, allow_reuse=True)(parse_duration)
 
     @root_validator
@@ -146,7 +152,16 @@ class ProfileRetry(CoreModel):
         return values
 
 
-class UtilizationPolicy(CoreModel):
+class UtilizationPolicyConfig(CoreConfig):
+    @staticmethod
+    def schema_extra(schema: Dict[str, Any]):
+        add_extra_schema_types(
+            schema["properties"]["time_window"],
+            extra_types=[{"type": "string"}],
+        )
+
+
+class UtilizationPolicy(generate_dual_core_model(UtilizationPolicyConfig)):
     _min_time_window = "5m"
 
     min_gpu_utilization: Annotated[
@@ -171,14 +186,6 @@ class UtilizationPolicy(CoreModel):
         ),
     ]
 
-    class Config(CoreModel.Config):
-        @staticmethod
-        def schema_extra(schema: Dict[str, Any]):
-            add_extra_schema_types(
-                schema["properties"]["time_window"],
-                extra_types=[{"type": "string"}],
-            )
-
     @validator("time_window", pre=True)
     def validate_time_window(cls, v: Union[int, str]) -> int:
         v = parse_duration(v)
@@ -219,6 +226,28 @@ class Schedule(CoreModel):
         return self.cron
 
 
+class ProfileParamsConfig(CoreConfig):
+    @staticmethod
+    def schema_extra(schema: Dict[str, Any]):
+        del schema["properties"]["pool_name"]
+        del schema["properties"]["instance_name"]
+        del schema["properties"]["retry_policy"]
+        del schema["properties"]["termination_policy"]
+        del schema["properties"]["termination_idle_time"]
+        add_extra_schema_types(
+            schema["properties"]["max_duration"],
+            extra_types=[{"type": "boolean"}, {"type": "string"}],
+        )
+        add_extra_schema_types(
+            schema["properties"]["stop_duration"],
+            extra_types=[{"type": "boolean"}, {"type": "string"}],
+        )
+        add_extra_schema_types(
+            schema["properties"]["idle_duration"],
+            extra_types=[{"type": "string"}],
+        )
+
+
 class ProfileParams(CoreModel):
     backends: Annotated[
         Optional[List[BackendType]],
@@ -358,27 +387,6 @@ class ProfileParams(CoreModel):
     termination_policy: Annotated[Optional[TerminationPolicy], Field(exclude=True)] = None
     termination_idle_time: Annotated[Optional[Union[str, int]], Field(exclude=True)] = None
 
-    class Config(CoreModel.Config):
-        @staticmethod
-        def schema_extra(schema: Dict[str, Any]) -> None:
-            del schema["properties"]["pool_name"]
-            del schema["properties"]["instance_name"]
-            del schema["properties"]["retry_policy"]
-            del schema["properties"]["termination_policy"]
-            del schema["properties"]["termination_idle_time"]
-            add_extra_schema_types(
-                schema["properties"]["max_duration"],
-                extra_types=[{"type": "boolean"}, {"type": "string"}],
-            )
-            add_extra_schema_types(
-                schema["properties"]["stop_duration"],
-                extra_types=[{"type": "boolean"}, {"type": "string"}],
-            )
-            add_extra_schema_types(
-                schema["properties"]["idle_duration"],
-                extra_types=[{"type": "string"}],
-            )
-
     _validate_max_duration = validator("max_duration", pre=True, allow_reuse=True)(
         parse_max_duration
     )
@@ -403,17 +411,28 @@ class ProfileProps(CoreModel):
     ] = False
 
 
-class Profile(ProfileProps, ProfileParams):
+class ProfileConfig(ProfileParamsConfig):
+    @staticmethod
+    def schema_extra(schema: Dict[str, Any]):
+        ProfileParamsConfig.schema_extra(schema)
+
+
+class Profile(
+    ProfileProps,
+    ProfileParams,
+    generate_dual_core_model(ProfileConfig),
+):
     pass
 
 
-class ProfilesConfig(CoreModel):
-    profiles: List[Profile]
+class ProfilesConfigConfig(CoreConfig):
+    json_loads = orjson.loads
+    json_dumps = pydantic_orjson_dumps_with_indent
+    schema_extra = {"$schema": "http://json-schema.org/draft-07/schema#"}
 
-    class Config(CoreModel.Config):
-        json_loads = orjson.loads
-        json_dumps = pydantic_orjson_dumps_with_indent
-        schema_extra = {"$schema": "http://json-schema.org/draft-07/schema#"}
+
+class ProfilesConfig(generate_dual_core_model(ProfilesConfigConfig)):
+    profiles: List[Profile]
 
     def default(self) -> Optional[Profile]:
         for p in self.profiles:

dstack/_internal/core/models/projects.py

@@ -26,3 +26,11 @@ class Project(CoreModel):
     backends: List[BackendInfo]
     members: List[Member]
     is_public: bool = False
+
+
+class ProjectHookConfig(CoreModel):
+    """
+    This class can be inherited to extend the project creation configuration passed to the hooks.
+    """
+
+    pass

dstack/_internal/core/models/repos/remote.py

@@ -11,7 +11,7 @@ from pydantic import Field
 from typing_extensions import Literal
 
 from dstack._internal.core.errors import DstackError
-from dstack._internal.core.models.common import CoreModel
+from dstack._internal.core.models.common import CoreConfig, generate_dual_core_model
 from dstack._internal.core.models.repos.base import BaseRepoInfo, Repo
 from dstack._internal.utils.hash import get_sha256, slugify
 from dstack._internal.utils.path import PathLike
@@ -24,21 +24,33 @@ class RepoError(DstackError):
     pass
 
 
-class RemoteRepoCreds(CoreModel):
+class RemoteRepoCredsConfig(CoreConfig):
+    @staticmethod
+    def schema_extra(schema: Dict[str, Any]):
+        del schema["properties"]["protocol"]
+
+
+class RemoteRepoCreds(generate_dual_core_model(RemoteRepoCredsConfig)):
     clone_url: str
-    private_key: Optional[str]
-    oauth_token: Optional[str]
+    private_key: Optional[str] = None
+    oauth_token: Optional[str] = None
 
     # TODO: remove in 0.20. Left for compatibility with CLI <=0.18.44
     protocol: Annotated[Optional[str], Field(exclude=True)] = None
 
-    class Config(CoreModel.Config):
-        @staticmethod
-        def schema_extra(schema: Dict[str, Any]) -> None:
-            del schema["properties"]["protocol"]
 
+class RemoteRepoInfoConfig(CoreConfig):
+    @staticmethod
+    def schema_extra(schema: Dict[str, Any]):
+        del schema["properties"]["repo_host_name"]
+        del schema["properties"]["repo_port"]
+        del schema["properties"]["repo_user_name"]
 
-class RemoteRepoInfo(BaseRepoInfo):
+
+class RemoteRepoInfo(
+    BaseRepoInfo,
+    generate_dual_core_model(RemoteRepoInfoConfig),
+):
     repo_type: Literal["remote"] = "remote"
     repo_name: str
 
@@ -47,13 +59,6 @@ class RemoteRepoInfo(BaseRepoInfo):
     repo_port: Annotated[Optional[int], Field(exclude=True)] = None
     repo_user_name: Annotated[Optional[str], Field(exclude=True)] = None
 
-    class Config(BaseRepoInfo.Config):
-        @staticmethod
-        def schema_extra(schema: Dict[str, Any]) -> None:
-            del schema["properties"]["repo_host_name"]
-            del schema["properties"]["repo_port"]
-            del schema["properties"]["repo_user_name"]
-
 
 class RemoteRunRepoData(RemoteRepoInfo):
     repo_branch: Optional[str] = None

dstack/_internal/core/models/resources.py

@@ -7,7 +7,7 @@ from pydantic import Field, parse_obj_as, root_validator, validator
 from pydantic.generics import GenericModel
 from typing_extensions import Annotated
 
-from dstack._internal.core.models.common import CoreModel
+from dstack._internal.core.models.common import CoreConfig, CoreModel, generate_dual_core_model
 from dstack._internal.utils.common import pretty_resources
 from dstack._internal.utils.json_schema import add_extra_schema_types
 from dstack._internal.utils.logging import get_logger
@@ -129,21 +129,22 @@ DEFAULT_MEMORY_SIZE = Range[Memory](min=Memory.parse("8GB"))
 DEFAULT_GPU_COUNT = Range[int](min=1)
 
 
-class CPUSpec(CoreModel):
+class CPUSpecConfig(CoreConfig):
+    @staticmethod
+    def schema_extra(schema: Dict[str, Any]):
+        add_extra_schema_types(
+            schema["properties"]["count"],
+            extra_types=[{"type": "integer"}, {"type": "string"}],
+        )
+
+
+class CPUSpec(generate_dual_core_model(CPUSpecConfig)):
     arch: Annotated[
         Optional[gpuhunt.CPUArchitecture],
         Field(description="The CPU architecture, one of: `x86`, `arm`"),
     ] = None
     count: Annotated[Range[int], Field(description="The number of CPU cores")] = DEFAULT_CPU_COUNT
 
-    class Config(CoreModel.Config):
-        @staticmethod
-        def schema_extra(schema: Dict[str, Any]):
-            add_extra_schema_types(
-                schema["properties"]["count"],
-                extra_types=[{"type": "integer"}, {"type": "string"}],
-            )
-
     @classmethod
     def __get_validators__(cls):
         yield cls.parse
@@ -190,7 +191,28 @@ class CPUSpec(CoreModel):
         return v
 
 
-class GPUSpec(CoreModel):
+class GPUSpecConfig(CoreConfig):
+    @staticmethod
+    def schema_extra(schema: Dict[str, Any]):
+        add_extra_schema_types(
+            schema["properties"]["count"],
+            extra_types=[{"type": "integer"}, {"type": "string"}],
+        )
+        add_extra_schema_types(
+            schema["properties"]["name"],
+            extra_types=[{"type": "string"}],
+        )
+        add_extra_schema_types(
+            schema["properties"]["memory"],
+            extra_types=[{"type": "integer"}, {"type": "string"}],
+        )
+        add_extra_schema_types(
+            schema["properties"]["total_memory"],
+            extra_types=[{"type": "integer"}, {"type": "string"}],
+        )
+
+
+class GPUSpec(generate_dual_core_model(GPUSpecConfig)):
     vendor: Annotated[
         Optional[gpuhunt.AcceleratorVendor],
         Field(
@@ -218,26 +240,6 @@ class GPUSpec(CoreModel):
         Field(description="The minimum compute capability of the GPU (e.g., `7.5`)"),
     ] = None
 
-    class Config(CoreModel.Config):
-        @staticmethod
-        def schema_extra(schema: Dict[str, Any]):
-            add_extra_schema_types(
-                schema["properties"]["count"],
-                extra_types=[{"type": "integer"}, {"type": "string"}],
-            )
-            add_extra_schema_types(
-                schema["properties"]["name"],
-                extra_types=[{"type": "string"}],
-            )
-            add_extra_schema_types(
-                schema["properties"]["memory"],
-                extra_types=[{"type": "integer"}, {"type": "string"}],
-            )
-            add_extra_schema_types(
-                schema["properties"]["total_memory"],
-                extra_types=[{"type": "integer"}, {"type": "string"}],
-            )
-
     @classmethod
     def __get_validators__(cls):
         yield cls.parse
@@ -317,16 +319,17 @@ class GPUSpec(CoreModel):
         return gpuhunt.AcceleratorVendor.cast(v)
 
 
-class DiskSpec(CoreModel):
-    size: Annotated[Range[Memory], Field(description="Disk size")]
+class DiskSpecConfig(CoreConfig):
+    @staticmethod
+    def schema_extra(schema: Dict[str, Any]):
+        add_extra_schema_types(
+            schema["properties"]["size"],
+            extra_types=[{"type": "integer"}, {"type": "string"}],
+        )
 
-    class Config(CoreModel.Config):
-        @staticmethod
-        def schema_extra(schema: Dict[str, Any]):
-            add_extra_schema_types(
-                schema["properties"]["size"],
-                extra_types=[{"type": "integer"}, {"type": "string"}],
-            )
+
+class DiskSpec(generate_dual_core_model(DiskSpecConfig)):
+    size: Annotated[Range[Memory], Field(description="Disk size")]
 
     @classmethod
     def __get_validators__(cls):
@@ -343,7 +346,32 @@ class DiskSpec(CoreModel):
 DEFAULT_DISK = DiskSpec(size=Range[Memory](min=Memory.parse("100GB"), max=None))
 
 
-class ResourcesSpec(CoreModel):
+class ResourcesSpecConfig(CoreConfig):
+    @staticmethod
+    def schema_extra(schema: Dict[str, Any]):
+        add_extra_schema_types(
+            schema["properties"]["cpu"],
+            extra_types=[{"type": "integer"}, {"type": "string"}],
+        )
+        add_extra_schema_types(
+            schema["properties"]["memory"],
+            extra_types=[{"type": "integer"}, {"type": "string"}],
+        )
+        add_extra_schema_types(
+            schema["properties"]["shm_size"],
+            extra_types=[{"type": "integer"}, {"type": "string"}],
+        )
+        add_extra_schema_types(
+            schema["properties"]["gpu"],
+            extra_types=[{"type": "integer"}, {"type": "string"}],
+        )
+        add_extra_schema_types(
+            schema["properties"]["disk"],
+            extra_types=[{"type": "integer"}, {"type": "string"}],
+        )
+
+
+class ResourcesSpec(generate_dual_core_model(ResourcesSpecConfig)):
     # TODO: Remove Range[int] in 0.20. Range[int] for backward compatibility only.
     cpu: Annotated[Union[CPUSpec, Range[int]], Field(description="The CPU requirements")] = (
         CPUSpec()
@@ -362,30 +390,6 @@ class ResourcesSpec(CoreModel):
     gpu: Annotated[Optional[GPUSpec], Field(description="The GPU requirements")] = None
     disk: Annotated[Optional[DiskSpec], Field(description="The disk resources")] = DEFAULT_DISK
 
-    class Config(CoreModel.Config):
-        @staticmethod
-        def schema_extra(schema: Dict[str, Any]):
-            add_extra_schema_types(
-                schema["properties"]["cpu"],
-                extra_types=[{"type": "integer"}, {"type": "string"}],
-            )
-            add_extra_schema_types(
-                schema["properties"]["memory"],
-                extra_types=[{"type": "integer"}, {"type": "string"}],
-            )
-            add_extra_schema_types(
-                schema["properties"]["shm_size"],
-                extra_types=[{"type": "integer"}, {"type": "string"}],
-            )
-            add_extra_schema_types(
-                schema["properties"]["gpu"],
-                extra_types=[{"type": "integer"}, {"type": "string"}],
-            )
-            add_extra_schema_types(
-                schema["properties"]["disk"],
-                extra_types=[{"type": "integer"}, {"type": "string"}],
-            )
-
     def pretty_format(self) -> str:
         # TODO: Remove in 0.20. Use self.cpu directly
         cpu = parse_obj_as(CPUSpec, self.cpu)

dstack/_internal/core/models/runs.py

@@ -1,13 +1,20 @@
 from datetime import datetime, timedelta
 from enum import Enum
-from typing import Any, Dict, List, Literal, Optional, Type
+from typing import Any, Dict, List, Literal, Optional
 from urllib.parse import urlparse
 
 from pydantic import UUID4, Field, root_validator
 from typing_extensions import Annotated
 
 from dstack._internal.core.models.backends.base import BackendType
-from dstack._internal.core.models.common import ApplyAction, CoreModel, NetworkMode, RegistryAuth
+from dstack._internal.core.models.common import (
+    ApplyAction,
+    CoreConfig,
+    CoreModel,
+    NetworkMode,
+    RegistryAuth,
+    generate_dual_core_model,
+)
 from dstack._internal.core.models.configurations import (
     DEFAULT_PROBE_METHOD,
     LEGACY_REPO_DIR,
@@ -385,7 +392,14 @@ class Job(CoreModel):
     job_submissions: List[JobSubmission]
 
 
-class RunSpec(CoreModel):
+class RunSpecConfig(CoreConfig):
+    @staticmethod
+    def schema_extra(schema: Dict[str, Any]):
+        prop = schema.get("properties", {})
+        prop.pop("merged_profile", None)
+
+
+class RunSpec(generate_dual_core_model(RunSpecConfig)):
     # TODO: run_name, working_dir are redundant here since they already passed in configuration
     run_name: Annotated[
         Optional[str],
@@ -458,12 +472,6 @@ class RunSpec(CoreModel):
     # TODO: make merged_profile a computed field after migrating to pydanticV2
     merged_profile: Annotated[Profile, Field(exclude=True)] = None
 
-    class Config(CoreModel.Config):
-        @staticmethod
-        def schema_extra(schema: Dict[str, Any], model: Type) -> None:
-            prop = schema.get("properties", {})
-            prop.pop("merged_profile", None)
-
     @root_validator
     def _merged_profile(cls, values) -> Dict:
         if values.get("profile") is None:

dstack/_internal/server/app.py

@@ -160,6 +160,11 @@ async def lifespan(app: FastAPI):
         logger.info("Background processing is disabled")
     PROBES_SCHEDULER.start()
     dstack_version = DSTACK_VERSION if DSTACK_VERSION else "(no version)"
+    logger.info(
+        "Job network mode: %s (%d)",
+        settings.JOB_NETWORK_MODE.name,
+        settings.JOB_NETWORK_MODE.value,
+    )
     logger.info(f"The admin token is {admin.token.get_plaintext_or_error()}", {"show_path": False})
     logger.info(
         f"The dstack server {dstack_version} is running at {SERVER_URL}",

dstack/_internal/server/background/tasks/process_fleets.py

@@ -177,6 +177,14 @@ def _maintain_fleet_nodes_min(
 
 
 def _autodelete_fleet(fleet_model: FleetModel) -> bool:
+    if fleet_model.project.deleted:
+        # It used to be possible to delete project with active resources:
+        # https://github.com/dstackai/dstack/issues/3077
+        fleet_model.status = FleetStatus.TERMINATED
+        fleet_model.deleted = True
+        logger.info("Fleet %s deleted due to deleted project", fleet_model.name)
+        return True
+
     if is_fleet_in_use(fleet_model) or not is_fleet_empty(fleet_model):
         return False
 

dstack/_internal/server/background/tasks/process_instances.py

@@ -578,7 +578,6 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No
                 if placement_group_model is None:  # error occurred
                     continue
                 session.add(placement_group_model)
-                await session.flush()
                 placement_group_models.append(placement_group_model)
         logger.debug(
             "Trying %s in %s/%s for $%0.4f per hour",
@@ -636,7 +635,9 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No
             },
         )
         if instance.fleet_id and _is_fleet_master_instance(instance):
-            # Clean up placement groups that did not end up being used
+            # Clean up placement groups that did not end up being used.
+            # Flush to update still uncommitted placement groups.
+            await session.flush()
             await schedule_fleet_placement_groups_deletion(
                 session=session,
                 fleet_id=instance.fleet_id,