dstack 0.19.26__py3-none-any.whl → 0.19.27__py3-none-any.whl

dstack/_internal/core/models/configurations.py

@@ -34,7 +34,7 @@ STRIP_PREFIX_DEFAULT = True
 RUN_PRIOTIRY_MIN = 0
 RUN_PRIOTIRY_MAX = 100
 RUN_PRIORITY_DEFAULT = 0
-DEFAULT_REPO_DIR = "/workflow"
+LEGACY_REPO_DIR = "/workflow"
 MIN_PROBE_TIMEOUT = 1
 MIN_PROBE_INTERVAL = 1
 DEFAULT_PROBE_URL = "/"
@@ -112,8 +112,15 @@ class RepoSpec(CoreModel):
         Optional[str],
         Field(description="The commit hash"),
     ] = None
-    # Not implemented, has no effect, hidden in the docs
-    path: str = DEFAULT_REPO_DIR
+    path: Annotated[
+        Optional[str],
+        Field(
+            description=(
+                "The repo path inside the run container. Relative paths are resolved"
+                f" relative to the working directory. Defaults to `{LEGACY_REPO_DIR}`"
+            )
+        ),
+    ] = None
 
     @classmethod
     def parse(cls, v: str) -> Self:
@@ -149,6 +156,14 @@ class RepoSpec(CoreModel):
             raise ValueError("Either `local_path` or `url` must be specified")
         return values
 
+    @validator("path")
+    def validate_path(cls, v: Optional[str]) -> Optional[str]:
+        if v is None:
+            return v
+        if v.startswith("~") and PurePosixPath(v).parts[0] != "~":
+            raise ValueError("`~username` syntax is not supported")
+        return v
+
 
 class ScalingSpec(CoreModel):
     metric: Annotated[
@@ -380,7 +395,7 @@ class BaseRunConfiguration(CoreModel):
         Field(
             description=(
                 "The user inside the container, `user_name_or_id[:group_name_or_id]`"
-                " (e.g., `ubuntu`, `1000:1000`). Defaults to the default `image` user"
+                " (e.g., `ubuntu`, `1000:1000`). Defaults to the default user from the `image`"
             )
         ),
     ] = None
@@ -390,9 +405,8 @@ class BaseRunConfiguration(CoreModel):
         Optional[str],
         Field(
             description=(
-                "The path to the working directory inside the container."
-                f" It's specified relative to the repository directory (`{DEFAULT_REPO_DIR}`) and should be inside it."
-                ' Defaults to `"."` '
+                "The absolute path to the working directory inside the container."
+                f" Defaults to `{LEGACY_REPO_DIR}`"
             )
         ),
     ] = None

dstack/_internal/core/models/files.py

@@ -28,7 +28,7 @@ class FilePathMapping(CoreModel):
         Field(
             description=(
                 "The path in the container. Relative paths are resolved relative to"
-                " the repo directory"
+                " the working directory"
             )
         ),
     ]

dstack/_internal/core/models/fleets.py

@@ -19,7 +19,7 @@ from dstack._internal.core.models.profiles import (
     TerminationPolicy,
     parse_idle_duration,
 )
-from dstack._internal.core.models.resources import Range, ResourcesSpec
+from dstack._internal.core.models.resources import ResourcesSpec
 from dstack._internal.utils.common import list_enum_values_for_annotation
 from dstack._internal.utils.json_schema import add_extra_schema_types
 from dstack._internal.utils.tags import tags_validator
@@ -141,6 +141,67 @@ class SSHParams(CoreModel):
         return value
 
 
+class FleetNodesSpec(CoreModel):
+    min: Annotated[
+        int, Field(description=("The minimum number of instances to maintain in the fleet"))
+    ]
+    target: Annotated[
+        int,
+        Field(
+            description=(
+                "The number of instances to provision on fleet apply. `min` <= `target` <= `max`"
+                " Defaults to `min`"
+            )
+        ),
+    ]
+    max: Annotated[
+        Optional[int],
+        Field(
+            description=(
+                "The maximum number of instances allowed in the fleet. Unlimited if not specified"
+            )
+        ),
+    ] = None
+
+    def dict(self, *args, **kwargs) -> Dict:
+        # super() does not work with pydantic-duality
+        res = CoreModel.dict(self, *args, **kwargs)
+        # For backward compatibility with old clients
+        # that do not ignore extra fields due to https://github.com/dstackai/dstack/issues/3066
+        if "target" in res and res["target"] == res["min"]:
+            del res["target"]
+        return res
+
+    @root_validator(pre=True)
+    def set_min_and_target_defaults(cls, values):
+        min_ = values.get("min")
+        target = values.get("target")
+        if min_ is None:
+            values["min"] = 0
+        if target is None:
+            values["target"] = values["min"]
+        return values
+
+    @validator("min")
+    def validate_min(cls, v: int) -> int:
+        if v < 0:
+            raise ValueError("min cannot be negative")
+        return v
+
+    @root_validator(skip_on_failure=True)
+    def _post_validate_ranges(cls, values):
+        min_ = values["min"]
+        target = values["target"]
+        max_ = values.get("max")
+        if target < min_:
+            raise ValueError("target must not be be less than min")
+        if max_ is not None and max_ < min_:
+            raise ValueError("max must not be less than min")
+        if max_ is not None and max_ < target:
+            raise ValueError("max must not be less than target")
+        return values
+
+
 class InstanceGroupParams(CoreModel):
     env: Annotated[
         Env,
@@ -151,7 +212,9 @@ class InstanceGroupParams(CoreModel):
         Field(description="The parameters for adding instances via SSH"),
     ] = None
 
-    nodes: Annotated[Optional[Range[int]], Field(description="The number of instances")] = None
+    nodes: Annotated[
+        Optional[FleetNodesSpec], Field(description="The number of instances in cloud fleet")
+    ] = None
     placement: Annotated[
         Optional[InstanceGroupPlacement],
         Field(description="The placement of instances: `any` or `cluster`"),
@@ -248,6 +311,16 @@ class InstanceGroupParams(CoreModel):
                 extra_types=[{"type": "string"}],
             )
 
+    @validator("nodes", pre=True)
+    def parse_nodes(cls, v: Optional[Union[dict, str]]) -> Optional[dict]:
+        if isinstance(v, str) and ".." in v:
+            v = v.replace(" ", "")
+            min, max = v.split("..")
+            return dict(min=min or None, max=max or None)
+        elif isinstance(v, str) or isinstance(v, int):
+            return dict(min=v, max=v)
+        return v
+
     _validate_idle_duration = validator("idle_duration", pre=True, allow_reuse=True)(
         parse_idle_duration
     )

dstack/_internal/core/models/runs.py

@@ -10,7 +10,7 @@ from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.common import ApplyAction, CoreModel, NetworkMode, RegistryAuth
 from dstack._internal.core.models.configurations import (
     DEFAULT_PROBE_METHOD,
-    DEFAULT_REPO_DIR,
+    LEGACY_REPO_DIR,
     AnyRunConfiguration,
     HTTPHeaderSpec,
     HTTPMethod,
@@ -259,6 +259,7 @@ class JobSpec(CoreModel):
     retry: Optional[Retry]
     volumes: Optional[List[MountPoint]] = None
     ssh_key: Optional[JobSSHKey] = None
+    # `working_dir` is always absolute (if not None) since 0.19.27
     working_dir: Optional[str]
     # `repo_data` is optional for client compatibility with pre-0.19.17 servers and for compatibility
     # with jobs submitted before 0.19.17. All new jobs are expected to have non-None `repo_data`.
@@ -268,6 +269,8 @@ class JobSpec(CoreModel):
     # submitted before 0.19.17. See `_get_repo_code_hash` on how to get the correct `repo_code_hash`
     # TODO: drop this comment when supporting jobs submitted before 0.19.17 is no longer relevant.
     repo_code_hash: Optional[str] = None
+    # `repo_dir` was added in 0.19.27. Default value is set for backward compatibility
+    repo_dir: str = LEGACY_REPO_DIR
     file_archives: list[FileArchiveMapping] = []
     # None for non-services and pre-0.19.19 services. See `get_service_port`
     service_port: Optional[int] = None
@@ -409,17 +412,27 @@ class RunSpec(CoreModel):
         Optional[str],
         Field(description="The hash of the repo diff. Can be omitted if there is no repo diff."),
     ] = None
+    repo_dir: Annotated[
+        Optional[str],
+        Field(
+            description=(
+                "The repo path inside the container. Relative paths are resolved"
+                f" relative to the working directory. Defaults to `{LEGACY_REPO_DIR}`."
+            )
+        ),
+    ] = None
     file_archives: Annotated[
         list[FileArchiveMapping],
-        Field(description="The list of file archive ID to container path mappings"),
+        Field(description="The list of file archive ID to container path mappings."),
     ] = []
+    # Server uses configuration.working_dir instead of this field since 0.19.27, but
+    # the field still exists for compatibility with older servers
     working_dir: Annotated[
         Optional[str],
         Field(
             description=(
-                "The path to the working directory inside the container."
-                f" It's specified relative to the repository directory (`{DEFAULT_REPO_DIR}`) and should be inside it."
-                ' Defaults to `"."`.'
+                "The absolute path to the working directory inside the container."
+                " Defaults to the default working directory from the `image`."
             )
         ),
     ] = None
@@ -506,10 +519,16 @@ class RunStatus(str, Enum):
         return self in self.finished_statuses()
 
 
+class RunFleet(CoreModel):
+    id: UUID4
+    name: str
+
+
 class Run(CoreModel):
     id: UUID4
     project_name: str
     user: str
+    fleet: Optional[RunFleet] = None
     submitted_at: datetime
     last_processed_at: datetime
     status: RunStatus

dstack/_internal/core/services/repos.py

@@ -1,9 +1,10 @@
 import os
+from contextlib import suppress
 from pathlib import Path
+from tempfile import NamedTemporaryFile
 from typing import Optional, Union
 
 import git.cmd
-import requests
 import yaml
 from git.exc import GitCommandError
 
@@ -13,135 +14,139 @@ from dstack._internal.core.models.repos import LocalRepo, RemoteRepo, RemoteRepo
 from dstack._internal.core.models.repos.remote import GitRepoURL
 from dstack._internal.utils.logging import get_logger
 from dstack._internal.utils.path import PathLike
-from dstack._internal.utils.ssh import (
-    get_host_config,
-    make_ssh_command_for_git,
-    try_ssh_key_passphrase,
-)
+from dstack._internal.utils.ssh import get_host_config, make_git_env, try_ssh_key_passphrase
 
 logger = get_logger(__name__)
 
 gh_config_path = os.path.expanduser("~/.config/gh/hosts.yml")
 default_ssh_key = os.path.expanduser("~/.ssh/id_rsa")
 
-no_prompt_env = dict(GIT_TERMINAL_PROMPT="0")
-
 
 class InvalidRepoCredentialsError(DstackError):
     pass
 
 
-def get_local_repo_credentials(
+def get_repo_creds_and_default_branch(
     repo_url: str,
     identity_file: Optional[PathLike] = None,
+    private_key: Optional[str] = None,
     oauth_token: Optional[str] = None,
-) -> RemoteRepoCreds:
+) -> tuple[RemoteRepoCreds, Optional[str]]:
     url = GitRepoURL.parse(repo_url, get_ssh_config=get_host_config)
 
     # no auth
-    r = requests.get(f"{url.as_https()}/info/refs?service=git-upload-pack", timeout=10)
-    if r.status_code == 200:
-        return RemoteRepoCreds(
-            clone_url=url.as_https(),
-            private_key=None,
-            oauth_token=None,
-        )
-
-    # user-provided ssh key
-    if identity_file is not None:
-        identity_file = os.path.expanduser(identity_file)
-        return check_remote_repo_credentials_ssh(url, identity_file)
-
-    # user-provided oauth token
+    with suppress(InvalidRepoCredentialsError):
+        return _get_repo_creds_and_default_branch_https(url)
+
+    # ssh key provided by the user or pulled from the server
+    if identity_file is not None or private_key is not None:
+        if identity_file is not None:
+            private_key = _read_private_key(identity_file)
+            return _get_repo_creds_and_default_branch_ssh(url, identity_file, private_key)
+        elif private_key is not None:
+            with NamedTemporaryFile("w+", 0o600) as f:
+                f.write(private_key)
+                f.flush()
+                return _get_repo_creds_and_default_branch_ssh(url, f.name, private_key)
+        else:
+            assert False, "should not reach here"
+
+    # oauth token provided by the user or pulled from the server
     if oauth_token is not None:
-        return check_remote_repo_credentials_https(url, oauth_token)
+        return _get_repo_creds_and_default_branch_https(url, oauth_token)
 
     # key from ssh config
     identities = get_host_config(url.original_host).get("identityfile")
     if identities:
-        return check_remote_repo_credentials_ssh(url, identities[0])
+        _identity_file = identities[0]
+        with suppress(InvalidRepoCredentialsError):
+            _private_key = _read_private_key(_identity_file)
+            return _get_repo_creds_and_default_branch_ssh(url, _identity_file, _private_key)
 
     # token from gh config
     if os.path.exists(gh_config_path):
         with open(gh_config_path, "r") as f:
             gh_hosts = yaml.load(f, Loader=yaml.FullLoader)
-        oauth_token = gh_hosts.get(url.host, {}).get("oauth_token")
-        if oauth_token is not None:
-            try:
-                return check_remote_repo_credentials_https(url, oauth_token)
-            except InvalidRepoCredentialsError:
-                pass
+        _oauth_token = gh_hosts.get(url.host, {}).get("oauth_token")
+        if _oauth_token is not None:
+            with suppress(InvalidRepoCredentialsError):
+                return _get_repo_creds_and_default_branch_https(url, _oauth_token)
 
     # default user key
     if os.path.exists(default_ssh_key):
-        try:
-            return check_remote_repo_credentials_ssh(url, default_ssh_key)
-        except InvalidRepoCredentialsError:
-            pass
+        with suppress(InvalidRepoCredentialsError):
+            _private_key = _read_private_key(default_ssh_key)
+            return _get_repo_creds_and_default_branch_ssh(url, default_ssh_key, _private_key)
 
     raise InvalidRepoCredentialsError(
         "No valid default Git credentials found. Pass valid `--token` or `--git-identity`."
     )
 
 
-def check_remote_repo_credentials_https(url: GitRepoURL, oauth_token: str) -> RemoteRepoCreds:
+def _get_repo_creds_and_default_branch_ssh(
+    url: GitRepoURL, identity_file: PathLike, private_key: str
+) -> tuple[RemoteRepoCreds, Optional[str]]:
+    _url = url.as_ssh()
     try:
-        git.cmd.Git().ls_remote(url.as_https(oauth_token), env=no_prompt_env)
-    except GitCommandError:
-        masked = len(oauth_token[:-4]) * "*" + oauth_token[-4:]
-        raise InvalidRepoCredentialsError(
-            f"Can't access `{url.as_https()}` using the `{masked}` token"
-        )
-    return RemoteRepoCreds(
-        clone_url=url.as_https(),
-        oauth_token=oauth_token,
+        default_branch = _get_repo_default_branch(_url, make_git_env(identity_file=identity_file))
+    except GitCommandError as e:
+        message = f"Cannot access `{_url}` using the `{identity_file}` private SSH key"
+        raise InvalidRepoCredentialsError(message) from e
+    creds = RemoteRepoCreds(
+        clone_url=_url,
+        private_key=private_key,
+        oauth_token=None,
+    )
+    return creds, default_branch
+
+
+def _get_repo_creds_and_default_branch_https(
+    url: GitRepoURL, oauth_token: Optional[str] = None
+) -> tuple[RemoteRepoCreds, Optional[str]]:
+    _url = url.as_https()
+    try:
+        default_branch = _get_repo_default_branch(url.as_https(oauth_token), make_git_env())
+    except GitCommandError as e:
+        message = f"Cannot access `{_url}`"
+        if oauth_token is not None:
+            masked_token = len(oauth_token[:-4]) * "*" + oauth_token[-4:]
+            message = f"{message} using the `{masked_token}` token"
+        raise InvalidRepoCredentialsError(message) from e
+    creds = RemoteRepoCreds(
+        clone_url=_url,
         private_key=None,
+        oauth_token=oauth_token,
     )
+    return creds, default_branch
 
 
-def check_remote_repo_credentials_ssh(url: GitRepoURL, identity_file: PathLike) -> RemoteRepoCreds:
+def _get_repo_default_branch(url: str, env: dict[str, str]) -> Optional[str]:
+    # output example: "ref: refs/heads/dev\tHEAD\n545344f77c0df78367085952a97fc3a058eb4c65\tHEAD"
+    output: str = git.cmd.Git().ls_remote("--symref", url, "HEAD", env=env)
+    for line in output.splitlines():
+        # line format: `<oid> TAB <ref> LF`
+        oid, _, ref = line.partition("\t")
+        if oid.startswith("ref:") and ref == "HEAD":
+            return oid.rsplit("/", maxsplit=1)[-1]
+    return None
+
+
+def _read_private_key(identity_file: PathLike) -> str:
+    identity_file = Path(identity_file).expanduser().resolve()
     if not Path(identity_file).exists():
-        raise InvalidRepoCredentialsError(f"The {identity_file} private SSH key doesn't exist")
+        raise InvalidRepoCredentialsError(f"The `{identity_file}` private SSH key doesn't exist")
     if not os.access(identity_file, os.R_OK):
-        raise InvalidRepoCredentialsError(f"Can't access the {identity_file} private SSH key")
+        raise InvalidRepoCredentialsError(f"Cannot access the `{identity_file}` private SSH key")
     if not try_ssh_key_passphrase(identity_file):
         raise InvalidRepoCredentialsError(
             f"Cannot use the `{identity_file}` private SSH key. "
             "Ensure that it is valid and passphrase-free"
         )
-    with open(identity_file, "r") as f:
-        private_key = f.read()
-
-    try:
-        git.cmd.Git().ls_remote(
-            url.as_ssh(), env=dict(GIT_SSH_COMMAND=make_ssh_command_for_git(identity_file))
-        )
-    except GitCommandError:
-        raise InvalidRepoCredentialsError(
-            f"Can't access `{url.as_ssh()}` using the `{identity_file}` private SSH key"
-        )
-
-    return RemoteRepoCreds(
-        clone_url=url.as_ssh(),
-        private_key=private_key,
-        oauth_token=None,
-    )
-
-
-def get_default_branch(remote_url: str) -> Optional[str]:
-    """
-    Get the default branch of a remote Git repository.
-    """
-    try:
-        output = git.cmd.Git().ls_remote("--symref", remote_url, "HEAD", env=no_prompt_env)
-        for line in output.splitlines():
-            if line.startswith("ref:"):
-                return line.split()[1].split("/")[-1]
-    except Exception as e:
-        logger.debug("Failed to get remote repo default branch: %s", repr(e))
-    return None
+    with open(identity_file, "r") as file:
+        return file.read()
 
 
+# Used for `config.yml` only, remove it with `repos` in `config.yml`
 def load_repo(config: RepoConfig) -> Union[RemoteRepo, LocalRepo]:
     if config.repo_type == "remote":
         return RemoteRepo(repo_id=config.repo_id, local_repo_dir=config.path)

dstack/_internal/server/background/tasks/process_fleets.py

@@ -1,11 +1,13 @@
 from datetime import timedelta
 from typing import List
+from uuid import UUID
 
 from sqlalchemy import select, update
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import joinedload, load_only
 
-from dstack._internal.core.models.fleets import FleetStatus
+from dstack._internal.core.models.fleets import FleetSpec, FleetStatus
+from dstack._internal.core.models.instances import InstanceStatus
 from dstack._internal.server.db import get_db, get_session_ctx
 from dstack._internal.server.models import (
     FleetModel,
@@ -15,7 +17,9 @@ from dstack._internal.server.models import (
     RunModel,
 )
 from dstack._internal.server.services.fleets import (
+    create_fleet_instance_model,
     get_fleet_spec,
+    get_next_instance_num,
     is_fleet_empty,
     is_fleet_in_use,
 )
@@ -65,31 +69,111 @@ async def _process_fleets(session: AsyncSession, fleet_models: List[FleetModel])
     res = await session.execute(
         select(FleetModel)
         .where(FleetModel.id.in_(fleet_ids))
-        .options(joinedload(FleetModel.instances).load_only(InstanceModel.deleted))
         .options(
-            joinedload(FleetModel.instances).joinedload(InstanceModel.jobs).load_only(JobModel.id)
+            joinedload(FleetModel.instances).joinedload(InstanceModel.jobs).load_only(JobModel.id),
+            joinedload(FleetModel.project),
         )
         .options(joinedload(FleetModel.runs).load_only(RunModel.status))
         .execution_options(populate_existing=True)
     )
     fleet_models = list(res.unique().scalars().all())
 
+    # TODO: Drop fleets auto-deletion after dropping fleets auto-creation.
     deleted_fleets_ids = []
-    now = get_current_datetime()
     for fleet_model in fleet_models:
+        _consolidate_fleet_state_with_spec(session, fleet_model)
         deleted = _autodelete_fleet(fleet_model)
         if deleted:
             deleted_fleets_ids.append(fleet_model.id)
-        fleet_model.last_processed_at = now
+        fleet_model.last_processed_at = get_current_datetime()
+    await _update_deleted_fleets_placement_groups(session, deleted_fleets_ids)
+    await session.commit()
 
-    await session.execute(
-        update(PlacementGroupModel)
-        .where(
-            PlacementGroupModel.fleet_id.in_(deleted_fleets_ids),
+
+def _consolidate_fleet_state_with_spec(session: AsyncSession, fleet_model: FleetModel):
+    if fleet_model.status == FleetStatus.TERMINATING:
+        return
+    fleet_spec = get_fleet_spec(fleet_model)
+    if fleet_spec.configuration.nodes is None or fleet_spec.autocreated:
+        # Only explicitly created cloud fleets are consolidated.
+        return
+    if not _is_fleet_ready_for_consolidation(fleet_model):
+        return
+    added_instances = _maintain_fleet_nodes_min(session, fleet_model, fleet_spec)
+    if added_instances:
+        fleet_model.consolidation_attempt += 1
+    else:
+        # The fleet is already consolidated or consolidation is in progress.
+        # We reset consolidation_attempt in both cases for simplicity.
+        # The second case does not need reset but is ok to do since
+        # it means consolidation is longer than delay, so it won't happen too often.
+        # TODO: Reset consolidation_attempt on fleet in-place update.
+        fleet_model.consolidation_attempt = 0
+    fleet_model.last_consolidated_at = get_current_datetime()
+
+
+def _is_fleet_ready_for_consolidation(fleet_model: FleetModel) -> bool:
+    consolidation_retry_delay = _get_consolidation_retry_delay(fleet_model.consolidation_attempt)
+    last_consolidated_at = fleet_model.last_consolidated_at or fleet_model.last_processed_at
+    duration_since_last_consolidation = get_current_datetime() - last_consolidated_at
+    return duration_since_last_consolidation >= consolidation_retry_delay
+
+
+# We use exponentially increasing consolidation retry delays so that
+# consolidation does not happen too often. In particular, this prevents
+# retrying instance provisioning constantly in case of no offers.
+# TODO: Adjust delays.
+_CONSOLIDATION_RETRY_DELAYS = [
+    timedelta(seconds=30),
+    timedelta(minutes=1),
+    timedelta(minutes=2),
+    timedelta(minutes=5),
+    timedelta(minutes=10),
+]
+
+
+def _get_consolidation_retry_delay(consolidation_attempt: int) -> timedelta:
+    if consolidation_attempt < len(_CONSOLIDATION_RETRY_DELAYS):
+        return _CONSOLIDATION_RETRY_DELAYS[consolidation_attempt]
+    return _CONSOLIDATION_RETRY_DELAYS[-1]
+
+
+def _maintain_fleet_nodes_min(
+    session: AsyncSession,
+    fleet_model: FleetModel,
+    fleet_spec: FleetSpec,
+) -> bool:
+    """
+    Ensures the fleet has at least `nodes.min` instances.
+    Returns `True` if retried or added new instances and `False` otherwise.
+    """
+    assert fleet_spec.configuration.nodes is not None
+    for instance in fleet_model.instances:
+        # Delete terminated but not deleted instances since
+        # they are going to be replaced with new pending instances.
+        if instance.status == InstanceStatus.TERMINATED and not instance.deleted:
+            # It's safe to modify instances without instance lock since
+            # no other task modifies already terminated instances.
+            instance.deleted = True
+            instance.deleted_at = get_current_datetime()
+    active_instances = [i for i in fleet_model.instances if not i.deleted]
+    active_instances_num = len(active_instances)
+    if active_instances_num >= fleet_spec.configuration.nodes.min:
+        return False
+    nodes_missing = fleet_spec.configuration.nodes.min - active_instances_num
+    for i in range(nodes_missing):
+        instance_model = create_fleet_instance_model(
+            session=session,
+            project=fleet_model.project,
+            # TODO: Store fleet.user and pass it instead of the project owner.
+            username=fleet_model.project.owner.name,
+            spec=fleet_spec,
+            instance_num=get_next_instance_num({i.instance_num for i in active_instances}),
         )
-        .values(fleet_deleted=True)
-    )
-    await session.commit()
+        active_instances.append(instance_model)
+        fleet_model.instances.append(instance_model)
+    logger.info("Added %s instances to fleet %s", nodes_missing, fleet_model.name)
+    return True
 
 
 def _autodelete_fleet(fleet_model: FleetModel) -> bool:
@@ -100,7 +184,7 @@ def _autodelete_fleet(fleet_model: FleetModel) -> bool:
     if (
         fleet_model.status != FleetStatus.TERMINATING
         and fleet_spec.configuration.nodes is not None
-        and (fleet_spec.configuration.nodes.min is None or fleet_spec.configuration.nodes.min == 0)
+        and fleet_spec.configuration.nodes.min == 0
     ):
         # Empty fleets that allow 0 nodes should not be auto-deleted
         return False
@@ -110,3 +194,15 @@ def _autodelete_fleet(fleet_model: FleetModel) -> bool:
     fleet_model.deleted = True
     logger.info("Fleet %s deleted", fleet_model.name)
     return True
+
+
+async def _update_deleted_fleets_placement_groups(session: AsyncSession, fleets_ids: list[UUID]):
+    if len(fleets_ids) == 0:
+        return
+    await session.execute(
+        update(PlacementGroupModel)
+        .where(
+            PlacementGroupModel.fleet_id.in_(fleets_ids),
+        )
+        .values(fleet_deleted=True)
+    )