dstack 0.19.26__py3-none-any.whl → 0.19.28__py3-none-any.whl

dstack/_internal/core/services/repos.py

@@ -1,9 +1,10 @@
 import os
+from contextlib import suppress
 from pathlib import Path
+from tempfile import NamedTemporaryFile
 from typing import Optional, Union
 
 import git.cmd
-import requests
 import yaml
 from git.exc import GitCommandError
 
@@ -13,135 +14,139 @@ from dstack._internal.core.models.repos import LocalRepo, RemoteRepo, RemoteRepo
 from dstack._internal.core.models.repos.remote import GitRepoURL
 from dstack._internal.utils.logging import get_logger
 from dstack._internal.utils.path import PathLike
-from dstack._internal.utils.ssh import (
-    get_host_config,
-    make_ssh_command_for_git,
-    try_ssh_key_passphrase,
-)
+from dstack._internal.utils.ssh import get_host_config, make_git_env, try_ssh_key_passphrase
 
 logger = get_logger(__name__)
 
 gh_config_path = os.path.expanduser("~/.config/gh/hosts.yml")
 default_ssh_key = os.path.expanduser("~/.ssh/id_rsa")
 
-no_prompt_env = dict(GIT_TERMINAL_PROMPT="0")
-
 
 class InvalidRepoCredentialsError(DstackError):
     pass
 
 
-def get_local_repo_credentials(
+def get_repo_creds_and_default_branch(
     repo_url: str,
     identity_file: Optional[PathLike] = None,
+    private_key: Optional[str] = None,
     oauth_token: Optional[str] = None,
-) -> RemoteRepoCreds:
+) -> tuple[RemoteRepoCreds, Optional[str]]:
     url = GitRepoURL.parse(repo_url, get_ssh_config=get_host_config)
 
     # no auth
-    r = requests.get(f"{url.as_https()}/info/refs?service=git-upload-pack", timeout=10)
-    if r.status_code == 200:
-        return RemoteRepoCreds(
-            clone_url=url.as_https(),
-            private_key=None,
-            oauth_token=None,
-        )
-
-    # user-provided ssh key
-    if identity_file is not None:
-        identity_file = os.path.expanduser(identity_file)
-        return check_remote_repo_credentials_ssh(url, identity_file)
-
-    # user-provided oauth token
+    with suppress(InvalidRepoCredentialsError):
+        return _get_repo_creds_and_default_branch_https(url)
+
+    # ssh key provided by the user or pulled from the server
+    if identity_file is not None or private_key is not None:
+        if identity_file is not None:
+            private_key = _read_private_key(identity_file)
+            return _get_repo_creds_and_default_branch_ssh(url, identity_file, private_key)
+        elif private_key is not None:
+            with NamedTemporaryFile("w+", 0o600) as f:
+                f.write(private_key)
+                f.flush()
+                return _get_repo_creds_and_default_branch_ssh(url, f.name, private_key)
+        else:
+            assert False, "should not reach here"
+
+    # oauth token provided by the user or pulled from the server
     if oauth_token is not None:
-        return check_remote_repo_credentials_https(url, oauth_token)
+        return _get_repo_creds_and_default_branch_https(url, oauth_token)
 
     # key from ssh config
     identities = get_host_config(url.original_host).get("identityfile")
     if identities:
-        return check_remote_repo_credentials_ssh(url, identities[0])
+        _identity_file = identities[0]
+        with suppress(InvalidRepoCredentialsError):
+            _private_key = _read_private_key(_identity_file)
+            return _get_repo_creds_and_default_branch_ssh(url, _identity_file, _private_key)
 
     # token from gh config
     if os.path.exists(gh_config_path):
         with open(gh_config_path, "r") as f:
             gh_hosts = yaml.load(f, Loader=yaml.FullLoader)
-        oauth_token = gh_hosts.get(url.host, {}).get("oauth_token")
-        if oauth_token is not None:
-            try:
-                return check_remote_repo_credentials_https(url, oauth_token)
-            except InvalidRepoCredentialsError:
-                pass
+        _oauth_token = gh_hosts.get(url.host, {}).get("oauth_token")
+        if _oauth_token is not None:
+            with suppress(InvalidRepoCredentialsError):
+                return _get_repo_creds_and_default_branch_https(url, _oauth_token)
 
     # default user key
     if os.path.exists(default_ssh_key):
-        try:
-            return check_remote_repo_credentials_ssh(url, default_ssh_key)
-        except InvalidRepoCredentialsError:
-            pass
+        with suppress(InvalidRepoCredentialsError):
+            _private_key = _read_private_key(default_ssh_key)
+            return _get_repo_creds_and_default_branch_ssh(url, default_ssh_key, _private_key)
 
     raise InvalidRepoCredentialsError(
         "No valid default Git credentials found. Pass valid `--token` or `--git-identity`."
     )
 
 
-def check_remote_repo_credentials_https(url: GitRepoURL, oauth_token: str) -> RemoteRepoCreds:
+def _get_repo_creds_and_default_branch_ssh(
+    url: GitRepoURL, identity_file: PathLike, private_key: str
+) -> tuple[RemoteRepoCreds, Optional[str]]:
+    _url = url.as_ssh()
     try:
-        git.cmd.Git().ls_remote(url.as_https(oauth_token), env=no_prompt_env)
-    except GitCommandError:
-        masked = len(oauth_token[:-4]) * "*" + oauth_token[-4:]
-        raise InvalidRepoCredentialsError(
-            f"Can't access `{url.as_https()}` using the `{masked}` token"
-        )
-    return RemoteRepoCreds(
-        clone_url=url.as_https(),
-        oauth_token=oauth_token,
+        default_branch = _get_repo_default_branch(_url, make_git_env(identity_file=identity_file))
+    except GitCommandError as e:
+        message = f"Cannot access `{_url}` using the `{identity_file}` private SSH key"
+        raise InvalidRepoCredentialsError(message) from e
+    creds = RemoteRepoCreds(
+        clone_url=_url,
+        private_key=private_key,
+        oauth_token=None,
+    )
+    return creds, default_branch
+
+
+def _get_repo_creds_and_default_branch_https(
+    url: GitRepoURL, oauth_token: Optional[str] = None
+) -> tuple[RemoteRepoCreds, Optional[str]]:
+    _url = url.as_https()
+    try:
+        default_branch = _get_repo_default_branch(url.as_https(oauth_token), make_git_env())
+    except GitCommandError as e:
+        message = f"Cannot access `{_url}`"
+        if oauth_token is not None:
+            masked_token = len(oauth_token[:-4]) * "*" + oauth_token[-4:]
+            message = f"{message} using the `{masked_token}` token"
+        raise InvalidRepoCredentialsError(message) from e
+    creds = RemoteRepoCreds(
+        clone_url=_url,
         private_key=None,
+        oauth_token=oauth_token,
     )
+    return creds, default_branch
 
 
-def check_remote_repo_credentials_ssh(url: GitRepoURL, identity_file: PathLike) -> RemoteRepoCreds:
+def _get_repo_default_branch(url: str, env: dict[str, str]) -> Optional[str]:
+    # output example: "ref: refs/heads/dev\tHEAD\n545344f77c0df78367085952a97fc3a058eb4c65\tHEAD"
+    output: str = git.cmd.Git().ls_remote("--symref", url, "HEAD", env=env)
+    for line in output.splitlines():
+        # line format: `<oid> TAB <ref> LF`
+        oid, _, ref = line.partition("\t")
+        if oid.startswith("ref:") and ref == "HEAD":
+            return oid.rsplit("/", maxsplit=1)[-1]
+    return None
+
+
+def _read_private_key(identity_file: PathLike) -> str:
+    identity_file = Path(identity_file).expanduser().resolve()
     if not Path(identity_file).exists():
-        raise InvalidRepoCredentialsError(f"The {identity_file} private SSH key doesn't exist")
+        raise InvalidRepoCredentialsError(f"The `{identity_file}` private SSH key doesn't exist")
     if not os.access(identity_file, os.R_OK):
-        raise InvalidRepoCredentialsError(f"Can't access the {identity_file} private SSH key")
+        raise InvalidRepoCredentialsError(f"Cannot access the `{identity_file}` private SSH key")
     if not try_ssh_key_passphrase(identity_file):
         raise InvalidRepoCredentialsError(
             f"Cannot use the `{identity_file}` private SSH key. "
             "Ensure that it is valid and passphrase-free"
         )
-    with open(identity_file, "r") as f:
-        private_key = f.read()
-
-    try:
-        git.cmd.Git().ls_remote(
-            url.as_ssh(), env=dict(GIT_SSH_COMMAND=make_ssh_command_for_git(identity_file))
-        )
-    except GitCommandError:
-        raise InvalidRepoCredentialsError(
-            f"Can't access `{url.as_ssh()}` using the `{identity_file}` private SSH key"
-        )
-
-    return RemoteRepoCreds(
-        clone_url=url.as_ssh(),
-        private_key=private_key,
-        oauth_token=None,
-    )
-
-
-def get_default_branch(remote_url: str) -> Optional[str]:
-    """
-    Get the default branch of a remote Git repository.
-    """
-    try:
-        output = git.cmd.Git().ls_remote("--symref", remote_url, "HEAD", env=no_prompt_env)
-        for line in output.splitlines():
-            if line.startswith("ref:"):
-                return line.split()[1].split("/")[-1]
-    except Exception as e:
-        logger.debug("Failed to get remote repo default branch: %s", repr(e))
-    return None
+    with open(identity_file, "r") as file:
+        return file.read()
 
 
+# Used for `config.yml` only, remove it with `repos` in `config.yml`
 def load_repo(config: RepoConfig) -> Union[RemoteRepo, LocalRepo]:
     if config.repo_type == "remote":
         return RemoteRepo(repo_id=config.repo_id, local_repo_dir=config.path)

dstack/_internal/server/app.py

@@ -160,6 +160,11 @@ async def lifespan(app: FastAPI):
         logger.info("Background processing is disabled")
     PROBES_SCHEDULER.start()
     dstack_version = DSTACK_VERSION if DSTACK_VERSION else "(no version)"
+    logger.info(
+        "Job network mode: %s (%d)",
+        settings.JOB_NETWORK_MODE.name,
+        settings.JOB_NETWORK_MODE.value,
+    )
     logger.info(f"The admin token is {admin.token.get_plaintext_or_error()}", {"show_path": False})
     logger.info(
         f"The dstack server {dstack_version} is running at {SERVER_URL}",

dstack/_internal/server/background/tasks/process_fleets.py

@@ -1,11 +1,13 @@
 from datetime import timedelta
 from typing import List
+from uuid import UUID
 
 from sqlalchemy import select, update
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import joinedload, load_only
 
-from dstack._internal.core.models.fleets import FleetStatus
+from dstack._internal.core.models.fleets import FleetSpec, FleetStatus
+from dstack._internal.core.models.instances import InstanceStatus
 from dstack._internal.server.db import get_db, get_session_ctx
 from dstack._internal.server.models import (
     FleetModel,
@@ -15,7 +17,9 @@ from dstack._internal.server.models import (
     RunModel,
 )
 from dstack._internal.server.services.fleets import (
+    create_fleet_instance_model,
     get_fleet_spec,
+    get_next_instance_num,
     is_fleet_empty,
     is_fleet_in_use,
 )
@@ -65,34 +69,122 @@ async def _process_fleets(session: AsyncSession, fleet_models: List[FleetModel])
     res = await session.execute(
         select(FleetModel)
         .where(FleetModel.id.in_(fleet_ids))
-        .options(joinedload(FleetModel.instances).load_only(InstanceModel.deleted))
         .options(
-            joinedload(FleetModel.instances).joinedload(InstanceModel.jobs).load_only(JobModel.id)
+            joinedload(FleetModel.instances).joinedload(InstanceModel.jobs).load_only(JobModel.id),
+            joinedload(FleetModel.project),
         )
         .options(joinedload(FleetModel.runs).load_only(RunModel.status))
         .execution_options(populate_existing=True)
     )
     fleet_models = list(res.unique().scalars().all())
 
+    # TODO: Drop fleets auto-deletion after dropping fleets auto-creation.
     deleted_fleets_ids = []
-    now = get_current_datetime()
     for fleet_model in fleet_models:
+        _consolidate_fleet_state_with_spec(session, fleet_model)
         deleted = _autodelete_fleet(fleet_model)
         if deleted:
             deleted_fleets_ids.append(fleet_model.id)
-        fleet_model.last_processed_at = now
+        fleet_model.last_processed_at = get_current_datetime()
+    await _update_deleted_fleets_placement_groups(session, deleted_fleets_ids)
+    await session.commit()
 
-    await session.execute(
-        update(PlacementGroupModel)
-        .where(
-            PlacementGroupModel.fleet_id.in_(deleted_fleets_ids),
+
+def _consolidate_fleet_state_with_spec(session: AsyncSession, fleet_model: FleetModel):
+    if fleet_model.status == FleetStatus.TERMINATING:
+        return
+    fleet_spec = get_fleet_spec(fleet_model)
+    if fleet_spec.configuration.nodes is None or fleet_spec.autocreated:
+        # Only explicitly created cloud fleets are consolidated.
+        return
+    if not _is_fleet_ready_for_consolidation(fleet_model):
+        return
+    added_instances = _maintain_fleet_nodes_min(session, fleet_model, fleet_spec)
+    if added_instances:
+        fleet_model.consolidation_attempt += 1
+    else:
+        # The fleet is already consolidated or consolidation is in progress.
+        # We reset consolidation_attempt in both cases for simplicity.
+        # The second case does not need reset but is ok to do since
+        # it means consolidation is longer than delay, so it won't happen too often.
+        # TODO: Reset consolidation_attempt on fleet in-place update.
+        fleet_model.consolidation_attempt = 0
+    fleet_model.last_consolidated_at = get_current_datetime()
+
+
+def _is_fleet_ready_for_consolidation(fleet_model: FleetModel) -> bool:
+    consolidation_retry_delay = _get_consolidation_retry_delay(fleet_model.consolidation_attempt)
+    last_consolidated_at = fleet_model.last_consolidated_at or fleet_model.last_processed_at
+    duration_since_last_consolidation = get_current_datetime() - last_consolidated_at
+    return duration_since_last_consolidation >= consolidation_retry_delay
+
+
+# We use exponentially increasing consolidation retry delays so that
+# consolidation does not happen too often. In particular, this prevents
+# retrying instance provisioning constantly in case of no offers.
+# TODO: Adjust delays.
+_CONSOLIDATION_RETRY_DELAYS = [
+    timedelta(seconds=30),
+    timedelta(minutes=1),
+    timedelta(minutes=2),
+    timedelta(minutes=5),
+    timedelta(minutes=10),
+]
+
+
+def _get_consolidation_retry_delay(consolidation_attempt: int) -> timedelta:
+    if consolidation_attempt < len(_CONSOLIDATION_RETRY_DELAYS):
+        return _CONSOLIDATION_RETRY_DELAYS[consolidation_attempt]
+    return _CONSOLIDATION_RETRY_DELAYS[-1]
+
+
+def _maintain_fleet_nodes_min(
+    session: AsyncSession,
+    fleet_model: FleetModel,
+    fleet_spec: FleetSpec,
+) -> bool:
+    """
+    Ensures the fleet has at least `nodes.min` instances.
+    Returns `True` if retried or added new instances and `False` otherwise.
+    """
+    assert fleet_spec.configuration.nodes is not None
+    for instance in fleet_model.instances:
+        # Delete terminated but not deleted instances since
+        # they are going to be replaced with new pending instances.
+        if instance.status == InstanceStatus.TERMINATED and not instance.deleted:
+            # It's safe to modify instances without instance lock since
+            # no other task modifies already terminated instances.
+            instance.deleted = True
+            instance.deleted_at = get_current_datetime()
+    active_instances = [i for i in fleet_model.instances if not i.deleted]
+    active_instances_num = len(active_instances)
+    if active_instances_num >= fleet_spec.configuration.nodes.min:
+        return False
+    nodes_missing = fleet_spec.configuration.nodes.min - active_instances_num
+    for i in range(nodes_missing):
+        instance_model = create_fleet_instance_model(
+            session=session,
+            project=fleet_model.project,
+            # TODO: Store fleet.user and pass it instead of the project owner.
+            username=fleet_model.project.owner.name,
+            spec=fleet_spec,
+            instance_num=get_next_instance_num({i.instance_num for i in active_instances}),
         )
-        .values(fleet_deleted=True)
-    )
-    await session.commit()
+        active_instances.append(instance_model)
+        fleet_model.instances.append(instance_model)
+    logger.info("Added %s instances to fleet %s", nodes_missing, fleet_model.name)
+    return True
 
 
 def _autodelete_fleet(fleet_model: FleetModel) -> bool:
+    if fleet_model.project.deleted:
+        # It used to be possible to delete project with active resources:
+        # https://github.com/dstackai/dstack/issues/3077
+        fleet_model.status = FleetStatus.TERMINATED
+        fleet_model.deleted = True
+        logger.info("Fleet %s deleted due to deleted project", fleet_model.name)
+        return True
+
     if is_fleet_in_use(fleet_model) or not is_fleet_empty(fleet_model):
         return False
 
@@ -100,7 +192,7 @@ def _autodelete_fleet(fleet_model: FleetModel) -> bool:
     if (
         fleet_model.status != FleetStatus.TERMINATING
         and fleet_spec.configuration.nodes is not None
-        and (fleet_spec.configuration.nodes.min is None or fleet_spec.configuration.nodes.min == 0)
+        and fleet_spec.configuration.nodes.min == 0
     ):
         # Empty fleets that allow 0 nodes should not be auto-deleted
         return False
@@ -110,3 +202,15 @@ def _autodelete_fleet(fleet_model: FleetModel) -> bool:
     fleet_model.deleted = True
     logger.info("Fleet %s deleted", fleet_model.name)
     return True
+
+
+async def _update_deleted_fleets_placement_groups(session: AsyncSession, fleets_ids: list[UUID]):
+    if len(fleets_ids) == 0:
+        return
+    await session.execute(
+        update(PlacementGroupModel)
+        .where(
+            PlacementGroupModel.fleet_id.in_(fleets_ids),
+        )
+        .values(fleet_deleted=True)
+    )

dstack/_internal/server/background/tasks/process_instances.py

@@ -53,14 +53,12 @@ from dstack._internal.core.models.placement import (
     PlacementStrategy,
 )
 from dstack._internal.core.models.profiles import (
-    RetryEvent,
     TerminationPolicy,
 )
 from dstack._internal.core.models.runs import (
     JobProvisioningData,
     Retry,
 )
-from dstack._internal.core.services.profiles import get_retry
 from dstack._internal.server import settings as server_settings
 from dstack._internal.server.background.tasks.common import get_provisioning_timeout
 from dstack._internal.server.db import get_db, get_session_ctx
@@ -327,7 +325,6 @@ async def _add_remote(instance: InstanceModel) -> None:
             e,
         )
         instance.status = InstanceStatus.PENDING
-        instance.last_retry_at = get_current_datetime()
         return
 
     instance_type = host_info_to_instance_type(host_info, cpu_arch)
@@ -426,7 +423,6 @@ async def _add_remote(instance: InstanceModel) -> None:
     instance.offer = instance_offer.json()
     instance.job_provisioning_data = jpd.json()
     instance.started_at = get_current_datetime()
-    instance.last_retry_at = get_current_datetime()
 
 
 def _deploy_instance(
@@ -493,29 +489,6 @@ def _deploy_instance(
 
 
 async def _create_instance(session: AsyncSession, instance: InstanceModel) -> None:
-    if instance.last_retry_at is not None:
-        last_retry = instance.last_retry_at
-        if get_current_datetime() < last_retry + timedelta(minutes=1):
-            return
-
-    if (
-        instance.profile is None
-        or instance.requirements is None
-        or instance.instance_configuration is None
-    ):
-        instance.status = InstanceStatus.TERMINATED
-        instance.termination_reason = "Empty profile, requirements or instance_configuration"
-        instance.last_retry_at = get_current_datetime()
-        logger.warning(
-            "Empty profile, requirements or instance_configuration. Terminate instance: %s",
-            instance.name,
-            extra={
-                "instance_name": instance.name,
-                "instance_status": InstanceStatus.TERMINATED.value,
-            },
-        )
-        return
-
     if _need_to_wait_fleet_provisioning(instance):
         logger.debug("Waiting for the first instance in the fleet to be provisioned")
         return
@@ -529,7 +502,6 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No
         instance.termination_reason = (
             f"Error to parse profile, requirements or instance_configuration: {e}"
         )
-        instance.last_retry_at = get_current_datetime()
         logger.warning(
             "Error to parse profile, requirements or instance_configuration. Terminate instance: %s",
             instance.name,
@@ -540,24 +512,6 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No
         )
         return
 
-    retry = get_retry(profile)
-    should_retry = retry is not None and RetryEvent.NO_CAPACITY in retry.on_events
-
-    if retry is not None:
-        retry_duration_deadline = _get_retry_duration_deadline(instance, retry)
-        if get_current_datetime() > retry_duration_deadline:
-            instance.status = InstanceStatus.TERMINATED
-            instance.termination_reason = "Retry duration expired"
-            logger.warning(
-                "Retry duration expired. Terminating instance %s",
-                instance.name,
-                extra={
-                    "instance_name": instance.name,
-                    "instance_status": InstanceStatus.TERMINATED.value,
-                },
-            )
-            return
-
     placement_group_models = []
     placement_group_model = None
     if instance.fleet_id:
@@ -595,15 +549,6 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No
         exclude_not_available=True,
     )
 
-    if not offers and should_retry:
-        instance.last_retry_at = get_current_datetime()
-        logger.debug(
-            "No offers for instance %s. Next retry",
-            instance.name,
-            extra={"instance_name": instance.name},
-        )
-        return
-
     # Limit number of offers tried to prevent long-running processing
     # in case all offers fail.
     for backend, instance_offer in offers[: server_settings.MAX_OFFERS_TRIED]:
@@ -681,7 +626,6 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No
         instance.offer = instance_offer.json()
         instance.total_blocks = instance_offer.total_blocks
         instance.started_at = get_current_datetime()
-        instance.last_retry_at = get_current_datetime()
 
         logger.info(
             "Created instance %s",
@@ -702,21 +646,18 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No
             )
         return
 
-    instance.last_retry_at = get_current_datetime()
-
-    if not should_retry:
-        _mark_terminated(instance, "All offers failed" if offers else "No offers found")
-        if (
-            instance.fleet
-            and _is_fleet_master_instance(instance)
-            and _is_cloud_cluster(instance.fleet)
-        ):
-            # Do not attempt to deploy other instances, as they won't determine the correct cluster
-            # backend, region, and placement group without a successfully deployed master instance
-            for sibling_instance in instance.fleet.instances:
-                if sibling_instance.id == instance.id:
-                    continue
-                _mark_terminated(sibling_instance, "Master instance failed to start")
+    _mark_terminated(instance, "All offers failed" if offers else "No offers found")
+    if (
+        instance.fleet
+        and _is_fleet_master_instance(instance)
+        and _is_cloud_cluster(instance.fleet)
+    ):
+        # Do not attempt to deploy other instances, as they won't determine the correct cluster
+        # backend, region, and placement group without a successfully deployed master instance
+        for sibling_instance in instance.fleet.instances:
+            if sibling_instance.id == instance.id:
+                continue
+            _mark_terminated(sibling_instance, "Master instance failed to start")
 
 
 def _mark_terminated(instance: InstanceModel, termination_reason: str) -> None:

dstack/_internal/server/background/tasks/process_running_jobs.py

@@ -41,6 +41,7 @@ from dstack._internal.core.models.volumes import InstanceMountPoint, Volume, Vol
 from dstack._internal.server.background.tasks.common import get_provisioning_timeout
 from dstack._internal.server.db import get_db, get_session_ctx
 from dstack._internal.server.models import (
+    FleetModel,
     InstanceModel,
     JobModel,
     ProbeModel,
@@ -151,6 +152,7 @@ async def _process_running_job(session: AsyncSession, job_model: JobModel):
         .options(joinedload(RunModel.project))
         .options(joinedload(RunModel.user))
         .options(joinedload(RunModel.repo))
+        .options(joinedload(RunModel.fleet).load_only(FleetModel.id, FleetModel.name))
         .options(joinedload(RunModel.jobs))
     )
     run_model = res.unique().scalar_one()

dstack/_internal/server/background/tasks/process_runs.py

@@ -21,6 +21,7 @@ from dstack._internal.core.models.runs import (
 )
 from dstack._internal.server.db import get_db, get_session_ctx
 from dstack._internal.server.models import (
+    FleetModel,
     InstanceModel,
     JobModel,
     ProjectModel,
@@ -145,6 +146,7 @@ async def _process_run(session: AsyncSession, run_model: RunModel):
         .execution_options(populate_existing=True)
         .options(joinedload(RunModel.project).load_only(ProjectModel.id, ProjectModel.name))
         .options(joinedload(RunModel.user).load_only(UserModel.name))
+        .options(joinedload(RunModel.fleet).load_only(FleetModel.id, FleetModel.name))
         .options(
             selectinload(RunModel.jobs)
             .joinedload(JobModel.instance)