dstack 0.19.18__py3-none-any.whl → 0.19.19__py3-none-any.whl

dstack/_internal/server/services/jobs/__init__.py

@@ -134,6 +134,8 @@ def job_model_to_job_submission(job_model: JobModel) -> JobSubmission:
     finished_at = None
     if job_model.status.is_finished():
         finished_at = last_processed_at
+    status_message = _get_job_status_message(job_model)
+    error = _get_job_error(job_model)
     return JobSubmission(
         id=job_model.id,
         submission_num=job_model.submission_num,
@@ -143,11 +145,13 @@ def job_model_to_job_submission(job_model: JobModel) -> JobSubmission:
         finished_at=finished_at,
         inactivity_secs=job_model.inactivity_secs,
         status=job_model.status,
+        status_message=status_message,
         termination_reason=job_model.termination_reason,
         termination_reason_message=job_model.termination_reason_message,
         exit_status=job_model.exit_status,
         job_provisioning_data=job_provisioning_data,
         job_runtime_data=get_job_runtime_data(job_model),
+        error=error,
     )
 
 
@@ -289,6 +293,19 @@ async def process_terminating_job(
     # so that stuck volumes don't prevent the instance from terminating.
     job_model.instance_id = None
     instance_model.last_job_processed_at = common.get_current_datetime()
+
+    volume_names = (
+        jrd.volume_names
+        if jrd and jrd.volume_names
+        else [va.volume.name for va in instance_model.volume_attachments]
+    )
+    if volume_names:
+        volumes = await list_project_volume_models(
+            session=session, project=instance_model.project, names=volume_names
+        )
+        for volume in volumes:
+            volume.last_job_processed_at = common.get_current_datetime()
+
     logger.info(
         "%s: instance '%s' has been released, new status is %s",
         fmt(job_model),
@@ -693,3 +710,31 @@ def _get_job_mount_point_attached_volume(
             continue
         return volume
     raise ServerClientError("Failed to find an eligible volume for the mount point")
+
+
+def _get_job_status_message(job_model: JobModel) -> str:
+    if job_model.status == JobStatus.DONE:
+        return "exited (0)"
+    elif job_model.status == JobStatus.FAILED:
+        if job_model.termination_reason == JobTerminationReason.CONTAINER_EXITED_WITH_ERROR:
+            return f"exited ({job_model.exit_status})"
+        elif (
+            job_model.termination_reason == JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY
+        ):
+            return "no offers"
+        elif job_model.termination_reason == JobTerminationReason.INTERRUPTED_BY_NO_CAPACITY:
+            return "interrupted"
+        else:
+            return "error"
+    elif job_model.status == JobStatus.TERMINATED:
+        if job_model.termination_reason == JobTerminationReason.TERMINATED_BY_USER:
+            return "stopped"
+        elif job_model.termination_reason == JobTerminationReason.ABORTED_BY_USER:
+            return "aborted"
+    return job_model.status.value
+
+
+def _get_job_error(job_model: JobModel) -> Optional[str]:
+    if job_model.termination_reason is None:
+        return None
+    return job_model.termination_reason.to_error()

dstack/_internal/server/services/jobs/configurators/base.py

@@ -15,6 +15,7 @@ from dstack._internal.core.models.configurations import (
     PortMapping,
     PythonVersion,
     RunConfigurationType,
+    ServiceConfiguration,
 )
 from dstack._internal.core.models.profiles import (
     DEFAULT_STOP_DURATION,
@@ -153,6 +154,7 @@ class JobConfigurator(ABC):
             repo_data=self.run_spec.repo_data,
             repo_code_hash=self.run_spec.repo_code_hash,
             file_archives=self.run_spec.file_archives,
+            service_port=self._service_port(),
         )
         return job_spec
 
@@ -306,6 +308,11 @@ class JobConfigurator(ABC):
             )
         return self._job_ssh_key
 
+    def _service_port(self) -> Optional[int]:
+        if isinstance(self.run_spec.configuration, ServiceConfiguration):
+            return self.run_spec.configuration.port.container_port
+        return None
+
 
 def interpolate_job_volumes(
     run_volumes: List[Union[MountPoint, str]],

dstack/_internal/server/services/locking.py

@@ -172,7 +172,7 @@ async def _wait_to_lock_many(
     The keys must be sorted to prevent deadlock.
     """
     left_to_lock = keys.copy()
-    while len(left_to_lock) > 0:
+    while True:
         async with lock:
             locked_now_num = 0
             for key in left_to_lock:
@@ -182,4 +182,6 @@ async def _wait_to_lock_many(
                 locked.add(key)
                 locked_now_num += 1
             left_to_lock = left_to_lock[locked_now_num:]
+        if not left_to_lock:
+            return
         await asyncio.sleep(delay)

dstack/_internal/server/services/logging.py

@@ -1,12 +1,14 @@
 from typing import Union
 
-from dstack._internal.server.models import JobModel, RunModel
+from dstack._internal.server.models import GatewayModel, JobModel, RunModel
 
 
-def fmt(model: Union[RunModel, JobModel]) -> str:
+def fmt(model: Union[RunModel, JobModel, GatewayModel]) -> str:
     """Consistent string representation of a model for logging."""
     if isinstance(model, RunModel):
         return f"run({model.id.hex[:6]}){model.run_name}"
     if isinstance(model, JobModel):
         return f"job({model.id.hex[:6]}){model.job_name}"
+    if isinstance(model, GatewayModel):
+        return f"gateway({model.id.hex[:6]}){model.name}"
     return str(model)

dstack/_internal/server/services/logs/__init__.py

@@ -8,7 +8,11 @@ from dstack._internal.server.models import ProjectModel
 from dstack._internal.server.schemas.logs import PollLogsRequest
 from dstack._internal.server.schemas.runner import LogEvent as RunnerLogEvent
 from dstack._internal.server.services.logs.aws import BOTO_AVAILABLE, CloudWatchLogStorage
-from dstack._internal.server.services.logs.base import LogStorage, LogStorageError
+from dstack._internal.server.services.logs.base import (
+    LogStorage,
+    LogStorageError,
+    b64encode_raw_message,
+)
 from dstack._internal.server.services.logs.filelog import FileLogStorage
 from dstack._internal.server.services.logs.gcp import GCP_LOGGING_AVAILABLE, GCPLogStorage
 from dstack._internal.utils.common import run_async
@@ -75,4 +79,13 @@ def write_logs(
 
 
 async def poll_logs_async(project: ProjectModel, request: PollLogsRequest) -> JobSubmissionLogs:
-    return await run_async(get_log_storage().poll_logs, project=project, request=request)
+    job_submission_logs = await run_async(
+        get_log_storage().poll_logs, project=project, request=request
+    )
+    # Logs are stored in plaintext but transmitted in base64 for API/CLI backward compatibility.
+    # Old logs stored in base64 are encoded twice for transmission and shown as base64 in CLI/UI.
+    # We live with that.
+    # TODO: Drop base64 encoding in 0.20.
+    for log_event in job_submission_logs.logs:
+        log_event.message = b64encode_raw_message(log_event.message.encode())
+    return job_submission_logs

dstack/_internal/server/services/logs/aws.py

@@ -17,7 +17,6 @@ from dstack._internal.server.schemas.runner import LogEvent as RunnerLogEvent
 from dstack._internal.server.services.logs.base import (
     LogStorage,
     LogStorageError,
-    b64encode_raw_message,
     datetime_to_unix_time_ms,
     unix_time_ms_to_datetime,
 )
@@ -238,8 +237,7 @@ class CloudWatchLogStorage(LogStorage):
                 skipped_future_events += 1
                 continue
             cw_event = self._runner_log_event_to_cloudwatch_event(event)
-            # as message is base64-encoded, length in bytes = length in code points.
-            message_size = len(cw_event["message"]) + self.MESSAGE_OVERHEAD_SIZE
+            message_size = len(event.message) + self.MESSAGE_OVERHEAD_SIZE
             if message_size > self.MESSAGE_MAX_SIZE:
                 # we should never hit this limit, as we use `io.Copy` to copy from pty to logs,
                 # which under the hood uses 32KiB buffer, see runner/internal/executor/executor.go,
@@ -271,7 +269,7 @@ class CloudWatchLogStorage(LogStorage):
     ) -> _CloudWatchLogEvent:
         return {
             "timestamp": runner_log_event.timestamp,
-            "message": b64encode_raw_message(runner_log_event.message),
+            "message": runner_log_event.message.decode(errors="replace"),
         }
 
     @contextmanager

dstack/_internal/server/services/logs/filelog.py

@@ -2,6 +2,7 @@ from pathlib import Path
 from typing import List, Union
 from uuid import UUID
 
+from dstack._internal.core.errors import ServerClientError
 from dstack._internal.core.models.logs import (
     JobSubmissionLogs,
     LogEvent,
@@ -14,8 +15,6 @@ from dstack._internal.server.schemas.logs import PollLogsRequest
 from dstack._internal.server.schemas.runner import LogEvent as RunnerLogEvent
 from dstack._internal.server.services.logs.base import (
     LogStorage,
-    LogStorageError,
-    b64encode_raw_message,
     unix_time_ms_to_datetime,
 )
 
@@ -30,9 +29,6 @@ class FileLogStorage(LogStorage):
             self.root = Path(root)
 
     def poll_logs(self, project: ProjectModel, request: PollLogsRequest) -> JobSubmissionLogs:
-        if request.descending:
-            raise LogStorageError("descending: true is not supported")
-
         log_producer = LogProducer.RUNNER if request.diagnose else LogProducer.JOB
         log_file_path = self._get_log_file_path(
             project_name=project.name,
@@ -46,11 +42,11 @@ class FileLogStorage(LogStorage):
             try:
                 start_line = int(request.next_token)
                 if start_line < 0:
-                    raise LogStorageError(
+                    raise ServerClientError(
                         f"Invalid next_token: {request.next_token}. Must be a non-negative integer."
                     )
             except ValueError:
-                raise LogStorageError(
+                raise ServerClientError(
                     f"Invalid next_token: {request.next_token}. Must be a valid integer."
                 )
 
@@ -60,31 +56,41 @@ class FileLogStorage(LogStorage):
 
         try:
             with open(log_file_path) as f:
-                lines = f.readlines()
-
-            for i, line in enumerate(lines):
-                if current_line < start_line:
+                # Skip to start_line if needed
+                for _ in range(start_line):
+                    if f.readline() == "":
+                        # File is shorter than start_line
+                        return JobSubmissionLogs(logs=logs, next_token=next_token)
                     current_line += 1
-                    continue
 
-                log_event = LogEvent.__response__.parse_raw(line)
-                current_line += 1
+                # Read lines one by one
+                while True:
+                    line = f.readline()
+                    if line == "":  # EOF
+                        break
+
+                    current_line += 1
 
-                if request.start_time and log_event.timestamp <= request.start_time:
-                    continue
-                if request.end_time is not None and log_event.timestamp >= request.end_time:
-                    break
+                    try:
+                        log_event = LogEvent.__response__.parse_raw(line)
+                    except Exception:
+                        # Skip malformed lines
+                        continue
 
-                logs.append(log_event)
+                    if request.start_time and log_event.timestamp <= request.start_time:
+                        continue
+                    if request.end_time is not None and log_event.timestamp >= request.end_time:
+                        break
 
-                if len(logs) >= request.limit:
-                    # Only set next_token if there are more lines to read
-                    if current_line < len(lines):
-                        next_token = str(current_line)
-                    break
+                    logs.append(log_event)
 
-        except IOError as e:
-            raise LogStorageError(f"Failed to read log file {log_file_path}: {e}")
+                    if len(logs) >= request.limit:
+                        # Check if there are more lines to read
+                        if f.readline() != "":
+                            next_token = str(current_line)
+                        break
+        except FileNotFoundError:
+            pass
 
         return JobSubmissionLogs(logs=logs, next_token=next_token)
 
@@ -140,5 +146,5 @@ class FileLogStorage(LogStorage):
         return LogEvent(
             timestamp=unix_time_ms_to_datetime(runner_log_event.timestamp),
             log_source=LogEventSource.STDOUT,
-            message=b64encode_raw_message(runner_log_event.message),
+            message=runner_log_event.message.decode(errors="replace"),
         )

dstack/_internal/server/services/logs/gcp.py

@@ -14,7 +14,6 @@ from dstack._internal.server.schemas.runner import LogEvent as RunnerLogEvent
 from dstack._internal.server.services.logs.base import (
     LogStorage,
     LogStorageError,
-    b64encode_raw_message,
     unix_time_ms_to_datetime,
 )
 from dstack._internal.utils.common import batched
@@ -137,15 +136,14 @@ class GCPLogStorage(LogStorage):
         with self.logger.batch() as batcher:
             for batch in batched(logs, self.MAX_BATCH_SIZE):
                 for log in batch:
-                    message = b64encode_raw_message(log.message)
+                    message = log.message.decode(errors="replace")
                     timestamp = unix_time_ms_to_datetime(log.timestamp)
-                    # as message is base64-encoded, length in bytes = length in code points
-                    if len(message) > self.MAX_RUNNER_MESSAGE_SIZE:
+                    if len(log.message) > self.MAX_RUNNER_MESSAGE_SIZE:
                         logger.error(
                             "Stream %s: skipping event at %s, message exceeds max size: %d > %d",
                             stream_name,
                             timestamp.isoformat(),
-                            len(message),
+                            len(log.message),
                             self.MAX_RUNNER_MESSAGE_SIZE,
                         )
                         continue

dstack/_internal/server/services/proxy/repo.py

@@ -12,10 +12,12 @@ from dstack._internal.core.models.configurations import ServiceConfiguration
 from dstack._internal.core.models.instances import RemoteConnectionInfo, SSHConnectionParams
 from dstack._internal.core.models.runs import (
     JobProvisioningData,
+    JobSpec,
     JobStatus,
     RunSpec,
     RunStatus,
     ServiceSpec,
+    get_service_port,
 )
 from dstack._internal.core.models.services import AnyModel
 from dstack._internal.proxy.lib.models import (
@@ -97,9 +99,10 @@ class ServerProxyRepo(BaseProxyRepo):
                 if rci.ssh_proxy is not None:
                     ssh_head_proxy = rci.ssh_proxy
                     ssh_head_proxy_private_key = get_or_error(rci.ssh_proxy_keys)[0].private
+            job_spec: JobSpec = JobSpec.__response__.parse_raw(job.job_spec_data)
             replica = Replica(
                 id=job.id.hex,
-                app_port=run_spec.configuration.port.container_port,
+                app_port=get_service_port(job_spec, run_spec.configuration),
                 ssh_destination=ssh_destination,
                 ssh_port=ssh_port,
                 ssh_proxy=ssh_proxy,

dstack/_internal/server/services/runs.py

@@ -24,6 +24,7 @@ from dstack._internal.core.models.instances import (
 )
 from dstack._internal.core.models.profiles import (
     CreationPolicy,
+    RetryEvent,
 )
 from dstack._internal.core.models.repos.virtual import DEFAULT_VIRTUAL_REPO_ID, VirtualRunRepoData
 from dstack._internal.core.models.runs import (
@@ -105,6 +106,8 @@ async def list_user_runs(
     repo_id: Optional[str],
     username: Optional[str],
     only_active: bool,
+    include_jobs: bool,
+    job_submissions_limit: Optional[int],
     prev_submitted_at: Optional[datetime],
     prev_run_id: Optional[uuid.UUID],
     limit: int,
@@ -148,7 +151,14 @@ async def list_user_runs(
     runs = []
     for r in run_models:
         try:
-            runs.append(run_model_to_run(r, return_in_api=True))
+            runs.append(
+                run_model_to_run(
+                    r,
+                    return_in_api=True,
+                    include_jobs=include_jobs,
+                    job_submissions_limit=job_submissions_limit,
+                )
+            )
         except pydantic.ValidationError:
             pass
     if len(run_models) > len(runs):
@@ -652,51 +662,33 @@ async def delete_runs(
 
 def run_model_to_run(
     run_model: RunModel,
-    include_job_submissions: bool = True,
+    include_jobs: bool = True,
+    job_submissions_limit: Optional[int] = None,
     return_in_api: bool = False,
     include_sensitive: bool = False,
 ) -> Run:
     jobs: List[Job] = []
-    run_jobs = sorted(run_model.jobs, key=lambda j: (j.replica_num, j.job_num, j.submission_num))
-    for replica_num, replica_submissions in itertools.groupby(
-        run_jobs, key=lambda j: j.replica_num
-    ):
-        for job_num, job_submissions in itertools.groupby(
-            replica_submissions, key=lambda j: j.job_num
-        ):
-            submissions = []
-            job_model = None
-            for job_model in job_submissions:
-                if include_job_submissions:
-                    job_submission = job_model_to_job_submission(job_model)
-                    if return_in_api:
-                        # Set default non-None values for 0.18 backward-compatibility
-                        # Remove in 0.19
-                        if job_submission.job_provisioning_data is not None:
-                            if job_submission.job_provisioning_data.hostname is None:
-                                job_submission.job_provisioning_data.hostname = ""
-                            if job_submission.job_provisioning_data.ssh_port is None:
-                                job_submission.job_provisioning_data.ssh_port = 22
-                    submissions.append(job_submission)
-            if job_model is not None:
-                # Use the spec from the latest submission. Submissions can have different specs
-                job_spec = JobSpec.__response__.parse_raw(job_model.job_spec_data)
-                if not include_sensitive:
-                    _remove_job_spec_sensitive_info(job_spec)
-                jobs.append(Job(job_spec=job_spec, job_submissions=submissions))
+    if include_jobs:
+        jobs = _get_run_jobs_with_submissions(
+            run_model=run_model,
+            job_submissions_limit=job_submissions_limit,
+            return_in_api=return_in_api,
+            include_sensitive=include_sensitive,
+        )
 
     run_spec = RunSpec.__response__.parse_raw(run_model.run_spec)
 
     latest_job_submission = None
-    if include_job_submissions:
+    if len(jobs) > 0 and len(jobs[0].job_submissions) > 0:
         # TODO(egor-s): does it make sense with replicas and multi-node?
-        if jobs:
-            latest_job_submission = jobs[0].job_submissions[-1]
+        latest_job_submission = jobs[0].job_submissions[-1]
 
     service_spec = None
     if run_model.service_spec is not None:
         service_spec = ServiceSpec.__response__.parse_raw(run_model.service_spec)
 
+    status_message = _get_run_status_message(run_model)
+    error = _get_run_error(run_model)
     run = Run(
         id=run_model.id,
         project_name=run_model.project.name,
@@ -704,18 +696,107 @@ def run_model_to_run(
         submitted_at=run_model.submitted_at.replace(tzinfo=timezone.utc),
         last_processed_at=run_model.last_processed_at.replace(tzinfo=timezone.utc),
         status=run_model.status,
+        status_message=status_message,
         termination_reason=run_model.termination_reason,
         run_spec=run_spec,
         jobs=jobs,
         latest_job_submission=latest_job_submission,
         service=service_spec,
         deployment_num=run_model.deployment_num,
+        error=error,
         deleted=run_model.deleted,
     )
     run.cost = _get_run_cost(run)
     return run
 
 
+def _get_run_jobs_with_submissions(
+    run_model: RunModel,
+    job_submissions_limit: Optional[int],
+    return_in_api: bool = False,
+    include_sensitive: bool = False,
+) -> List[Job]:
+    jobs: List[Job] = []
+    run_jobs = sorted(run_model.jobs, key=lambda j: (j.replica_num, j.job_num, j.submission_num))
+    for replica_num, replica_submissions in itertools.groupby(
+        run_jobs, key=lambda j: j.replica_num
+    ):
+        for job_num, job_models in itertools.groupby(replica_submissions, key=lambda j: j.job_num):
+            submissions = []
+            job_model = None
+            if job_submissions_limit is not None:
+                if job_submissions_limit == 0:
+                    # Take latest job submission to return its job_spec
+                    job_models = list(job_models)[-1:]
+                else:
+                    job_models = list(job_models)[-job_submissions_limit:]
+            for job_model in job_models:
+                if job_submissions_limit != 0:
+                    job_submission = job_model_to_job_submission(job_model)
+                    if return_in_api:
+                        # Set default non-None values for 0.18 backward-compatibility
+                        # Remove in 0.19
+                        if job_submission.job_provisioning_data is not None:
+                            if job_submission.job_provisioning_data.hostname is None:
+                                job_submission.job_provisioning_data.hostname = ""
+                            if job_submission.job_provisioning_data.ssh_port is None:
+                                job_submission.job_provisioning_data.ssh_port = 22
+                    submissions.append(job_submission)
+            if job_model is not None:
+                # Use the spec from the latest submission. Submissions can have different specs
+                job_spec = JobSpec.__response__.parse_raw(job_model.job_spec_data)
+                if not include_sensitive:
+                    _remove_job_spec_sensitive_info(job_spec)
+                jobs.append(Job(job_spec=job_spec, job_submissions=submissions))
+    return jobs
+
+
+def _get_run_status_message(run_model: RunModel) -> str:
+    if len(run_model.jobs) == 0:
+        return run_model.status.value
+
+    sorted_job_models = sorted(
+        run_model.jobs, key=lambda j: (j.replica_num, j.job_num, j.submission_num)
+    )
+    job_models_grouped_by_job = list(
+        list(jm)
+        for _, jm in itertools.groupby(sorted_job_models, key=lambda j: (j.replica_num, j.job_num))
+    )
+
+    if all(job_models[-1].status == JobStatus.PULLING for job_models in job_models_grouped_by_job):
+        # Show `pulling`` if last job submission of all jobs is pulling
+        return "pulling"
+
+    if run_model.status in [RunStatus.SUBMITTED, RunStatus.PENDING]:
+        # Show `retrying` if any job caused the run to retry
+        for job_models in job_models_grouped_by_job:
+            last_job_spec = JobSpec.__response__.parse_raw(job_models[-1].job_spec_data)
+            retry_on_events = last_job_spec.retry.on_events if last_job_spec.retry else []
+            last_job_termination_reason = _get_last_job_termination_reason(job_models)
+            if (
+                last_job_termination_reason
+                == JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY
+                and RetryEvent.NO_CAPACITY in retry_on_events
+            ):
+                # TODO: Show `retrying` for other retry events
+                return "retrying"
+
+    return run_model.status.value
+
+
+def _get_last_job_termination_reason(job_models: List[JobModel]) -> Optional[JobTerminationReason]:
+    for job_model in reversed(job_models):
+        if job_model.termination_reason is not None:
+            return job_model.termination_reason
+    return None
+
+
+def _get_run_error(run_model: RunModel) -> Optional[str]:
+    if run_model.termination_reason is None:
+        return None
+    return run_model.termination_reason.to_error()
+
+
 async def _get_pool_offers(
     session: AsyncSession,
     project: ProjectModel,
@@ -914,6 +995,8 @@ _TYPE_SPECIFIC_CONF_UPDATABLE_FIELDS = {
         "replicas",
         "scaling",
         # rolling deployment
+        # NOTE: keep this list in sync with the "Rolling deployment" section in services.md
+        "port",
         "resources",
         "volumes",
         "docker",

dstack/_internal/server/services/services/__init__.py

@@ -22,7 +22,7 @@ from dstack._internal.core.errors import (
 from dstack._internal.core.models.configurations import SERVICE_HTTPS_DEFAULT, ServiceConfiguration
 from dstack._internal.core.models.gateways import GatewayConfiguration, GatewayStatus
 from dstack._internal.core.models.instances import SSHConnectionParams
-from dstack._internal.core.models.runs import Run, RunSpec, ServiceModelSpec, ServiceSpec
+from dstack._internal.core.models.runs import JobSpec, Run, RunSpec, ServiceModelSpec, ServiceSpec
 from dstack._internal.server import settings
 from dstack._internal.server.models import GatewayModel, JobModel, ProjectModel, RunModel
 from dstack._internal.server.services.gateways import (
@@ -179,6 +179,7 @@ async def register_replica(
         async with conn.client() as client:
             await client.register_replica(
                 run=run,
+                job_spec=JobSpec.__response__.parse_raw(job_model.job_spec_data),
                 job_submission=job_submission,
                 ssh_head_proxy=ssh_head_proxy,
                 ssh_head_proxy_private_key=ssh_head_proxy_private_key,

dstack/_internal/server/services/users.py

@@ -44,7 +44,9 @@ async def list_users_for_user(
     session: AsyncSession,
     user: UserModel,
 ) -> List[User]:
-    return await list_all_users(session=session)
+    if user.global_role == GlobalRole.ADMIN:
+        return await list_all_users(session=session)
+    return [user_model_to_user(user)]
 
 
 async def list_all_users(

dstack/_internal/server/services/volumes.py

@@ -401,6 +401,19 @@ def _validate_volume_configuration(configuration: VolumeConfiguration):
     if configuration.name is not None:
         validate_dstack_resource_name(configuration.name)
 
+    if configuration.volume_id is not None and configuration.auto_cleanup_duration is not None:
+        if (
+            isinstance(configuration.auto_cleanup_duration, int)
+            and configuration.auto_cleanup_duration > 0
+        ) or (
+            isinstance(configuration.auto_cleanup_duration, str)
+            and configuration.auto_cleanup_duration not in ("off", "-1")
+        ):
+            raise ServerClientError(
+                "External volumes (with volume_id) do not support auto_cleanup_duration. "
+                "Auto-cleanup only works for volumes created and managed by dstack."
+            )
+
 
 async def _delete_volume(session: AsyncSession, project: ProjectModel, volume_model: VolumeModel):
     volume = volume_model_to_volume(volume_model)

dstack/_internal/server/settings.py

@@ -42,6 +42,11 @@ SERVER_BACKGROUND_PROCESSING_FACTOR = int(
     os.getenv("DSTACK_SERVER_BACKGROUND_PROCESSING_FACTOR", 1)
 )
 
+SERVER_BACKGROUND_PROCESSING_DISABLED = (
+    os.getenv("DSTACK_SERVER_BACKGROUND_PROCESSING_DISABLED") is not None
+)
+SERVER_BACKGROUND_PROCESSING_ENABLED = not SERVER_BACKGROUND_PROCESSING_DISABLED
+
 SERVER_EXECUTOR_MAX_WORKERS = int(os.getenv("DSTACK_SERVER_EXECUTOR_MAX_WORKERS", 128))
 
 MAX_OFFERS_TRIED = int(os.getenv("DSTACK_SERVER_MAX_OFFERS_TRIED", 25))
@@ -113,5 +118,5 @@ SERVER_PROFILING_ENABLED = os.getenv("DSTACK_SERVER_PROFILING_ENABLED") is not N
 
 UPDATE_DEFAULT_PROJECT = os.getenv("DSTACK_UPDATE_DEFAULT_PROJECT") is not None
 DO_NOT_UPDATE_DEFAULT_PROJECT = os.getenv("DSTACK_DO_NOT_UPDATE_DEFAULT_PROJECT") is not None
-SKIP_GATEWAY_UPDATE = os.getenv("DSTACK_SKIP_GATEWAY_UPDATE", None) is not None
-ENABLE_PROMETHEUS_METRICS = os.getenv("DSTACK_ENABLE_PROMETHEUS_METRICS", None) is not None
+SKIP_GATEWAY_UPDATE = os.getenv("DSTACK_SKIP_GATEWAY_UPDATE") is not None
+ENABLE_PROMETHEUS_METRICS = os.getenv("DSTACK_ENABLE_PROMETHEUS_METRICS") is not None