dstack 0.19.9__py3-none-any.whl → 0.19.10__py3-none-any.whl

dstack/_internal/cli/commands/config.py

@@ -14,7 +14,7 @@ logger = get_logger(__name__)
 
 class ConfigCommand(BaseCommand):
     NAME = "config"
-    DESCRIPTION = "Configure CLI"
+    DESCRIPTION = "Configure CLI (deprecated; use `dstack project`)"
 
     def _register(self):
         super()._register()

dstack/_internal/cli/commands/project.py

@@ -0,0 +1,161 @@
+import argparse
+
+from requests import HTTPError
+from rich.table import Table
+
+import dstack.api.server
+from dstack._internal.cli.commands import BaseCommand
+from dstack._internal.cli.utils.common import confirm_ask, console
+from dstack._internal.core.errors import ClientError, CLIError
+from dstack._internal.core.services.configs import ConfigManager
+from dstack._internal.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+class ProjectCommand(BaseCommand):
+    NAME = "project"
+    DESCRIPTION = "Manage projects configs"
+
+    def _register(self):
+        super()._register()
+        subparsers = self._parser.add_subparsers(dest="subcommand", help="Command to execute")
+
+        # Add subcommand
+        add_parser = subparsers.add_parser("add", help="Add or update a project config")
+        add_parser.add_argument(
+            "--name", type=str, help="The name of the project to configure", required=True
+        )
+        add_parser.add_argument("--url", type=str, help="Server url", required=True)
+        add_parser.add_argument("--token", type=str, help="User token", required=True)
+        add_parser.add_argument(
+            "-y",
+            "--yes",
+            help="Don't ask for confirmation (e.g. update the config)",
+            action="store_true",
+        )
+        add_parser.add_argument(
+            "-n",
+            "--no",
+            help="Don't ask for confirmation (e.g. do not update the config)",
+            action="store_true",
+        )
+        add_parser.set_defaults(subfunc=self._add)
+
+        # Delete subcommand
+        delete_parser = subparsers.add_parser("delete", help="Delete a project config")
+        delete_parser.add_argument(
+            "--name", type=str, help="The name of the project to delete", required=True
+        )
+        delete_parser.add_argument(
+            "-y",
+            "--yes",
+            help="Don't ask for confirmation",
+            action="store_true",
+        )
+        delete_parser.set_defaults(subfunc=self._delete)
+
+        # List subcommand
+        list_parser = subparsers.add_parser("list", help="List configured projects")
+        list_parser.set_defaults(subfunc=self._list)
+
+        # Set default subcommand
+        set_default_parser = subparsers.add_parser("set-default", help="Set default project")
+        set_default_parser.add_argument(
+            "name", type=str, help="The name of the project to set as default"
+        )
+        set_default_parser.set_defaults(subfunc=self._set_default)
+
+    def _command(self, args: argparse.Namespace):
+        if not hasattr(args, "subfunc"):
+            args.subfunc = self._list
+        args.subfunc(args)
+
+    def _add(self, args: argparse.Namespace):
+        config_manager = ConfigManager()
+        api_client = dstack.api.server.APIClient(base_url=args.url, token=args.token)
+        try:
+            api_client.projects.get(args.name)
+        except HTTPError as e:
+            if e.response.status_code == 403:
+                raise CLIError("Forbidden. Ensure the token is valid.")
+            elif e.response.status_code == 404:
+                raise CLIError(f"Project '{args.name}' not found.")
+            else:
+                raise e
+        default_project = config_manager.get_project_config()
+        if (
+            default_project is None
+            or default_project.name != args.name
+            or default_project.url != args.url
+            or default_project.token != args.token
+        ):
+            set_it_as_default = (
+                (
+                    args.yes
+                    or not default_project
+                    or confirm_ask(f"Set '{args.name}' as your default project?")
+                )
+                if not args.no
+                else False
+            )
+            config_manager.configure_project(
+                name=args.name, url=args.url, token=args.token, default=set_it_as_default
+            )
+            config_manager.save()
+        logger.info(
+            f"Configuration updated at {config_manager.config_filepath}", {"show_path": False}
+        )
+
+    def _delete(self, args: argparse.Namespace):
+        config_manager = ConfigManager()
+        if args.yes or confirm_ask(f"Are you sure you want to delete project '{args.name}'?"):
+            config_manager.delete_project(args.name)
+            config_manager.save()
+            console.print("[grey58]OK[/]")
+
+    def _list(self, args: argparse.Namespace):
+        config_manager = ConfigManager()
+        default_project = config_manager.get_project_config()
+
+        table = Table(box=None)
+        table.add_column("PROJECT", style="bold", no_wrap=True)
+        table.add_column("URL", style="grey58")
+        table.add_column("USER", style="grey58")
+        table.add_column("DEFAULT", justify="center")
+
+        for project_name in config_manager.list_projects():
+            project_config = config_manager.get_project_config(project_name)
+            is_default = project_name == default_project.name if default_project else False
+
+            # Get username from API
+            try:
+                api_client = dstack.api.server.APIClient(
+                    base_url=project_config.url, token=project_config.token
+                )
+                user_info = api_client.users.get_my_user()
+                username = user_info.username
+            except ClientError:
+                username = "(invalid token)"
+
+            table.add_row(
+                project_name,
+                project_config.url,
+                username,
+                "✓" if is_default else "",
+                style="bold" if is_default else None,
+            )
+
+        console.print(table)
+
+    def _set_default(self, args: argparse.Namespace):
+        config_manager = ConfigManager()
+        project_config = config_manager.get_project_config(args.name)
+        if project_config is None:
+            raise CLIError(f"Project '{args.name}' not found")
+
+        config_manager.configure_project(
+            name=args.name, url=project_config.url, token=project_config.token, default=True
+        )
+        config_manager.save()
+        console.print("[grey58]OK[/]")

dstack/_internal/cli/commands/ps.py

@@ -36,10 +36,17 @@ class PsCommand(APIBaseCommand):
             help="Watch statuses of runs in realtime",
             action="store_true",
         )
+        self._parser.add_argument(
+            "-n",
+            "--last",
+            help="Show only the last N runs. Implies --all",
+            type=int,
+            default=None,
+        )
 
     def _command(self, args: argparse.Namespace):
         super()._command(args)
-        runs = self.api.runs.list(all=args.all)
+        runs = self.api.runs.list(all=args.all, limit=args.last)
         if not args.watch:
             console.print(run_utils.get_runs_table(runs, verbose=args.verbose))
             return
@@ -49,6 +56,6 @@ class PsCommand(APIBaseCommand):
                 while True:
                     live.update(run_utils.get_runs_table(runs, verbose=args.verbose))
                     time.sleep(LIVE_TABLE_PROVISION_INTERVAL_SECS)
-                    runs = self.api.runs.list(all=args.all)
+                    runs = self.api.runs.list(all=args.all, limit=args.last)
         except KeyboardInterrupt:
             pass

dstack/_internal/cli/main.py

@@ -15,6 +15,7 @@ from dstack._internal.cli.commands.init import InitCommand
 from dstack._internal.cli.commands.logs import LogsCommand
 from dstack._internal.cli.commands.metrics import MetricsCommand
 from dstack._internal.cli.commands.offer import OfferCommand
+from dstack._internal.cli.commands.project import ProjectCommand
 from dstack._internal.cli.commands.ps import PsCommand
 from dstack._internal.cli.commands.server import ServerCommand
 from dstack._internal.cli.commands.stats import StatsCommand
@@ -69,6 +70,7 @@ def main():
     OfferCommand.register(subparsers)
     LogsCommand.register(subparsers)
     MetricsCommand.register(subparsers)
+    ProjectCommand.register(subparsers)
     PsCommand.register(subparsers)
     ServerCommand.register(subparsers)
     StatsCommand.register(subparsers)

dstack/_internal/core/backends/azure/compute.py

@@ -391,9 +391,12 @@ class VMImageVariant(enum.Enum):
 
 
 _SUPPORTED_VM_SERIES_PATTERNS = [
-    r"D(\d+)s_v3",  # Dsv3-series
-    r"E(\d+)i?s_v4",  # Esv4-series
+    # TODO: Support newer CPU series (Dsv6, Esv6).
+    # They are NVMe-only and require marking the VM image as NVMe.
+    r"D(\d+)s_v3",  # Dsv3-series (general purpose)
+    r"E(\d+)i?s_v4",  # Esv4-series (memory optimized)
     r"E(\d+)-(\d+)s_v4",  # Esv4-series (constrained vCPU)
+    r"F(\d+)s_v2",  # Fsv2-series (compute optimized)
     r"NC(\d+)s_v3",  # NCv3-series [V100 16GB]
     r"NC(\d+)as_T4_v3",  # NCasT4_v3-series [T4]
     r"ND(\d+)rs_v2",  # NDv2-series [8xV100 32GB]

dstack/_internal/core/models/configurations.py

@@ -23,6 +23,9 @@ ValidPort = conint(gt=0, le=65536)
 MAX_INT64 = 2**63 - 1
 SERVICE_HTTPS_DEFAULT = True
 STRIP_PREFIX_DEFAULT = True
+RUN_PRIOTIRY_MIN = 0
+RUN_PRIOTIRY_MAX = 100
+RUN_PRIORITY_DEFAULT = 0
 
 
 class RunConfigurationType(str, Enum):
@@ -77,7 +80,8 @@ class ScalingSpec(CoreModel):
         Field(
             description="The target value of the metric. "
             "The number of replicas is calculated based on this number and automatically adjusts "
-            "(scales up or down) as this metric changes"
+            "(scales up or down) as this metric changes",
+            gt=0,
         ),
     ]
     scale_up_delay: Annotated[
@@ -221,14 +225,26 @@ class BaseRunConfiguration(CoreModel):
             )
         ),
     ] = None
-    # deprecated since 0.18.31; task, service -- no effect; dev-environment -- executed right before `init`
-    setup: CommandsList = []
     resources: Annotated[
         ResourcesSpec, Field(description="The resources requirements to run the configuration")
     ] = ResourcesSpec()
+    priority: Annotated[
+        Optional[int],
+        Field(
+            ge=RUN_PRIOTIRY_MIN,
+            le=RUN_PRIOTIRY_MAX,
+            description=(
+                f"The priority of the run, an integer between `{RUN_PRIOTIRY_MIN}` and `{RUN_PRIOTIRY_MAX}`."
+                " `dstack` tries to provision runs with higher priority first."
+                f" Defaults to `{RUN_PRIORITY_DEFAULT}`"
+            ),
+        ),
+    ] = None
     volumes: Annotated[
         List[Union[MountPoint, str]], Field(description="The volumes mount points")
     ] = []
+    # deprecated since 0.18.31; task, service -- no effect; dev-environment -- executed right before `init`
+    setup: CommandsList = []
 
     @validator("python", pre=True, always=True)
     def convert_python(cls, v, values) -> Optional[PythonVersion]:

dstack/_internal/server/background/tasks/process_metrics.py

@@ -2,7 +2,7 @@ import asyncio
 import json
 from typing import Dict, List, Optional
 
-from sqlalchemy import delete, select
+from sqlalchemy import Delete, delete, select
 from sqlalchemy.orm import joinedload
 
 from dstack._internal.core.consts import DSTACK_RUNNER_HTTP_PORT
@@ -49,27 +49,29 @@ async def delete_metrics():
     finished_timestamp_micro_cutoff = (
         now_timestamp_micro - settings.SERVER_METRICS_FINISHED_TTL_SECONDS * 1_000_000
     )
+    await asyncio.gather(
+        _execute_delete_statement(
+            delete(JobMetricsPoint).where(
+                JobMetricsPoint.job_id.in_(
+                    select(JobModel.id).where(JobModel.status.in_([JobStatus.RUNNING]))
+                ),
+                JobMetricsPoint.timestamp_micro < running_timestamp_micro_cutoff,
+            )
+        ),
+        _execute_delete_statement(
+            delete(JobMetricsPoint).where(
+                JobMetricsPoint.job_id.in_(
+                    select(JobModel.id).where(JobModel.status.in_(JobStatus.finished_statuses()))
+                ),
+                JobMetricsPoint.timestamp_micro < finished_timestamp_micro_cutoff,
+            )
+        ),
+    )
+
+
+async def _execute_delete_statement(stmt: Delete) -> None:
     async with get_session_ctx() as session:
-        await asyncio.gather(
-            session.execute(
-                delete(JobMetricsPoint).where(
-                    JobMetricsPoint.job_id.in_(
-                        select(JobModel.id).where(JobModel.status.in_([JobStatus.RUNNING]))
-                    ),
-                    JobMetricsPoint.timestamp_micro < running_timestamp_micro_cutoff,
-                )
-            ),
-            session.execute(
-                delete(JobMetricsPoint).where(
-                    JobMetricsPoint.job_id.in_(
-                        select(JobModel.id).where(
-                            JobModel.status.in_(JobStatus.finished_statuses())
-                        )
-                    ),
-                    JobMetricsPoint.timestamp_micro < finished_timestamp_micro_cutoff,
-                )
-            ),
-        )
+        await session.execute(stmt)
         await session.commit()
 
 

dstack/_internal/server/background/tasks/process_submitted_jobs.py

@@ -93,11 +93,20 @@ async def _process_next_submitted_job():
         async with lock:
             res = await session.execute(
                 select(JobModel)
+                .join(JobModel.run)
                 .where(
                     JobModel.status == JobStatus.SUBMITTED,
                     JobModel.id.not_in(lockset),
                 )
-                .order_by(JobModel.last_processed_at.asc())
+                # Jobs are process in FIFO sorted by priority globally,
+                # thus runs from different project can "overtake" each other by using higher priorities.
+                # That's not a big problem as long as projects do not compete for the same compute resources.
+                # Jobs with lower priorities from other projects will be processed without major lag
+                # as long as new higher priority runs are not constantly submitted.
+                # TODO: Consider processing jobs from different projects fairly/round-robin
+                # Fully fair processing can be tricky to implement via the current DB queue as
+                # there can be many projects and we are limited by the max DB connections.
+                .order_by(RunModel.priority.desc(), JobModel.last_processed_at.asc())
                 .limit(1)
                 .with_for_update(skip_locked=True)
             )
@@ -360,16 +369,16 @@ async def _assign_job_to_pool_instance(
         (instance, common_utils.get_or_error(get_instance_offer(instance)))
         for instance in nonshared_instances
     ]
-    if not multinode:
-        shared_instances_with_offers = get_shared_pool_instances_with_offers(
-            pool_instances=pool_instances,
-            profile=profile,
-            requirements=job.job_spec.requirements,
-            idle_only=True,
-            fleet_model=fleet_model,
-            volumes=volumes,
-        )
-        instances_with_offers.extend(shared_instances_with_offers)
+    shared_instances_with_offers = get_shared_pool_instances_with_offers(
+        pool_instances=pool_instances,
+        profile=profile,
+        requirements=job.job_spec.requirements,
+        idle_only=True,
+        fleet_model=fleet_model,
+        multinode=multinode,
+        volumes=volumes,
+    )
+    instances_with_offers.extend(shared_instances_with_offers)
 
     if len(instances_with_offers) == 0:
         return None
@@ -572,7 +581,7 @@ def _create_instance_model_for_job(
 
 
 def _prepare_job_runtime_data(offer: InstanceOfferWithAvailability) -> JobRuntimeData:
-    if offer.total_blocks == 1:
+    if offer.blocks == offer.total_blocks:
         if env_utils.get_bool("DSTACK_FORCE_BRIDGE_NETWORK"):
             network_mode = NetworkMode.BRIDGE
         else:

dstack/_internal/server/migrations/versions/bca2fdf130bf_add_runmodel_priority.py

@@ -0,0 +1,34 @@
+"""Add RunModel.priority
+
+Revision ID: bca2fdf130bf
+Revises: 20166748b60c
+Create Date: 2025-05-14 15:24:21.269775
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "bca2fdf130bf"
+down_revision = "20166748b60c"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("runs", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("priority", sa.Integer(), nullable=True))
+    batch_op.execute("UPDATE runs SET priority = 0")
+    with op.batch_alter_table("runs", schema=None) as batch_op:
+        batch_op.alter_column("priority", nullable=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("runs", schema=None) as batch_op:
+        batch_op.drop_column("priority")
+
+    # ### end Alembic commands ###

dstack/_internal/server/models.py

@@ -348,6 +348,7 @@ class RunModel(BaseModel):
     resubmission_attempt: Mapped[int] = mapped_column(Integer, default=0)
     run_spec: Mapped[str] = mapped_column(Text)
     service_spec: Mapped[Optional[str]] = mapped_column(Text)
+    priority: Mapped[int] = mapped_column(Integer, default=0)
 
     jobs: Mapped[List["JobModel"]] = relationship(
         back_populates="run", lazy="selectin", order_by="[JobModel.replica_num, JobModel.job_num]"

dstack/_internal/server/routers/repos.py

@@ -1,6 +1,7 @@
 from typing import List, Tuple
 
 from fastapi import APIRouter, Depends, Request, UploadFile
+from humanize import naturalsize
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from dstack._internal.core.errors import ResourceNotExistsError, ServerClientError
@@ -14,9 +15,10 @@ from dstack._internal.server.schemas.repos import (
 )
 from dstack._internal.server.security.permissions import ProjectMember
 from dstack._internal.server.services import repos
+from dstack._internal.server.settings import SERVER_CODE_UPLOAD_LIMIT
 from dstack._internal.server.utils.routers import (
     get_base_api_additional_responses,
-    request_size_exceeded,
+    get_request_size,
 )
 
 router = APIRouter(
@@ -94,10 +96,12 @@ async def upload_code(
     session: AsyncSession = Depends(get_session),
     user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectMember()),
 ):
-    if request_size_exceeded(request, limit=2 * 2**20):
+    request_size = get_request_size(request)
+    if SERVER_CODE_UPLOAD_LIMIT > 0 and request_size > SERVER_CODE_UPLOAD_LIMIT:
         raise ServerClientError(
-            "Repo diff size exceeds the limit of 2MB. "
-            "Use .gitignore to exclude large files from the repo."
+            f"Repo diff size is {naturalsize(request_size)}, which exceeds the limit of "
+            f"{naturalsize(SERVER_CODE_UPLOAD_LIMIT)}. Use .gitignore to exclude large files from the repo. This "
+            f"limit can be modified by setting the DSTACK_SERVER_CODE_UPLOAD_LIMIT_BYTES environment variable"
         )
     _, project = user_project
     await repos.upload_code(

dstack/_internal/server/services/instances.py

@@ -235,6 +235,7 @@ def get_shared_pool_instances_with_offers(
     *,
     idle_only: bool = False,
     fleet_model: Optional[FleetModel] = None,
+    multinode: bool = False,
     volumes: Optional[List[List[Volume]]] = None,
 ) -> list[tuple[InstanceModel, InstanceOfferWithAvailability]]:
     instances_with_offers: list[tuple[InstanceModel, InstanceOfferWithAvailability]] = []
@@ -243,19 +244,22 @@ def get_shared_pool_instances_with_offers(
         pool_instances=pool_instances,
         profile=profile,
         fleet_model=fleet_model,
-        multinode=False,
+        multinode=multinode,
         volumes=volumes,
         shared=True,
     )
     for instance in filtered_instances:
         if idle_only and instance.status not in [InstanceStatus.IDLE, InstanceStatus.BUSY]:
             continue
+        if multinode and instance.busy_blocks > 0:
+            continue
         offer = get_instance_offer(instance)
         if offer is None:
             continue
         total_blocks = common_utils.get_or_error(instance.total_blocks)
         idle_blocks = total_blocks - instance.busy_blocks
-        for blocks in range(1, total_blocks + 1):
+        min_blocks = total_blocks if multinode else 1
+        for blocks in range(min_blocks, total_blocks + 1):
             shared_offer = generate_shared_offer(offer, blocks, total_blocks)
             catalog_item = offer_to_catalog_item(shared_offer)
             if gpuhunt.matches(catalog_item, query_filter):

dstack/_internal/server/services/jobs/configurators/base.py

@@ -6,7 +6,7 @@ from typing import Dict, List, Optional, Union
 
 from cachetools import TTLCache, cached
 
-import dstack.version as version
+from dstack._internal import settings
 from dstack._internal.core.errors import DockerRegistryError, ServerClientError
 from dstack._internal.core.models.common import RegistryAuth
 from dstack._internal.core.models.configurations import (
@@ -53,14 +53,14 @@ def get_default_image(python_version: str, nvcc: bool = False) -> str:
     suffix = ""
     if nvcc:
         suffix = "-devel"
-    return f"dstackai/base:py{python_version}-{version.base_image}-cuda-12.1{suffix}"
+    return f"{settings.DSTACK_BASE_IMAGE}:py{python_version}-{settings.DSTACK_BASE_IMAGE_VERSION}-cuda-12.1{suffix}"
 
 
 class JobConfigurator(ABC):
     TYPE: RunConfigurationType
 
     _image_config: Optional[ImageConfig] = None
-    # JobSSHKey should be shared for all jobs in a replica for inter-node communitation.
+    # JobSSHKey should be shared for all jobs in a replica for inter-node communication.
     _job_ssh_key: Optional[JobSSHKey] = None
 
     def __init__(self, run_spec: RunSpec):

dstack/_internal/server/services/runs.py

@@ -16,7 +16,7 @@ from dstack._internal.core.errors import (
     ServerClientError,
 )
 from dstack._internal.core.models.common import ApplyAction
-from dstack._internal.core.models.configurations import AnyRunConfiguration
+from dstack._internal.core.models.configurations import RUN_PRIORITY_DEFAULT, AnyRunConfiguration
 from dstack._internal.core.models.instances import (
     InstanceAvailability,
     InstanceOfferWithAvailability,
@@ -434,7 +434,12 @@ async def apply_plan(
     # FIXME: potentially long write transaction
     # Avoid getting run_model after update
     await session.execute(
-        update(RunModel).where(RunModel.id == current_resource.id).values(run_spec=run_spec.json())
+        update(RunModel)
+        .where(RunModel.id == current_resource.id)
+        .values(
+            run_spec=run_spec.json(),
+            priority=run_spec.configuration.priority,
+        )
     )
     run = await get_run_by_name(
         session=session,
@@ -495,6 +500,7 @@ async def submit_run(
             status=RunStatus.SUBMITTED,
             run_spec=run_spec.json(),
             last_processed_at=submitted_at,
+            priority=run_spec.configuration.priority,
         )
         session.add(run_model)
 
@@ -721,15 +727,15 @@ async def _get_pool_offers(
     pool_instances = [i for i in pool_instances if i.id not in detaching_instances_ids]
     multinode = job.job_spec.jobs_per_replica > 1
 
-    if not multinode:
-        shared_instances_with_offers = get_shared_pool_instances_with_offers(
-            pool_instances=pool_instances,
-            profile=run_spec.merged_profile,
-            requirements=job.job_spec.requirements,
-            volumes=volumes,
-        )
-        for _, offer in shared_instances_with_offers:
-            pool_offers.append(offer)
+    shared_instances_with_offers = get_shared_pool_instances_with_offers(
+        pool_instances=pool_instances,
+        profile=run_spec.merged_profile,
+        requirements=job.job_spec.requirements,
+        volumes=volumes,
+        multinode=multinode,
+    )
+    for _, offer in shared_instances_with_offers:
+        pool_offers.append(offer)
 
     nonshared_instances = filter_pool_instances(
         pool_instances=pool_instances,
@@ -852,6 +858,13 @@ def _get_job_submission_cost(job_submission: JobSubmission) -> float:
 
 
 def _validate_run_spec_and_set_defaults(run_spec: RunSpec):
+    # This function may set defaults for null run_spec values,
+    # although most defaults are resolved when building job_spec
+    # so that we can keep both the original user-supplied value (null in run_spec)
+    # and the default in job_spec.
+    # If a property is stored in job_spec - resolve the default there.
+    # Server defaults are preferable over client defaults so that
+    # the defaults depend on the server version, not the client version.
     if run_spec.run_name is not None:
         validate_dstack_resource_name(run_spec.run_name)
     for mount_point in run_spec.configuration.volumes:
@@ -875,11 +888,14 @@ def _validate_run_spec_and_set_defaults(run_spec: RunSpec):
         raise ServerClientError(
             f"Maximum utilization_policy.time_window is {settings.SERVER_METRICS_RUNNING_TTL_SECONDS}s"
         )
+    if run_spec.configuration.priority is None:
+        run_spec.configuration.priority = RUN_PRIORITY_DEFAULT
     set_resources_defaults(run_spec.configuration.resources)
 
 
 _UPDATABLE_SPEC_FIELDS = ["repo_code_hash", "configuration"]
-_CONF_TYPE_TO_UPDATABLE_FIELDS = {
+_CONF_UPDATABLE_FIELDS = ["priority"]
+_TYPE_SPECIFIC_CONF_UPDATABLE_FIELDS = {
     "dev-environment": ["inactivity_duration"],
     # Most service fields can be updated via replica redeployment.
     # TODO: Allow updating other fields when rolling deployment is supported.
@@ -915,12 +931,9 @@ def _check_can_update_configuration(
         raise ServerClientError(
             f"Configuration type changed from {current.type} to {new.type}, cannot update"
         )
-    updatable_fields = _CONF_TYPE_TO_UPDATABLE_FIELDS.get(new.type)
-    if updatable_fields is None:
-        raise ServerClientError(
-            f"Can only update {', '.join(_CONF_TYPE_TO_UPDATABLE_FIELDS)} configurations."
-            f" Not {new.type}"
-        )
+    updatable_fields = _CONF_UPDATABLE_FIELDS + _TYPE_SPECIFIC_CONF_UPDATABLE_FIELDS.get(
+        new.type, []
+    )
     diff = diff_models(current, new)
     changed_fields = list(diff.keys())
     for key in changed_fields:

dstack/_internal/server/settings.py

@@ -85,6 +85,7 @@ DEFAULT_SERVICE_CLIENT_MAX_BODY_SIZE = int(
 USER_PROJECT_DEFAULT_QUOTA = int(os.getenv("DSTACK_USER_PROJECT_DEFAULT_QUOTA", 10))
 FORBID_SERVICES_WITHOUT_GATEWAY = os.getenv("DSTACK_FORBID_SERVICES_WITHOUT_GATEWAY") is not None
 
+SERVER_CODE_UPLOAD_LIMIT = int(os.getenv("DSTACK_SERVER_CODE_UPLOAD_LIMIT", 2 * 2**20))
 
 # Development settings