dstack 0.19.12rc1__py3-none-any.whl → 0.19.14__py3-none-any.whl

dstack/_internal/server/routers/projects.py

@@ -14,7 +14,7 @@ from dstack._internal.server.schemas.projects import (
 from dstack._internal.server.security.permissions import (
     Authenticated,
     ProjectManager,
-    ProjectMember,
+    ProjectMemberOrPublicAccess,
 )
 from dstack._internal.server.services import projects
 from dstack._internal.server.utils.routers import get_base_api_additional_responses
@@ -36,7 +36,7 @@ async def list_projects(
 
     `members` and `backends` are always empty - call `/api/projects/{project_name}/get` to retrieve them.
     """
-    return await projects.list_user_projects(session=session, user=user)
+    return await projects.list_user_accessible_projects(session=session, user=user)
 
 
 @router.post("/create")
@@ -49,6 +49,7 @@ async def create_project(
         session=session,
         user=user,
         project_name=body.project_name,
+        is_public=body.is_public,
     )
 
 
@@ -68,7 +69,7 @@ async def delete_projects(
 @router.post("/{project_name}/get")
 async def get_project(
     session: AsyncSession = Depends(get_session),
-    user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectMember()),
+    user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectMemberOrPublicAccess()),
 ) -> Project:
     _, project = user_project
     return projects.project_model_to_project(project)

dstack/_internal/server/routers/prometheus.py

@@ -3,6 +3,7 @@ from typing import Annotated
 
 from fastapi import APIRouter, Depends
 from fastapi.responses import PlainTextResponse
+from prometheus_client import generate_latest
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from dstack._internal.server import settings
@@ -26,4 +27,6 @@ async def get_prometheus_metrics(
 ) -> str:
     if not settings.ENABLE_PROMETHEUS_METRICS:
         raise error_not_found()
-    return await prometheus.get_metrics(session=session)
+    custom_metrics = await prometheus.get_metrics(session=session)
+    prometheus_metrics = generate_latest()
+    return custom_metrics + prometheus_metrics.decode()

dstack/_internal/server/schemas/projects.py

@@ -8,6 +8,7 @@ from dstack._internal.core.models.users import ProjectRole
 
 class CreateProjectRequest(CoreModel):
     project_name: str
+    is_public: bool = False
 
 
 class DeleteProjectsRequest(CoreModel):

dstack/_internal/server/security/permissions.py

@@ -99,6 +99,42 @@ class ProjectMember:
         return await get_project_member(session, project_name, token.credentials)
 
 
+class ProjectMemberOrPublicAccess:
+    """
+    Allows access to project for:
+    - Global admins
+    - Project members
+    - Any authenticated user if the project is public
+    """
+
+    async def __call__(
+        self,
+        *,
+        session: AsyncSession = Depends(get_session),
+        project_name: str,
+        token: HTTPAuthorizationCredentials = Security(HTTPBearer()),
+    ) -> Tuple[UserModel, ProjectModel]:
+        user = await log_in_with_token(session=session, token=token.credentials)
+        if user is None:
+            raise error_invalid_token()
+
+        project = await get_project_model_by_name(session=session, project_name=project_name)
+        if project is None:
+            raise error_not_found()
+
+        if user.global_role == GlobalRole.ADMIN:
+            return user, project
+
+        project_role = get_user_project_role(user=user, project=project)
+        if project_role is not None:
+            return user, project
+
+        if project.is_public:
+            return user, project
+
+        raise error_forbidden()
+
+
 class OptionalServiceAccount:
     def __init__(self, token: Optional[str]) -> None:
         self._token = token

dstack/_internal/server/services/jobs/__init__.py

@@ -128,6 +128,7 @@ def job_model_to_job_submission(job_model: JobModel) -> JobSubmission:
     return JobSubmission(
         id=job_model.id,
         submission_num=job_model.submission_num,
+        deployment_num=job_model.deployment_num,
         submitted_at=job_model.submitted_at.replace(tzinfo=timezone.utc),
         last_processed_at=last_processed_at,
         finished_at=finished_at,

dstack/_internal/server/services/jobs/configurators/base.py

@@ -50,11 +50,15 @@ def get_default_python_verison() -> str:
         )
 
 
-def get_default_image(python_version: str, nvcc: bool = False) -> str:
-    suffix = ""
-    if nvcc:
-        suffix = "-devel"
-    return f"{settings.DSTACK_BASE_IMAGE}:py{python_version}-{settings.DSTACK_BASE_IMAGE_VERSION}-cuda-12.1{suffix}"
+def get_default_image(nvcc: bool = False) -> str:
+    """
+    Note: May be overridden by dstack (e.g., EFA-enabled version for AWS EFA-capable instances).
+    See `dstack._internal.server.background.tasks.process_running_jobs._patch_base_image_for_aws_efa` for details.
+
+    Args:
+        nvcc: If True, returns 'devel' variant, otherwise 'base'.
+    """
+    return f"{settings.DSTACK_BASE_IMAGE}:{settings.DSTACK_BASE_IMAGE_VERSION}-{'devel' if nvcc else 'base'}-ubuntu{settings.DSTACK_BASE_IMAGE_UBUNTU_VERSION}"
 
 
 class JobConfigurator(ABC):
@@ -173,7 +177,7 @@ class JobConfigurator(ABC):
         ):
             return []
         return [
-            f"uv venv --prompt workflow --seed {DEFAULT_REPO_DIR}/.venv > /dev/null 2>&1",
+            f"uv venv --python {self._python()} --prompt workflow --seed {DEFAULT_REPO_DIR}/.venv > /dev/null 2>&1",
             f"echo 'source {DEFAULT_REPO_DIR}/.venv/bin/activate' >> ~/.bashrc",
             f"source {DEFAULT_REPO_DIR}/.venv/bin/activate",
         ]
@@ -199,7 +203,7 @@ class JobConfigurator(ABC):
     def _image_name(self) -> str:
         if self.run_spec.configuration.image is not None:
             return self.run_spec.configuration.image
-        return get_default_image(self._python(), nvcc=bool(self.run_spec.configuration.nvcc))
+        return get_default_image(nvcc=bool(self.run_spec.configuration.nvcc))
 
     async def _user(self) -> Optional[UnixUser]:
         user = self.run_spec.configuration.user

dstack/_internal/server/services/projects.py

@@ -53,10 +53,37 @@ async def list_user_projects(
     session: AsyncSession,
     user: UserModel,
 ) -> List[Project]:
+    """
+    Returns projects where the user is a member.
+    """
     if user.global_role == GlobalRole.ADMIN:
         projects = await list_project_models(session=session)
     else:
         projects = await list_user_project_models(session=session, user=user)
+
+    projects = sorted(projects, key=lambda p: p.created_at)
+    return [
+        project_model_to_project(p, include_backends=False, include_members=False)
+        for p in projects
+    ]
+
+
+async def list_user_accessible_projects(
+    session: AsyncSession,
+    user: UserModel,
+) -> List[Project]:
+    """
+    Returns all projects accessible to the user:
+    - For global admins: ALL projects in the system
+    - For regular users: Projects where user is a member + public projects where user is NOT a member
+    """
+    if user.global_role == GlobalRole.ADMIN:
+        projects = await list_project_models(session=session)
+    else:
+        member_projects = await list_user_project_models(session=session, user=user)
+        public_projects = await list_public_non_member_project_models(session=session, user=user)
+        projects = member_projects + public_projects
+
     projects = sorted(projects, key=lambda p: p.created_at)
     return [
         project_model_to_project(p, include_backends=False, include_members=False)
@@ -86,6 +113,7 @@ async def create_project(
     session: AsyncSession,
     user: UserModel,
     project_name: str,
+    is_public: bool = False,
 ) -> Project:
     user_permissions = users.get_user_permissions(user)
     if not user_permissions.can_create_projects:
@@ -100,6 +128,7 @@ async def create_project(
         session=session,
         owner=user,
         project_name=project_name,
+        is_public=is_public,
     )
     await add_project_member(
         session=session,
@@ -233,6 +262,9 @@ async def list_user_project_models(
     user: UserModel,
     include_members: bool = False,
 ) -> List[ProjectModel]:
+    """
+    List project models for a user where they are a member.
+    """
     options = []
     if include_members:
         options.append(joinedload(ProjectModel.members))
@@ -248,6 +280,25 @@ async def list_user_project_models(
     return list(res.scalars().unique().all())
 
 
+async def list_public_non_member_project_models(
+    session: AsyncSession,
+    user: UserModel,
+) -> List[ProjectModel]:
+    """
+    List public project models where user is NOT a member.
+    """
+    res = await session.execute(
+        select(ProjectModel).where(
+            ProjectModel.deleted == False,
+            ProjectModel.is_public == True,
+            ProjectModel.id.notin_(
+                select(MemberModel.project_id).where(MemberModel.user_id == user.id)
+            ),
+        )
+    )
+    return list(res.scalars().all())
+
+
 async def list_user_owned_project_models(
     session: AsyncSession, user: UserModel, include_deleted: bool = False
 ) -> List[ProjectModel]:
@@ -323,7 +374,7 @@ async def get_project_model_by_id_or_error(
 
 
 async def create_project_model(
-    session: AsyncSession, owner: UserModel, project_name: str
+    session: AsyncSession, owner: UserModel, project_name: str, is_public: bool = False
 ) -> ProjectModel:
     private_bytes, public_bytes = await run_async(
         generate_rsa_key_pair_bytes, f"{project_name}@dstack"
@@ -334,6 +385,7 @@ async def create_project_model(
         name=project_name,
         ssh_private_key=private_bytes.decode(),
         ssh_public_key=public_bytes.decode(),
+        is_public=is_public,
     )
     session.add(project)
     await session.commit()
@@ -407,6 +459,7 @@ def project_model_to_project(
         created_at=project_model.created_at.replace(tzinfo=timezone.utc),
         backends=backends,
         members=members,
+        is_public=project_model.is_public,
     )
 
 

dstack/_internal/server/services/runner/client.py

@@ -32,6 +32,7 @@ from dstack._internal.utils.common import get_or_error
 from dstack._internal.utils.logging import get_logger
 
 REQUEST_TIMEOUT = 9
+UPLOAD_CODE_REQUEST_TIMEOUT = 60
 
 logger = get_logger(__name__)
 
@@ -109,7 +110,9 @@ class RunnerClient:
         resp.raise_for_status()
 
     def upload_code(self, file: Union[BinaryIO, bytes]):
-        resp = requests.post(self._url("/api/upload_code"), data=file, timeout=REQUEST_TIMEOUT)
+        resp = requests.post(
+            self._url("/api/upload_code"), data=file, timeout=UPLOAD_CODE_REQUEST_TIMEOUT
+        )
         resp.raise_for_status()
 
     def run_job(self):

dstack/_internal/server/services/runs.py

@@ -439,6 +439,7 @@ async def apply_plan(
         .values(
             run_spec=run_spec.json(),
             priority=run_spec.configuration.priority,
+            deployment_num=current_resource.deployment_num + 1,
         )
     )
     run = await get_run_by_name(
@@ -501,6 +502,8 @@ async def submit_run(
             run_spec=run_spec.json(),
             last_processed_at=submitted_at,
             priority=run_spec.configuration.priority,
+            deployment_num=0,
+            desired_replica_count=1,  # a relevant value will be set in process_runs.py
         )
         session.add(run_model)
 
@@ -539,6 +542,7 @@ def create_job_model_for_new_submission(
         job_num=job.job_spec.job_num,
         job_name=f"{job.job_spec.job_name}",
         replica_num=job.job_spec.replica_num,
+        deployment_num=run_model.deployment_num,
         submission_num=len(job.job_submissions),
         submitted_at=now,
         last_processed_at=now,
@@ -662,13 +666,9 @@ def run_model_to_run(
         for job_num, job_submissions in itertools.groupby(
             replica_submissions, key=lambda j: j.job_num
         ):
-            job_spec = None
             submissions = []
+            job_model = None
             for job_model in job_submissions:
-                if job_spec is None:
-                    job_spec = JobSpec.__response__.parse_raw(job_model.job_spec_data)
-                    if not include_sensitive:
-                        _remove_job_spec_sensitive_info(job_spec)
                 if include_job_submissions:
                     job_submission = job_model_to_job_submission(job_model)
                     if return_in_api:
@@ -680,7 +680,11 @@ def run_model_to_run(
                             if job_submission.job_provisioning_data.ssh_port is None:
                                 job_submission.job_provisioning_data.ssh_port = 22
                     submissions.append(job_submission)
-            if job_spec is not None:
+            if job_model is not None:
+                # Use the spec from the latest submission. Submissions can have different specs
+                job_spec = JobSpec.__response__.parse_raw(job_model.job_spec_data)
+                if not include_sensitive:
+                    _remove_job_spec_sensitive_info(job_spec)
                 jobs.append(Job(job_spec=job_spec, job_submissions=submissions))
 
     run_spec = RunSpec.__response__.parse_raw(run_model.run_spec)
@@ -707,6 +711,7 @@ def run_model_to_run(
         jobs=jobs,
         latest_job_submission=latest_job_submission,
         service=service_spec,
+        deployment_num=run_model.deployment_num,
         deleted=run_model.deleted,
     )
     run.cost = _get_run_cost(run)
@@ -897,9 +902,24 @@ _UPDATABLE_SPEC_FIELDS = ["repo_code_hash", "configuration"]
 _CONF_UPDATABLE_FIELDS = ["priority"]
 _TYPE_SPECIFIC_CONF_UPDATABLE_FIELDS = {
     "dev-environment": ["inactivity_duration"],
-    # Most service fields can be updated via replica redeployment.
-    # TODO: Allow updating other fields when rolling deployment is supported.
-    "service": ["replicas", "scaling", "strip_prefix"],
+    "service": [
+        # in-place
+        "replicas",
+        "scaling",
+        # rolling deployment
+        "resources",
+        "volumes",
+        "image",
+        "user",
+        "privileged",
+        "entrypoint",
+        "python",
+        "nvcc",
+        "single_branch",
+        "env",
+        "shell",
+        "commands",
+    ],
 }
 
 
@@ -1004,34 +1024,33 @@ async def scale_run_replicas(session: AsyncSession, run_model: RunModel, replica
         abs(replicas_diff),
     )
 
-    # lists of (importance, replica_num, jobs)
+    # lists of (importance, is_out_of_date, replica_num, jobs)
     active_replicas = []
     inactive_replicas = []
 
     for replica_num, replica_jobs in group_jobs_by_replica_latest(run_model.jobs):
         statuses = set(job.status for job in replica_jobs)
+        deployment_num = replica_jobs[0].deployment_num  # same for all jobs
+        is_out_of_date = deployment_num < run_model.deployment_num
         if {JobStatus.TERMINATING, *JobStatus.finished_statuses()} & statuses:
             # if there are any terminating or finished jobs, the replica is inactive
-            inactive_replicas.append((0, replica_num, replica_jobs))
+            inactive_replicas.append((0, is_out_of_date, replica_num, replica_jobs))
         elif JobStatus.SUBMITTED in statuses:
             # if there are any submitted jobs, the replica is active and has the importance of 0
-            active_replicas.append((0, replica_num, replica_jobs))
+            active_replicas.append((0, is_out_of_date, replica_num, replica_jobs))
         elif {JobStatus.PROVISIONING, JobStatus.PULLING} & statuses:
             # if there are any provisioning or pulling jobs, the replica is active and has the importance of 1
-            active_replicas.append((1, replica_num, replica_jobs))
+            active_replicas.append((1, is_out_of_date, replica_num, replica_jobs))
         else:
             # all jobs are running, the replica is active and has the importance of 2
-            active_replicas.append((2, replica_num, replica_jobs))
+            active_replicas.append((2, is_out_of_date, replica_num, replica_jobs))
 
-    # sort by importance (desc) and replica_num (asc)
-    active_replicas.sort(key=lambda r: (-r[0], r[1]))
+    # sort by is_out_of_date (up-to-date first), importance (desc), and replica_num (asc)
+    active_replicas.sort(key=lambda r: (r[1], -r[0], r[2]))
     run_spec = RunSpec.__response__.parse_raw(run_model.run_spec)
 
     if replicas_diff < 0:
-        if len(active_replicas) + replicas_diff < run_spec.configuration.replicas.min:
-            raise ServerClientError("Can't scale down below the minimum number of replicas")
-
-        for _, _, replica_jobs in reversed(active_replicas[-abs(replicas_diff) :]):
+        for _, _, _, replica_jobs in reversed(active_replicas[-abs(replicas_diff) :]):
             # scale down the less important replicas first
             for job in replica_jobs:
                 if job.status.is_finished() or job.status == JobStatus.TERMINATING:
@@ -1040,18 +1059,15 @@ async def scale_run_replicas(session: AsyncSession, run_model: RunModel, replica
                 job.termination_reason = JobTerminationReason.SCALED_DOWN
                 # background task will process the job later
     else:
-        if len(active_replicas) + replicas_diff > run_spec.configuration.replicas.max:
-            raise ServerClientError("Can't scale up above the maximum number of replicas")
         scheduled_replicas = 0
 
         # rerun inactive replicas
-        for _, _, replica_jobs in inactive_replicas:
+        for _, _, _, replica_jobs in inactive_replicas:
             if scheduled_replicas == replicas_diff:
                 break
             await retry_run_replica_jobs(session, run_model, replica_jobs, only_failed=False)
             scheduled_replicas += 1
 
-        # create new replicas
         for replica_num in range(
             len(active_replicas) + scheduled_replicas, len(active_replicas) + replicas_diff
         ):
@@ -1068,7 +1084,14 @@ async def scale_run_replicas(session: AsyncSession, run_model: RunModel, replica
 async def retry_run_replica_jobs(
     session: AsyncSession, run_model: RunModel, latest_jobs: List[JobModel], *, only_failed: bool
 ):
-    for job_model in latest_jobs:
+    new_jobs = await get_jobs_from_run_spec(
+        RunSpec.__response__.parse_raw(run_model.run_spec),
+        replica_num=latest_jobs[0].replica_num,
+    )
+    assert len(new_jobs) == len(latest_jobs), (
+        "Changing the number of jobs within a replica is not yet supported"
+    )
+    for job_model, new_job in zip(latest_jobs, new_jobs):
         if not (job_model.status.is_finished() or job_model.status == JobStatus.TERMINATING):
             if only_failed:
                 # No need to resubmit, skip
@@ -1079,10 +1102,7 @@ async def retry_run_replica_jobs(
 
         new_job_model = create_job_model_for_new_submission(
             run_model=run_model,
-            job=Job(
-                job_spec=JobSpec.__response__.parse_raw(job_model.job_spec_data),
-                job_submissions=[],
-            ),
+            job=new_job,
             status=JobStatus.SUBMITTED,
         )
         # dirty hack to avoid passing all job submissions

dstack/_internal/server/services/services/__init__.py

@@ -30,6 +30,7 @@ from dstack._internal.server.services.gateways import (
     get_project_gateway_model_by_name,
 )
 from dstack._internal.server.services.logging import fmt
+from dstack._internal.server.services.services.autoscalers import get_service_scaler
 from dstack._internal.server.services.services.options import get_service_options
 from dstack._internal.utils.logging import get_logger
 
@@ -258,3 +259,21 @@ def _get_gateway_https(configuration: GatewayConfiguration) -> bool:
     if configuration.certificate is not None and configuration.certificate.type == "lets-encrypt":
         return True
     return False
+
+
+async def update_service_desired_replica_count(
+    session: AsyncSession,
+    run_model: RunModel,
+    configuration: ServiceConfiguration,
+    last_scaled_at: Optional[int],
+) -> None:
+    scaler = get_service_scaler(configuration)
+    stats = None
+    if run_model.gateway_id is not None:
+        conn = await get_or_add_gateway_connection(session, run_model.gateway_id)
+        stats = await conn.get_stats(run_model.project.name, run_model.run_name)
+    run_model.desired_replica_count = scaler.get_desired_count(
+        current_desired_count=run_model.desired_replica_count,
+        stats=stats,
+        last_scaled_at=last_scaled_at,
+    )

dstack/_internal/server/services/services/autoscalers.py

@@ -1,7 +1,7 @@
 import datetime
 import math
 from abc import ABC, abstractmethod
-from typing import List, Optional
+from typing import Optional
 
 from pydantic import BaseModel
 
@@ -23,14 +23,20 @@ class ReplicaInfo(BaseModel):
 
 class BaseServiceScaler(ABC):
     @abstractmethod
-    def scale(self, replicas: List[ReplicaInfo], stats: Optional[PerWindowStats]) -> int:
+    def get_desired_count(
+        self,
+        current_desired_count: int,
+        stats: Optional[PerWindowStats],
+        last_scaled_at: Optional[datetime.datetime],
+    ) -> int:
         """
         Args:
-            replicas: list of all replicas
             stats: service usage stats
+            current_desired_count: currently used desired count
+            last_scaled_at: last time service was scaled, None if it was never scaled yet
 
         Returns:
-            diff: number of replicas to add or remove
+            desired_count: desired count of replicas
         """
         pass
 
@@ -49,12 +55,14 @@ class ManualScaler(BaseServiceScaler):
         self.min_replicas = min_replicas
         self.max_replicas = max_replicas
 
-    def scale(self, replicas: List[ReplicaInfo], stats: Optional[PerWindowStats]) -> int:
-        active_replicas = [r for r in replicas if r.active]
-        target_replicas = len(active_replicas)
-        # clip the target replicas to the min and max values
-        target_replicas = min(max(target_replicas, self.min_replicas), self.max_replicas)
-        return target_replicas - len(active_replicas)
+    def get_desired_count(
+        self,
+        current_desired_count: int,
+        stats: Optional[PerWindowStats],
+        last_scaled_at: Optional[datetime.datetime],
+    ) -> int:
+        # clip the desired count to the min and max values
+        return min(max(current_desired_count, self.min_replicas), self.max_replicas)
 
 
 class RPSAutoscaler(BaseServiceScaler):
@@ -72,40 +80,43 @@ class RPSAutoscaler(BaseServiceScaler):
         self.scale_up_delay = scale_up_delay
         self.scale_down_delay = scale_down_delay
 
-    def scale(self, replicas: List[ReplicaInfo], stats: Optional[PerWindowStats]) -> int:
+    def get_desired_count(
+        self,
+        current_desired_count: int,
+        stats: Optional[PerWindowStats],
+        last_scaled_at: Optional[datetime.datetime],
+    ) -> int:
         if not stats:
-            return 0
+            return current_desired_count
 
         now = common_utils.get_current_datetime()
-        active_replicas = [r for r in replicas if r.active]
-        last_scaled_at = max((r.timestamp for r in replicas), default=None)
 
         # calculate the average RPS over the last minute
         rps = stats[60].requests / 60
-        target_replicas = math.ceil(rps / self.target)
-        # clip the target replicas to the min and max values
-        target_replicas = min(max(target_replicas, self.min_replicas), self.max_replicas)
+        new_desired_count = math.ceil(rps / self.target)
+        # clip the desired count to the min and max values
+        new_desired_count = min(max(new_desired_count, self.min_replicas), self.max_replicas)
 
-        if target_replicas > len(active_replicas):
-            if len(active_replicas) == 0:
+        if new_desired_count > current_desired_count:
+            if current_desired_count == 0:
                 # no replicas, scale up immediately
-                return target_replicas
+                return new_desired_count
             if (
                 last_scaled_at is not None
                 and (now - last_scaled_at).total_seconds() < self.scale_up_delay
             ):
                 # too early to scale up, wait for the delay
-                return 0
-            return target_replicas - len(active_replicas)
-        elif target_replicas < len(active_replicas):
+                return current_desired_count
+            return new_desired_count
+        elif new_desired_count < current_desired_count:
             if (
                 last_scaled_at is not None
                 and (now - last_scaled_at).total_seconds() < self.scale_down_delay
             ):
                 # too early to scale down, wait for the delay
-                return 0
-            return target_replicas - len(active_replicas)
-        return 0
+                return current_desired_count
+            return new_desired_count
+        return new_desired_count
 
 
 def get_service_scaler(conf: ServiceConfiguration) -> BaseServiceScaler:

dstack/_internal/server/services/storage/__init__.py

@@ -0,0 +1,38 @@
+from typing import Optional
+
+from dstack._internal.server import settings
+from dstack._internal.server.services.storage.base import BaseStorage
+from dstack._internal.server.services.storage.gcs import GCS_AVAILABLE, GCSStorage
+from dstack._internal.server.services.storage.s3 import BOTO_AVAILABLE, S3Storage
+
+_default_storage = None
+
+
+def init_default_storage():
+    global _default_storage
+    if settings.SERVER_S3_BUCKET is None and settings.SERVER_GCS_BUCKET is None:
+        raise ValueError(
+            "Either settings.SERVER_S3_BUCKET or settings.SERVER_GCS_BUCKET must be set"
+        )
+    if settings.SERVER_S3_BUCKET and settings.SERVER_GCS_BUCKET:
+        raise ValueError(
+            "Only one of settings.SERVER_S3_BUCKET or settings.SERVER_GCS_BUCKET can be set"
+        )
+
+    if settings.SERVER_S3_BUCKET:
+        if not BOTO_AVAILABLE:
+            raise ValueError("AWS dependencies are not installed")
+        _default_storage = S3Storage(
+            bucket=settings.SERVER_S3_BUCKET,
+            region=settings.SERVER_S3_BUCKET_REGION,
+        )
+    elif settings.SERVER_GCS_BUCKET:
+        if not GCS_AVAILABLE:
+            raise ValueError("GCS dependencies are not installed")
+        _default_storage = GCSStorage(
+            bucket=settings.SERVER_GCS_BUCKET,
+        )
+
+
+def get_default_storage() -> Optional[BaseStorage]:
+    return _default_storage

dstack/_internal/server/services/storage/base.py

@@ -0,0 +1,27 @@
+from abc import ABC, abstractmethod
+from typing import Optional
+
+
+class BaseStorage(ABC):
+    @abstractmethod
+    def upload_code(
+        self,
+        project_id: str,
+        repo_id: str,
+        code_hash: str,
+        blob: bytes,
+    ):
+        pass
+
+    @abstractmethod
+    def get_code(
+        self,
+        project_id: str,
+        repo_id: str,
+        code_hash: str,
+    ) -> Optional[bytes]:
+        pass
+
+    @staticmethod
+    def _get_code_key(project_id: str, repo_id: str, code_hash: str) -> str:
+        return f"data/projects/{project_id}/codes/{repo_id}/{code_hash}"