dstack 0.19.12__py3-none-any.whl → 0.19.13__py3-none-any.whl

dstack/_internal/cli/services/configurators/run.py

@@ -166,12 +166,7 @@ class BaseRunConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator):
             # We can attach to run multiple times if it goes from running to pending (retried).
             while True:
                 with MultiItemStatus(f"Launching [code]{run.name}[/]...", console=console) as live:
-                    while run.status in (
-                        RunStatus.SUBMITTED,
-                        RunStatus.PENDING,
-                        RunStatus.PROVISIONING,
-                        RunStatus.TERMINATING,
-                    ):
+                    while not _is_ready_to_attach(run):
                         table = get_runs_table([run])
                         live.update(table)
                         time.sleep(5)

dstack/_internal/cli/utils/run.py

@@ -166,11 +166,15 @@ def get_runs_table(
         run_row: Dict[Union[str, int], Any] = {
             "NAME": run.run_spec.run_name,
             "SUBMITTED": format_date(run.submitted_at),
+            "STATUS": (
+                run.latest_job_submission.status_message
+                if run.status.is_finished() and run.latest_job_submission
+                else run.status_message
+            ),
         }
         if run.error:
             run_row["ERROR"] = run.error
         if len(run.jobs) != 1:
-            run_row["STATUS"] = run.status
             add_row_from_dict(table, run_row)
 
         for job in run.jobs:

dstack/_internal/core/backends/aws/compute.py

@@ -132,7 +132,8 @@ class AWSCompute(
         availability_offers = []
         for offer in offers:
             availability = InstanceAvailability.UNKNOWN
-            if not _has_quota(regions_to_quotas[offer.region], offer.instance.name):
+            quota = _has_quota(regions_to_quotas[offer.region], offer.instance.name)
+            if quota is not None and not quota:
                 availability = InstanceAvailability.NO_QUOTA
             availability_offers.append(
                 InstanceOfferWithAvailability(
@@ -231,6 +232,7 @@ class AWSCompute(
                 image_id, username = aws_resources.get_image_id_and_username(
                     ec2_client=ec2_client,
                     cuda=len(instance_offer.instance.resources.gpus) > 0,
+                    instance_type=instance_offer.instance.name,
                     image_config=self.config.os_images,
                 )
                 response = ec2_resource.create_instances(
@@ -781,10 +783,18 @@ def _get_regions_to_quotas(
 ) -> Dict[str, Dict[str, int]]:
     def get_region_quotas(client: botocore.client.BaseClient) -> Dict[str, int]:
         region_quotas = {}
-        for page in client.get_paginator("list_service_quotas").paginate(ServiceCode="ec2"):
-            for q in page["Quotas"]:
-                if "On-Demand" in q["QuotaName"]:
-                    region_quotas[q["UsageMetric"]["MetricDimensions"]["Class"]] = q["Value"]
+        try:
+            for page in client.get_paginator("list_service_quotas").paginate(ServiceCode="ec2"):
+                for q in page["Quotas"]:
+                    if "On-Demand" in q["QuotaName"]:
+                        region_quotas[q["UsageMetric"]["MetricDimensions"]["Class"]] = q["Value"]
+        except botocore.exceptions.ClientError as e:
+            if len(e.args) > 0 and "TooManyRequestsException" in e.args[0]:
+                logger.warning(
+                    "Failed to get quotas due to rate limits. Quotas won't be accounted for."
+                )
+            else:
+                logger.exception(e)
         return region_quotas
 
     regions_to_quotas = {}
@@ -800,12 +810,15 @@ def _get_regions_to_quotas(
     return regions_to_quotas
 
 
-def _has_quota(quotas: Dict[str, int], instance_name: str) -> bool:
+def _has_quota(quotas: Dict[str, int], instance_name: str) -> Optional[bool]:
+    quota = quotas.get("Standard/OnDemand")
     if instance_name.startswith("p"):
-        return quotas.get("P/OnDemand", 0) > 0
+        quota = quotas.get("P/OnDemand")
     if instance_name.startswith("g"):
-        return quotas.get("G/OnDemand", 0) > 0
-    return quotas.get("Standard/OnDemand", 0) > 0
+        quota = quotas.get("G/OnDemand")
+    if quota is None:
+        return None
+    return quota > 0
 
 
 def _get_regions_to_zones(session: boto3.Session, regions: List[str]) -> Dict[str, List[str]]:

dstack/_internal/core/backends/aws/resources.py

@@ -12,11 +12,13 @@ from dstack._internal.utils.logging import get_logger
 logger = get_logger(__name__)
 
 DSTACK_ACCOUNT_ID = "142421590066"
+DLAMI_OWNER_ACCOUNT_ID = "898082745236"
 
 
 def get_image_id_and_username(
     ec2_client: botocore.client.BaseClient,
     cuda: bool,
+    instance_type: str,
     image_config: Optional[AWSOSImageConfig] = None,
 ) -> tuple[str, str]:
     if image_config is not None:
@@ -27,6 +29,11 @@ def get_image_id_and_username(
         image_name = image.name
         image_owner = image.owner
         username = image.user
+    elif _supported_by_dlami(instance_type):
+        # TODO: Update DLAMI image version from time to time
+        image_name = "Deep Learning Base OSS Nvidia Driver GPU AMI (Ubuntu 22.04) 20250516"
+        image_owner = DLAMI_OWNER_ACCOUNT_ID
+        username = "ubuntu"
     else:
         image_name = (
             f"dstack-{version.base_image}" if not cuda else f"dstack-cuda-{version.base_image}"
@@ -628,6 +635,25 @@ def _is_private_subnet_with_internet_egress(
     return False
 
 
+def _supported_by_dlami(instance_type: str) -> bool:
+    # Currently only p3. instances are not supported by DLAMI among GPU instances.
+    return any(
+        instance_type.startswith(family)
+        for family in [
+            "g4dn.",
+            "g5.",
+            "g6.",
+            "gr6.",
+            "g6e.",
+            "p4d.",
+            "p4de.",
+            "p5.",
+            "p5e.",
+            "p6-b200.",
+        ]
+    )
+
+
 def get_reservation(
     ec2_client: botocore.client.BaseClient,
     reservation_id: str,

dstack/_internal/core/backends/base/offers.py

@@ -80,7 +80,6 @@ def catalog_item_to_offer(
         spot=item.spot,
         disk=Disk(size_mib=disk_size_mib),
     )
-    resources.description = resources.pretty_format()
     return InstanceOffer(
         backend=backend,
         instance=InstanceType(

dstack/_internal/core/compatibility/fleets.py

@@ -0,0 +1,72 @@
+from typing import Any, Dict, Optional
+
+from dstack._internal.core.models.fleets import ApplyFleetPlanInput, FleetSpec
+from dstack._internal.core.models.instances import Instance
+
+
+def get_get_plan_excludes(fleet_spec: FleetSpec) -> Dict:
+    get_plan_excludes = {}
+    spec_excludes = get_fleet_spec_excludes(fleet_spec)
+    if spec_excludes:
+        get_plan_excludes["spec"] = spec_excludes
+    return get_plan_excludes
+
+
+def get_apply_plan_excludes(plan_input: ApplyFleetPlanInput) -> Dict:
+    apply_plan_excludes = {}
+    spec_excludes = get_fleet_spec_excludes(plan_input.spec)
+    if spec_excludes:
+        apply_plan_excludes["spec"] = spec_excludes
+    current_resource = plan_input.current_resource
+    if current_resource is not None:
+        current_resource_excludes = {}
+        apply_plan_excludes["current_resource"] = current_resource_excludes
+        if all(map(_should_exclude_instance_cpu_arch, current_resource.instances)):
+            current_resource_excludes["instances"] = {
+                "__all__": {"instance_type": {"resources": {"cpu_arch"}}}
+            }
+    return {"plan": apply_plan_excludes}
+
+
+def get_create_fleet_excludes(fleet_spec: FleetSpec) -> Dict:
+    create_fleet_excludes = {}
+    spec_excludes = get_fleet_spec_excludes(fleet_spec)
+    if spec_excludes:
+        create_fleet_excludes["spec"] = spec_excludes
+    return create_fleet_excludes
+
+
+def get_fleet_spec_excludes(fleet_spec: FleetSpec) -> Optional[Dict]:
+    """
+    Returns `fleet_spec` exclude mapping to exclude certain fields from the request.
+    Use this method to exclude new fields when they are not set to keep
+    clients backward-compatibility with older servers.
+    """
+    spec_excludes: Dict[str, Any] = {}
+    configuration_excludes: Dict[str, Any] = {}
+    profile_excludes: set[str] = set()
+    profile = fleet_spec.profile
+    if profile.fleets is None:
+        profile_excludes.add("fleets")
+    if fleet_spec.configuration.tags is None:
+        configuration_excludes["tags"] = True
+    if profile.tags is None:
+        profile_excludes.add("tags")
+    if profile.startup_order is None:
+        profile_excludes.add("startup_order")
+    if profile.stop_criteria is None:
+        profile_excludes.add("stop_criteria")
+    if configuration_excludes:
+        spec_excludes["configuration"] = configuration_excludes
+    if profile_excludes:
+        spec_excludes["profile"] = profile_excludes
+    if spec_excludes:
+        return spec_excludes
+    return None
+
+
+def _should_exclude_instance_cpu_arch(instance: Instance) -> bool:
+    try:
+        return instance.instance_type.resources.cpu_arch is None
+    except AttributeError:
+        return True

dstack/_internal/core/compatibility/gateways.py

@@ -0,0 +1,34 @@
+from typing import Dict
+
+from dstack._internal.core.models.gateways import GatewayConfiguration, GatewaySpec
+
+
+def get_gateway_spec_excludes(gateway_spec: GatewaySpec) -> Dict:
+    """
+    Returns `gateway_spec` exclude mapping to exclude certain fields from the request.
+    Use this method to exclude new fields when they are not set to keep
+    clients backward-compatibility with older servers.
+    """
+    spec_excludes = {}
+    spec_excludes["configuration"] = _get_gateway_configuration_excludes(
+        gateway_spec.configuration
+    )
+    return spec_excludes
+
+
+def get_create_gateway_excludes(configuration: GatewayConfiguration) -> Dict:
+    """
+    Returns an exclude mapping to exclude certain fields from the create gateway request.
+    Use this method to exclude new fields when they are not set to keep
+    clients backward-compatibility with older servers.
+    """
+    create_gateway_excludes = {}
+    create_gateway_excludes["configuration"] = _get_gateway_configuration_excludes(configuration)
+    return create_gateway_excludes
+
+
+def _get_gateway_configuration_excludes(configuration: GatewayConfiguration) -> Dict:
+    configuration_excludes = {}
+    if configuration.tags is None:
+        configuration_excludes["tags"] = True
+    return configuration_excludes

dstack/_internal/core/compatibility/runs.py

@@ -0,0 +1,125 @@
+from typing import Any, Dict, Optional
+
+from dstack._internal.core.models.configurations import ServiceConfiguration
+from dstack._internal.core.models.runs import ApplyRunPlanInput, JobSubmission, RunSpec
+from dstack._internal.server.schemas.runs import GetRunPlanRequest
+
+
+def get_apply_plan_excludes(plan: ApplyRunPlanInput) -> Optional[Dict]:
+    """
+    Returns `plan` exclude mapping to exclude certain fields from the request.
+    Use this method to exclude new fields when they are not set to keep
+    clients backward-compatibility with older servers.
+    """
+    apply_plan_excludes = {}
+    run_spec_excludes = get_run_spec_excludes(plan.run_spec)
+    if run_spec_excludes is not None:
+        apply_plan_excludes["run_spec"] = run_spec_excludes
+    current_resource = plan.current_resource
+    if current_resource is not None:
+        current_resource_excludes = {}
+        current_resource_excludes["status_message"] = True
+        apply_plan_excludes["current_resource"] = current_resource_excludes
+        current_resource_excludes["run_spec"] = get_run_spec_excludes(current_resource.run_spec)
+        job_submissions_excludes = {}
+        current_resource_excludes["jobs"] = {
+            "__all__": {"job_submissions": {"__all__": job_submissions_excludes}}
+        }
+        job_submissions = [js for j in current_resource.jobs for js in j.job_submissions]
+        if all(map(_should_exclude_job_submission_jpd_cpu_arch, job_submissions)):
+            job_submissions_excludes["job_provisioning_data"] = {
+                "instance_type": {"resources": {"cpu_arch"}}
+            }
+        if all(map(_should_exclude_job_submission_jrd_cpu_arch, job_submissions)):
+            job_submissions_excludes["job_runtime_data"] = {
+                "offer": {"instance": {"resources": {"cpu_arch"}}}
+            }
+        if all(js.exit_status is None for js in job_submissions):
+            job_submissions_excludes["exit_status"] = True
+        latest_job_submission = current_resource.latest_job_submission
+        if latest_job_submission is not None:
+            latest_job_submission_excludes = {}
+            current_resource_excludes["latest_job_submission"] = latest_job_submission_excludes
+            if _should_exclude_job_submission_jpd_cpu_arch(latest_job_submission):
+                latest_job_submission_excludes["job_provisioning_data"] = {
+                    "instance_type": {"resources": {"cpu_arch"}}
+                }
+            if _should_exclude_job_submission_jrd_cpu_arch(latest_job_submission):
+                latest_job_submission_excludes["job_runtime_data"] = {
+                    "offer": {"instance": {"resources": {"cpu_arch"}}}
+                }
+            if latest_job_submission.exit_status is None:
+                latest_job_submission_excludes["exit_status"] = True
+    return {"plan": apply_plan_excludes}
+
+
+def get_get_plan_excludes(request: GetRunPlanRequest) -> Optional[Dict]:
+    """
+    Excludes new fields when they are not set to keep
+    clients backward-compatibility with older servers.
+    """
+    get_plan_excludes = {}
+    run_spec_excludes = get_run_spec_excludes(request.run_spec)
+    if run_spec_excludes is not None:
+        get_plan_excludes["run_spec"] = run_spec_excludes
+    if request.max_offers is None:
+        get_plan_excludes["max_offers"] = True
+    return get_plan_excludes
+
+
+def get_run_spec_excludes(run_spec: RunSpec) -> Optional[Dict]:
+    """
+    Returns `run_spec` exclude mapping to exclude certain fields from the request.
+    Use this method to exclude new fields when they are not set to keep
+    clients backward-compatibility with older servers.
+    """
+    spec_excludes: dict[str, Any] = {}
+    configuration_excludes: dict[str, Any] = {}
+    profile_excludes: set[str] = set()
+    configuration = run_spec.configuration
+    profile = run_spec.profile
+
+    if configuration.fleets is None:
+        configuration_excludes["fleets"] = True
+    if profile is not None and profile.fleets is None:
+        profile_excludes.add("fleets")
+    if configuration.tags is None:
+        configuration_excludes["tags"] = True
+    if profile is not None and profile.tags is None:
+        profile_excludes.add("tags")
+    if isinstance(configuration, ServiceConfiguration) and not configuration.rate_limits:
+        configuration_excludes["rate_limits"] = True
+    if configuration.shell is None:
+        configuration_excludes["shell"] = True
+    if configuration.priority is None:
+        configuration_excludes["priority"] = True
+    if configuration.startup_order is None:
+        configuration_excludes["startup_order"] = True
+    if profile is not None and profile.startup_order is None:
+        profile_excludes.add("startup_order")
+    if configuration.stop_criteria is None:
+        configuration_excludes["stop_criteria"] = True
+    if profile is not None and profile.stop_criteria is None:
+        profile_excludes.add("stop_criteria")
+
+    if configuration_excludes:
+        spec_excludes["configuration"] = configuration_excludes
+    if profile_excludes:
+        spec_excludes["profile"] = profile_excludes
+    if spec_excludes:
+        return spec_excludes
+    return None
+
+
+def _should_exclude_job_submission_jpd_cpu_arch(job_submission: JobSubmission) -> bool:
+    try:
+        return job_submission.job_provisioning_data.instance_type.resources.cpu_arch is None
+    except AttributeError:
+        return True
+
+
+def _should_exclude_job_submission_jrd_cpu_arch(job_submission: JobSubmission) -> bool:
+    try:
+        return job_submission.job_runtime_data.offer.instance.resources.cpu_arch is None
+    except AttributeError:
+        return True

dstack/_internal/core/compatibility/volumes.py

@@ -0,0 +1,32 @@
+from typing import Dict
+
+from dstack._internal.core.models.volumes import VolumeConfiguration, VolumeSpec
+
+
+def get_volume_spec_excludes(volume_spec: VolumeSpec) -> Dict:
+    """
+    Returns `volume_spec` exclude mapping to exclude certain fields from the request.
+    Use this method to exclude new fields when they are not set to keep
+    clients backward-compatibility with older servers.
+    """
+    spec_excludes = {}
+    spec_excludes["configuration"] = _get_volume_configuration_excludes(volume_spec.configuration)
+    return spec_excludes
+
+
+def get_create_volume_excludes(configuration: VolumeConfiguration) -> Dict:
+    """
+    Returns an exclude mapping to exclude certain fields from the create volume request.
+    Use this method to exclude new fields when they are not set to keep
+    clients backward-compatibility with older servers.
+    """
+    create_volume_excludes = {}
+    create_volume_excludes["configuration"] = _get_volume_configuration_excludes(configuration)
+    return create_volume_excludes
+
+
+def _get_volume_configuration_excludes(configuration: VolumeConfiguration) -> Dict:
+    configuration_excludes = {}
+    if configuration.tags is None:
+        configuration_excludes["tags"] = True
+    return configuration_excludes

dstack/_internal/core/models/instances.py

@@ -48,29 +48,68 @@ class Resources(CoreModel):
     gpus: List[Gpu]
     spot: bool
     disk: Disk = Disk(size_mib=102400)  # the default value (100GB) for backward compatibility
+    # TODO: make description a computed field after migrating to pydanticV2
     description: str = ""
     cpu_arch: Optional[gpuhunt.CPUArchitecture] = None
 
-    def pretty_format(self, include_spot: bool = False) -> str:
+    @root_validator
+    def _description(cls, values) -> Dict:
+        try:
+            description = values["description"]
+            if not description:
+                cpus = values["cpus"]
+                memory_mib = values["memory_mib"]
+                gpus = values["gpus"]
+                disk_size_mib = values["disk"].size_mib
+                spot = values["spot"]
+                cpu_arch = values["cpu_arch"]
+                values["description"] = Resources._pretty_format(
+                    cpus, cpu_arch, memory_mib, disk_size_mib, gpus, spot, include_spot=True
+                )
+        except KeyError:
+            return values
+        return values
+
+    @staticmethod
+    def _pretty_format(
+        cpus: int,
+        cpu_arch: Optional[gpuhunt.CPUArchitecture],
+        memory_mib: int,
+        disk_size_mib: int,
+        gpus: List[Gpu],
+        spot: bool,
+        include_spot: bool = False,
+    ) -> str:
         resources = {}
-        if self.cpus > 0:
-            resources["cpus"] = self.cpus
-            resources["cpu_arch"] = self.cpu_arch
-        if self.memory_mib > 0:
-            resources["memory"] = f"{self.memory_mib / 1024:.0f}GB"
-        if self.disk.size_mib > 0:
-            resources["disk_size"] = f"{self.disk.size_mib / 1024:.0f}GB"
-        if self.gpus:
-            gpu = self.gpus[0]
+        if cpus > 0:
+            resources["cpus"] = cpus
+            resources["cpu_arch"] = cpu_arch
+        if memory_mib > 0:
+            resources["memory"] = f"{memory_mib / 1024:.0f}GB"
+        if disk_size_mib > 0:
+            resources["disk_size"] = f"{disk_size_mib / 1024:.0f}GB"
+        if gpus:
+            gpu = gpus[0]
             resources["gpu_name"] = gpu.name
-            resources["gpu_count"] = len(self.gpus)
+            resources["gpu_count"] = len(gpus)
             if gpu.memory_mib > 0:
                 resources["gpu_memory"] = f"{gpu.memory_mib / 1024:.0f}GB"
         output = pretty_resources(**resources)
-        if include_spot and self.spot:
+        if include_spot and spot:
             output += " (spot)"
         return output
 
+    def pretty_format(self, include_spot: bool = False) -> str:
+        return Resources._pretty_format(
+            self.cpus,
+            self.cpu_arch,
+            self.memory_mib,
+            self.disk.size_mib,
+            self.gpus,
+            self.spot,
+            include_spot,
+        )
+
 
 class InstanceType(CoreModel):
     name: str

dstack/_internal/core/models/runs.py

@@ -496,6 +496,7 @@ class Run(CoreModel):
     submitted_at: datetime
     last_processed_at: datetime
     status: RunStatus
+    status_message: Optional[str] = None
     termination_reason: Optional[RunTerminationReason]
     run_spec: RunSpec
     jobs: List[Job]
@@ -524,6 +525,46 @@ class Run(CoreModel):
         else:
             return None
 
+    @root_validator
+    def _status_message(cls, values) -> Dict:
+        try:
+            status = values["status"]
+            jobs: List[Job] = values["jobs"]
+            retry_on_events = (
+                jobs[0].job_spec.retry.on_events if jobs and jobs[0].job_spec.retry else []
+            )
+            termination_reason = Run.get_last_termination_reason(jobs[0]) if jobs else None
+        except KeyError:
+            return values
+        values["status_message"] = Run._get_status_message(
+            status=status,
+            retry_on_events=retry_on_events,
+            termination_reason=termination_reason,
+        )
+        return values
+
+    @staticmethod
+    def get_last_termination_reason(job: "Job") -> Optional[JobTerminationReason]:
+        for submission in reversed(job.job_submissions):
+            if submission.termination_reason is not None:
+                return submission.termination_reason
+        return None
+
+    @staticmethod
+    def _get_status_message(
+        status: RunStatus,
+        retry_on_events: List[RetryEvent],
+        termination_reason: Optional[JobTerminationReason],
+    ) -> str:
+        # Currently, `retrying` is shown only for `no-capacity` events
+        if (
+            status in [RunStatus.SUBMITTED, RunStatus.PENDING]
+            and termination_reason == JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY
+            and RetryEvent.NO_CAPACITY in retry_on_events
+        ):
+            return "retrying"
+        return status.value
+
 
 class JobPlan(CoreModel):
     job_spec: JobSpec

dstack/_internal/server/app.py

@@ -128,7 +128,7 @@ async def lifespan(app: FastAPI):
         yes=UPDATE_DEFAULT_PROJECT,
         no=DO_NOT_UPDATE_DEFAULT_PROJECT,
     )
-    if settings.SERVER_BUCKET is not None:
+    if settings.SERVER_S3_BUCKET is not None or settings.SERVER_GCS_BUCKET is not None:
         init_default_storage()
     scheduler = start_background_tasks()
     dstack_version = DSTACK_VERSION if DSTACK_VERSION else "(no version)"

dstack/_internal/server/services/storage/__init__.py

@@ -0,0 +1,38 @@
+from typing import Optional
+
+from dstack._internal.server import settings
+from dstack._internal.server.services.storage.base import BaseStorage
+from dstack._internal.server.services.storage.gcs import GCS_AVAILABLE, GCSStorage
+from dstack._internal.server.services.storage.s3 import BOTO_AVAILABLE, S3Storage
+
+_default_storage = None
+
+
+def init_default_storage():
+    global _default_storage
+    if settings.SERVER_S3_BUCKET is None and settings.SERVER_GCS_BUCKET is None:
+        raise ValueError(
+            "Either settings.SERVER_S3_BUCKET or settings.SERVER_GCS_BUCKET must be set"
+        )
+    if settings.SERVER_S3_BUCKET and settings.SERVER_GCS_BUCKET:
+        raise ValueError(
+            "Only one of settings.SERVER_S3_BUCKET or settings.SERVER_GCS_BUCKET can be set"
+        )
+
+    if settings.SERVER_S3_BUCKET:
+        if not BOTO_AVAILABLE:
+            raise ValueError("AWS dependencies are not installed")
+        _default_storage = S3Storage(
+            bucket=settings.SERVER_S3_BUCKET,
+            region=settings.SERVER_S3_BUCKET_REGION,
+        )
+    elif settings.SERVER_GCS_BUCKET:
+        if not GCS_AVAILABLE:
+            raise ValueError("GCS dependencies are not installed")
+        _default_storage = GCSStorage(
+            bucket=settings.SERVER_GCS_BUCKET,
+        )
+
+
+def get_default_storage() -> Optional[BaseStorage]:
+    return _default_storage

dstack/_internal/server/services/storage/base.py

@@ -0,0 +1,27 @@
+from abc import ABC, abstractmethod
+from typing import Optional
+
+
+class BaseStorage(ABC):
+    @abstractmethod
+    def upload_code(
+        self,
+        project_id: str,
+        repo_id: str,
+        code_hash: str,
+        blob: bytes,
+    ):
+        pass
+
+    @abstractmethod
+    def get_code(
+        self,
+        project_id: str,
+        repo_id: str,
+        code_hash: str,
+    ) -> Optional[bytes]:
+        pass
+
+    @staticmethod
+    def _get_code_key(project_id: str, repo_id: str, code_hash: str) -> str:
+        return f"data/projects/{project_id}/codes/{repo_id}/{code_hash}"