dstack 0.19.25__py3-none-any.whl → 0.19.26__py3-none-any.whl

dstack/_internal/server/services/logs/filelog.py

@@ -48,7 +48,7 @@ class FileLogStorage(LogStorage):
     ) -> JobSubmissionLogs:
         start_line = 0
         if request.next_token:
-            start_line = self._next_token(request)
+            start_line = self._parse_next_token(request.next_token)
 
         logs = []
         next_token = None
@@ -97,7 +97,9 @@ class FileLogStorage(LogStorage):
     def _poll_logs_descending(
         self, log_file_path: Path, request: PollLogsRequest
     ) -> JobSubmissionLogs:
-        start_offset = self._next_token(request)
+        start_offset = None
+        if request.next_token is not None:
+            start_offset = self._parse_next_token(request.next_token)
 
         candidate_logs = []
 
@@ -123,12 +125,12 @@ class FileLogStorage(LogStorage):
         except FileNotFoundError:
             return JobSubmissionLogs(logs=[], next_token=None)
 
-        logs = [log for log, offset in candidate_logs[: request.limit]]
+        logs = [log for log, _ in candidate_logs[: request.limit]]
         next_token = None
         if len(candidate_logs) > request.limit:
             # We fetched one more than the limit, so there are more pages.
             # The next token should point to the start of the last log we are returning.
-            _last_log_event, last_log_offset = candidate_logs[request.limit - 1]
+            _, last_log_offset = candidate_logs[request.limit - 1]
             next_token = str(last_log_offset)
 
         return JobSubmissionLogs(logs=logs, next_token=next_token)
@@ -245,8 +247,7 @@ class FileLogStorage(LogStorage):
             message=runner_log_event.message.decode(errors="replace"),
         )
 
-    def _next_token(self, request: PollLogsRequest) -> Optional[int]:
-        next_token = request.next_token
+    def _parse_next_token(self, next_token: str) -> int:
         if next_token is None:
             return None
         try:

dstack/_internal/server/services/logs/gcp.py

@@ -20,6 +20,9 @@ from dstack._internal.server.services.logs.base import (
 from dstack._internal.utils.common import batched
 from dstack._internal.utils.logging import get_logger
 
+logger = get_logger(__name__)
+
+
 GCP_LOGGING_AVAILABLE = True
 try:
     import google.api_core.exceptions
@@ -28,152 +31,151 @@ try:
     from google.cloud.logging_v2.types import ListLogEntriesRequest
 except ImportError:
     GCP_LOGGING_AVAILABLE = False
-
-
-logger = get_logger(__name__)
-
-
-class GCPLogStorage(LogStorage):
-    # Max expected message size from runner is 32KB.
-    # Max expected LogEntry size is 32KB + metadata < 50KB < 256KB limit.
-    # With MAX_BATCH_SIZE = 100, max write request size < 5MB < 10 MB limit.
-    # See: https://cloud.google.com/logging/quotas.
-    MAX_RUNNER_MESSAGE_SIZE = 32 * 1024
-    MAX_BATCH_SIZE = 100
-
-    # Use the same log name for all run logs so that it's easy to manage all dstack-related logs.
-    LOG_NAME = "dstack-run-logs"
-    # Logs from different jobs belong to different "streams".
-    # GCP Logging has no built-in concepts of streams, so we implement them with labels.
-    # It should be fast to filter by labels since labels are indexed by default
-    # (https://cloud.google.com/logging/docs/analyze/custom-index).
-
-    def __init__(self, project_id: str):
-        self.project_id = project_id
-        try:
-            self.client = logging_v2.Client(project=project_id)
-            self.logger = self.client.logger(name=self.LOG_NAME)
-            self.logger.list_entries(max_results=1)
-            # Python client doesn't seem to support dry_run,
-            # so emit an empty log to check permissions.
-            self.logger.log_empty()
-        except google.auth.exceptions.DefaultCredentialsError:
-            raise LogStorageError("Default credentials not found")
-        except google.api_core.exceptions.NotFound:
-            raise LogStorageError(f"Project {project_id} not found")
-        except google.api_core.exceptions.PermissionDenied:
-            raise LogStorageError("Insufficient permissions")
-
-    def poll_logs(self, project: ProjectModel, request: PollLogsRequest) -> JobSubmissionLogs:
-        # TODO: GCP may return logs in random order when events have the same timestamp.
-        producer = LogProducer.RUNNER if request.diagnose else LogProducer.JOB
-        stream_name = self._get_stream_name(
-            project_name=project.name,
-            run_name=request.run_name,
-            job_submission_id=request.job_submission_id,
-            producer=producer,
-        )
-        log_filters = [f'labels.stream = "{stream_name}"']
-        if request.start_time:
-            log_filters.append(f'timestamp > "{request.start_time.isoformat()}"')
-        if request.end_time:
-            log_filters.append(f'timestamp < "{request.end_time.isoformat()}"')
-        log_filter = " AND ".join(log_filters)
-
-        order_by = logging_v2.DESCENDING if request.descending else logging_v2.ASCENDING
-        try:
-            # Use low-level API to get access to next_page_token
-            request_obj = ListLogEntriesRequest(
-                resource_names=[f"projects/{self.client.project}"],
-                filter=log_filter,
-                order_by=order_by,
-                page_size=request.limit,
-                page_token=request.next_token,
-            )
-            response = self.client._logging_api._gapic_api.list_log_entries(request=request_obj)
-
-            logs = [
-                LogEvent(
-                    timestamp=entry.timestamp,
-                    message=entry.json_payload.get("message"),
-                    log_source=LogEventSource.STDOUT,
-                )
-                for entry in response.entries
-            ]
-            next_token = response.next_page_token or None
-        except google.api_core.exceptions.ResourceExhausted as e:
-            logger.warning("GCP Logging exception: %s", repr(e))
-            # GCP Logging has severely low quota of 60 reads/min for entries.list
-            raise ServerClientError(
-                "GCP Logging read request limit exceeded."
-                " It's recommended to increase default entries.list request quota from 60 per minute."
-            )
-        return JobSubmissionLogs(
-            logs=logs,
-            external_url=self._get_stream_extrnal_url(stream_name),
-            next_token=next_token if len(logs) > 0 else None,
-        )
-
-    def write_logs(
-        self,
-        project: ProjectModel,
-        run_name: str,
-        job_submission_id: UUID,
-        runner_logs: List[RunnerLogEvent],
-        job_logs: List[RunnerLogEvent],
-    ):
-        producers_with_logs = [(LogProducer.RUNNER, runner_logs), (LogProducer.JOB, job_logs)]
-        for producer, producer_logs in producers_with_logs:
+else:
+
+    class GCPLogStorage(LogStorage):
+        # Max expected message size from runner is 32KB.
+        # Max expected LogEntry size is 32KB + metadata < 50KB < 256KB limit.
+        # With MAX_BATCH_SIZE = 100, max write request size < 5MB < 10 MB limit.
+        # See: https://cloud.google.com/logging/quotas.
+        MAX_RUNNER_MESSAGE_SIZE = 32 * 1024
+        MAX_BATCH_SIZE = 100
+
+        # Use the same log name for all run logs so that it's easy to manage all dstack-related logs.
+        LOG_NAME = "dstack-run-logs"
+        # Logs from different jobs belong to different "streams".
+        # GCP Logging has no built-in concepts of streams, so we implement them with labels.
+        # It should be fast to filter by labels since labels are indexed by default
+        # (https://cloud.google.com/logging/docs/analyze/custom-index).
+
+        def __init__(self, project_id: str):
+            self.project_id = project_id
+            try:
+                self.client = logging_v2.Client(project=project_id)
+                self.logger = self.client.logger(name=self.LOG_NAME)
+                self.logger.list_entries(max_results=1)
+                # Python client doesn't seem to support dry_run,
+                # so emit an empty log to check permissions.
+                self.logger.log_empty()
+            except google.auth.exceptions.DefaultCredentialsError:
+                raise LogStorageError("Default credentials not found")
+            except google.api_core.exceptions.NotFound:
+                raise LogStorageError(f"Project {project_id} not found")
+            except google.api_core.exceptions.PermissionDenied:
+                raise LogStorageError("Insufficient permissions")
+
+        def poll_logs(self, project: ProjectModel, request: PollLogsRequest) -> JobSubmissionLogs:
+            # TODO: GCP may return logs in random order when events have the same timestamp.
+            producer = LogProducer.RUNNER if request.diagnose else LogProducer.JOB
             stream_name = self._get_stream_name(
                 project_name=project.name,
-                run_name=run_name,
-                job_submission_id=job_submission_id,
+                run_name=request.run_name,
+                job_submission_id=request.job_submission_id,
                 producer=producer,
             )
-            self._write_logs_to_stream(
-                stream_name=stream_name,
-                logs=producer_logs,
+            log_filters = [f'labels.stream = "{stream_name}"']
+            if request.start_time:
+                log_filters.append(f'timestamp > "{request.start_time.isoformat()}"')
+            if request.end_time:
+                log_filters.append(f'timestamp < "{request.end_time.isoformat()}"')
+            log_filter = " AND ".join(log_filters)
+
+            order_by = logging_v2.DESCENDING if request.descending else logging_v2.ASCENDING
+            try:
+                # Use low-level API to get access to next_page_token
+                request_obj = ListLogEntriesRequest(
+                    resource_names=[f"projects/{self.client.project}"],
+                    filter=log_filter,
+                    order_by=order_by,
+                    page_size=request.limit,
+                    page_token=request.next_token,
+                )
+                response = self.client._logging_api._gapic_api.list_log_entries(  # type: ignore[attr-defined]
+                    request=request_obj
+                )
+
+                logs = [
+                    LogEvent(
+                        timestamp=entry.timestamp,
+                        message=entry.json_payload.get("message"),
+                        log_source=LogEventSource.STDOUT,
+                    )
+                    for entry in response.entries
+                ]
+                next_token = response.next_page_token or None
+            except google.api_core.exceptions.ResourceExhausted as e:
+                logger.warning("GCP Logging exception: %s", repr(e))
+                # GCP Logging has severely low quota of 60 reads/min for entries.list
+                raise ServerClientError(
+                    "GCP Logging read request limit exceeded."
+                    " It's recommended to increase default entries.list request quota from 60 per minute."
+                )
+            return JobSubmissionLogs(
+                logs=logs,
+                external_url=self._get_stream_extrnal_url(stream_name),
+                next_token=next_token if len(logs) > 0 else None,
             )
 
-    def close(self):
-        self.client.close()
-
-    def _write_logs_to_stream(self, stream_name: str, logs: List[RunnerLogEvent]):
-        with self.logger.batch() as batcher:
-            for batch in batched(logs, self.MAX_BATCH_SIZE):
-                for log in batch:
-                    message = log.message.decode(errors="replace")
-                    timestamp = unix_time_ms_to_datetime(log.timestamp)
-                    if len(log.message) > self.MAX_RUNNER_MESSAGE_SIZE:
-                        logger.error(
-                            "Stream %s: skipping event at %s, message exceeds max size: %d > %d",
-                            stream_name,
-                            timestamp.isoformat(),
-                            len(log.message),
-                            self.MAX_RUNNER_MESSAGE_SIZE,
+        def write_logs(
+            self,
+            project: ProjectModel,
+            run_name: str,
+            job_submission_id: UUID,
+            runner_logs: List[RunnerLogEvent],
+            job_logs: List[RunnerLogEvent],
+        ):
+            producers_with_logs = [(LogProducer.RUNNER, runner_logs), (LogProducer.JOB, job_logs)]
+            for producer, producer_logs in producers_with_logs:
+                stream_name = self._get_stream_name(
+                    project_name=project.name,
+                    run_name=run_name,
+                    job_submission_id=job_submission_id,
+                    producer=producer,
+                )
+                self._write_logs_to_stream(
+                    stream_name=stream_name,
+                    logs=producer_logs,
+                )
+
+        def close(self):
+            self.client.close()
+
+        def _write_logs_to_stream(self, stream_name: str, logs: List[RunnerLogEvent]):
+            with self.logger.batch() as batcher:
+                for batch in batched(logs, self.MAX_BATCH_SIZE):
+                    for log in batch:
+                        message = log.message.decode(errors="replace")
+                        timestamp = unix_time_ms_to_datetime(log.timestamp)
+                        if len(log.message) > self.MAX_RUNNER_MESSAGE_SIZE:
+                            logger.error(
+                                "Stream %s: skipping event at %s, message exceeds max size: %d > %d",
+                                stream_name,
+                                timestamp.isoformat(),
+                                len(log.message),
+                                self.MAX_RUNNER_MESSAGE_SIZE,
+                            )
+                            continue
+                        batcher.log_struct(
+                            {
+                                "message": message,
+                            },
+                            labels={
+                                "stream": stream_name,
+                            },
+                            timestamp=timestamp,
                         )
-                        continue
-                    batcher.log_struct(
-                        {
-                            "message": message,
-                        },
-                        labels={
-                            "stream": stream_name,
-                        },
-                        timestamp=timestamp,
-                    )
-                batcher.commit()
+                    batcher.commit()
 
-    def _get_stream_name(
-        self, project_name: str, run_name: str, job_submission_id: UUID, producer: LogProducer
-    ) -> str:
-        return f"{project_name}-{run_name}-{job_submission_id}-{producer.value}"
+        def _get_stream_name(
+            self, project_name: str, run_name: str, job_submission_id: UUID, producer: LogProducer
+        ) -> str:
+            return f"{project_name}-{run_name}-{job_submission_id}-{producer.value}"
 
-    def _get_stream_extrnal_url(self, stream_name: str) -> str:
-        log_name_resource_name = self._get_log_name_resource_name()
-        query = f'logName="{log_name_resource_name}" AND labels.stream="{stream_name}"'
-        quoted_query = urllib.parse.quote(query, safe="")
-        return f"https://console.cloud.google.com/logs/query;query={quoted_query}?project={self.project_id}"
+        def _get_stream_extrnal_url(self, stream_name: str) -> str:
+            log_name_resource_name = self._get_log_name_resource_name()
+            query = f'logName="{log_name_resource_name}" AND labels.stream="{stream_name}"'
+            quoted_query = urllib.parse.quote(query, safe="")
+            return f"https://console.cloud.google.com/logs/query;query={quoted_query}?project={self.project_id}"
 
-    def _get_log_name_resource_name(self) -> str:
-        return f"projects/{self.project_id}/logs/{self.LOG_NAME}"
+        def _get_log_name_resource_name(self) -> str:
+            return f"projects/{self.project_id}/logs/{self.LOG_NAME}"

dstack/_internal/server/services/plugins.py

@@ -60,7 +60,7 @@ def load_plugins(enabled_plugins: list[str]):
     _PLUGINS.clear()
     entrypoints: dict[str, PluginEntrypoint] = {}
     plugins_to_load = enabled_plugins.copy()
-    for entrypoint in entry_points(group="dstack.plugins"):
+    for entrypoint in entry_points(group="dstack.plugins"):  # type: ignore[call-arg]
         if entrypoint.name not in enabled_plugins:
             logger.info(
                 ("Found not enabled plugin %s. Plugin will not be loaded."),

dstack/_internal/server/services/projects.py

@@ -19,7 +19,7 @@ from dstack._internal.server.models import MemberModel, ProjectModel, UserModel
 from dstack._internal.server.schemas.projects import MemberSetting
 from dstack._internal.server.services import users
 from dstack._internal.server.services.backends import (
-    get_backend_config_from_backend_model,
+    get_backend_config_without_creds_from_backend_model,
 )
 from dstack._internal.server.services.permissions import get_default_permissions
 from dstack._internal.server.settings import DEFAULT_PROJECT_NAME
@@ -313,7 +313,6 @@ async def add_project_members(
                 member_num=None,
                 commit=False,
             )
-            member_by_user_id[user_to_add.id] = None
 
     await session.commit()
 
@@ -544,9 +543,7 @@ def project_model_to_project(
                     b.type.value,
                 )
                 continue
-            backend_config = get_backend_config_from_backend_model(
-                configurator, b, include_creds=False
-            )
+            backend_config = get_backend_config_without_creds_from_backend_model(configurator, b)
             if isinstance(backend_config, DstackBackendConfig):
                 for backend_type in backend_config.base_backends:
                     backends.append(

dstack/_internal/server/services/proxy/repo.py

@@ -74,6 +74,8 @@ class ServerProxyRepo(BaseProxyRepo):
             jpd: JobProvisioningData = JobProvisioningData.__response__.parse_raw(
                 job.job_provisioning_data
             )
+            assert jpd.hostname is not None
+            assert jpd.ssh_port is not None
             if not jpd.dockerized:
                 ssh_destination = f"{jpd.username}@{jpd.hostname}"
                 ssh_port = jpd.ssh_port
@@ -140,7 +142,7 @@ class ServerProxyRepo(BaseProxyRepo):
             model_options_obj = service_spec.options.get("openai", {}).get("model")
             if model_spec is None or model_options_obj is None:
                 continue
-            model_options = pydantic.parse_obj_as(AnyModel, model_options_obj)
+            model_options = pydantic.parse_obj_as(AnyModel, model_options_obj)  # type: ignore[arg-type]
             model = ChatModel(
                 project_name=project_name,
                 name=model_spec.name,
@@ -175,6 +177,8 @@ def _model_options_to_format_spec(model: AnyModel) -> AnyModelFormat:
         if model.format == "openai":
             return OpenAIChatModelFormat(prefix=model.prefix)
         elif model.format == "tgi":
+            assert model.chat_template is not None
+            assert model.eos_token is not None
             return TGIChatModelFormat(
                 chat_template=model.chat_template,
                 eos_token=model.eos_token,

dstack/_internal/server/services/requirements/combine.py

@@ -0,0 +1,259 @@
+from typing import Callable, Optional, Protocol, TypeVar
+
+from pydantic import BaseModel
+from typing_extensions import Self
+
+from dstack._internal.core.models.profiles import Profile, SpotPolicy
+from dstack._internal.core.models.resources import (
+    CPUSpec,
+    DiskSpec,
+    GPUSpec,
+    Memory,
+    Range,
+    ResourcesSpec,
+)
+from dstack._internal.core.models.runs import Requirements
+from dstack._internal.utils.typing import SupportsRichComparison
+
+
+class CombineError(ValueError):
+    pass
+
+
+def combine_fleet_and_run_profiles(
+    fleet_profile: Profile, run_profile: Profile
+) -> Optional[Profile]:
+    """
+    Combines fleet and run profile parameters that affect offer selection or provisioning.
+    """
+    try:
+        return Profile(
+            backends=_intersect_lists_optional(fleet_profile.backends, run_profile.backends),
+            regions=_intersect_lists_optional(fleet_profile.regions, run_profile.regions),
+            availability_zones=_intersect_lists_optional(
+                fleet_profile.availability_zones, run_profile.availability_zones
+            ),
+            instance_types=_intersect_lists_optional(
+                fleet_profile.instance_types, run_profile.instance_types
+            ),
+            reservation=_get_single_value_optional(
+                fleet_profile.reservation, run_profile.reservation
+            ),
+            spot_policy=_combine_spot_policy_optional(
+                fleet_profile.spot_policy, run_profile.spot_policy
+            ),
+            max_price=_get_min_optional(fleet_profile.max_price, run_profile.max_price),
+            idle_duration=_combine_idle_duration_optional(
+                fleet_profile.idle_duration, run_profile.idle_duration
+            ),
+            tags=_combine_tags_optional(fleet_profile.tags, run_profile.tags),
+        )
+    except CombineError:
+        return None
+
+
+def combine_fleet_and_run_requirements(
+    fleet_requirements: Requirements, run_requirements: Requirements
+) -> Optional[Requirements]:
+    try:
+        return Requirements(
+            resources=_combine_resources(fleet_requirements.resources, run_requirements.resources),
+            max_price=_get_min_optional(fleet_requirements.max_price, run_requirements.max_price),
+            spot=_combine_spot_optional(fleet_requirements.spot, run_requirements.spot),
+            reservation=_get_single_value_optional(
+                fleet_requirements.reservation, run_requirements.reservation
+            ),
+        )
+    except CombineError:
+        return None
+
+
+_T = TypeVar("_T")
+_ModelT = TypeVar("_ModelT", bound=BaseModel)
+_CompT = TypeVar("_CompT", bound=SupportsRichComparison)
+
+
+class _SupportsCopy(Protocol):
+    def copy(self) -> Self: ...
+
+
+_CopyT = TypeVar("_CopyT", bound=_SupportsCopy)
+
+
+def _intersect_lists_optional(
+    list1: Optional[list[_T]], list2: Optional[list[_T]]
+) -> Optional[list[_T]]:
+    if list1 is None:
+        if list2 is None:
+            return None
+        return list2.copy()
+    if list2 is None:
+        return list1.copy()
+    return [x for x in list1 if x in list2]
+
+
+def _get_min(value1: _CompT, value2: _CompT) -> _CompT:
+    return min(value1, value2)
+
+
+def _get_min_optional(value1: Optional[_CompT], value2: Optional[_CompT]) -> Optional[_CompT]:
+    return _combine_optional(value1, value2, _get_min)
+
+
+def _get_single_value(value1: _T, value2: _T) -> _T:
+    if value1 == value2:
+        return value1
+    raise CombineError(f"Values {value1} and {value2} cannot be combined")
+
+
+def _get_single_value_optional(value1: Optional[_T], value2: Optional[_T]) -> Optional[_T]:
+    return _combine_optional(value1, value2, _get_single_value)
+
+
+def _combine_spot_policy(value1: SpotPolicy, value2: SpotPolicy) -> SpotPolicy:
+    if value1 == SpotPolicy.AUTO:
+        return value2
+    if value2 == SpotPolicy.AUTO:
+        return value1
+    if value1 == value2:
+        return value1
+    raise CombineError(f"spot_policy values {value1} and {value2} cannot be combined")
+
+
+def _combine_spot_policy_optional(
+    value1: Optional[SpotPolicy], value2: Optional[SpotPolicy]
+) -> Optional[SpotPolicy]:
+    return _combine_optional(value1, value2, _combine_spot_policy)
+
+
+def _combine_idle_duration(value1: int, value2: int) -> int:
+    if value1 < 0 and value2 >= 0 or value2 < 0 and value1 >= 0:
+        raise CombineError(f"idle_duration values {value1} and {value2} cannot be combined")
+    return min(value1, value2)
+
+
+def _combine_idle_duration_optional(value1: Optional[int], value2: Optional[int]) -> Optional[int]:
+    return _combine_optional(value1, value2, _combine_idle_duration)
+
+
+def _combine_tags_optional(
+    value1: Optional[dict[str, str]], value2: Optional[dict[str, str]]
+) -> Optional[dict[str, str]]:
+    return _combine_copy_optional(value1, value2, _combine_tags)
+
+
+def _combine_tags(value1: dict[str, str], value2: dict[str, str]) -> dict[str, str]:
+    return value1 | value2
+
+
+def _combine_resources(value1: ResourcesSpec, value2: ResourcesSpec) -> ResourcesSpec:
+    return ResourcesSpec(
+        cpu=_combine_cpu(value1.cpu, value2.cpu),  # type: ignore[attr-defined]
+        memory=_combine_memory(value1.memory, value2.memory),
+        shm_size=_combine_shm_size_optional(value1.shm_size, value2.shm_size),
+        gpu=_combine_gpu_optional(value1.gpu, value2.gpu),
+        disk=_combine_disk_optional(value1.disk, value2.disk),
+    )
+
+
+def _combine_cpu(value1: CPUSpec, value2: CPUSpec) -> CPUSpec:
+    return CPUSpec(
+        arch=_get_single_value_optional(value1.arch, value2.arch),
+        count=_combine_range(value1.count, value2.count),
+    )
+
+
+def _combine_memory(value1: Range[Memory], value2: Range[Memory]) -> Range[Memory]:
+    return _combine_range(value1, value2)
+
+
+def _combine_shm_size_optional(
+    value1: Optional[Memory], value2: Optional[Memory]
+) -> Optional[Memory]:
+    return _get_min_optional(value1, value2)
+
+
+def _combine_gpu(value1: GPUSpec, value2: GPUSpec) -> GPUSpec:
+    return GPUSpec(
+        vendor=_get_single_value_optional(value1.vendor, value2.vendor),
+        name=_intersect_lists_optional(value1.name, value2.name),
+        count=_combine_range(value1.count, value2.count),
+        memory=_combine_range_optional(value1.memory, value2.memory),
+        total_memory=_combine_range_optional(value1.total_memory, value2.total_memory),
+        compute_capability=_get_min_optional(value1.compute_capability, value2.compute_capability),
+    )
+
+
+def _combine_gpu_optional(
+    value1: Optional[GPUSpec], value2: Optional[GPUSpec]
+) -> Optional[GPUSpec]:
+    return _combine_models_optional(value1, value2, _combine_gpu)
+
+
+def _combine_disk(value1: DiskSpec, value2: DiskSpec) -> DiskSpec:
+    return DiskSpec(size=_combine_range(value1.size, value2.size))
+
+
+def _combine_disk_optional(
+    value1: Optional[DiskSpec], value2: Optional[DiskSpec]
+) -> Optional[DiskSpec]:
+    return _combine_models_optional(value1, value2, _combine_disk)
+
+
+def _combine_spot(value1: bool, value2: bool) -> bool:
+    if value1 != value2:
+        raise CombineError(f"spot values {value1} and {value2} cannot be combined")
+    return value1
+
+
+def _combine_spot_optional(value1: Optional[bool], value2: Optional[bool]) -> Optional[bool]:
+    return _combine_optional(value1, value2, _combine_spot)
+
+
+def _combine_range(value1: Range, value2: Range) -> Range:
+    res = value1.intersect(value2)
+    if res is None:
+        raise CombineError(f"Ranges {value1} and {value2} cannot be combined")
+    return res
+
+
+def _combine_range_optional(value1: Optional[Range], value2: Optional[Range]) -> Optional[Range]:
+    return _combine_models_optional(value1, value2, _combine_range)
+
+
+def _combine_optional(
+    value1: Optional[_T], value2: Optional[_T], combiner: Callable[[_T, _T], _T]
+) -> Optional[_T]:
+    if value1 is None:
+        return value2
+    if value2 is None:
+        return value1
+    return combiner(value1, value2)
+
+
+def _combine_models_optional(
+    value1: Optional[_ModelT],
+    value2: Optional[_ModelT],
+    combiner: Callable[[_ModelT, _ModelT], _ModelT],
+) -> Optional[_ModelT]:
+    if value1 is None:
+        if value2 is not None:
+            return value2.copy(deep=True)
+        return None
+    if value2 is None:
+        return value1.copy(deep=True)
+    return combiner(value1, value2)
+
+
+def _combine_copy_optional(
+    value1: Optional[_CopyT],
+    value2: Optional[_CopyT],
+    combiner: Callable[[_CopyT, _CopyT], _CopyT],
+) -> Optional[_CopyT]:
+    if value1 is None:
+        if value2 is not None:
+            return value2.copy()
+        return None
+    if value2 is None:
+        return value1.copy()
+    return combiner(value1, value2)