dagster-cloud 1.8.2__py3-none-any.whl → 1.12.6__py3-none-any.whl

dagster_cloud/workspace/user_code_launcher/user_code_launcher.py

@@ -2,6 +2,7 @@
 # ruff: noqa: PLE1205
 import asyncio
 import functools
+import hashlib
 import json
 import logging
 import os
@@ -11,57 +12,66 @@ import threading
 import time
 import zlib
 from abc import abstractmethod, abstractproperty
+from collections import defaultdict
+from collections.abc import Collection, Mapping, Sequence
 from concurrent.futures import ThreadPoolExecutor, as_completed, wait
 from contextlib import AbstractContextManager
-from typing import (
-    Any,
-    Callable,
-    Collection,
-    DefaultDict,
-    Dict,
-    Generic,
-    List,
-    Mapping,
-    NamedTuple,
-    Optional,
-    Sequence,
-    Set,
-    Tuple,
-    TypeVar,
-    Union,
-    cast,
-)
+from io import BytesIO
+from typing import Any, Callable, Generic, NamedTuple, Optional, TypeVar, Union, cast
 
 import dagster._check as check
 import grpc
 from dagster import BoolSource, Field, IntSource
-from dagster._api.list_repositories import sync_list_repositories_grpc
+from dagster._api.list_repositories import gen_list_repositories_grpc
 from dagster._core.definitions.selector import JobSelector
 from dagster._core.errors import DagsterUserCodeUnreachableError
 from dagster._core.instance import MayHaveInstanceWeakref
 from dagster._core.launcher import RunLauncher
-from dagster._core.remote_representation import RemoteRepositoryOrigin
-from dagster._core.remote_representation.origin import (
+from dagster._core.remote_origin import (
     CodeLocationOrigin,
     RegisteredCodeLocationOrigin,
+    RemoteRepositoryOrigin,
+)
+from dagster._core.remote_representation.external_data import (
+    extract_serialized_job_snap_from_serialized_job_data_snap,
 )
 from dagster._grpc.client import DagsterGrpcClient
-from dagster._grpc.types import GetCurrentImageResult
-from dagster._serdes import deserialize_value, serialize_value, whitelist_for_serdes
+from dagster._grpc.types import GetCurrentImageResult, ListRepositoriesResponse
+from dagster._serdes import (
+    deserialize_value,
+    pack_value,
+    serialize_value,
+    unpack_value,
+    whitelist_for_serdes,
+)
 from dagster._time import get_current_timestamp
-from dagster._utils.error import SerializableErrorInfo, serializable_error_info_from_exc_info
+from dagster._utils.error import (
+    SerializableErrorInfo,
+    serializable_error_info_from_exc_info,
+    truncate_serialized_error,
+)
 from dagster._utils.merger import merge_dicts
 from dagster._utils.typed_dict import init_optional_typeddict
 from dagster_cloud_cli.core.errors import raise_http_error
 from dagster_cloud_cli.core.workspace import CodeLocationDeployData
 from typing_extensions import Self, TypeAlias
 
+from dagster_cloud.agent.instrumentation.constants import DAGSTER_CLOUD_AGENT_METRIC_PREFIX
 from dagster_cloud.agent.queries import GET_AGENTS_QUERY
 from dagster_cloud.api.dagster_cloud_api import (
+    CheckSnapshotResult,
+    ConfirmUploadResult,
+    DagsterCloudCodeLocationManifest,
+    DagsterCloudCodeLocationUpdateResponse,
+    DagsterCloudCodeLocationUpdateResult,
+    DagsterCloudRepositoryManifest,
     DagsterCloudUploadLocationData,
     DagsterCloudUploadRepositoryData,
     DagsterCloudUploadWorkspaceEntry,
     DagsterCloudUploadWorkspaceResponse,
+    FileFormat,
+    SnapshotType,
+    StoredSnapshot,
     UserCodeDeploymentType,
 )
 from dagster_cloud.execution.monitoring import (
@@ -75,6 +85,8 @@ from dagster_cloud.execution.monitoring import (
     start_run_worker_monitoring_thread,
 )
 from dagster_cloud.instance import DagsterCloudAgentInstance
+from dagster_cloud.opentelemetry.controller import OpenTelemetryController
+from dagster_cloud.opentelemetry.observers.execution_observer import observe_execution
 from dagster_cloud.pex.grpc.client import MultiPexGrpcClient
 from dagster_cloud.pex.grpc.types import (
     CreatePexServerArgs,
@@ -83,7 +95,6 @@ from dagster_cloud.pex.grpc.types import (
     ShutdownPexServerArgs,
 )
 from dagster_cloud.util import diff_serializable_namedtuple_map
-from dagster_cloud.util.errors import truncate_serialized_error
 
 DEFAULT_SERVER_PROCESS_STARTUP_TIMEOUT = 180
 DEFAULT_MAX_TTL_SERVERS = 25
@@ -155,7 +166,7 @@ class UserCodeLauncherEntry(
         code_location_deploy_data,
         update_timestamp,
     ):
-        return super(UserCodeLauncherEntry, cls).__new__(
+        return super().__new__(
             cls,
             check.inst_param(
                 code_location_deploy_data, "code_location_deploy_data", CodeLocationDeployData
@@ -234,10 +245,13 @@ SHARED_USER_CODE_LAUNCHER_CONFIG = {
         BoolSource,
         is_required=False,
         default_value=True,
-        description=(
-            "Do not include full job snapshots in the workspace "
-            "snapshot, upload them separately if they have not been previously uploaded."
-        ),
+        description=("Deprecated - no longer used"),
+    ),
+    "direct_snapshot_uploads": Field(
+        BoolSource,
+        is_required=False,
+        default_value=True,
+        description=("Opt-out for uploading definition snapshots directly to blob storage."),
     ),
     "upload_snapshots_on_startup": Field(
         BoolSource,
@@ -260,8 +274,8 @@ SHARED_USER_CODE_LAUNCHER_CONFIG = {
     ),
 }
 
-DeploymentAndLocation: TypeAlias = Tuple[str, str]
-UserCodeLauncherEntryMap: TypeAlias = Dict[DeploymentAndLocation, UserCodeLauncherEntry]
+DeploymentAndLocation: TypeAlias = tuple[str, str]
+UserCodeLauncherEntryMap: TypeAlias = dict[DeploymentAndLocation, UserCodeLauncherEntry]
 
 
 class ServerEndpoint(
@@ -271,12 +285,12 @@ class ServerEndpoint(
             ("host", str),
             ("port", Optional[int]),
             ("socket", Optional[str]),
-            ("metadata", Optional[List[Tuple[str, str]]]),
+            ("metadata", Optional[list[tuple[str, str]]]),
         ],
     )
 ):
     def __new__(cls, host, port, socket, metadata=None):
-        return super(ServerEndpoint, cls).__new__(
+        return super().__new__(
             cls,
             check.str_param(host, "host"),
             check.opt_int_param(port, "port"),
@@ -292,7 +306,7 @@ class ServerEndpoint(
     def create_multipex_client(self) -> MultiPexGrpcClient:
         return MultiPexGrpcClient(port=self.port, socket=self.socket, host=self.host)
 
-    def with_metadata(self, metadata: Optional[List[Tuple[str, str]]]):
+    def with_metadata(self, metadata: Optional[list[tuple[str, str]]]):
         return self._replace(metadata=metadata)
 
 
@@ -312,7 +326,7 @@ class DagsterCloudGrpcServer(
         server_endpoint: ServerEndpoint,
         code_location_deploy_data: CodeLocationDeployData,
     ):
-        return super(DagsterCloudGrpcServer, cls).__new__(
+        return super().__new__(
             cls,
             server_handle,
             check.inst_param(server_endpoint, "server_endpoint", ServerEndpoint),
@@ -322,57 +336,72 @@ class DagsterCloudGrpcServer(
         )
 
 
+_SUPPORTED_FILE_FORMATS = [FileFormat.GZIPPED_JSON, FileFormat.JSON]
+
+
+def _file_for_format(obj_bytes: bytes, fmt: str):
+    if fmt == FileFormat.JSON:
+        return BytesIO(obj_bytes)
+    elif fmt == FileFormat.GZIPPED_JSON:
+        return BytesIO(zlib.compress(obj_bytes))
+    else:
+        check.failed(f"Unexpected file format {fmt}")
+
+
 class DagsterCloudUserCodeLauncher(
     AbstractContextManager, MayHaveInstanceWeakref[DagsterCloudAgentInstance], Generic[ServerHandle]
 ):
     def __init__(
         self,
         server_ttl: Optional[dict] = None,
-        defer_job_snapshots: bool = True,
         server_process_startup_timeout=None,
         upload_snapshots_on_startup: bool = True,
         requires_healthcheck: bool = False,
         code_server_metrics: Optional[Mapping[str, Any]] = None,
         agent_metrics: Optional[Mapping[str, Any]] = None,
+        direct_snapshot_uploads: bool = False,
+        # ignored old setting, allowed to flow through to avoid breakage
+        defer_job_snapshots: bool = True,
     ):
-        self._grpc_servers: Dict[
+        self._grpc_servers: dict[
             DeploymentAndLocation, Union[DagsterCloudGrpcServer, SerializableErrorInfo]
         ] = {}
-        self._first_unavailable_times: Dict[DeploymentAndLocation, float] = {}
+        self._first_unavailable_times: dict[DeploymentAndLocation, float] = {}
 
-        self._pending_delete_grpc_server_handles: Set[ServerHandle] = set()
+        self._pending_delete_grpc_server_handles: set[ServerHandle] = set()
         self._grpc_servers_lock = threading.Lock()
-        self._per_location_metrics: Dict[
+        self._per_location_metrics: dict[
             DeploymentAndLocation, CloudCodeServerUtilizationMetrics
-        ] = DefaultDict(lambda: init_optional_typeddict(CloudCodeServerUtilizationMetrics))
+        ] = defaultdict(lambda: init_optional_typeddict(CloudCodeServerUtilizationMetrics))
 
-        self._multipex_servers: Dict[DeploymentAndLocation, DagsterCloudGrpcServer] = {}
+        self._multipex_servers: dict[DeploymentAndLocation, DagsterCloudGrpcServer] = {}
 
         self._server_ttl_config = check.opt_dict_param(server_ttl, "server_ttl")
-        self._defer_job_snapshots = defer_job_snapshots
+        self._direct_snapshot_uploads = direct_snapshot_uploads
         self.upload_snapshots_on_startup = check.bool_param(
             upload_snapshots_on_startup, "upload_snapshots_on_startup"
         )
         self._requires_healthcheck = check.bool_param(requires_healthcheck, "requires_healthcheck")
 
         # periodically reconciles to make desired = actual
-        self._desired_entries: Dict[DeploymentAndLocation, UserCodeLauncherEntry] = {}
-        self._actual_entries: Dict[DeploymentAndLocation, UserCodeLauncherEntry] = {}
+        self._desired_entries: dict[DeploymentAndLocation, UserCodeLauncherEntry] = {}
+        self._actual_entries: dict[DeploymentAndLocation, UserCodeLauncherEntry] = {}
         self._last_refreshed_actual_entries = 0
         self._last_cleaned_up_dangling_code_servers = 0
         self._metadata_lock = threading.Lock()
 
-        self._upload_locations: Set[DeploymentAndLocation] = set()
+        self._upload_locations: set[DeploymentAndLocation] = set()
 
         self._logger = logging.getLogger("dagster_cloud.user_code_launcher")
         self._event_logger = logging.getLogger("cloud-events")
         self._started: bool = False
         self._run_worker_monitoring_thread = None
         self._run_worker_monitoring_thread_shutdown_event = None
-        self._run_worker_deployments_to_check: Set[str] = set()
-        self._run_worker_statuses_dict: Dict[str, List[CloudRunWorkerStatus]] = {}
+        self._run_worker_deployments_to_check: set[str] = set()
+        self._run_worker_statuses_dict: dict[str, list[CloudRunWorkerStatus]] = {}
         self._run_worker_monitoring_lock = threading.Lock()
 
+        self._in_progress_reconcile_start_time = time.time()
         self._reconcile_count = 0
         self._reconcile_grpc_metadata_shutdown_event = threading.Event()
         self._reconcile_grpc_metadata_thread = None
@@ -390,7 +419,7 @@ class DagsterCloudUserCodeLauncher(
         self._agent_metrics_config = agent_metrics
         super().__init__()
 
-    def get_active_grpc_server_handles(self) -> List[ServerHandle]:
+    def get_active_grpc_server_handles(self) -> list[ServerHandle]:
         with self._grpc_servers_lock:
             return [
                 s.server_handle
@@ -398,7 +427,7 @@ class DagsterCloudUserCodeLauncher(
                 if not isinstance(s, SerializableErrorInfo)
             ] + list(self._pending_delete_grpc_server_handles)
 
-    def get_active_agent_ids(self) -> Optional[Set[str]]:
+    def get_active_agent_ids(self) -> Optional[set[str]]:
         try:
             result = self._instance.organization_scoped_graphql_client().execute(
                 GET_AGENTS_QUERY,
@@ -569,6 +598,193 @@ class DagsterCloudUserCodeLauncher(
         workspace_entry: DagsterCloudUploadWorkspaceEntry,
         server_or_error: Union[DagsterCloudGrpcServer, SerializableErrorInfo],
     ) -> None:
+        if self._direct_snapshot_uploads:
+            self._update_workspace_entry_direct_upload(
+                deployment_name,
+                workspace_entry,
+                server_or_error,
+            )
+        else:
+            self._update_workspace_entry_server_upload(
+                deployment_name,
+                workspace_entry,
+                server_or_error,
+            )
+
+    def _ensure_snapshot_uploaded(
+        self,
+        deployment_name: str,
+        snapshot_type: str,
+        serialized_object: str,
+    ) -> StoredSnapshot:
+        object_bytes = serialized_object.encode("utf-8")
+        sha1 = hashlib.sha1(object_bytes).hexdigest()
+        byte_count = len(object_bytes)
+        response = self._instance.requests_managed_retries_session.get(
+            self._instance.dagster_cloud_check_snapshot_url,
+            headers=self._instance.headers_for_deployment(deployment_name),
+            params={
+                "type": snapshot_type,
+                "sha1": sha1,
+                "size": byte_count,
+                "formats": _SUPPORTED_FILE_FORMATS,
+            },
+            timeout=self._instance.dagster_cloud_api_timeout,
+            proxies=self._instance.dagster_cloud_api_proxies,
+        )
+        raise_http_error(response)
+
+        result = unpack_value(response.json(), CheckSnapshotResult)
+
+        if not result.stored_snapshot:
+            upload_data = check.not_none(
+                result.upload_data,
+                "upload_data expected when stored_snapshot is None",
+            )
+            file = _file_for_format(object_bytes, upload_data.format)
+            response = self._instance.requests_managed_retries_session.put(
+                url=upload_data.presigned_put_url,
+                data=file,
+                timeout=self._instance.dagster_cloud_api_timeout,
+            )
+            raise_http_error(response)
+
+            response = self._instance.requests_managed_retries_session.put(
+                self._instance.dagster_cloud_confirm_upload_url,
+                headers=self._instance.headers_for_deployment(deployment_name),
+                json=pack_value(upload_data),
+                timeout=self._instance.dagster_cloud_api_timeout,
+                proxies=self._instance.dagster_cloud_api_proxies,
+            )
+            raise_http_error(response)
+            result = unpack_value(response.json(), ConfirmUploadResult)
+            return result.stored_snapshot
+
+        return result.stored_snapshot
+
+    def _update_workspace_entry_direct_upload(
+        self,
+        deployment_name: str,
+        workspace_entry: DagsterCloudUploadWorkspaceEntry,
+        server_or_error: Union[DagsterCloudGrpcServer, SerializableErrorInfo],
+    ) -> None:
+        # updated scheme, uploading definitions to blob storage via signed urls
+        error_snap = None
+        manifest = None
+        if workspace_entry.serialized_error_info:
+            error_snap = self._ensure_snapshot_uploaded(
+                deployment_name,
+                SnapshotType.ERROR,
+                serialize_value(workspace_entry.serialized_error_info),
+            )
+        elif isinstance(server_or_error, SerializableErrorInfo):
+            error_snap = self._ensure_snapshot_uploaded(
+                deployment_name,
+                SnapshotType.ERROR,
+                serialize_value(server_or_error),
+            )
+        elif workspace_entry.upload_location_data:
+            repos = []
+            for repo_data in workspace_entry.upload_location_data.upload_repository_datas:
+                stored_snapshot = self._ensure_snapshot_uploaded(
+                    deployment_name,
+                    SnapshotType.REPOSITORY,
+                    repo_data.serialized_repository_data,
+                )
+                repos.append(
+                    DagsterCloudRepositoryManifest(
+                        name=repo_data.repository_name,
+                        code_pointer=repo_data.code_pointer,
+                        stored_snapshot=stored_snapshot,
+                    )
+                )
+
+            manifest = DagsterCloudCodeLocationManifest(
+                repositories=repos,
+                executable_path=workspace_entry.upload_location_data.executable_path,
+                container_image=workspace_entry.upload_location_data.container_image,
+                dagster_library_versions=workspace_entry.upload_location_data.dagster_library_versions,
+                code_location_deploy_data=workspace_entry.code_location_deploy_data,
+            )
+        else:
+            check.failed(
+                "Expected DagsterCloudUploadWorkspaceEntry to have either location data or error, had neither."
+            )
+
+        result = DagsterCloudCodeLocationUpdateResult(
+            location_name=workspace_entry.location_name,
+            error_snapshot=error_snap,
+            manifest=manifest,
+        )
+
+        res = self._instance.requests_managed_retries_session.put(
+            self._instance.dagster_cloud_code_location_update_result_url,
+            headers=self._instance.headers_for_deployment(deployment_name),
+            json=pack_value(result),
+            timeout=self._instance.dagster_cloud_api_timeout,
+            proxies=self._instance.dagster_cloud_api_proxies,
+        )
+        raise_http_error(res)
+        first_response = unpack_value(res.json(), DagsterCloudCodeLocationUpdateResponse)
+        if first_response.updated:
+            self._logger.info(
+                "Code location update result for"
+                f" {deployment_name}:{workspace_entry.location_name} - {first_response.message}"
+            )
+            return
+
+        missing = check.not_none(
+            first_response.missing_job_snapshots,
+            "Expected missing_job_snapshots when updated is false.",
+        )
+        server = check.inst(
+            server_or_error,
+            DagsterCloudGrpcServer,
+            "Server should not be in error state if there are missing snapshots.",
+        )
+        self._logger.info(f"Uploading {len(missing)} job snapshots.")
+        with ThreadPoolExecutor() as executor:
+            _ = list(
+                executor.map(
+                    lambda job_selector: self.upload_job_snap_direct(
+                        deployment_name,
+                        job_selector,
+                        server,
+                    ),
+                    missing,
+                )
+            )
+        res = self._instance.requests_managed_retries_session.put(
+            self._instance.dagster_cloud_code_location_update_result_url,
+            headers=self._instance.headers_for_deployment(deployment_name),
+            json=pack_value(result),
+            timeout=self._instance.dagster_cloud_api_timeout,
+            proxies=self._instance.dagster_cloud_api_proxies,
+        )
+        raise_http_error(res)
+        second_response = unpack_value(res.json(), DagsterCloudCodeLocationUpdateResponse)
+        if not second_response.updated:
+            if second_response.missing_job_snapshots:
+                # this condition is expected to be extremely unlikely
+                raise Exception(
+                    "Code location update failed, job definitions changed while uploading:"
+                    f" {second_response.missing_job_snapshots}"
+                )
+            else:
+                raise Exception(f"Code location update failed: {second_response.message}")
+
+        self._logger.info(
+            "Code location update result for"
+            f" {deployment_name}:{workspace_entry.location_name} - {second_response.message}"
+        )
+
+    def _update_workspace_entry_server_upload(
+        self,
+        deployment_name: str,
+        workspace_entry: DagsterCloudUploadWorkspaceEntry,
+        server_or_error: Union[DagsterCloudGrpcServer, SerializableErrorInfo],
+    ) -> None:
+        # legacy scheme, uploading definitions blobs to web server
         with tempfile.TemporaryDirectory() as temp_dir:
             dst = os.path.join(temp_dir, "workspace_entry.tmp")
             with open(dst, "wb") as f:
@@ -647,7 +863,16 @@ class DagsterCloudUserCodeLauncher(
                 f" {deployment_name}:{workspace_entry.location_name} {response.message}"
             )
 
-    def _get_upload_location_data(
+    async def gen_list_repositories_response(
+        self,
+        client: DagsterGrpcClient,
+    ) -> "ListRepositoriesResponse":
+        return await gen_list_repositories_grpc(
+            client,
+            timeout=int(os.getenv("DAGSTER_CLOUD_LIST_REPOSITORIES_GRPC_TIMEOUT", "180")),
+        )
+
+    async def _get_upload_location_data(
         self,
         deployment_name: str,
         location_name: str,
@@ -656,30 +881,40 @@ class DagsterCloudUserCodeLauncher(
         location_origin = self._get_code_location_origin(location_name)
         client = server.server_endpoint.create_client()
 
-        list_repositories_response = sync_list_repositories_grpc(client)
+        list_repositories_response = await self.gen_list_repositories_response(client)
 
-        upload_repo_datas: List[DagsterCloudUploadRepositoryData] = []
+        upload_repo_datas: list[DagsterCloudUploadRepositoryData] = []
 
         for (
             repository_name,
             code_pointer,
         ) in list_repositories_response.repository_code_pointer_dict.items():
-            external_repository_chunks = list(
-                client.streaming_external_repository(
-                    external_repository_origin=RemoteRepositoryOrigin(
+            if os.getenv("DAGSTER_CLOUD_USE_STREAMING_EXTERNAL_REPOSITORY"):
+                external_repository_chunks = [
+                    chunk
+                    async for chunk in client.gen_streaming_external_repository(
+                        remote_repository_origin=RemoteRepositoryOrigin(
+                            location_origin,
+                            repository_name,
+                        ),
+                        defer_snapshots=True,
+                    )
+                ]
+
+                serialized_repository_data = "".join(
+                    [
+                        chunk["serialized_external_repository_chunk"]
+                        for chunk in external_repository_chunks
+                    ]
+                )
+            else:
+                serialized_repository_data = await client.gen_external_repository(
+                    remote_repository_origin=RemoteRepositoryOrigin(
                         location_origin,
                         repository_name,
                     ),
-                    defer_snapshots=self._defer_job_snapshots,
+                    defer_snapshots=True,
                 )
-            )
-
-            serialized_repository_data = "".join(
-                [
-                    chunk["serialized_external_repository_chunk"]
-                    for chunk in external_repository_chunks
-                ]
-            )
 
             # Don't deserialize in case there are breaking changes - let the server do it
             upload_repo_datas.append(
@@ -729,7 +964,27 @@ class DagsterCloudUserCodeLauncher(
             deployment_name, errored_workspace_entry, server_or_error=error_info
         )
 
-    def _update_location_data(
+    async def _try_update_location_data(
+        self,
+        deployment_name: str,
+        location_name: str,
+        server_or_error: Union[DagsterCloudGrpcServer, SerializableErrorInfo],
+        metadata: CodeLocationDeployData,
+    ):
+        try:
+            await self._update_location_data(
+                deployment_name,
+                location_name,
+                server_or_error,
+                metadata,
+            )
+        except Exception:
+            self._logger.error(
+                f"Error while writing location data for {deployment_name}:{location_name}:"
+                f" {serializable_error_info_from_exc_info(sys.exc_info())}"
+            )
+
+    async def _update_location_data(
         self,
         deployment_name: str,
         location_name: str,
@@ -756,7 +1011,7 @@ class DagsterCloudUserCodeLauncher(
             loaded_workspace_entry = DagsterCloudUploadWorkspaceEntry(
                 location_name=location_name,
                 code_location_deploy_data=metadata,
-                upload_location_data=self._get_upload_location_data(
+                upload_location_data=await self._get_upload_location_data(
                     deployment_name,
                     location_name,
                     server_or_error,
@@ -789,7 +1044,7 @@ class DagsterCloudUserCodeLauncher(
 
     def _get_existing_pex_servers(
         self, deployment_name: str, location_name: str
-    ) -> List[PexServerHandle]:
+    ) -> list[PexServerHandle]:
         server = self._multipex_servers.get((deployment_name, location_name))
 
         if not server:
@@ -870,6 +1125,11 @@ class DagsterCloudUserCodeLauncher(
     ):
         deployment_name, location_name = to_update_key
 
+        attributes = {
+            "deployment": deployment_name,
+            "location": location_name,
+        }
+
         code_location_deploy_data = desired_entry.code_location_deploy_data
         pex_metadata = code_location_deploy_data.pex_metadata
         deployment_info = (
@@ -882,13 +1142,19 @@ class DagsterCloudUserCodeLauncher(
                 self._logger.info(
                     f"Waiting for new grpc server for {deployment_name}:{location_name} for {deployment_info} to be ready..."
                 )
-                await self._wait_for_new_server_ready(
-                    deployment_name,
-                    location_name,
-                    desired_entry,
-                    server_or_error.server_handle,
-                    server_or_error.server_endpoint,
-                )
+                with observe_execution(
+                    opentelemetry=self.opentelemetry,
+                    event_key=f"{DAGSTER_CLOUD_AGENT_METRIC_PREFIX}.user_code.server.process_wait",
+                    short_description="waiting for new server process to be ready",
+                    attributes=attributes,
+                ):
+                    await self._wait_for_new_server_ready(
+                        deployment_name,
+                        location_name,
+                        desired_entry,
+                        server_or_error.server_handle,
+                        server_or_error.server_endpoint,
+                    )
             except Exception:
                 error_info = serializable_error_info_from_exc_info(sys.exc_info())
                 self._logger.error(
@@ -898,21 +1164,18 @@ class DagsterCloudUserCodeLauncher(
                 server_or_error = error_info
 
         if should_upload:
-            try:
-                self._update_location_data(
+            with observe_execution(
+                opentelemetry=self.opentelemetry,
+                event_key=f"{DAGSTER_CLOUD_AGENT_METRIC_PREFIX}.user_code.upload",
+                short_description="uploading user code data",
+                attributes=attributes,
+            ):
+                await self._try_update_location_data(
                     deployment_name,
                     location_name,
                     server_or_error,
                     desired_entry.code_location_deploy_data,
                 )
-            except Exception:
-                # If there was a failure uploading snapshots, log it but don't block other code locations
-                # from updating (and still use the new server to serve new requests)
-                error_info = serializable_error_info_from_exc_info(sys.exc_info())
-                self._logger.error(
-                    f"Error while writing location data for {deployment_name}:{location_name}:"
-                    f" {error_info}"
-                )
 
         # Once we've verified that the new server has uploaded its data successfully, swap in
         # the server to start serving new requests
@@ -986,7 +1249,7 @@ class DagsterCloudUserCodeLauncher(
                 self._pending_delete_grpc_server_handles.discard(server_handle)
 
     def _cleanup_servers(
-        self, active_agent_ids: Optional[Set[str]], include_own_servers: bool
+        self, active_agent_ids: Optional[set[str]], include_own_servers: bool
     ) -> None:
         """Remove all servers, across all deployments and locations."""
         with ThreadPoolExecutor() as executor:
@@ -1009,7 +1272,7 @@ class DagsterCloudUserCodeLauncher(
                     self._logger.exception("Error cleaning up server")
 
     @abstractmethod
-    def _list_server_handles(self) -> List[ServerHandle]:
+    def _list_server_handles(self) -> list[ServerHandle]:
         """Return a list of all server handles across all deployments and locations."""
 
     @abstractmethod
@@ -1021,7 +1284,7 @@ class DagsterCloudUserCodeLauncher(
         """Returns the update_timestamp value from the given code server."""
 
     def _can_cleanup_server(
-        self, handle: ServerHandle, active_agent_ids: Optional[Set[str]], include_own_servers: bool
+        self, handle: ServerHandle, active_agent_ids: Optional[set[str]], include_own_servers: bool
     ) -> bool:
         """Returns true if we can clean up the server identified by the handle without issues (server was started by this agent, or agent is no longer active)."""
         agent_id_for_server = self.get_agent_id_for_server(handle)
@@ -1057,7 +1320,7 @@ class DagsterCloudUserCodeLauncher(
             return False
 
         return (active_agent_ids is not None) and (
-            agent_id_for_server not in cast(Set[str], active_agent_ids)
+            agent_id_for_server not in cast("set[str]", active_agent_ids)
         )
 
     def _graceful_cleanup_servers(self, include_own_servers: bool):  # ServerHandles
@@ -1066,7 +1329,7 @@ class DagsterCloudUserCodeLauncher(
             return self._cleanup_servers(active_agent_ids, include_own_servers=include_own_servers)
 
         handles = self._list_server_handles()
-        servers_to_remove: List[ServerHandle] = []
+        servers_to_remove: list[ServerHandle] = []
         with self._grpc_servers_lock:
             for handle in handles:
                 if self._can_cleanup_server(
@@ -1090,7 +1353,7 @@ class DagsterCloudUserCodeLauncher(
             self._reconcile_grpc_metadata_thread.join()
 
         if self._run_worker_monitoring_thread:
-            self._run_worker_monitoring_thread_shutdown_event.set()
+            self._run_worker_monitoring_thread_shutdown_event.set()  # pyright: ignore[reportOptionalMemberAccess]
             self._run_worker_monitoring_thread.join()
 
         if self._reconcile_location_utilization_metrics_thread:
@@ -1100,10 +1363,10 @@ class DagsterCloudUserCodeLauncher(
         if self._started:
             self._graceful_cleanup_servers(include_own_servers=True)
 
-        super().__exit__(exception_value, exception_value, traceback)
+        super().__exit__(exception_value, exception_value, traceback)  # pyright: ignore[reportAbstractUsage]
 
     def add_upload_metadata(
-        self, upload_metadata: Dict[DeploymentAndLocation, UserCodeLauncherEntry]
+        self, upload_metadata: dict[DeploymentAndLocation, UserCodeLauncherEntry]
     ):
         """Add a set of locations to be uploaded in the next reconcilation loop."""
         with self._metadata_lock:
@@ -1112,7 +1375,7 @@ class DagsterCloudUserCodeLauncher(
 
     def update_grpc_metadata(
         self,
-        desired_metadata: Dict[DeploymentAndLocation, UserCodeLauncherEntry],
+        desired_metadata: dict[DeploymentAndLocation, UserCodeLauncherEntry],
     ):
         check.dict_param(
             desired_metadata,
@@ -1152,7 +1415,7 @@ class DagsterCloudUserCodeLauncher(
         endpoints_or_errors = self.get_grpc_endpoints()
         for (deployment_name, location_name), endpoint_or_error in endpoints_or_errors.items():
             if isinstance(endpoint_or_error, ServerEndpoint):
-                endpoint = cast(ServerEndpoint, endpoint_or_error)
+                endpoint = cast("ServerEndpoint", endpoint_or_error)
                 raw_metrics_str = (
                     endpoint.create_client().ping("").get("serialized_server_utilization_metrics")
                 )
@@ -1217,7 +1480,7 @@ class DagsterCloudUserCodeLauncher(
             try:
                 self._graceful_cleanup_servers(include_own_servers=False)
             except:
-                self._logger.exception("Failed to clean up dangling code serverrs.")
+                self._logger.exception("Failed to clean up dangling code servers.")
             self._last_cleaned_up_dangling_code_servers = now
 
         if now - self._last_refreshed_actual_entries > ACTUAL_ENTRIES_REFRESH_INTERVAL:
@@ -1227,6 +1490,8 @@ class DagsterCloudUserCodeLauncher(
                 self._logger.exception("Failed to refresh actual entries.")
             self._last_refreshed_actual_entries = now
 
+        self._in_progress_reconcile_start_time = time.time()
+
         self._reconcile(
             desired_entries,
             upload_locations,
@@ -1243,6 +1508,7 @@ class DagsterCloudUserCodeLauncher(
                 f"Started polling for requests from {self._instance.dagster_cloud_url}"
             )
 
+        self._in_progress_reconcile_start_time = None
         self._reconcile_count += 1
 
     def _update_metrics_thread(self, shutdown_event):
@@ -1267,19 +1533,39 @@ class DagsterCloudUserCodeLauncher(
         # thread-safe since reconcile_count is an integer
         return self._reconcile_count > 0
 
-    def _make_check_on_running_server_endpoint(self, server_endpoint: ServerEndpoint):
-        # Ensure that server_endpoint is bound correctly
-        return lambda: server_endpoint.create_client().ping("")
+    @property
+    def in_progress_reconcile_start_time(self) -> Optional[float]:
+        return self._in_progress_reconcile_start_time
+
+    def _make_check_on_running_server_endpoint(
+        self, server_endpoint: ServerEndpoint
+    ) -> Callable[[], Union[ListRepositoriesResponse, SerializableErrorInfo]]:
+        return lambda: deserialize_value(
+            server_endpoint.create_client().list_repositories(),
+            (ListRepositoriesResponse, SerializableErrorInfo),
+        )
+
+    def _trigger_recovery_server_restart(self, deployment_location: DeploymentAndLocation):
+        del self._actual_entries[deployment_location]
+
+        if deployment_location in self._first_unavailable_times:
+            del self._first_unavailable_times[deployment_location]
+
+        # redeploy the multipex server in this case as well to ensure a fresh start
+        # if it resource contrained (and ensure that we don't try to create the same
+        # PexServerHandle again and delete the code location in a loop)
+        if deployment_location in self._multipex_servers:
+            del self._multipex_servers[deployment_location]
 
     def _refresh_actual_entries(self) -> None:
-        for deployment_location, server in self._multipex_servers.items():
+        for deployment_location, multipex_server in self._multipex_servers.items():
             if deployment_location in self._actual_entries:
                 # If a multipex server exists, we query it over gRPC
                 # to make sure the pex server is still available.
 
                 # First verify that the multipex server is running
                 try:
-                    server.server_endpoint.create_multipex_client().ping("")
+                    multipex_server.server_endpoint.create_multipex_client().ping("")
                 except:
                     # If it isn't, this is expected if ECS is currently spinning up this service
                     # after it crashed. In this case, we want to wait for it to fully come up
@@ -1291,14 +1577,20 @@ class DagsterCloudUserCodeLauncher(
                     )
                     return
                 deployment_name, location_name = deployment_location
+
+                # If we expect there to be a running code location here but there is none,
                 if not self._get_existing_pex_servers(deployment_name, location_name):
-                    self._logger.warning(
-                        "Pex servers disappeared for %s:%s. Removing actual entries to"
-                        " activate reconciliation logic.",
-                        deployment_name,
-                        location_name,
-                    )
-                    del self._actual_entries[deployment_location]
+                    with self._grpc_servers_lock:
+                        grpc_server_or_error = self._grpc_servers.get(deployment_location)
+
+                    if isinstance(grpc_server_or_error, DagsterCloudGrpcServer):
+                        self._logger.warning(
+                            "Pex servers disappeared for running code location %s:%s. Removing actual entries to"
+                            " activate reconciliation logic and deploy a new code server and multipex server.",
+                            deployment_name,
+                            location_name,
+                        )
+                        self._trigger_recovery_server_restart(deployment_location)
 
         # Check to see if any servers have become unresponsive
         unavailable_server_timeout = int(
@@ -1332,8 +1624,6 @@ class DagsterCloudUserCodeLauncher(
         ) as executor:
             futures = {}
             for deployment_location, endpoint_or_error in running_locations.items():
-                deployment_name, location_name = deployment_location
-
                 futures[
                     executor.submit(self._make_check_on_running_server_endpoint(endpoint_or_error))
                 ] = deployment_location
@@ -1343,15 +1633,22 @@ class DagsterCloudUserCodeLauncher(
 
                 deployment_name, location_name = deployment_location
                 try:
-                    future.result()
-
+                    response_or_error = future.result()
                     # Successful ping resets the tracked last unavailable time for this code server, if set
                     self._first_unavailable_times.pop(deployment_location, None)
+                    if isinstance(response_or_error, SerializableErrorInfo):
+                        # This can happen if the server was previously healthy but restarted
+                        # and moved into an error state - attempt to recover
+                        self._logger.exception(
+                            f"Code server for {deployment_name}:{location_name} unexpectedly moved into an error state. Deploying a new code server. Observed error: \n{response_or_error.to_string()}"
+                        )
+                        self._trigger_recovery_server_restart(deployment_location)
                 except Exception as e:
                     if (
                         isinstance(e, DagsterUserCodeUnreachableError)
                         and isinstance(e.__cause__, grpc.RpcError)
-                        and cast(grpc.RpcError, e.__cause__).code() == grpc.StatusCode.UNAVAILABLE
+                        and cast("grpc.RpcError", e.__cause__).code()
+                        in {grpc.StatusCode.UNAVAILABLE, grpc.StatusCode.UNKNOWN}
                     ):
                         first_unavailable_time = self._first_unavailable_times.get(
                             deployment_location
@@ -1369,8 +1666,8 @@ class DagsterCloudUserCodeLauncher(
                             self._logger.warning(
                                 f"Code server for {deployment_name}:{location_name} has been unresponsive for more than {unavailable_server_timeout} seconds. Deploying a new code server."
                             )
-                            del self._actual_entries[deployment_location]
-                            del self._first_unavailable_times[deployment_location]
+                            self._trigger_recovery_server_restart(deployment_location)
+
                     else:
                         self._logger.exception(
                             f"Code server for {deployment_name}:{location_name} health check failed, but the error did not indicate that the server was unavailable."
@@ -1406,15 +1703,15 @@ class DagsterCloudUserCodeLauncher(
 
     def _deployments_and_locations_to_string(
         self,
-        deployments_and_locations: Set[DeploymentAndLocation],
-        entries: Dict[DeploymentAndLocation, UserCodeLauncherEntry],
+        deployments_and_locations: set[DeploymentAndLocation],
+        entries: dict[DeploymentAndLocation, UserCodeLauncherEntry],
     ):
         return (
             "{"
             + ", ".join(
                 sorted(
                     [
-                        f"({dep}, {loc}, {entries[(dep,loc)].update_timestamp})"
+                        f"({dep}, {loc}, {entries[(dep, loc)].update_timestamp})"
                         for dep, loc in deployments_and_locations
                     ]
                 )
@@ -1431,8 +1728,8 @@ class DagsterCloudUserCodeLauncher(
 
     def _reconcile(
         self,
-        desired_entries: Dict[DeploymentAndLocation, UserCodeLauncherEntry],
-        upload_locations: Set[DeploymentAndLocation],
+        desired_entries: dict[DeploymentAndLocation, UserCodeLauncherEntry],
+        upload_locations: set[DeploymentAndLocation],
         check_on_pending_delete_servers: bool,
     ):
         if check_on_pending_delete_servers:
@@ -1443,7 +1740,11 @@ class DagsterCloudUserCodeLauncher(
             for handle in handles:
                 self._graceful_remove_server_handle(handle)
 
-        diff = diff_serializable_namedtuple_map(desired_entries, self._actual_entries)
+        diff = diff_serializable_namedtuple_map(
+            desired_entries,
+            self._actual_entries,
+            update_key_fn=lambda entry: entry.update_timestamp,
+        )
         has_changes = diff.to_add or diff.to_update or diff.to_remove or upload_locations
 
         if not has_changes:
@@ -1469,26 +1770,26 @@ class DagsterCloudUserCodeLauncher(
         to_update_keys = diff.to_add.union(diff.to_update)
 
         # Handles for all running standalone Dagster GRPC servers
-        existing_standalone_dagster_server_handles: Dict[
+        existing_standalone_dagster_server_handles: dict[
             DeploymentAndLocation, Collection[ServerHandle]
         ] = {}
 
         # Handles for all running Dagster multipex servers (which can each host multiple grpc subprocesses)
-        existing_multipex_server_handles: Dict[DeploymentAndLocation, Collection[ServerHandle]] = {}
+        existing_multipex_server_handles: dict[DeploymentAndLocation, Collection[ServerHandle]] = {}
 
         # For each location, all currently running pex servers on the current multipex server
-        existing_pex_server_handles: Dict[DeploymentAndLocation, List[PexServerHandle]] = {}
+        existing_pex_server_handles: dict[DeploymentAndLocation, list[PexServerHandle]] = {}
 
         # Dagster grpc servers created in this loop (including both standalone grpc servers
         # and pex servers on a multipex server) - or an error that explains why it couldn't load
-        new_dagster_servers: Dict[
+        new_dagster_servers: dict[
             DeploymentAndLocation, Union[DagsterCloudGrpcServer, SerializableErrorInfo]
         ] = {}
 
         # Multipex servers created in this loop (a new multipex server might not always
         # be created on each loop even if the code has changed, as long as the base image
         # is the same)
-        new_multipex_servers: Dict[DeploymentAndLocation, DagsterCloudGrpcServer] = {}
+        new_multipex_servers: dict[DeploymentAndLocation, DagsterCloudGrpcServer] = {}
 
         for to_update_key in to_update_keys:
             deployment_name, location_name = to_update_key
@@ -1534,16 +1835,29 @@ class DagsterCloudUserCodeLauncher(
                         )
                         # confirm it's a valid image since _start_new_server_spinup will launch a container
                         self._check_for_image(desired_entry.code_location_deploy_data)
-                        multipex_server = self._start_new_server_spinup(
-                            deployment_name, location_name, desired_entry
-                        )
-                        self._multipex_servers[to_update_key] = multipex_server
-                        assert self._get_multipex_server(
-                            deployment_name,
-                            location_name,
-                            desired_entry.code_location_deploy_data,
-                        )
-                        new_multipex_servers[to_update_key] = multipex_server
+
+                        attributes = {
+                            "deployment": deployment_name,
+                            "location": location_name,
+                            "image": desired_entry.code_location_deploy_data.image,
+                            "python_version": desired_python_version,
+                        }
+                        with observe_execution(
+                            opentelemetry=self.opentelemetry,
+                            event_key=f"{DAGSTER_CLOUD_AGENT_METRIC_PREFIX}.user_code.multipex_server.start",
+                            short_description="starting new multipex server",
+                            attributes=attributes,
+                        ):
+                            multipex_server = self._start_new_server_spinup(
+                                deployment_name, location_name, desired_entry
+                            )
+                            self._multipex_servers[to_update_key] = multipex_server
+                            assert self._get_multipex_server(
+                                deployment_name,
+                                location_name,
+                                desired_entry.code_location_deploy_data,
+                            )
+                            new_multipex_servers[to_update_key] = multipex_server
                     else:
                         self._logger.info(
                             f"Found running multipex server for {multipex_server_repr}"
@@ -1569,8 +1883,7 @@ class DagsterCloudUserCodeLauncher(
             deployment_name, location_name = to_update_key
 
             self._logger.info(
-                f"Waiting for new multipex server for {deployment_name}:{location_name} to be"
-                " ready"
+                f"Waiting for new multipex server for {deployment_name}:{location_name} to be ready"
             )
             tasks[to_update_key] = self._wait_for_new_multipex_server(
                 deployment_name,
@@ -1681,6 +1994,7 @@ class DagsterCloudUserCodeLauncher(
 
             for server_handle in server_handles:
                 try:
+                    # TODO - telemetry of removing standalone servers
                     self._graceful_remove_server_handle(server_handle)
                 except Exception:
                     self._logger.error(
@@ -1711,6 +2025,7 @@ class DagsterCloudUserCodeLauncher(
                     )
 
                     try:
+                        # TODO - telemetry of removing multipex server
                         self._graceful_remove_server_handle(multipex_server_handle)
                     except Exception:
                         self._logger.error(
@@ -1728,6 +2043,7 @@ class DagsterCloudUserCodeLauncher(
                 )
                 for pex_server_handle in pex_server_handles:
                     try:
+                        # TODO - telemetry of removing pex server
                         self._remove_pex_server_handle(
                             deployment_name,
                             location_name,
@@ -1752,6 +2068,7 @@ class DagsterCloudUserCodeLauncher(
         for to_remove_key in diff.to_remove:
             deployment_name, location_name = to_remove_key
             try:
+                # TODO - telemetry of removing location's server
                 self._remove_server(deployment_name, location_name)
             except Exception:
                 self._logger.error(
@@ -1763,25 +2080,27 @@ class DagsterCloudUserCodeLauncher(
                 del self._grpc_servers[to_remove_key]
             del self._actual_entries[to_remove_key]
 
+            if to_remove_key in self._multipex_servers:
+                del self._multipex_servers[to_remove_key]
+
         # Upload any locations that were requested to be uploaded, but weren't updated
         # as part of this reconciliation loop
+
+        tasks = {}
         for location in upload_locations:
             with self._grpc_servers_lock:
                 server_or_error = self._grpc_servers[location]
 
             deployment_name, location_name = location
-            try:
-                self._update_location_data(
-                    deployment_name,
-                    location_name,
-                    server_or_error,
-                    self._actual_entries[location].code_location_deploy_data,
-                )
-            except Exception:
-                self._logger.error(
-                    f"Error while writing location data for {deployment_name}:{location_name}:"
-                    f" {serializable_error_info_from_exc_info(sys.exc_info())}"
-                )
+            tasks[location] = self._try_update_location_data(
+                deployment_name,
+                location_name,
+                server_or_error,
+                self._actual_entries[location].code_location_deploy_data,
+            )
+
+        if tasks:
+            results = asyncio.run(self._gather_tasks(tasks.values()))
 
         seconds = time.time() - start_time
         self._logger.info(f"Finished reconciling in {seconds} seconds.")
@@ -1854,6 +2173,11 @@ class DagsterCloudUserCodeLauncher(
     def _start_new_dagster_server(
         self, deployment_name: str, location_name: str, desired_entry: UserCodeLauncherEntry
     ) -> DagsterCloudGrpcServer:
+        attributes = {
+            "deployment": deployment_name,
+            "location": location_name,
+        }
+
         if desired_entry.code_location_deploy_data.pex_metadata:
             multipex_server = self._get_multipex_server(
                 deployment_name, location_name, desired_entry.code_location_deploy_data
@@ -1861,26 +2185,45 @@ class DagsterCloudUserCodeLauncher(
 
             assert multipex_server  # should have been started earlier or we should never reach here
 
-            self._create_pex_server(deployment_name, location_name, desired_entry, multipex_server)
+            if desired_entry.code_location_deploy_data.pex_metadata.python_version:
+                attributes["python_version"] = (
+                    desired_entry.code_location_deploy_data.pex_metadata.python_version
+                )
 
-            server_handle = multipex_server.server_handle
-            multipex_endpoint = multipex_server.server_endpoint
+            with observe_execution(
+                opentelemetry=self.opentelemetry,
+                event_key=f"{DAGSTER_CLOUD_AGENT_METRIC_PREFIX}.user_code.pex_server.start",
+                short_description="starting new pex server",
+                attributes=attributes,
+            ):
+                self._create_pex_server(
+                    deployment_name, location_name, desired_entry, multipex_server
+                )
 
-            # start a new pex server on the multipexer, which we can count on already existing
-            return DagsterCloudGrpcServer(
-                server_handle,
-                multipex_endpoint.with_metadata(
-                    [
-                        ("has_pex", "1"),
-                        ("deployment", deployment_name),
-                        ("location", location_name),
-                        ("timestamp", str(int(desired_entry.update_timestamp))),
-                    ],
-                ),
-                desired_entry.code_location_deploy_data,
-            )
+                server_handle = multipex_server.server_handle
+                multipex_endpoint = multipex_server.server_endpoint
+
+                # start a new pex server on the multipexer, which we can count on already existing
+                return DagsterCloudGrpcServer(
+                    server_handle,
+                    multipex_endpoint.with_metadata(
+                        [
+                            ("has_pex", "1"),
+                            ("deployment", deployment_name),
+                            ("location", location_name),
+                            ("timestamp", str(int(desired_entry.update_timestamp))),
+                        ],
+                    ),
+                    desired_entry.code_location_deploy_data,
+                )
         else:
-            return self._start_new_server_spinup(deployment_name, location_name, desired_entry)
+            with observe_execution(
+                opentelemetry=self.opentelemetry,
+                event_key=f"{DAGSTER_CLOUD_AGENT_METRIC_PREFIX}.user_code.code_server.start",
+                short_description="new code server spin up",
+                attributes=attributes,
+            ):
+                return self._start_new_server_spinup(deployment_name, location_name, desired_entry)
 
     def get_grpc_endpoint(
         self,
@@ -1924,12 +2267,12 @@ class DagsterCloudUserCodeLauncher(
 
         return server
 
-    def get_grpc_server_heartbeats(self) -> Dict[str, List[CloudCodeServerHeartbeat]]:
+    def get_grpc_server_heartbeats(self) -> dict[str, list[CloudCodeServerHeartbeat]]:
         endpoint_or_errors = self.get_grpc_endpoints()
         with self._metadata_lock:
             desired_entries = set(self._desired_entries.keys())
 
-        heartbeats: Dict[str, List[CloudCodeServerHeartbeat]] = {}
+        heartbeats: dict[str, list[CloudCodeServerHeartbeat]] = {}
         for entry_key in desired_entries:
             deployment_name, location_name = entry_key
             endpoint_or_error = endpoint_or_errors.get(entry_key)
@@ -1979,7 +2322,7 @@ class DagsterCloudUserCodeLauncher(
 
     def get_grpc_endpoints(
         self,
-    ) -> Dict[DeploymentAndLocation, Union[ServerEndpoint, SerializableErrorInfo]]:
+    ) -> dict[DeploymentAndLocation, Union[ServerEndpoint, SerializableErrorInfo]]:
         with self._grpc_servers_lock:
             return {
                 key: val if isinstance(val, SerializableErrorInfo) else val.server_endpoint
@@ -2011,7 +2354,7 @@ class DagsterCloudUserCodeLauncher(
             client, timeout, additional_check, get_timeout_debug_info=get_timeout_debug_info
         )
         # Call a method that raises an exception if there was an error importing the code
-        sync_list_repositories_grpc(client)
+        await self.gen_list_repositories_response(client)
 
     async def _wait_for_server_process(
         self,
@@ -2070,6 +2413,7 @@ class DagsterCloudUserCodeLauncher(
             response = client.external_job(
                 RemoteRepositoryOrigin(location_origin, job_selector.repository_name),
                 job_selector.job_name,
+                timeout=int(os.getenv("DAGSTER_CLOUD_EXTERNAL_JOB_GRPC_TIMEOUT", "180")),
             )
             if not response.serialized_job_data:
                 error = (
@@ -2098,3 +2442,39 @@ class DagsterCloudUserCodeLauncher(
                     f" {job_selector.job_name}@{job_selector.repository_name} ({os.path.getsize(dst)} bytes)"
                 )
                 return response
+
+    @property
+    def opentelemetry(self) -> Optional[OpenTelemetryController]:
+        if not self.has_instance:
+            return None
+        else:
+            return self._instance.opentelemetry
+
+    def upload_job_snap_direct(
+        self,
+        deployment_name: str,
+        job_selector: JobSelector,
+        server: DagsterCloudGrpcServer,
+    ):
+        client = server.server_endpoint.create_client()
+        location_origin = self._get_code_location_origin(job_selector.location_name)
+        response = client.external_job(
+            RemoteRepositoryOrigin(location_origin, job_selector.repository_name),
+            job_selector.job_name,
+        )
+        if not response.serialized_job_data:
+            error = (
+                deserialize_value(response.serialized_error, SerializableErrorInfo)
+                if response.serialized_error
+                else "no captured error"
+            )
+            raise Exception(f"Error fetching job data in code server:\n{error}")
+
+        job_snapshot = extract_serialized_job_snap_from_serialized_job_data_snap(
+            response.serialized_job_data
+        )
+        return self._ensure_snapshot_uploaded(
+            deployment_name,
+            SnapshotType.JOB,
+            job_snapshot,
+        )