dagster-cloud 1.8.2__py3-none-any.whl → 1.12.6__py3-none-any.whl

dagster_cloud/agent/dagster_cloud_agent.py

@@ -2,33 +2,49 @@ import logging
 import os
 import sys
 import time
-from collections import deque
+from collections import defaultdict, deque
+from collections.abc import Iterator
 from concurrent.futures import Future, ThreadPoolExecutor
 from contextlib import ExitStack
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Set, Tuple, Union, cast
+from typing import TYPE_CHECKING, Any, Optional, Union, cast
 
 import dagster._check as check
 from dagster import DagsterInstance
 from dagster._core.launcher.base import LaunchRunContext
-from dagster._core.remote_representation import CodeLocationOrigin
-from dagster._core.remote_representation.origin import RegisteredCodeLocationOrigin
+from dagster._core.remote_origin import CodeLocationOrigin, RegisteredCodeLocationOrigin
 from dagster._core.utils import FuturesAwareThreadPoolExecutor
 from dagster._grpc.client import DagsterGrpcClient
 from dagster._grpc.types import CancelExecutionRequest
 from dagster._serdes import deserialize_value, serialize_value
 from dagster._time import get_current_datetime, get_current_timestamp
+from dagster._utils.cached_method import cached_method
 from dagster._utils.container import retrieve_containerized_utilization_metrics
-from dagster._utils.error import SerializableErrorInfo, serializable_error_info_from_exc_info
+from dagster._utils.error import (
+    SerializableErrorInfo,
+    serializable_error_info_from_exc_info,
+    truncate_serialized_error,
+)
 from dagster._utils.interrupts import raise_interrupts_as
 from dagster._utils.merger import merge_dicts
 from dagster._utils.typed_dict import init_optional_typeddict
 from dagster_cloud_cli.core.errors import DagsterCloudHTTPError, raise_http_error
-from dagster_cloud_cli.core.workspace import CodeLocationDeployData, get_instance_ref_for_user_code
+from dagster_cloud_cli.core.workspace import CodeLocationDeployData
 
+from dagster_cloud.agent.instrumentation.constants import DAGSTER_CLOUD_AGENT_METRIC_PREFIX
+from dagster_cloud.agent.instrumentation.run_launch import extract_run_attributes
+from dagster_cloud.agent.instrumentation.schedule import inspect_schedule_result
+from dagster_cloud.agent.instrumentation.sensor import inspect_sensor_result
+from dagster_cloud.agent.queries import (
+    ADD_AGENT_HEARTBEATS_MUTATION,
+    DEPLOYMENTS_QUERY,
+    GET_USER_CLOUD_REQUESTS_QUERY,
+    WORKSPACE_ENTRIES_QUERY,
+)
 from dagster_cloud.api.dagster_cloud_api import (
     AgentHeartbeat,
     AgentUtilizationMetrics,
+    BatchDagsterCloudUploadApiResponse,
     DagsterCloudApi,
     DagsterCloudApiErrorResponse,
     DagsterCloudApiGrpcResponse,
@@ -40,21 +56,16 @@ from dagster_cloud.api.dagster_cloud_api import (
     DagsterCloudUploadApiResponse,
     TimestampedError,
 )
+from dagster_cloud.batching import Batcher
 from dagster_cloud.instance import DagsterCloudAgentInstance
-from dagster_cloud.util.errors import truncate_serialized_error
+from dagster_cloud.opentelemetry.observers.execution_observer import observe_execution
+from dagster_cloud.util import SERVER_HANDLE_TAG, compressed_namedtuple_upload_file, is_isolated_run
+from dagster_cloud.version import __version__
 from dagster_cloud.workspace.user_code_launcher import (
     DagsterCloudUserCodeLauncher,
     UserCodeLauncherEntry,
 )
-
-from ..util import SERVER_HANDLE_TAG, compressed_namedtuple_upload_file, is_isolated_run
-from ..version import __version__
-from .queries import (
-    ADD_AGENT_HEARTBEATS_MUTATION,
-    DEPLOYMENTS_QUERY,
-    GET_USER_CLOUD_REQUESTS_QUERY,
-    WORKSPACE_ENTRIES_QUERY,
-)
+from dagster_cloud.workspace.user_code_launcher.utils import get_instance_ref_for_user_code
 
 if TYPE_CHECKING:
     import datetime
@@ -73,6 +84,11 @@ DEFAULT_PENDING_REQUESTS_LIMIT = 100
 
 SLEEP_INTERVAL_SECONDS = float(os.getenv("DAGSTER_CLOUD_AGENT_SLEEP_INTERVAL_SECONDS", "0.5"))
 
+
+def UPLOAD_API_RESPONSE_BATCHING_ENABLED():
+    return os.getenv("DAGSTER_CLOUD_AGENT_UPLOAD_API_RESPONSE_BATCHING_ENABLED") == "true"
+
+
 DEPLOYMENT_INFO_QUERY = """
     query DeploymentInfo {
          deploymentInfo {
@@ -96,10 +112,22 @@ LIVENESS_CHECK_INTERVAL_SECONDS = float(
 
 
 class DagsterCloudAgent:
-    def __init__(self, pending_requests_limit: int = DEFAULT_PENDING_REQUESTS_LIMIT):
+    def __init__(
+        self,
+        instance: DagsterCloudAgentInstance,
+        pending_requests_limit: int = DEFAULT_PENDING_REQUESTS_LIMIT,
+    ):
         self._logger = logging.getLogger("dagster_cloud.agent")
+        self._instance: DagsterCloudAgentInstance = instance
 
-        self._logger.info("Starting Dagster Cloud agent...")
+        self._batcher: defaultdict[
+            str, Batcher[tuple[str, DagsterCloudUploadApiResponse], None]
+        ] = defaultdict(self._batcher_factory)
+
+        if self._logger.isEnabledFor(logging.DEBUG):
+            self._logger.info("Starting Dagster Cloud agent with debug logging...")
+        else:
+            self._logger.info("Starting Dagster Cloud agent...")
 
         self._exit_stack = ExitStack()
         self._iteration = 0
@@ -110,7 +138,7 @@ class DagsterCloudAgent:
                 thread_name_prefix="dagster_cloud_agent_worker",
             )
         )
-        self._request_ids_to_futures: Dict[str, Future] = {}
+        self._request_ids_to_futures: dict[str, Future] = {}
         self._utilization_metrics = init_optional_typeddict(AgentUtilizationMetrics)
 
         self._last_heartbeat_time: Optional[datetime.datetime] = None
@@ -121,26 +149,38 @@ class DagsterCloudAgent:
             maxlen=AGENT_HEARTBEAT_ERROR_LIMIT
         )  # (SerializableErrorInfo, timestamp) tuples
 
-        self._pending_requests: List[Dict[str, Any]] = []
-        self._locations_with_pending_requests: Set[Tuple[str, str, bool]] = set()
-        self._ready_requests: List[Dict[str, Any]] = []
+        self._pending_requests: list[dict[str, Any]] = []
+        self._locations_with_pending_requests: set[tuple[str, str, bool]] = set()
+        self._ready_requests: list[dict[str, Any]] = []
 
-        self._location_query_times: Dict[Tuple[str, str, bool], float] = {}
+        self._location_query_times: dict[tuple[str, str, bool], float] = {}
         self._pending_requests_limit = check.int_param(
             pending_requests_limit, "pending_requests_limit"
         )
-        self._active_deployments: Set[Tuple[str, bool]] = (  # deployment_name, is_branch_deployment
+        self._active_deployments: set[tuple[str, bool]] = (  # deployment_name, is_branch_deployment
             set()
         )
 
         self._last_liveness_check_time = None
 
+        self._warned_about_long_in_progress_reconcile = False
+
     def __enter__(self):
         return self
 
     def __exit__(self, _exception_type, _exception_value, _traceback):
         self._exit_stack.close()
 
+    def _batcher_factory(
+        self,
+    ) -> Batcher[tuple[str, DagsterCloudUploadApiResponse], None]:
+        return Batcher(
+            "upload_api_response",
+            self._batch_upload_api_response,
+            max_wait_ms=50,
+            max_batch_size=32,
+        )
+
     @property
     def _active_deployment_names(self):
         return [deployment[0] for deployment in self._active_deployments]
@@ -149,14 +189,15 @@ class DagsterCloudAgent:
     def _active_full_deployment_names(self):
         return [deployment[0] for deployment in self._active_deployments if not deployment[1]]
 
-    def _check_initial_deployment_names(self, instance: DagsterCloudAgentInstance):
-        if instance.deployment_names:
-            result = instance.organization_scoped_graphql_client().execute(
-                DEPLOYMENTS_QUERY, variable_values={"deploymentNames": instance.deployment_names}
+    def _check_initial_deployment_names(self):
+        if self._instance.deployment_names:
+            result = self._instance.organization_scoped_graphql_client().execute(
+                DEPLOYMENTS_QUERY,
+                variable_values={"deploymentNames": self._instance.deployment_names},
             )
             deployments = result["data"]["deployments"]
             existing_deployment_names = {deployment["deploymentName"] for deployment in deployments}
-            requested_deployment_names = set(instance.deployment_names)
+            requested_deployment_names = set(self._instance.deployment_names)
             missing_deployment_names = requested_deployment_names.difference(
                 existing_deployment_names
             )
@@ -171,13 +212,40 @@ class DagsterCloudAgent:
     def _update_agent_resource_limits(
         self, user_code_launcher: DagsterCloudUserCodeLauncher
     ) -> None:
-        # If the env var DAGSTER_CLOUD_AGENT_MEMORY_LIMIT is set, use that as the memory limit.
-        memory_limit = os.getenv("DAGSTER_CLOUD_AGENT_MEMORY_LIMIT")
-        # If the env var DAGSTER_CLOUD_AGENT_CPU_LIMIT is set, use that as the cpu limit.
-        cpu_limit = os.getenv("DAGSTER_CLOUD_AGENT_CPU_LIMIT")
+        # The agent should have environment variables defining its resource requests and limits.
+        # However, the agent may be running in a container with resource limits that are different
+        # For example, on k8s there are ways to effect change on the cpu limit, like mutating admission webhooks.
+        # Since the effective cgroup limits precede actual hosts resources when it comes to actual behavior for
+        # throttling and oom kills, we attempt to obtain these and fallback on the environment variables.
+        container_utilization_metrics = self._utilization_metrics.get("container_utilization", {})
+        memory_limit = container_utilization_metrics.get("memory_limit")
+        if not memory_limit:
+            memory_limit = os.getenv("DAGSTER_CLOUD_AGENT_MEMORY_LIMIT")
+            self._logger.info(
+                "Cannot obtain cgroup memory limit, using environment value: "
+                f"DAGSTER_CLOUD_AGENT_MEMORY_LIMIT={memory_limit}"
+            )
+
+        cpu_cfs_period_us = container_utilization_metrics.get("cpu_cfs_period_us")
+        cpu_cfs_quota_us = container_utilization_metrics.get("cpu_cfs_quota_us")
+
+        cpu_limit = None
+        if cpu_cfs_quota_us and cpu_cfs_period_us:
+            cpu_limit = (
+                1000.0 * cpu_cfs_quota_us
+            ) / cpu_cfs_period_us  # cpu_limit expressed in milliseconds of cpu
+
+        if not cpu_limit:
+            cpu_limit = os.getenv("DAGSTER_CLOUD_AGENT_CPU_LIMIT")
+            self._logger.info(
+                "Cannot obtain CPU CFS values, using environment value: "
+                f"DAGSTER_CLOUD_AGENT_CPU_LIMIT={cpu_limit}"
+            )
+
         if not user_code_launcher.user_code_deployment_type.supports_utilization_metrics:
             self._logger.info(
-                f"Cannot interpret resource limits for agent type {user_code_launcher.user_code_deployment_type.value}. Skipping utilization metrics retrieval."
+                f"Cannot interpret resource limits for agent type {user_code_launcher.user_code_deployment_type.value}."
+                "Skipping utilization metrics retrieval."
             )
             return
 
@@ -185,6 +253,7 @@ class DagsterCloudAgent:
             "cpu_limit": cpu_limit,
             "memory_limit": memory_limit,
         }
+
         cpu_request = os.getenv("DAGSTER_CLOUD_AGENT_CPU_REQUEST")
         memory_request = os.getenv("DAGSTER_CLOUD_AGENT_MEMORY_REQUEST")
         if cpu_request:
@@ -192,7 +261,6 @@ class DagsterCloudAgent:
         if memory_request:
             limits["memory_request"] = memory_request
 
-        # At this point, the only agent types possible are serverless, ecs, and k8s, all of which are supported. The linter isn't smart enough to realize this, so we disable it.
         self._utilization_metrics["resource_limits"][
             user_code_launcher.user_code_deployment_type.value
         ] = limits  # type: ignore
@@ -216,16 +284,15 @@ class DagsterCloudAgent:
 
     def run_loop(
         self,
-        instance: DagsterCloudAgentInstance,
         user_code_launcher,
         agent_uuid,
     ):
         heartbeat_interval_seconds = AGENT_HEARTBEAT_INTERVAL_SECONDS
 
         if (
-            not instance.includes_branch_deployments
-            and not instance.deployment_names
-            and not instance.include_all_serverless_deployments
+            not self._instance.includes_branch_deployments
+            and not self._instance.deployment_names
+            and not self._instance.include_all_serverless_deployments
         ):
             self._logger.info(
                 "Deployment name was not set - checking to see if it can be fetched from the"
@@ -233,20 +300,34 @@ class DagsterCloudAgent:
             )
             # Fetch the deployment name from the server if it isn't set (only true
             # for old agents, and only will work if there's a single deployment in the org)
-            result = instance.graphql_client.execute(DEPLOYMENT_INFO_QUERY)
+            result = self._instance.graphql_client.execute(DEPLOYMENT_INFO_QUERY)
             deployment_name = result["data"]["deploymentInfo"]["deploymentName"]
-            instance = self._exit_stack.enter_context(
-                DagsterInstance.from_ref(instance.ref_for_deployment(deployment_name))  # type: ignore  # (instance subclass)
+            self._instance = self._exit_stack.enter_context(
+                DagsterInstance.from_ref(self._instance.ref_for_deployment(deployment_name))  # type: ignore  # (instance subclass)
             )
 
-        self._check_initial_deployment_names(instance)
+        self._check_initial_deployment_names()
+
+        serving = []
+        queues = list(filter(None, self._instance.agent_queues_config.queues))
+        if queues:
+            serving.append(f"queues{queues}")
+        if self._instance.deployment_names:
+            serving.append(f"deployments{self._instance.deployment_names}")
+        if self._instance.include_all_serverless_deployments:
+            serving.append("all serverless deployments")
+        if self._instance.includes_branch_deployments:
+            serving.append("branch deployments")
+
+        self._logger.info(f"Agent is serving: {', '.join(serving)}")
 
         self._check_update_workspace(
-            instance, user_code_launcher, upload_all=user_code_launcher.upload_snapshots_on_startup
+            user_code_launcher,
+            upload_all=user_code_launcher.upload_snapshots_on_startup,
         )
 
         self._logger.info(
-            f"Will start polling for requests from {instance.dagster_cloud_url} once user code has"
+            f"Will start polling for requests from {self._instance.dagster_cloud_url} once user code has"
             " been loaded."
         )
 
@@ -254,7 +335,7 @@ class DagsterCloudAgent:
 
         while True:
             try:
-                for error in self.run_iteration(instance, user_code_launcher):
+                for error in self.run_iteration(user_code_launcher):
                     if error:
                         self._logger.error(str(error))
                         self._errors.appendleft(
@@ -283,16 +364,18 @@ class DagsterCloudAgent:
 
             if user_code_launcher.ready_to_serve_requests:
                 try:
-                    self._check_add_heartbeat(instance, agent_uuid, heartbeat_interval_seconds)
+                    self._check_add_heartbeat(agent_uuid, heartbeat_interval_seconds)
                 except Exception:
                     self._logger.exception("Failed to add heartbeat")
 
+            self._check_for_long_running_reconcile(user_code_launcher)
+
             # Check for any received interrupts
             with raise_interrupts_as(KeyboardInterrupt):
                 pass
 
             try:
-                self._check_update_workspace(instance, user_code_launcher, upload_all=False)
+                self._check_update_workspace(user_code_launcher, upload_all=False)
 
             except Exception:
                 self._logger.error(
@@ -329,7 +412,27 @@ class DagsterCloudAgent:
             self._logger.error(f"Failed to write liveness sentinel and disabling it: {e}")
             self._last_liveness_check_time = False
 
-    def _check_update_workspace(self, instance, user_code_launcher, upload_all):
+    def _check_for_long_running_reconcile(self, user_code_launcher):
+        """Detect from the main thread if the background reconcile thread is running behind or has gotten stuck."""
+        in_progress_reconcile_start_time = user_code_launcher.in_progress_reconcile_start_time
+
+        reconcile_start_time_warning = int(
+            os.getenv("DAGSTER_CLOUD_AGENT_RECONCILE_START_TIME_WARNING", "3600")
+        )
+
+        if (
+            in_progress_reconcile_start_time is not None
+            and (time.time() - in_progress_reconcile_start_time) >= reconcile_start_time_warning
+        ):
+            if not self._warned_about_long_in_progress_reconcile:
+                self._logger.warning(
+                    f"Agent has been redeploying code servers for more than {reconcile_start_time_warning} seconds. This may indicate the background thread that performs the redeploys is stuck."
+                )
+                self._warned_about_long_in_progress_reconcile = True
+        else:
+            self._warned_about_long_in_progress_reconcile = False
+
+    def _check_update_workspace(self, user_code_launcher, upload_all):
         curr_time = get_current_datetime()
 
         if (
@@ -340,11 +443,10 @@ class DagsterCloudAgent:
             return
 
         self._last_workspace_check_time = curr_time
-        self._query_for_workspace_updates(instance, user_code_launcher, upload_all=upload_all)
+        self._query_for_workspace_updates(user_code_launcher, upload_all=upload_all)
 
     def _check_add_heartbeat(
         self,
-        instance: DagsterCloudAgentInstance,
         agent_uuid,
         heartbeat_interval_seconds,
     ):
@@ -357,19 +459,22 @@ class DagsterCloudAgent:
             return
 
         errors = [
-            TimestampedError(timestamp.timestamp(), error)
+            TimestampedError(
+                timestamp=timestamp.timestamp(),
+                error=error,
+            )
             for (error, timestamp) in self._errors
             if timestamp.timestamp() > curr_time.timestamp() - 60 * 60 * 24
         ]
 
-        run_worker_statuses_dict = instance.user_code_launcher.get_cloud_run_worker_statuses(
+        run_worker_statuses_dict = self._instance.user_code_launcher.get_cloud_run_worker_statuses(
             self._active_deployment_names
         )
 
-        code_server_heartbeats_dict = instance.user_code_launcher.get_grpc_server_heartbeats()
+        code_server_heartbeats_dict = self._instance.user_code_launcher.get_grpc_server_heartbeats()
 
         agent_image_tag = os.getenv("DAGSTER_CLOUD_AGENT_IMAGE_TAG")
-        if instance.user_code_launcher.agent_metrics_enabled:
+        if self._instance.user_code_launcher.agent_metrics_enabled:
             num_running_requests = self._utilization_metrics["request_utilization"][
                 "num_running_requests"
             ]
@@ -386,10 +491,10 @@ class DagsterCloudAgent:
             deployment_name: AgentHeartbeat(
                 timestamp=curr_time.timestamp(),
                 agent_id=agent_uuid,
-                agent_label=instance.dagster_cloud_api_agent_label,
+                agent_label=self._instance.dagster_cloud_api_agent_label,
                 agent_type=(
-                    type(instance.user_code_launcher).__name__
-                    if instance.user_code_launcher
+                    type(self._instance.user_code_launcher).__name__
+                    if self._instance.user_code_launcher
                     else None
                 ),
                 metadata=merge_dicts(
@@ -397,13 +502,13 @@ class DagsterCloudAgent:
                     {"image_tag": agent_image_tag} if agent_image_tag else {},
                     {
                         "utilization_metrics": self._utilization_metrics
-                        if instance.user_code_launcher.agent_metrics_enabled
+                        if self._instance.user_code_launcher.agent_metrics_enabled
                         else {}
                     },
                 ),
                 run_worker_statuses=run_worker_statuses_dict[deployment_name],
                 code_server_heartbeats=code_server_heartbeats_dict.get(deployment_name, []),
-                agent_queues_config=instance.agent_queues_config,
+                agent_queues_config=self._instance.agent_queues_config,
             )
             for deployment_name in self._active_deployment_names
         }
@@ -418,7 +523,7 @@ class DagsterCloudAgent:
 
         serialized_errors = [serialize_value(error) for error in errors]
         try:
-            instance.organization_scoped_graphql_client().execute(
+            self._instance.organization_scoped_graphql_client().execute(
                 ADD_AGENT_HEARTBEATS_MUTATION,
                 variable_values={
                     "serializedAgentHeartbeats": serialized_agent_heartbeats,
@@ -444,15 +549,15 @@ class DagsterCloudAgent:
                     for deployment_name, heartbeat in heartbeats.items()
                 ]
 
-                instance.organization_scoped_graphql_client().execute(
+                self._instance.organization_scoped_graphql_client().execute(
                     ADD_AGENT_HEARTBEATS_MUTATION,
                     variable_values={
                         "serializedAgentHeartbeats": serialized_agent_heartbeats,
                         "serializedErrors": [
                             serialize_value(
                                 TimestampedError(
-                                    curr_time.timestamp(),
-                                    SerializableErrorInfo(
+                                    timestamp=curr_time.timestamp(),
+                                    error=SerializableErrorInfo(
                                         error_message,
                                         stack=[],
                                         cls_name=None,
@@ -471,26 +576,24 @@ class DagsterCloudAgent:
         return self._executor
 
     @property
-    def request_ids_to_futures(self) -> Dict[str, Future]:
+    def request_ids_to_futures(self) -> dict[str, Future]:
         return self._request_ids_to_futures
 
     def _upload_outdated_workspace_entries(
         self,
-        instance: DagsterCloudAgentInstance,
         deployment_name: str,
         is_branch_deployment: bool,
         user_code_launcher: DagsterCloudUserCodeLauncher,
     ):
-        result = instance.graphql_client_for_deployment(deployment_name).execute(
+        result = self._instance.graphql_client_for_deployment(deployment_name).execute(
             WORKSPACE_ENTRIES_QUERY,
             variable_values={
                 "deploymentNames": [deployment_name],
                 "includeAllServerlessDeployments": False,
-                "agentQueues": instance.agent_queues_config.queues,
+                "agentQueues": self._instance.agent_queues_config.queues,
             },
         )
         entries = result["data"]["deployments"][0]["workspaceEntries"]
-        now = time.time()
 
         upload_metadata = {}
 
@@ -501,10 +604,11 @@ class DagsterCloudAgent:
             )
             if entry["hasOutdatedData"]:
                 # Spin up a server for this location and upload its metadata to Cloud
-                # (Bump the TTL counter as well to leave the server up)
+                # (Bump the TTL counter as well to leave the server up - ensure that a slighty
+                # different timestamp is chosen for each location to break ties)
                 self._location_query_times[
                     (deployment_name, location_name, is_branch_deployment)
-                ] = now
+                ] = time.time()
                 upload_metadata[(deployment_name, location_name)] = UserCodeLauncherEntry(
                     code_location_deploy_data=code_location_deploy_data,
                     update_timestamp=float(entry["metadataTimestamp"]),
@@ -516,16 +620,14 @@ class DagsterCloudAgent:
         # branch deployments always have TTLs, other deployments only if you asked for it specifically
         return is_branch_deployment or user_code_launcher.server_ttl_enabled_for_full_deployments
 
-    def _get_ttl_seconds(self, instance, is_branch_deployment):
+    def _get_ttl_seconds(self, is_branch_deployment):
         return (
-            instance.user_code_launcher.branch_deployment_ttl_seconds
+            self._instance.user_code_launcher.branch_deployment_ttl_seconds
             if is_branch_deployment
-            else instance.user_code_launcher.full_deployment_ttl_seconds
+            else self._instance.user_code_launcher.full_deployment_ttl_seconds
         )
 
-    def _get_locations_with_ttl_to_query(
-        self, instance, user_code_launcher
-    ) -> List[Tuple[str, str]]:
+    def _get_locations_with_ttl_to_query(self, user_code_launcher) -> list[tuple[str, str]]:
         now = time.time()
 
         # For the deployments with TTLs, decide which locations to consider
@@ -533,13 +635,13 @@ class DagsterCloudAgent:
         # - a) There's a pending request in the queue for it
         # - b) It's TTL hasn't expired since the last time somebody asked for it
         # Always include locations in a), and add locations from b) until you hit a limit
-        location_candidates: Set[Tuple[str, str, float]] = {
+        location_candidates: set[tuple[str, str, float]] = {
             (deployment, location, -1.0)  # Score below 0 so that they're at the front of the list
             for deployment, location, is_branch_deployment in self._locations_with_pending_requests
             if self._has_ttl(user_code_launcher, is_branch_deployment)
         }
 
-        num_locations_to_query = instance.user_code_launcher.server_ttl_max_servers
+        num_locations_to_query = self._instance.user_code_launcher.server_ttl_max_servers
 
         if len(location_candidates) > num_locations_to_query:
             self._logger.warning(
@@ -561,7 +663,7 @@ class DagsterCloudAgent:
 
             time_since_last_query = now - query_time
 
-            if time_since_last_query >= self._get_ttl_seconds(instance, is_branch_deployment):
+            if time_since_last_query >= self._get_ttl_seconds(is_branch_deployment):
                 continue
 
             location_candidates.add((deployment_name, location, time_since_last_query))
@@ -583,13 +685,10 @@ class DagsterCloudAgent:
 
     def _query_for_workspace_updates(
         self,
-        instance: DagsterCloudAgentInstance,
         user_code_launcher: DagsterCloudUserCodeLauncher,
         upload_all: bool,
     ):
-        locations_with_ttl_to_query = self._get_locations_with_ttl_to_query(
-            instance, user_code_launcher
-        )
+        locations_with_ttl_to_query = self._get_locations_with_ttl_to_query(user_code_launcher)
 
         deployments_to_query = {key[0] for key in locations_with_ttl_to_query}
 
@@ -600,23 +699,23 @@ class DagsterCloudAgent:
             self._logger.debug(f"Querying for the following locations with TTL: {locations_str}")
 
         # If you have specified a non-branch deployment and no TTL, always consider it
-        if instance.deployment_names:
-            deployments_to_query = deployments_to_query.union(set(instance.deployment_names))
+        if self._instance.deployment_names:
+            deployments_to_query = deployments_to_query.union(set(self._instance.deployment_names))
 
         # Create mapping of
         # - location name => deployment metadata
-        deployment_map: Dict[Tuple[str, str], UserCodeLauncherEntry] = {}
-        all_locations: Set[Tuple[str, str]] = set()
+        deployment_map: dict[tuple[str, str], UserCodeLauncherEntry] = {}
+        all_locations: set[tuple[str, str]] = set()
 
         self._active_deployments = set()
 
-        if deployments_to_query or instance.include_all_serverless_deployments:
-            result = instance.organization_scoped_graphql_client().execute(
+        if deployments_to_query or self._instance.include_all_serverless_deployments:
+            result = self._instance.organization_scoped_graphql_client().execute(
                 WORKSPACE_ENTRIES_QUERY,
                 variable_values={
                     "deploymentNames": list(deployments_to_query),
-                    "includeAllServerlessDeployments": instance.include_all_serverless_deployments,
-                    "agentQueues": instance.agent_queues_config.queues,
+                    "includeAllServerlessDeployments": self._instance.include_all_serverless_deployments,
+                    "agentQueues": self._instance.agent_queues_config.queues,
                 },
             )
 
@@ -645,7 +744,7 @@ class DagsterCloudAgent:
                     # only include the locations within locations_with_ttl_to_query.
                     if not self._has_ttl(
                         user_code_launcher, is_branch_deployment
-                    ) or location_key in cast(Set[Tuple[str, str]], locations_with_ttl_to_query):
+                    ) or location_key in cast("set[tuple[str, str]]", locations_with_ttl_to_query):
                         deployment_map[location_key] = UserCodeLauncherEntry(
                             code_location_deploy_data=code_location_deploy_data,
                             update_timestamp=float(entry["metadataTimestamp"]),
@@ -718,17 +817,15 @@ class DagsterCloudAgent:
         else:
             return None
 
-    def _get_user_code_instance_ref(
-        self, instance: DagsterCloudAgentInstance, deployment_name: str
-    ):
-        return get_instance_ref_for_user_code(instance.ref_for_deployment(deployment_name))
+    @cached_method
+    def _get_user_code_instance_ref(self, deployment_name: str):
+        return get_instance_ref_for_user_code(self._instance.ref_for_deployment(deployment_name))
 
     def _handle_api_request(
         self,
         request: DagsterCloudApiRequest,
         deployment_name: str,
         is_branch_deployment: bool,
-        instance: DagsterCloudAgentInstance,
         user_code_launcher: DagsterCloudUserCodeLauncher,
     ) -> Union[DagsterCloudApiSuccess, DagsterCloudApiGrpcResponse]:
         api_name = request.request_api
@@ -743,107 +840,153 @@ class DagsterCloudAgent:
             # Dagster Cloud has requested that we upload new metadata for any out of date locations in
             # the workspace
             self._upload_outdated_workspace_entries(
-                instance, deployment_name, is_branch_deployment, user_code_launcher
+                deployment_name, is_branch_deployment, user_code_launcher
             )
             return DagsterCloudApiSuccess()
         elif api_name == DagsterCloudApi.GET_EXTERNAL_EXECUTION_PLAN:
             client = self._get_grpc_client(
-                user_code_launcher, deployment_name, cast(str, location_name)
+                user_code_launcher, deployment_name, cast("str", location_name)
             )
             serialized_snapshot_or_error = client.execution_plan_snapshot(
                 execution_plan_snapshot_args=request.request_args._replace(
-                    instance_ref=self._get_user_code_instance_ref(instance, deployment_name)
+                    instance_ref=self._get_user_code_instance_ref(deployment_name)
                 )
             )
-            return DagsterCloudApiGrpcResponse(serialized_snapshot_or_error)
+            return DagsterCloudApiGrpcResponse(
+                serialized_response_or_error=serialized_snapshot_or_error
+            )
 
         elif api_name == DagsterCloudApi.GET_SUBSET_EXTERNAL_PIPELINE_RESULT:
             client = self._get_grpc_client(
-                user_code_launcher, deployment_name, cast(str, location_name)
+                user_code_launcher, deployment_name, cast("str", location_name)
             )
 
             serialized_subset_result_or_error = client.external_pipeline_subset(
                 pipeline_subset_snapshot_args=request.request_args
             )
 
-            return DagsterCloudApiGrpcResponse(serialized_subset_result_or_error)
+            return DagsterCloudApiGrpcResponse(
+                serialized_response_or_error=serialized_subset_result_or_error
+            )
         elif api_name == DagsterCloudApi.GET_EXTERNAL_PARTITION_CONFIG:
             client = self._get_grpc_client(
-                user_code_launcher, deployment_name, cast(str, location_name)
+                user_code_launcher, deployment_name, cast("str", location_name)
             )
             serialized_partition_config_or_error = client.external_partition_config(
                 partition_args=request.request_args,
             )
-            return DagsterCloudApiGrpcResponse(serialized_partition_config_or_error)
+            return DagsterCloudApiGrpcResponse(
+                serialized_response_or_error=serialized_partition_config_or_error
+            )
         elif api_name == DagsterCloudApi.GET_EXTERNAL_PARTITION_TAGS:
             client = self._get_grpc_client(
-                user_code_launcher, deployment_name, cast(str, location_name)
+                user_code_launcher, deployment_name, cast("str", location_name)
             )
             serialized_partition_tags_or_error = client.external_partition_tags(
                 partition_args=request.request_args,
             )
-            return DagsterCloudApiGrpcResponse(serialized_partition_tags_or_error)
+            return DagsterCloudApiGrpcResponse(
+                serialized_response_or_error=serialized_partition_tags_or_error
+            )
         elif api_name == DagsterCloudApi.GET_EXTERNAL_PARTITION_NAMES:
             client = self._get_grpc_client(
-                user_code_launcher, deployment_name, cast(str, location_name)
+                user_code_launcher, deployment_name, cast("str", location_name)
             )
             serialized_partition_names_or_error = client.external_partition_names(
                 partition_names_args=request.request_args,
             )
-            return DagsterCloudApiGrpcResponse(serialized_partition_names_or_error)
+            return DagsterCloudApiGrpcResponse(
+                serialized_response_or_error=serialized_partition_names_or_error
+            )
         elif api_name == DagsterCloudApi.GET_EXTERNAL_PARTITION_SET_EXECUTION_PARAM_DATA:
             client = self._get_grpc_client(
-                user_code_launcher, deployment_name, cast(str, location_name)
+                user_code_launcher, deployment_name, cast("str", location_name)
             )
             serialized_partition_execution_params_or_error = (
                 client.external_partition_set_execution_params(
                     partition_set_execution_param_args=request.request_args
                 )
             )
-            return DagsterCloudApiGrpcResponse(serialized_partition_execution_params_or_error)
+            return DagsterCloudApiGrpcResponse(
+                serialized_response_or_error=serialized_partition_execution_params_or_error
+            )
         elif api_name == DagsterCloudApi.GET_EXTERNAL_SCHEDULE_EXECUTION_DATA:
             client = self._get_grpc_client(
-                user_code_launcher, deployment_name, cast(str, location_name)
+                user_code_launcher, deployment_name, cast("str", location_name)
             )
 
             args = request.request_args._replace(
-                instance_ref=self._get_user_code_instance_ref(instance, deployment_name)
+                instance_ref=self._get_user_code_instance_ref(deployment_name)
             )
 
-            serialized_schedule_data_or_error = client.external_schedule_execution(
-                external_schedule_execution_args=args,
-            )
+            schedule_attributes = {
+                "schedule": args.schedule_name,
+                "repository": args.repository_origin.repository_name,
+                "location": args.repository_origin.code_location_origin.location_name,
+                "deployment": deployment_name,
+            }
 
-            return DagsterCloudApiGrpcResponse(serialized_schedule_data_or_error)
+            with observe_execution(
+                opentelemetry=self._instance.opentelemetry,
+                event_key=f"{DAGSTER_CLOUD_AGENT_METRIC_PREFIX}.schedule.evaluation",
+                short_description="schedule evaluation requests",
+                attributes=schedule_attributes,
+                result_evaluator_callback=inspect_schedule_result,
+            ) as observer:
+                serialized_schedule_data_or_error = client.external_schedule_execution(
+                    external_schedule_execution_args=args,
+                )
+                observer.evaluate_result(
+                    serialized_data_or_error=serialized_schedule_data_or_error,
+                )
+                return DagsterCloudApiGrpcResponse(
+                    serialized_response_or_error=serialized_schedule_data_or_error
+                )
 
         elif api_name == DagsterCloudApi.GET_EXTERNAL_SENSOR_EXECUTION_DATA:
             client = self._get_grpc_client(
-                user_code_launcher, deployment_name, cast(str, location_name)
+                user_code_launcher, deployment_name, cast("str", location_name)
             )
 
             args = request.request_args._replace(
-                instance_ref=self._get_user_code_instance_ref(instance, deployment_name)
+                instance_ref=self._get_user_code_instance_ref(deployment_name)
             )
 
-            serialized_sensor_data_or_error = client.external_sensor_execution(
-                sensor_execution_args=args,
-            )
+            sensor_attributes = {
+                "sensor": args.sensor_name,
+                "repository": args.repository_origin.repository_name,
+                "location": args.repository_origin.code_location_origin.location_name,
+                "deployment": deployment_name,
+            }
 
-            return DagsterCloudApiGrpcResponse(serialized_sensor_data_or_error)
+            with observe_execution(
+                opentelemetry=self._instance.opentelemetry,
+                event_key=f"{DAGSTER_CLOUD_AGENT_METRIC_PREFIX}.sensor.evaluation",
+                short_description="sensor evaluation requests",
+                attributes=sensor_attributes,
+                result_evaluator_callback=inspect_sensor_result,
+            ) as observer:
+                serialized_sensor_data_or_error = client.external_sensor_execution(
+                    sensor_execution_args=args,
+                )
+                observer.evaluate_result(serialized_sensor_data_or_error)
+                return DagsterCloudApiGrpcResponse(
+                    serialized_response_or_error=serialized_sensor_data_or_error
+                )
         elif api_name == DagsterCloudApi.GET_EXTERNAL_NOTEBOOK_DATA:
             client = self._get_grpc_client(
-                user_code_launcher, deployment_name, cast(str, location_name)
+                user_code_launcher, deployment_name, cast("str", location_name)
             )
             response = client.external_notebook_data(request.request_args.notebook_path)
-            return DagsterCloudApiGrpcResponse(response.decode())
+            return DagsterCloudApiGrpcResponse(serialized_response_or_error=response.decode())
         elif api_name == DagsterCloudApi.LAUNCH_RUN:
             run = request.request_args.dagster_run
 
             with DagsterInstance.from_ref(
-                self._get_user_code_instance_ref(instance, deployment_name)
+                self._get_user_code_instance_ref(deployment_name)
             ) as scoped_instance:
                 scoped_instance.report_engine_event(
-                    f"{instance.agent_display_name} is launching run {run.run_id}",
+                    f"{self._instance.agent_display_name} is launching run {run.run_id}",
                     run,
                     cls=self.__class__,
                 )
@@ -852,40 +995,50 @@ class DagsterCloudAgent:
                     run.run_id,
                     merge_dicts(
                         (
-                            {"dagster/agent_label": instance.dagster_cloud_api_agent_label}
-                            if instance.dagster_cloud_api_agent_label
+                            {"dagster/agent_label": self._instance.dagster_cloud_api_agent_label}
+                            if self._instance.dagster_cloud_api_agent_label
                             else {}
                         ),
-                        {"dagster/agent_id": instance.instance_uuid},
+                        {"dagster/agent_id": self._instance.instance_uuid},
                     ),
                 )
 
-                launcher = scoped_instance.get_run_launcher_for_run(run)  # type: ignore  # (instance subclass)
+                run_attributes = extract_run_attributes(deployment_name, run)
+                with observe_execution(
+                    opentelemetry=self._instance.opentelemetry,
+                    event_key=f"{DAGSTER_CLOUD_AGENT_METRIC_PREFIX}.run.launches",
+                    short_description="run execution requests",
+                    attributes=run_attributes,
+                ) as observer:
+                    launcher = scoped_instance.get_run_launcher_for_run(run)  # type: ignore  # (instance subclass)
 
-                if is_isolated_run(run):
-                    launcher.launch_run(LaunchRunContext(dagster_run=run, workspace=None))
-                else:
-                    scoped_instance.report_engine_event(
-                        f"Launching {run.run_id} without an isolated run environment.",
-                        run,
-                        cls=self.__class__,
-                    )
+                    if is_isolated_run(run):
+                        launcher.launch_run(LaunchRunContext(dagster_run=run, workspace=None))
+                    else:
+                        scoped_instance.report_engine_event(
+                            f"Launching {run.run_id} without an isolated run environment.",
+                            run,
+                            cls=self.__class__,
+                        )
 
-                    run_location_name = cast(
-                        str,
-                        run.external_job_origin.repository_origin.code_location_origin.location_name,
-                    )
+                        run_location_name = cast(
+                            "str",
+                            run.remote_job_origin.repository_origin.code_location_origin.location_name,
+                        )
 
-                    server = user_code_launcher.get_grpc_server(deployment_name, run_location_name)
+                        server = user_code_launcher.get_grpc_server(
+                            deployment_name, run_location_name
+                        )
 
-                    # Record the server handle that we launched it on to for run monitoring
-                    scoped_instance.add_run_tags(
-                        run.run_id, new_tags={SERVER_HANDLE_TAG: str(server.server_handle)}
-                    )
+                        # Record the server handle that we launched it on to for run monitoring
+                        scoped_instance.add_run_tags(
+                            run.run_id, new_tags={SERVER_HANDLE_TAG: str(server.server_handle)}
+                        )
 
-                    launcher.launch_run_from_grpc_client(
-                        scoped_instance, run, server.server_endpoint.create_client()
-                    )
+                        launcher.launch_run_from_grpc_client(
+                            scoped_instance, run, server.server_endpoint.create_client()
+                        )
+                    observer.evaluate_result(run=run)
 
                 return DagsterCloudApiSuccess()
         elif api_name == DagsterCloudApi.TERMINATE_RUN:
@@ -896,18 +1049,18 @@ class DagsterCloudAgent:
             run = request.request_args.dagster_run
 
             with DagsterInstance.from_ref(
-                instance.ref_for_deployment(deployment_name)
+                self._instance.ref_for_deployment(deployment_name)
             ) as scoped_instance:
-                if instance.is_using_isolated_agents:
+                if self._instance.is_using_isolated_agents:
                     scoped_instance.report_engine_event(
-                        f"{instance.agent_display_name} received request to mark run as canceling",
+                        f"{self._instance.agent_display_name} received request to mark run as canceling",
                         run,
                         cls=self.__class__,
                     )
                     scoped_instance.report_run_canceling(run)
                 else:
                     scoped_instance.report_engine_event(
-                        f"{instance.agent_display_name} received request to terminate run",
+                        f"{self._instance.agent_display_name} received request to terminate run",
                         run,
                         cls=self.__class__,
                     )
@@ -916,8 +1069,8 @@ class DagsterCloudAgent:
                         launcher.terminate(run.run_id)
                     else:
                         run_location_name = cast(
-                            str,
-                            run.external_job_origin.repository_origin.code_location_origin.location_name,
+                            "str",
+                            run.remote_job_origin.repository_origin.code_location_origin.location_name,
                         )
 
                         server = user_code_launcher.get_grpc_server(
@@ -929,15 +1082,20 @@ class DagsterCloudAgent:
                         client.cancel_execution(CancelExecutionRequest(run_id=run.run_id))
 
             return DagsterCloudApiSuccess()
-
+        elif api_name in (
+            DagsterCloudApi.CHECK_STEP_HEALTH,
+            DagsterCloudApi.TERMINATE_STEP,
+            DagsterCloudApi.LAUNCH_STEP,
+            DagsterCloudApi.CHECK_RUN_HEALTH,
+            DagsterCloudApi.LOAD_REPOSITORIES,
+        ):
+            check.failed(f"Unexpected deprecated request type {api_name}")
         else:
             check.assert_never(api_name)
-            raise Exception(f"Unexpected dagster cloud api call {api_name}")
 
     def _process_api_request(
         self,
-        json_request: Dict,
-        instance: DagsterCloudAgentInstance,
+        json_request: dict,
         user_code_launcher: DagsterCloudUserCodeLauncher,
         submitted_to_executor_timestamp: float,
     ) -> Optional[SerializableErrorInfo]:
@@ -956,7 +1114,7 @@ class DagsterCloudAgent:
         )
 
         if request_api not in DagsterCloudApi.__members__:
-            api_result = DagsterCloudApiUnknownCommandResponse(request_api)
+            api_result = DagsterCloudApiUnknownCommandResponse(request_api=request_api)
             self._logger.warning(
                 f"Ignoring request {json_request}: Unknown command. This is likely due to running an "
                 "older version of the agent."
@@ -966,7 +1124,7 @@ class DagsterCloudAgent:
                 request = deserialize_value(request_body, DagsterCloudApiRequest)
                 self._logger.info(f"Received request {request}.")
                 api_result = self._handle_api_request(
-                    request, deployment_name, is_branch_deployment, instance, user_code_launcher
+                    request, deployment_name, is_branch_deployment, user_code_launcher
                 )
             except Exception:
                 error_info = serializable_error_info_from_exc_info(sys.exc_info())
@@ -995,13 +1153,28 @@ class DagsterCloudAgent:
 
         self._logger.info(f"Uploading response for request {request}.")
 
-        upload_api_response(instance, deployment_name, upload_response)
+        if UPLOAD_API_RESPONSE_BATCHING_ENABLED():
+            self._batcher[deployment_name].submit((deployment_name, upload_response))
+        else:
+            upload_api_response(self._instance, deployment_name, upload_response)
 
         self._logger.info(f"Finished uploading response for request {request}.")
 
         return error_info
 
-    def _get_location_from_request(self, json_request: Dict[str, Any]) -> Optional[str]:
+    def _batch_upload_api_response(
+        self, upload_response_batch: list[tuple[str, DagsterCloudUploadApiResponse]]
+    ) -> list[None]:
+        deployment_names = set(deployment_name for deployment_name, _ in upload_response_batch)
+        assert len(deployment_names) == 1
+        batch_upload_api_response(
+            self._instance,
+            next(iter(deployment_names)),
+            [resp for _, resp in upload_response_batch],
+        )
+        return [None for _ in upload_response_batch]
+
+    def _get_location_from_request(self, json_request: dict[str, Any]) -> Optional[str]:
         request_api = json_request["requestApi"]
         request_body = json_request["requestBody"]
         if request_api not in DagsterCloudApi.__members__:
@@ -1015,7 +1188,7 @@ class DagsterCloudAgent:
         return location_origin.location_name
 
     def run_iteration(
-        self, instance: DagsterCloudAgentInstance, user_code_launcher: DagsterCloudUserCodeLauncher
+        self, user_code_launcher: DagsterCloudUserCodeLauncher
     ) -> Iterator[Optional[SerializableErrorInfo]]:
         if not user_code_launcher.ready_to_serve_requests:
             return
@@ -1024,12 +1197,12 @@ class DagsterCloudAgent:
 
         if num_pending_requests < self._pending_requests_limit:
             # limit (implicit default 10) applied separately for requests to branch deployments, and for each full deployment
-            result = instance.organization_scoped_graphql_client().execute(
+            result = self._instance.organization_scoped_graphql_client().execute(
                 GET_USER_CLOUD_REQUESTS_QUERY,
                 {
-                    "forBranchDeployments": instance.includes_branch_deployments,
+                    "forBranchDeployments": self._instance.includes_branch_deployments,
                     "forFullDeployments": self._active_full_deployment_names,
-                    "agentQueues": instance.agent_queues_config.queues,
+                    "agentQueues": self._instance.agent_queues_config.queues,
                 },
             )
             json_requests = result["data"]["userCloudAgent"]["popUserCloudAgentRequests"]
@@ -1082,7 +1255,6 @@ class DagsterCloudAgent:
             future = self._executor.submit(
                 self._process_api_request,
                 json_request,
-                instance,
                 user_code_launcher,
                 submitted_to_executor_timestamp,
             )
@@ -1109,7 +1281,7 @@ class DagsterCloudAgent:
                 if response:
                     yield response
 
-        if instance.user_code_launcher.agent_metrics_enabled and (
+        if self._instance.user_code_launcher.agent_metrics_enabled and (
             self._utilization_metrics["container_utilization"]["measurement_timestamp"] is None
             or (
                 get_current_timestamp()
@@ -1123,6 +1295,22 @@ class DagsterCloudAgent:
         yield None
 
 
+def batch_upload_api_response(
+    instance: DagsterCloudAgentInstance,
+    deployment_name: str,
+    batch: list[DagsterCloudUploadApiResponse],
+):
+    with compressed_namedtuple_upload_file(BatchDagsterCloudUploadApiResponse(batch=batch)) as f:
+        resp = instance.requests_managed_retries_session.put(
+            instance.dagster_cloud_upload_api_response_url,
+            headers=instance.headers_for_deployment(deployment_name),
+            files={"api_response_batch.tmp": f},
+            timeout=instance.dagster_cloud_api_timeout,
+            proxies=instance.dagster_cloud_api_proxies,
+        )
+        raise_http_error(resp)
+
+
 def upload_api_response(
     instance: DagsterCloudAgentInstance,
     deployment_name: str,