dagster-cloud 1.8.2__py3-none-any.whl → 1.12.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. dagster_cloud/__init__.py +3 -3
  2. dagster_cloud/agent/__init__.py +4 -4
  3. dagster_cloud/agent/cli/__init__.py +56 -17
  4. dagster_cloud/agent/dagster_cloud_agent.py +360 -172
  5. dagster_cloud/agent/instrumentation/__init__.py +0 -0
  6. dagster_cloud/agent/instrumentation/constants.py +2 -0
  7. dagster_cloud/agent/instrumentation/run_launch.py +23 -0
  8. dagster_cloud/agent/instrumentation/schedule.py +34 -0
  9. dagster_cloud/agent/instrumentation/sensor.py +34 -0
  10. dagster_cloud/anomaly_detection/__init__.py +2 -2
  11. dagster_cloud/anomaly_detection/defs.py +17 -12
  12. dagster_cloud/anomaly_detection/types.py +3 -3
  13. dagster_cloud/api/dagster_cloud_api.py +209 -293
  14. dagster_cloud/auth/constants.py +21 -5
  15. dagster_cloud/batching/__init__.py +1 -0
  16. dagster_cloud/batching/batcher.py +210 -0
  17. dagster_cloud/dagster_insights/__init__.py +12 -6
  18. dagster_cloud/dagster_insights/bigquery/bigquery_utils.py +3 -2
  19. dagster_cloud/dagster_insights/bigquery/dbt_wrapper.py +39 -12
  20. dagster_cloud/dagster_insights/bigquery/insights_bigquery_resource.py +8 -6
  21. dagster_cloud/dagster_insights/insights_utils.py +18 -8
  22. dagster_cloud/dagster_insights/metrics_utils.py +12 -12
  23. dagster_cloud/dagster_insights/snowflake/dagster_snowflake_insights.py +5 -12
  24. dagster_cloud/dagster_insights/snowflake/dbt_wrapper.py +34 -8
  25. dagster_cloud/dagster_insights/snowflake/definitions.py +38 -12
  26. dagster_cloud/dagster_insights/snowflake/insights_snowflake_resource.py +11 -23
  27. dagster_cloud/definitions/__init__.py +0 -0
  28. dagster_cloud/definitions/job_selection.py +36 -0
  29. dagster_cloud/execution/cloud_run_launcher/k8s.py +1 -1
  30. dagster_cloud/execution/cloud_run_launcher/process.py +3 -3
  31. dagster_cloud/execution/monitoring/__init__.py +27 -33
  32. dagster_cloud/execution/utils/process.py +3 -3
  33. dagster_cloud/instance/__init__.py +125 -38
  34. dagster_cloud/instrumentation/__init__.py +32 -0
  35. dagster_cloud/metadata/source_code.py +13 -8
  36. dagster_cloud/metrics/__init__.py +0 -0
  37. dagster_cloud/metrics/tracer.py +59 -0
  38. dagster_cloud/opentelemetry/__init__.py +0 -0
  39. dagster_cloud/opentelemetry/config/__init__.py +73 -0
  40. dagster_cloud/opentelemetry/config/exporter.py +81 -0
  41. dagster_cloud/opentelemetry/config/log_record_processor.py +40 -0
  42. dagster_cloud/opentelemetry/config/logging_handler.py +14 -0
  43. dagster_cloud/opentelemetry/config/meter_provider.py +9 -0
  44. dagster_cloud/opentelemetry/config/metric_reader.py +39 -0
  45. dagster_cloud/opentelemetry/controller.py +319 -0
  46. dagster_cloud/opentelemetry/enum.py +58 -0
  47. dagster_cloud/opentelemetry/factories/__init__.py +1 -0
  48. dagster_cloud/opentelemetry/factories/logs.py +113 -0
  49. dagster_cloud/opentelemetry/factories/metrics.py +121 -0
  50. dagster_cloud/opentelemetry/metrics/__init__.py +0 -0
  51. dagster_cloud/opentelemetry/metrics/meter.py +140 -0
  52. dagster_cloud/opentelemetry/observers/__init__.py +0 -0
  53. dagster_cloud/opentelemetry/observers/dagster_exception_handler.py +40 -0
  54. dagster_cloud/opentelemetry/observers/execution_observer.py +178 -0
  55. dagster_cloud/pex/grpc/__generated__/multi_pex_api_pb2.pyi +175 -0
  56. dagster_cloud/pex/grpc/__init__.py +2 -2
  57. dagster_cloud/pex/grpc/client.py +4 -4
  58. dagster_cloud/pex/grpc/compile.py +2 -2
  59. dagster_cloud/pex/grpc/server/__init__.py +2 -2
  60. dagster_cloud/pex/grpc/server/cli/__init__.py +31 -19
  61. dagster_cloud/pex/grpc/server/manager.py +60 -42
  62. dagster_cloud/pex/grpc/server/registry.py +28 -21
  63. dagster_cloud/pex/grpc/server/server.py +23 -14
  64. dagster_cloud/pex/grpc/types.py +5 -5
  65. dagster_cloud/py.typed +0 -0
  66. dagster_cloud/secrets/__init__.py +1 -1
  67. dagster_cloud/secrets/loader.py +3 -3
  68. dagster_cloud/serverless/__init__.py +1 -1
  69. dagster_cloud/serverless/io_manager.py +36 -53
  70. dagster_cloud/storage/client.py +54 -17
  71. dagster_cloud/storage/compute_logs/__init__.py +3 -1
  72. dagster_cloud/storage/compute_logs/compute_log_manager.py +22 -17
  73. dagster_cloud/storage/defs_state/__init__.py +3 -0
  74. dagster_cloud/storage/defs_state/queries.py +15 -0
  75. dagster_cloud/storage/defs_state/storage.py +113 -0
  76. dagster_cloud/storage/event_logs/__init__.py +3 -1
  77. dagster_cloud/storage/event_logs/queries.py +102 -4
  78. dagster_cloud/storage/event_logs/storage.py +266 -73
  79. dagster_cloud/storage/event_logs/utils.py +88 -7
  80. dagster_cloud/storage/runs/__init__.py +1 -1
  81. dagster_cloud/storage/runs/queries.py +17 -2
  82. dagster_cloud/storage/runs/storage.py +88 -42
  83. dagster_cloud/storage/schedules/__init__.py +1 -1
  84. dagster_cloud/storage/schedules/storage.py +6 -8
  85. dagster_cloud/storage/tags.py +66 -1
  86. dagster_cloud/util/__init__.py +10 -12
  87. dagster_cloud/util/errors.py +49 -64
  88. dagster_cloud/version.py +1 -1
  89. dagster_cloud/workspace/config_schema/__init__.py +55 -13
  90. dagster_cloud/workspace/docker/__init__.py +76 -25
  91. dagster_cloud/workspace/docker/utils.py +1 -1
  92. dagster_cloud/workspace/ecs/__init__.py +1 -1
  93. dagster_cloud/workspace/ecs/client.py +51 -33
  94. dagster_cloud/workspace/ecs/launcher.py +76 -22
  95. dagster_cloud/workspace/ecs/run_launcher.py +3 -3
  96. dagster_cloud/workspace/ecs/utils.py +14 -5
  97. dagster_cloud/workspace/kubernetes/__init__.py +1 -1
  98. dagster_cloud/workspace/kubernetes/launcher.py +61 -29
  99. dagster_cloud/workspace/kubernetes/utils.py +34 -22
  100. dagster_cloud/workspace/user_code_launcher/__init__.py +5 -3
  101. dagster_cloud/workspace/user_code_launcher/process.py +16 -14
  102. dagster_cloud/workspace/user_code_launcher/user_code_launcher.py +552 -172
  103. dagster_cloud/workspace/user_code_launcher/utils.py +105 -1
  104. {dagster_cloud-1.8.2.dist-info → dagster_cloud-1.12.6.dist-info}/METADATA +48 -42
  105. dagster_cloud-1.12.6.dist-info/RECORD +134 -0
  106. {dagster_cloud-1.8.2.dist-info → dagster_cloud-1.12.6.dist-info}/WHEEL +1 -1
  107. dagster_cloud-1.8.2.dist-info/RECORD +0 -100
  108. {dagster_cloud-1.8.2.dist-info → dagster_cloud-1.12.6.dist-info}/top_level.txt +0 -0
@@ -1,15 +1,17 @@
1
1
  import enum
2
+ from collections.abc import Mapping, Sequence
2
3
  from datetime import timedelta
3
4
  from enum import Enum
4
- from typing import Any, List, Mapping, NamedTuple, Optional, Sequence, TypedDict, Union, cast
5
+ from typing import Any, Optional, TypedDict, Union
5
6
 
6
- import dagster._check as check
7
7
  from dagster._core.code_pointer import CodePointer
8
8
  from dagster._core.definitions.selector import JobSelector
9
9
  from dagster._core.events.log import EventLogEntry
10
- from dagster._core.remote_representation import CodeLocationOrigin, ExternalRepositoryData
10
+ from dagster._core.remote_origin import CodeLocationOrigin
11
+ from dagster._core.remote_representation.external_data import RepositorySnap
11
12
  from dagster._core.storage.dagster_run import DagsterRun
12
13
  from dagster._core.utils import RequestUtilizationMetrics
14
+ from dagster._record import IHaveNew, copy, record, record_custom
13
15
  from dagster._serdes import whitelist_for_serdes
14
16
  from dagster._time import get_current_datetime, get_current_timestamp
15
17
  from dagster._utils.container import ContainerUtilizationMetrics
@@ -29,7 +31,8 @@ DEFAULT_EXPIRATION_MILLISECONDS = 10 * 60 * 1000
29
31
 
30
32
 
31
33
  @whitelist_for_serdes
32
- class DagsterCloudUploadRepositoryData(NamedTuple):
34
+ @record
35
+ class DagsterCloudUploadRepositoryData:
33
36
  """Serialized object uploaded by the Dagster Cloud agent with information pulled
34
37
  from a gRPC server about an individual repository - the data field is serialized since the
35
38
  agent may be running older code that doesn't know how to deserialize it, so it passes
@@ -42,21 +45,23 @@ class DagsterCloudUploadRepositoryData(NamedTuple):
42
45
 
43
46
 
44
47
  @whitelist_for_serdes
45
- class DagsterCloudUploadLocationData(NamedTuple):
48
+ @record
49
+ class DagsterCloudUploadLocationData:
46
50
  """Serialized object uploaded by the Dagster Cloud agent with information pulled
47
51
  about a successfully loaded repository location, including information about
48
52
  each repository as well as shared metadata like the image to use when launching
49
53
  runs in this location.
50
54
  """
51
55
 
52
- upload_repository_datas: List[DagsterCloudUploadRepositoryData]
56
+ upload_repository_datas: list[DagsterCloudUploadRepositoryData]
53
57
  container_image: Optional[str]
54
58
  executable_path: Optional[str]
55
59
  dagster_library_versions: Optional[Mapping[str, str]] = None
56
60
 
57
61
 
58
62
  @whitelist_for_serdes(storage_field_names={"code_location_deploy_data": "deployment_metadata"})
59
- class DagsterCloudUploadWorkspaceEntry(NamedTuple):
63
+ @record
64
+ class DagsterCloudUploadWorkspaceEntry:
60
65
  """Serialized object uploaded by the Dagster Cloud agent with information about
61
66
  a repository location - either the serialized DagsterCloudUploadLocationData
62
67
  if the location loaded succesfully, or a SerializableErrorInfo describing the
@@ -70,7 +75,8 @@ class DagsterCloudUploadWorkspaceEntry(NamedTuple):
70
75
 
71
76
 
72
77
  @whitelist_for_serdes
73
- class DagsterCloudUploadWorkspaceResponse(NamedTuple):
78
+ @record
79
+ class DagsterCloudUploadWorkspaceResponse:
74
80
  updated: bool
75
81
  message: str
76
82
  missing_job_snapshots: Optional[Sequence[JobSelector]]
@@ -79,7 +85,6 @@ class DagsterCloudUploadWorkspaceResponse(NamedTuple):
79
85
  @whitelist_for_serdes
80
86
  class DagsterCloudApi(Enum):
81
87
  CHECK_FOR_WORKSPACE_UPDATES = "CHECK_FOR_WORKSPACE_UPDATES"
82
- LOAD_REPOSITORIES = "LOAD_REPOSITORIES"
83
88
  GET_EXTERNAL_EXECUTION_PLAN = "GET_EXTERNAL_EXECUTION_PLAN"
84
89
  GET_SUBSET_EXTERNAL_PIPELINE_RESULT = "GET_SUBSET_EXTERNAL_PIPELINE_RESULT"
85
90
  GET_EXTERNAL_PARTITION_CONFIG = "GET_EXTERNAL_PARTITION_CONFIG"
@@ -93,43 +98,26 @@ class DagsterCloudApi(Enum):
93
98
  GET_EXTERNAL_NOTEBOOK_DATA = "GET_EXTERNAL_NOTEBOOK_DATA"
94
99
 
95
100
  LAUNCH_RUN = "LAUNCH_RUN"
96
- CHECK_RUN_HEALTH = "CHECK_RUN_HEALTH" # deprecated, agents now surface this in heartbeats
97
101
  TERMINATE_RUN = "TERMINATE_RUN"
102
+
103
+ PING_LOCATION = "PING_LOCATION" # Signal that a location is in use and should keep servers up
104
+
105
+ CHECK_RUN_HEALTH = "CHECK_RUN_HEALTH" # deprecated, agents now surface this in heartbeats
98
106
  LAUNCH_STEP = "LAUNCH_STEP" # deprecated with cloud executor
99
107
  CHECK_STEP_HEALTH = "CHECK_STEP_HEALTH" # deprecated with cloud executor
100
108
  TERMINATE_STEP = "TERMINATE_STEP" # deprecated with cloud executor
101
-
102
- PING_LOCATION = "PING_LOCATION" # Signal that a location is in use and should keep servers up
109
+ LOAD_REPOSITORIES = "LOAD_REPOSITORIES" # deprecated
103
110
 
104
111
  def __structlog__(self):
105
112
  return self.name
106
113
 
107
114
 
108
115
  @whitelist_for_serdes
109
- class DagsterCloudApiThreadTelemetry(
110
- NamedTuple(
111
- "_DagsterCloudApiThreadTelemetry",
112
- [
113
- ("submitted_to_executor_timestamp", float),
114
- ("thread_start_run_timestamp", float),
115
- ("thread_end_handle_api_request_timestamp", float),
116
- ],
117
- )
118
- ):
119
- def __new__(
120
- cls,
121
- submitted_to_executor_timestamp: float,
122
- thread_start_run_timestamp: float,
123
- thread_end_handle_api_request_timestamp: float,
124
- ):
125
- return super(cls, DagsterCloudApiThreadTelemetry).__new__(
126
- cls,
127
- check.float_param(submitted_to_executor_timestamp, "submitted_to_executor_timestamp"),
128
- check.float_param(thread_start_run_timestamp, "thread_start_run_timestamp"),
129
- check.float_param(
130
- thread_end_handle_api_request_timestamp, "thread_end_handle_api_request_timestamp"
131
- ),
132
- )
116
+ @record
117
+ class DagsterCloudApiThreadTelemetry:
118
+ submitted_to_executor_timestamp: float
119
+ thread_start_run_timestamp: float
120
+ thread_end_handle_api_request_timestamp: float
133
121
 
134
122
  @property
135
123
  def time_to_thread_initialization_seconds(self) -> float:
@@ -140,214 +128,102 @@ class DagsterCloudApiThreadTelemetry(
140
128
  return self.thread_end_handle_api_request_timestamp - self.thread_start_run_timestamp
141
129
 
142
130
 
143
- @whitelist_for_serdes(old_storage_names={"CheckForWorkspaceUpdatesRequest"})
144
- class DagsterCloudApiSuccess(
145
- NamedTuple(
146
- "_DagsterCloudApiSuccess", [("thread_telemetry", Optional[DagsterCloudApiThreadTelemetry])]
147
- )
148
- ):
149
- def __new__(cls, thread_telemetry: Optional[DagsterCloudApiThreadTelemetry] = None):
150
- return super(cls, DagsterCloudApiSuccess).__new__(
151
- cls,
152
- check.opt_inst_param(
153
- thread_telemetry, "thread_telemetry", DagsterCloudApiThreadTelemetry
154
- ),
155
- )
131
+ @whitelist_for_serdes(old_storage_names={"CheckForCodeLocationUpdatesRequest"})
132
+ @record
133
+ class DagsterCloudApiSuccess:
134
+ thread_telemetry: Optional[DagsterCloudApiThreadTelemetry] = None
156
135
 
157
136
  def with_thread_telemetry(self, thread_telemetry: DagsterCloudApiThreadTelemetry):
158
- return self._replace(thread_telemetry=thread_telemetry)
137
+ return copy(self, thread_telemetry=thread_telemetry)
159
138
 
160
139
 
161
140
  @whitelist_for_serdes
162
- class DagsterCloudApiUnknownCommandResponse(
163
- NamedTuple(
164
- "_DagsterCloudApiUnknownCommandResponse",
165
- [("request_api", str), ("thread_telemetry", Optional[DagsterCloudApiThreadTelemetry])],
166
- )
167
- ):
168
- def __new__(
169
- cls, request_api: str, thread_telemetry: Optional[DagsterCloudApiThreadTelemetry] = None
170
- ):
171
- return super(cls, DagsterCloudApiUnknownCommandResponse).__new__(
172
- cls,
173
- check.str_param(request_api, "request_api"),
174
- check.opt_inst_param(
175
- thread_telemetry, "thread_telemetry", DagsterCloudApiThreadTelemetry
176
- ),
177
- )
141
+ @record
142
+ class DagsterCloudApiUnknownCommandResponse:
143
+ request_api: str
144
+ thread_telemetry: Optional[DagsterCloudApiThreadTelemetry] = None
178
145
 
179
146
  def with_thread_telemetry(self, thread_telemetry: DagsterCloudApiThreadTelemetry):
180
- return self._replace(thread_telemetry=thread_telemetry)
147
+ return copy(self, thread_telemetry=thread_telemetry)
181
148
 
182
149
 
183
150
  @whitelist_for_serdes
184
- class DagsterCloudApiErrorResponse(
185
- NamedTuple(
186
- "_DagsterCloudApiErrorResponse",
187
- [
188
- ("error_infos", List[SerializableErrorInfo]),
189
- ("thread_telemetry", Optional[DagsterCloudApiThreadTelemetry]),
190
- ],
191
- )
192
- ):
193
- def __new__(
194
- cls,
195
- error_infos: List[SerializableErrorInfo],
196
- thread_telemetry: Optional[DagsterCloudApiThreadTelemetry] = None,
197
- ):
198
- return super(cls, DagsterCloudApiErrorResponse).__new__(
199
- cls,
200
- check.list_param(error_infos, "error_infos", of_type=SerializableErrorInfo),
201
- check.opt_inst_param(
202
- thread_telemetry, "thread_telemetry", DagsterCloudApiThreadTelemetry
203
- ),
204
- )
151
+ @record
152
+ class DagsterCloudApiErrorResponse:
153
+ error_infos: list[SerializableErrorInfo]
154
+ thread_telemetry: Optional[DagsterCloudApiThreadTelemetry] = None
205
155
 
206
156
  def with_thread_telemetry(self, thread_telemetry: DagsterCloudApiThreadTelemetry):
207
- return self._replace(thread_telemetry=thread_telemetry)
157
+ return copy(self, thread_telemetry=thread_telemetry)
208
158
 
209
159
 
210
160
  @whitelist_for_serdes
211
- class DagsterCloudApiGrpcResponse(
212
- NamedTuple(
213
- "_DagsterCloudApiGrpcResponse",
214
- [
215
- ("serialized_response_or_error", str),
216
- ("thread_telemetry", Optional[DagsterCloudApiThreadTelemetry]),
217
- ],
218
- )
219
- ):
161
+ @record
162
+ class DagsterCloudApiGrpcResponse:
220
163
  # Class that DagsterCloudApi methods can use to pass along the result of
221
164
  # a gRPC call against the user code server. The field here is passed in
222
165
  # serialized as a string, because we can't guarantee that the agent code will
223
166
  # be up-to-date enough to know how to deserialize it (but the host cloud always
224
167
  # should, since it will always be up to date).
225
- def __new__(
226
- cls,
227
- serialized_response_or_error: str,
228
- thread_telemetry: Optional[DagsterCloudApiThreadTelemetry] = None,
229
- ):
230
- return super(cls, DagsterCloudApiGrpcResponse).__new__(
231
- cls,
232
- check.str_param(serialized_response_or_error, "serialized_response_or_error"),
233
- check.opt_inst_param(
234
- thread_telemetry, "thread_telemetry", DagsterCloudApiThreadTelemetry
235
- ),
236
- )
168
+ serialized_response_or_error: str
169
+ thread_telemetry: Optional[DagsterCloudApiThreadTelemetry] = None
237
170
 
238
171
  def with_thread_telemetry(self, thread_telemetry: DagsterCloudApiThreadTelemetry):
239
- return self._replace(thread_telemetry=thread_telemetry)
172
+ return copy(self, thread_telemetry=thread_telemetry)
240
173
 
241
174
 
242
175
  @whitelist_for_serdes
243
- class LoadRepositoriesArgs(
244
- NamedTuple("_LoadRepositoryArgs", [("location_origin", CodeLocationOrigin)])
245
- ):
246
- def __new__(cls, location_origin):
247
- return super(cls, LoadRepositoriesArgs).__new__(
248
- cls,
249
- check.inst_param(location_origin, "location_origin", CodeLocationOrigin),
250
- )
176
+ @record
177
+ class LoadRepositoriesArgs:
178
+ location_origin: CodeLocationOrigin
251
179
 
252
180
 
253
181
  @whitelist_for_serdes
254
- class DagsterCloudRepositoryData(
255
- NamedTuple(
256
- "_DagsterCloudRepositoryData",
257
- [
258
- ("repo_name", str),
259
- ("code_pointer", CodePointer),
260
- ("external_repository_data", ExternalRepositoryData),
261
- ],
262
- )
263
- ):
264
- def __new__(cls, repo_name, code_pointer, external_repository_data):
265
- return super(cls, DagsterCloudRepositoryData).__new__(
266
- cls,
267
- check.str_param(repo_name, "repo_name"),
268
- check.inst_param(code_pointer, "code_pointer", CodePointer),
269
- check.inst_param(
270
- external_repository_data,
271
- "external_repository_data",
272
- ExternalRepositoryData,
273
- ),
274
- )
182
+ @record
183
+ class DagsterCloudRepositoryData:
184
+ repo_name: str
185
+ code_pointer: CodePointer
186
+ external_repository_data: RepositorySnap
275
187
 
276
188
 
277
189
  @whitelist_for_serdes(storage_field_names={"code_location_deploy_data": "code_deployment_metadata"})
278
- class LoadRepositoriesResponse(
279
- NamedTuple(
280
- "_LoadRepositoriesResponse",
281
- [
282
- ("repository_datas", Sequence[DagsterCloudRepositoryData]),
283
- ("container_image", Optional[str]),
284
- ("executable_path", Optional[str]),
285
- ("code_location_deploy_data", Optional[CodeLocationDeployData]),
286
- ("dagster_library_versions", Optional[Mapping[str, str]]),
287
- ],
288
- )
289
- ):
290
- def __new__(
291
- cls,
292
- repository_datas,
293
- container_image,
294
- executable_path,
295
- code_location_deploy_data=None,
296
- dagster_library_versions: Optional[Mapping[str, str]] = None,
297
- ):
298
- return super(cls, LoadRepositoriesResponse).__new__(
299
- cls,
300
- check.list_param(
301
- repository_datas,
302
- "repository_datas",
303
- of_type=DagsterCloudRepositoryData,
304
- ),
305
- check.opt_str_param(container_image, "container_image"),
306
- check.opt_str_param(executable_path, "executable_path"),
307
- check.opt_inst_param(
308
- code_location_deploy_data, "code_location_deploy_data", CodeLocationDeployData
309
- ),
310
- check.opt_nullable_mapping_param(dagster_library_versions, "dagster_library_versions"),
311
- )
190
+ @record
191
+ class LoadRepositoriesResponse:
192
+ repository_datas: Sequence[DagsterCloudRepositoryData]
193
+ container_image: Optional[str]
194
+ executable_path: Optional[str]
195
+ code_location_deploy_data: Optional[CodeLocationDeployData] = None
196
+ dagster_library_versions: Optional[Mapping[str, str]] = None
312
197
 
313
198
 
314
199
  @whitelist_for_serdes
315
- class PingLocationArgs(NamedTuple("_PingLocationArgs", [("location_name", str)])):
316
- pass
200
+ @record
201
+ class PingLocationArgs:
202
+ location_name: str
317
203
 
318
204
 
319
205
  @whitelist_for_serdes(storage_field_names={"dagster_run": "pipeline_run"})
320
- class LaunchRunArgs(NamedTuple("_LaunchRunArgs", [("dagster_run", DagsterRun)])):
321
- def __new__(cls, dagster_run):
322
- return super(cls, LaunchRunArgs).__new__(
323
- cls,
324
- check.inst_param(dagster_run, "dagster_run", DagsterRun),
325
- )
206
+ @record
207
+ class LaunchRunArgs:
208
+ dagster_run: DagsterRun
326
209
 
327
210
 
328
211
  @whitelist_for_serdes(storage_field_names={"dagster_run": "pipeline_run"})
329
- class TerminateRunArgs(NamedTuple("_TerminateRunArgs", [("dagster_run", DagsterRun)])):
330
- def __new__(cls, dagster_run):
331
- return super(cls, TerminateRunArgs).__new__(
332
- cls,
333
- check.inst_param(dagster_run, "dagster_run", DagsterRun),
334
- )
212
+ @record
213
+ class TerminateRunArgs:
214
+ dagster_run: DagsterRun
335
215
 
336
216
 
337
217
  @whitelist_for_serdes
338
- class DagsterCloudApiRequest(
339
- NamedTuple(
340
- "_DagsterCloudApiRequest",
341
- [
342
- ("request_id", str),
343
- ("request_api", DagsterCloudApi),
344
- ("request_args", Any),
345
- ("deployment_name", str),
346
- ("expire_at", float),
347
- ("is_branch_deployment", bool),
348
- ],
349
- )
350
- ):
218
+ @record_custom
219
+ class DagsterCloudApiRequest(IHaveNew):
220
+ request_id: str
221
+ request_api: DagsterCloudApi
222
+ request_args: Any
223
+ deployment_name: str
224
+ expire_at: float
225
+ is_branch_deployment: bool
226
+
351
227
  def __new__(
352
228
  cls,
353
229
  request_id: str,
@@ -357,20 +233,18 @@ class DagsterCloudApiRequest(
357
233
  expire_at: Optional[float] = None,
358
234
  is_branch_deployment: Optional[bool] = None,
359
235
  ):
360
- return super(cls, DagsterCloudApiRequest).__new__(
236
+ return super().__new__(
361
237
  cls,
362
- check.str_param(request_id, "request_id"),
363
- check.inst_param(request_api, "request_api", DagsterCloudApi),
364
- request_args,
365
- check.str_param(deployment_name, "deployment_name"),
366
- check.opt_float_param(
367
- expire_at,
368
- "expire_at",
369
- default=(
370
- get_current_datetime() + timedelta(milliseconds=DEFAULT_EXPIRATION_MILLISECONDS)
371
- ).timestamp(),
372
- ),
373
- check.opt_bool_param(is_branch_deployment, "is_branch_deployment", default=False),
238
+ request_id=request_id,
239
+ request_api=request_api,
240
+ request_args=request_args,
241
+ deployment_name=deployment_name,
242
+ expire_at=expire_at
243
+ if expire_at is not None
244
+ else (
245
+ get_current_datetime() + timedelta(milliseconds=DEFAULT_EXPIRATION_MILLISECONDS)
246
+ ).timestamp(),
247
+ is_branch_deployment=False if is_branch_deployment is None else is_branch_deployment,
374
248
  )
375
249
 
376
250
  @property
@@ -401,54 +275,30 @@ DagsterCloudApiResponseTypesTuple = (
401
275
 
402
276
 
403
277
  @whitelist_for_serdes
404
- class StoreEventBatchRequest(
405
- NamedTuple("_StoreEventBatchRequest", [("event_log_entries", Sequence[EventLogEntry])])
406
- ):
407
- def __new__(cls, event_log_entries: Sequence[EventLogEntry]):
408
- return super().__new__(cls, event_log_entries=event_log_entries)
278
+ @record
279
+ class StoreEventBatchRequest:
280
+ event_log_entries: Sequence[EventLogEntry]
409
281
 
410
282
 
411
283
  @whitelist_for_serdes
412
- class DagsterCloudUploadApiResponse(
413
- NamedTuple(
414
- "_DagsterCloudUploadApiResponse",
415
- [
416
- ("request_id", str),
417
- ("request_api", str),
418
- ("response", DagsterCloudApiResponse),
419
- ],
420
- )
421
- ):
422
- def __new__(
423
- cls,
424
- request_id: str,
425
- request_api: str,
426
- response: DagsterCloudApiResponse,
427
- ):
428
- return super().__new__(
429
- cls,
430
- request_id=check.str_param(request_id, "request_id"),
431
- request_api=check.str_param(request_api, "request_api"),
432
- response=check.inst_param(response, "response", DagsterCloudApiResponseTypesTuple),
433
- )
284
+ @record
285
+ class DagsterCloudUploadApiResponse:
286
+ request_id: str
287
+ request_api: str
288
+ response: DagsterCloudApiResponse
434
289
 
435
290
 
436
291
  @whitelist_for_serdes
437
- class TimestampedError(
438
- NamedTuple(
439
- "_TimestampedError",
440
- [
441
- ("timestamp", Optional[float]),
442
- ("error", SerializableErrorInfo),
443
- ],
444
- )
445
- ):
446
- def __new__(cls, timestamp, error):
447
- return super(TimestampedError, cls).__new__(
448
- cls,
449
- timestamp=check.opt_float_param(timestamp, "timestamp"),
450
- error=check.inst_param(error, "error", SerializableErrorInfo),
451
- )
292
+ @record
293
+ class BatchDagsterCloudUploadApiResponse:
294
+ batch: list[DagsterCloudUploadApiResponse]
295
+
296
+
297
+ @whitelist_for_serdes
298
+ @record
299
+ class TimestampedError:
300
+ timestamp: Optional[float]
301
+ error: SerializableErrorInfo
452
302
 
453
303
 
454
304
  class UserCodeDeploymentType(enum.Enum):
@@ -479,26 +329,22 @@ class AgentHeartbeatMetadata(TypedDict):
479
329
  version: NotRequired[str]
480
330
  image_tag: NotRequired[str]
481
331
  type: NotRequired[str]
482
- queues: NotRequired[List[str]]
332
+ queues: NotRequired[list[str]]
483
333
 
484
334
 
485
335
  @whitelist_for_serdes
486
- class AgentHeartbeat(
487
- NamedTuple(
488
- "_AgentHeartbeat",
489
- [
490
- ("timestamp", float),
491
- ("agent_id", str),
492
- ("agent_label", Optional[str]),
493
- ("agent_type", Optional[str]),
494
- ("errors", Optional[Sequence[TimestampedError]]),
495
- ("metadata", AgentHeartbeatMetadata),
496
- ("run_worker_statuses", Optional[CloudRunWorkerStatuses]),
497
- ("code_server_heartbeats", Sequence[CloudCodeServerHeartbeat]),
498
- ("agent_queues_config", AgentQueuesConfig),
499
- ],
500
- )
501
- ):
336
+ @record_custom
337
+ class AgentHeartbeat(IHaveNew):
338
+ timestamp: float
339
+ agent_id: str
340
+ agent_label: Optional[str]
341
+ agent_type: Optional[str]
342
+ errors: Optional[Sequence[TimestampedError]]
343
+ metadata: AgentHeartbeatMetadata
344
+ run_worker_statuses: Optional[CloudRunWorkerStatuses]
345
+ code_server_heartbeats: Sequence[CloudCodeServerHeartbeat]
346
+ agent_queues_config: AgentQueuesConfig
347
+
502
348
  def __new__(
503
349
  cls,
504
350
  timestamp: float,
@@ -511,30 +357,22 @@ class AgentHeartbeat(
511
357
  code_server_heartbeats: Optional[Sequence[CloudCodeServerHeartbeat]] = None,
512
358
  agent_queues_config: Optional[AgentQueuesConfig] = None,
513
359
  ):
514
- return super(AgentHeartbeat, cls).__new__(
360
+ return super().__new__(
515
361
  cls,
516
- timestamp=check.float_param(timestamp, "timestamp"),
517
- agent_id=check.str_param(agent_id, "agent_id"),
518
- agent_label=check.opt_str_param(agent_label, "agent_label"),
519
- agent_type=check.opt_str_param(agent_type, "agent_type"),
520
- errors=check.opt_list_param(errors, "errors", of_type=TimestampedError),
521
- metadata=cast(
522
- AgentHeartbeatMetadata, check.opt_mapping_param(metadata, "metadata", key_type=str)
523
- ),
524
- run_worker_statuses=check.opt_inst_param(
525
- run_worker_statuses, "run_worker_statuses", CloudRunWorkerStatuses
526
- ),
527
- code_server_heartbeats=check.opt_list_param(
528
- code_server_heartbeats, "code_server_heartbeats", of_type=CloudCodeServerHeartbeat
529
- ),
530
- agent_queues_config=(
531
- check.opt_inst_param(agent_queues_config, "agent_queues_config", AgentQueuesConfig)
532
- or AgentQueuesConfig()
533
- ),
362
+ timestamp=timestamp,
363
+ agent_id=agent_id,
364
+ agent_label=agent_label,
365
+ agent_type=agent_type,
366
+ errors=errors,
367
+ metadata=metadata or {},
368
+ run_worker_statuses=run_worker_statuses,
369
+ code_server_heartbeats=code_server_heartbeats or [],
370
+ agent_queues_config=agent_queues_config or AgentQueuesConfig(),
534
371
  )
535
372
 
536
373
  def without_messages_and_errors(self) -> "AgentHeartbeat":
537
- return self._replace(
374
+ return copy(
375
+ self,
538
376
  errors=[],
539
377
  run_worker_statuses=self.run_worker_statuses.without_messages_and_errors()
540
378
  if self.run_worker_statuses
@@ -552,3 +390,81 @@ class AgentHeartbeat(
552
390
  if metrics and keys_not_none(["container_utilization", "request_utilization"], metrics):
553
391
  return metrics
554
392
  return None
393
+
394
+
395
+ class FileFormat:
396
+ JSON = "json"
397
+ GZIPPED_JSON = "json.gz"
398
+
399
+
400
+ class SnapshotType:
401
+ ERROR = "error"
402
+ JOB = "job"
403
+ REPOSITORY = "repository"
404
+
405
+
406
+ @whitelist_for_serdes
407
+ @record
408
+ class StoredSnapshot:
409
+ sha1: str
410
+ format: str
411
+ uri: str
412
+ decompressed_size: int
413
+
414
+
415
+ @whitelist_for_serdes
416
+ @record
417
+ class SnapshotUploadData:
418
+ sha1: str
419
+ format: str
420
+ uri: str
421
+ presigned_put_url: str
422
+ id: str
423
+ type: str
424
+
425
+
426
+ @whitelist_for_serdes
427
+ @record
428
+ class CheckSnapshotResult:
429
+ stored_snapshot: Optional[StoredSnapshot]
430
+ upload_data: Optional[SnapshotUploadData]
431
+
432
+
433
+ @whitelist_for_serdes
434
+ @record
435
+ class ConfirmUploadResult:
436
+ stored_snapshot: StoredSnapshot
437
+
438
+
439
+ @whitelist_for_serdes
440
+ @record
441
+ class DagsterCloudRepositoryManifest:
442
+ name: str
443
+ code_pointer: CodePointer
444
+ stored_snapshot: StoredSnapshot
445
+
446
+
447
+ @whitelist_for_serdes
448
+ @record
449
+ class DagsterCloudCodeLocationManifest:
450
+ repositories: Sequence[DagsterCloudRepositoryManifest]
451
+ container_image: Optional[str]
452
+ executable_path: Optional[str]
453
+ dagster_library_versions: Optional[Mapping[str, str]]
454
+ code_location_deploy_data: CodeLocationDeployData
455
+
456
+
457
+ @whitelist_for_serdes
458
+ @record
459
+ class DagsterCloudCodeLocationUpdateResult:
460
+ location_name: str
461
+ manifest: Optional[DagsterCloudCodeLocationManifest]
462
+ error_snapshot: Optional[StoredSnapshot]
463
+
464
+
465
+ @whitelist_for_serdes
466
+ @record
467
+ class DagsterCloudCodeLocationUpdateResponse:
468
+ updated: bool
469
+ message: str
470
+ missing_job_snapshots: Optional[Sequence[JobSelector]]