dagster-cloud 1.8.2__py3-none-any.whl → 1.12.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dagster_cloud/__init__.py +3 -3
- dagster_cloud/agent/__init__.py +4 -4
- dagster_cloud/agent/cli/__init__.py +56 -17
- dagster_cloud/agent/dagster_cloud_agent.py +360 -172
- dagster_cloud/agent/instrumentation/__init__.py +0 -0
- dagster_cloud/agent/instrumentation/constants.py +2 -0
- dagster_cloud/agent/instrumentation/run_launch.py +23 -0
- dagster_cloud/agent/instrumentation/schedule.py +34 -0
- dagster_cloud/agent/instrumentation/sensor.py +34 -0
- dagster_cloud/anomaly_detection/__init__.py +2 -2
- dagster_cloud/anomaly_detection/defs.py +17 -12
- dagster_cloud/anomaly_detection/types.py +3 -3
- dagster_cloud/api/dagster_cloud_api.py +209 -293
- dagster_cloud/auth/constants.py +21 -5
- dagster_cloud/batching/__init__.py +1 -0
- dagster_cloud/batching/batcher.py +210 -0
- dagster_cloud/dagster_insights/__init__.py +12 -6
- dagster_cloud/dagster_insights/bigquery/bigquery_utils.py +3 -2
- dagster_cloud/dagster_insights/bigquery/dbt_wrapper.py +39 -12
- dagster_cloud/dagster_insights/bigquery/insights_bigquery_resource.py +8 -6
- dagster_cloud/dagster_insights/insights_utils.py +18 -8
- dagster_cloud/dagster_insights/metrics_utils.py +12 -12
- dagster_cloud/dagster_insights/snowflake/dagster_snowflake_insights.py +5 -12
- dagster_cloud/dagster_insights/snowflake/dbt_wrapper.py +34 -8
- dagster_cloud/dagster_insights/snowflake/definitions.py +38 -12
- dagster_cloud/dagster_insights/snowflake/insights_snowflake_resource.py +11 -23
- dagster_cloud/definitions/__init__.py +0 -0
- dagster_cloud/definitions/job_selection.py +36 -0
- dagster_cloud/execution/cloud_run_launcher/k8s.py +1 -1
- dagster_cloud/execution/cloud_run_launcher/process.py +3 -3
- dagster_cloud/execution/monitoring/__init__.py +27 -33
- dagster_cloud/execution/utils/process.py +3 -3
- dagster_cloud/instance/__init__.py +125 -38
- dagster_cloud/instrumentation/__init__.py +32 -0
- dagster_cloud/metadata/source_code.py +13 -8
- dagster_cloud/metrics/__init__.py +0 -0
- dagster_cloud/metrics/tracer.py +59 -0
- dagster_cloud/opentelemetry/__init__.py +0 -0
- dagster_cloud/opentelemetry/config/__init__.py +73 -0
- dagster_cloud/opentelemetry/config/exporter.py +81 -0
- dagster_cloud/opentelemetry/config/log_record_processor.py +40 -0
- dagster_cloud/opentelemetry/config/logging_handler.py +14 -0
- dagster_cloud/opentelemetry/config/meter_provider.py +9 -0
- dagster_cloud/opentelemetry/config/metric_reader.py +39 -0
- dagster_cloud/opentelemetry/controller.py +319 -0
- dagster_cloud/opentelemetry/enum.py +58 -0
- dagster_cloud/opentelemetry/factories/__init__.py +1 -0
- dagster_cloud/opentelemetry/factories/logs.py +113 -0
- dagster_cloud/opentelemetry/factories/metrics.py +121 -0
- dagster_cloud/opentelemetry/metrics/__init__.py +0 -0
- dagster_cloud/opentelemetry/metrics/meter.py +140 -0
- dagster_cloud/opentelemetry/observers/__init__.py +0 -0
- dagster_cloud/opentelemetry/observers/dagster_exception_handler.py +40 -0
- dagster_cloud/opentelemetry/observers/execution_observer.py +178 -0
- dagster_cloud/pex/grpc/__generated__/multi_pex_api_pb2.pyi +175 -0
- dagster_cloud/pex/grpc/__init__.py +2 -2
- dagster_cloud/pex/grpc/client.py +4 -4
- dagster_cloud/pex/grpc/compile.py +2 -2
- dagster_cloud/pex/grpc/server/__init__.py +2 -2
- dagster_cloud/pex/grpc/server/cli/__init__.py +31 -19
- dagster_cloud/pex/grpc/server/manager.py +60 -42
- dagster_cloud/pex/grpc/server/registry.py +28 -21
- dagster_cloud/pex/grpc/server/server.py +23 -14
- dagster_cloud/pex/grpc/types.py +5 -5
- dagster_cloud/py.typed +0 -0
- dagster_cloud/secrets/__init__.py +1 -1
- dagster_cloud/secrets/loader.py +3 -3
- dagster_cloud/serverless/__init__.py +1 -1
- dagster_cloud/serverless/io_manager.py +36 -53
- dagster_cloud/storage/client.py +54 -17
- dagster_cloud/storage/compute_logs/__init__.py +3 -1
- dagster_cloud/storage/compute_logs/compute_log_manager.py +22 -17
- dagster_cloud/storage/defs_state/__init__.py +3 -0
- dagster_cloud/storage/defs_state/queries.py +15 -0
- dagster_cloud/storage/defs_state/storage.py +113 -0
- dagster_cloud/storage/event_logs/__init__.py +3 -1
- dagster_cloud/storage/event_logs/queries.py +102 -4
- dagster_cloud/storage/event_logs/storage.py +266 -73
- dagster_cloud/storage/event_logs/utils.py +88 -7
- dagster_cloud/storage/runs/__init__.py +1 -1
- dagster_cloud/storage/runs/queries.py +17 -2
- dagster_cloud/storage/runs/storage.py +88 -42
- dagster_cloud/storage/schedules/__init__.py +1 -1
- dagster_cloud/storage/schedules/storage.py +6 -8
- dagster_cloud/storage/tags.py +66 -1
- dagster_cloud/util/__init__.py +10 -12
- dagster_cloud/util/errors.py +49 -64
- dagster_cloud/version.py +1 -1
- dagster_cloud/workspace/config_schema/__init__.py +55 -13
- dagster_cloud/workspace/docker/__init__.py +76 -25
- dagster_cloud/workspace/docker/utils.py +1 -1
- dagster_cloud/workspace/ecs/__init__.py +1 -1
- dagster_cloud/workspace/ecs/client.py +51 -33
- dagster_cloud/workspace/ecs/launcher.py +76 -22
- dagster_cloud/workspace/ecs/run_launcher.py +3 -3
- dagster_cloud/workspace/ecs/utils.py +14 -5
- dagster_cloud/workspace/kubernetes/__init__.py +1 -1
- dagster_cloud/workspace/kubernetes/launcher.py +61 -29
- dagster_cloud/workspace/kubernetes/utils.py +34 -22
- dagster_cloud/workspace/user_code_launcher/__init__.py +5 -3
- dagster_cloud/workspace/user_code_launcher/process.py +16 -14
- dagster_cloud/workspace/user_code_launcher/user_code_launcher.py +552 -172
- dagster_cloud/workspace/user_code_launcher/utils.py +105 -1
- {dagster_cloud-1.8.2.dist-info → dagster_cloud-1.12.6.dist-info}/METADATA +48 -42
- dagster_cloud-1.12.6.dist-info/RECORD +134 -0
- {dagster_cloud-1.8.2.dist-info → dagster_cloud-1.12.6.dist-info}/WHEEL +1 -1
- dagster_cloud-1.8.2.dist-info/RECORD +0 -100
- {dagster_cloud-1.8.2.dist-info → dagster_cloud-1.12.6.dist-info}/top_level.txt +0 -0
|
@@ -1,15 +1,17 @@
|
|
|
1
1
|
import enum
|
|
2
|
+
from collections.abc import Mapping, Sequence
|
|
2
3
|
from datetime import timedelta
|
|
3
4
|
from enum import Enum
|
|
4
|
-
from typing import Any,
|
|
5
|
+
from typing import Any, Optional, TypedDict, Union
|
|
5
6
|
|
|
6
|
-
import dagster._check as check
|
|
7
7
|
from dagster._core.code_pointer import CodePointer
|
|
8
8
|
from dagster._core.definitions.selector import JobSelector
|
|
9
9
|
from dagster._core.events.log import EventLogEntry
|
|
10
|
-
from dagster._core.
|
|
10
|
+
from dagster._core.remote_origin import CodeLocationOrigin
|
|
11
|
+
from dagster._core.remote_representation.external_data import RepositorySnap
|
|
11
12
|
from dagster._core.storage.dagster_run import DagsterRun
|
|
12
13
|
from dagster._core.utils import RequestUtilizationMetrics
|
|
14
|
+
from dagster._record import IHaveNew, copy, record, record_custom
|
|
13
15
|
from dagster._serdes import whitelist_for_serdes
|
|
14
16
|
from dagster._time import get_current_datetime, get_current_timestamp
|
|
15
17
|
from dagster._utils.container import ContainerUtilizationMetrics
|
|
@@ -29,7 +31,8 @@ DEFAULT_EXPIRATION_MILLISECONDS = 10 * 60 * 1000
|
|
|
29
31
|
|
|
30
32
|
|
|
31
33
|
@whitelist_for_serdes
|
|
32
|
-
|
|
34
|
+
@record
|
|
35
|
+
class DagsterCloudUploadRepositoryData:
|
|
33
36
|
"""Serialized object uploaded by the Dagster Cloud agent with information pulled
|
|
34
37
|
from a gRPC server about an individual repository - the data field is serialized since the
|
|
35
38
|
agent may be running older code that doesn't know how to deserialize it, so it passes
|
|
@@ -42,21 +45,23 @@ class DagsterCloudUploadRepositoryData(NamedTuple):
|
|
|
42
45
|
|
|
43
46
|
|
|
44
47
|
@whitelist_for_serdes
|
|
45
|
-
|
|
48
|
+
@record
|
|
49
|
+
class DagsterCloudUploadLocationData:
|
|
46
50
|
"""Serialized object uploaded by the Dagster Cloud agent with information pulled
|
|
47
51
|
about a successfully loaded repository location, including information about
|
|
48
52
|
each repository as well as shared metadata like the image to use when launching
|
|
49
53
|
runs in this location.
|
|
50
54
|
"""
|
|
51
55
|
|
|
52
|
-
upload_repository_datas:
|
|
56
|
+
upload_repository_datas: list[DagsterCloudUploadRepositoryData]
|
|
53
57
|
container_image: Optional[str]
|
|
54
58
|
executable_path: Optional[str]
|
|
55
59
|
dagster_library_versions: Optional[Mapping[str, str]] = None
|
|
56
60
|
|
|
57
61
|
|
|
58
62
|
@whitelist_for_serdes(storage_field_names={"code_location_deploy_data": "deployment_metadata"})
|
|
59
|
-
|
|
63
|
+
@record
|
|
64
|
+
class DagsterCloudUploadWorkspaceEntry:
|
|
60
65
|
"""Serialized object uploaded by the Dagster Cloud agent with information about
|
|
61
66
|
a repository location - either the serialized DagsterCloudUploadLocationData
|
|
62
67
|
if the location loaded succesfully, or a SerializableErrorInfo describing the
|
|
@@ -70,7 +75,8 @@ class DagsterCloudUploadWorkspaceEntry(NamedTuple):
|
|
|
70
75
|
|
|
71
76
|
|
|
72
77
|
@whitelist_for_serdes
|
|
73
|
-
|
|
78
|
+
@record
|
|
79
|
+
class DagsterCloudUploadWorkspaceResponse:
|
|
74
80
|
updated: bool
|
|
75
81
|
message: str
|
|
76
82
|
missing_job_snapshots: Optional[Sequence[JobSelector]]
|
|
@@ -79,7 +85,6 @@ class DagsterCloudUploadWorkspaceResponse(NamedTuple):
|
|
|
79
85
|
@whitelist_for_serdes
|
|
80
86
|
class DagsterCloudApi(Enum):
|
|
81
87
|
CHECK_FOR_WORKSPACE_UPDATES = "CHECK_FOR_WORKSPACE_UPDATES"
|
|
82
|
-
LOAD_REPOSITORIES = "LOAD_REPOSITORIES"
|
|
83
88
|
GET_EXTERNAL_EXECUTION_PLAN = "GET_EXTERNAL_EXECUTION_PLAN"
|
|
84
89
|
GET_SUBSET_EXTERNAL_PIPELINE_RESULT = "GET_SUBSET_EXTERNAL_PIPELINE_RESULT"
|
|
85
90
|
GET_EXTERNAL_PARTITION_CONFIG = "GET_EXTERNAL_PARTITION_CONFIG"
|
|
@@ -93,43 +98,26 @@ class DagsterCloudApi(Enum):
|
|
|
93
98
|
GET_EXTERNAL_NOTEBOOK_DATA = "GET_EXTERNAL_NOTEBOOK_DATA"
|
|
94
99
|
|
|
95
100
|
LAUNCH_RUN = "LAUNCH_RUN"
|
|
96
|
-
CHECK_RUN_HEALTH = "CHECK_RUN_HEALTH" # deprecated, agents now surface this in heartbeats
|
|
97
101
|
TERMINATE_RUN = "TERMINATE_RUN"
|
|
102
|
+
|
|
103
|
+
PING_LOCATION = "PING_LOCATION" # Signal that a location is in use and should keep servers up
|
|
104
|
+
|
|
105
|
+
CHECK_RUN_HEALTH = "CHECK_RUN_HEALTH" # deprecated, agents now surface this in heartbeats
|
|
98
106
|
LAUNCH_STEP = "LAUNCH_STEP" # deprecated with cloud executor
|
|
99
107
|
CHECK_STEP_HEALTH = "CHECK_STEP_HEALTH" # deprecated with cloud executor
|
|
100
108
|
TERMINATE_STEP = "TERMINATE_STEP" # deprecated with cloud executor
|
|
101
|
-
|
|
102
|
-
PING_LOCATION = "PING_LOCATION" # Signal that a location is in use and should keep servers up
|
|
109
|
+
LOAD_REPOSITORIES = "LOAD_REPOSITORIES" # deprecated
|
|
103
110
|
|
|
104
111
|
def __structlog__(self):
|
|
105
112
|
return self.name
|
|
106
113
|
|
|
107
114
|
|
|
108
115
|
@whitelist_for_serdes
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
("thread_start_run_timestamp", float),
|
|
115
|
-
("thread_end_handle_api_request_timestamp", float),
|
|
116
|
-
],
|
|
117
|
-
)
|
|
118
|
-
):
|
|
119
|
-
def __new__(
|
|
120
|
-
cls,
|
|
121
|
-
submitted_to_executor_timestamp: float,
|
|
122
|
-
thread_start_run_timestamp: float,
|
|
123
|
-
thread_end_handle_api_request_timestamp: float,
|
|
124
|
-
):
|
|
125
|
-
return super(cls, DagsterCloudApiThreadTelemetry).__new__(
|
|
126
|
-
cls,
|
|
127
|
-
check.float_param(submitted_to_executor_timestamp, "submitted_to_executor_timestamp"),
|
|
128
|
-
check.float_param(thread_start_run_timestamp, "thread_start_run_timestamp"),
|
|
129
|
-
check.float_param(
|
|
130
|
-
thread_end_handle_api_request_timestamp, "thread_end_handle_api_request_timestamp"
|
|
131
|
-
),
|
|
132
|
-
)
|
|
116
|
+
@record
|
|
117
|
+
class DagsterCloudApiThreadTelemetry:
|
|
118
|
+
submitted_to_executor_timestamp: float
|
|
119
|
+
thread_start_run_timestamp: float
|
|
120
|
+
thread_end_handle_api_request_timestamp: float
|
|
133
121
|
|
|
134
122
|
@property
|
|
135
123
|
def time_to_thread_initialization_seconds(self) -> float:
|
|
@@ -140,214 +128,102 @@ class DagsterCloudApiThreadTelemetry(
|
|
|
140
128
|
return self.thread_end_handle_api_request_timestamp - self.thread_start_run_timestamp
|
|
141
129
|
|
|
142
130
|
|
|
143
|
-
@whitelist_for_serdes(old_storage_names={"
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
)
|
|
148
|
-
):
|
|
149
|
-
def __new__(cls, thread_telemetry: Optional[DagsterCloudApiThreadTelemetry] = None):
|
|
150
|
-
return super(cls, DagsterCloudApiSuccess).__new__(
|
|
151
|
-
cls,
|
|
152
|
-
check.opt_inst_param(
|
|
153
|
-
thread_telemetry, "thread_telemetry", DagsterCloudApiThreadTelemetry
|
|
154
|
-
),
|
|
155
|
-
)
|
|
131
|
+
@whitelist_for_serdes(old_storage_names={"CheckForCodeLocationUpdatesRequest"})
|
|
132
|
+
@record
|
|
133
|
+
class DagsterCloudApiSuccess:
|
|
134
|
+
thread_telemetry: Optional[DagsterCloudApiThreadTelemetry] = None
|
|
156
135
|
|
|
157
136
|
def with_thread_telemetry(self, thread_telemetry: DagsterCloudApiThreadTelemetry):
|
|
158
|
-
return self
|
|
137
|
+
return copy(self, thread_telemetry=thread_telemetry)
|
|
159
138
|
|
|
160
139
|
|
|
161
140
|
@whitelist_for_serdes
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
)
|
|
167
|
-
):
|
|
168
|
-
def __new__(
|
|
169
|
-
cls, request_api: str, thread_telemetry: Optional[DagsterCloudApiThreadTelemetry] = None
|
|
170
|
-
):
|
|
171
|
-
return super(cls, DagsterCloudApiUnknownCommandResponse).__new__(
|
|
172
|
-
cls,
|
|
173
|
-
check.str_param(request_api, "request_api"),
|
|
174
|
-
check.opt_inst_param(
|
|
175
|
-
thread_telemetry, "thread_telemetry", DagsterCloudApiThreadTelemetry
|
|
176
|
-
),
|
|
177
|
-
)
|
|
141
|
+
@record
|
|
142
|
+
class DagsterCloudApiUnknownCommandResponse:
|
|
143
|
+
request_api: str
|
|
144
|
+
thread_telemetry: Optional[DagsterCloudApiThreadTelemetry] = None
|
|
178
145
|
|
|
179
146
|
def with_thread_telemetry(self, thread_telemetry: DagsterCloudApiThreadTelemetry):
|
|
180
|
-
return self
|
|
147
|
+
return copy(self, thread_telemetry=thread_telemetry)
|
|
181
148
|
|
|
182
149
|
|
|
183
150
|
@whitelist_for_serdes
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
("error_infos", List[SerializableErrorInfo]),
|
|
189
|
-
("thread_telemetry", Optional[DagsterCloudApiThreadTelemetry]),
|
|
190
|
-
],
|
|
191
|
-
)
|
|
192
|
-
):
|
|
193
|
-
def __new__(
|
|
194
|
-
cls,
|
|
195
|
-
error_infos: List[SerializableErrorInfo],
|
|
196
|
-
thread_telemetry: Optional[DagsterCloudApiThreadTelemetry] = None,
|
|
197
|
-
):
|
|
198
|
-
return super(cls, DagsterCloudApiErrorResponse).__new__(
|
|
199
|
-
cls,
|
|
200
|
-
check.list_param(error_infos, "error_infos", of_type=SerializableErrorInfo),
|
|
201
|
-
check.opt_inst_param(
|
|
202
|
-
thread_telemetry, "thread_telemetry", DagsterCloudApiThreadTelemetry
|
|
203
|
-
),
|
|
204
|
-
)
|
|
151
|
+
@record
|
|
152
|
+
class DagsterCloudApiErrorResponse:
|
|
153
|
+
error_infos: list[SerializableErrorInfo]
|
|
154
|
+
thread_telemetry: Optional[DagsterCloudApiThreadTelemetry] = None
|
|
205
155
|
|
|
206
156
|
def with_thread_telemetry(self, thread_telemetry: DagsterCloudApiThreadTelemetry):
|
|
207
|
-
return self
|
|
157
|
+
return copy(self, thread_telemetry=thread_telemetry)
|
|
208
158
|
|
|
209
159
|
|
|
210
160
|
@whitelist_for_serdes
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
"_DagsterCloudApiGrpcResponse",
|
|
214
|
-
[
|
|
215
|
-
("serialized_response_or_error", str),
|
|
216
|
-
("thread_telemetry", Optional[DagsterCloudApiThreadTelemetry]),
|
|
217
|
-
],
|
|
218
|
-
)
|
|
219
|
-
):
|
|
161
|
+
@record
|
|
162
|
+
class DagsterCloudApiGrpcResponse:
|
|
220
163
|
# Class that DagsterCloudApi methods can use to pass along the result of
|
|
221
164
|
# a gRPC call against the user code server. The field here is passed in
|
|
222
165
|
# serialized as a string, because we can't guarantee that the agent code will
|
|
223
166
|
# be up-to-date enough to know how to deserialize it (but the host cloud always
|
|
224
167
|
# should, since it will always be up to date).
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
serialized_response_or_error: str,
|
|
228
|
-
thread_telemetry: Optional[DagsterCloudApiThreadTelemetry] = None,
|
|
229
|
-
):
|
|
230
|
-
return super(cls, DagsterCloudApiGrpcResponse).__new__(
|
|
231
|
-
cls,
|
|
232
|
-
check.str_param(serialized_response_or_error, "serialized_response_or_error"),
|
|
233
|
-
check.opt_inst_param(
|
|
234
|
-
thread_telemetry, "thread_telemetry", DagsterCloudApiThreadTelemetry
|
|
235
|
-
),
|
|
236
|
-
)
|
|
168
|
+
serialized_response_or_error: str
|
|
169
|
+
thread_telemetry: Optional[DagsterCloudApiThreadTelemetry] = None
|
|
237
170
|
|
|
238
171
|
def with_thread_telemetry(self, thread_telemetry: DagsterCloudApiThreadTelemetry):
|
|
239
|
-
return self
|
|
172
|
+
return copy(self, thread_telemetry=thread_telemetry)
|
|
240
173
|
|
|
241
174
|
|
|
242
175
|
@whitelist_for_serdes
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
def __new__(cls, location_origin):
|
|
247
|
-
return super(cls, LoadRepositoriesArgs).__new__(
|
|
248
|
-
cls,
|
|
249
|
-
check.inst_param(location_origin, "location_origin", CodeLocationOrigin),
|
|
250
|
-
)
|
|
176
|
+
@record
|
|
177
|
+
class LoadRepositoriesArgs:
|
|
178
|
+
location_origin: CodeLocationOrigin
|
|
251
179
|
|
|
252
180
|
|
|
253
181
|
@whitelist_for_serdes
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
("code_pointer", CodePointer),
|
|
260
|
-
("external_repository_data", ExternalRepositoryData),
|
|
261
|
-
],
|
|
262
|
-
)
|
|
263
|
-
):
|
|
264
|
-
def __new__(cls, repo_name, code_pointer, external_repository_data):
|
|
265
|
-
return super(cls, DagsterCloudRepositoryData).__new__(
|
|
266
|
-
cls,
|
|
267
|
-
check.str_param(repo_name, "repo_name"),
|
|
268
|
-
check.inst_param(code_pointer, "code_pointer", CodePointer),
|
|
269
|
-
check.inst_param(
|
|
270
|
-
external_repository_data,
|
|
271
|
-
"external_repository_data",
|
|
272
|
-
ExternalRepositoryData,
|
|
273
|
-
),
|
|
274
|
-
)
|
|
182
|
+
@record
|
|
183
|
+
class DagsterCloudRepositoryData:
|
|
184
|
+
repo_name: str
|
|
185
|
+
code_pointer: CodePointer
|
|
186
|
+
external_repository_data: RepositorySnap
|
|
275
187
|
|
|
276
188
|
|
|
277
189
|
@whitelist_for_serdes(storage_field_names={"code_location_deploy_data": "code_deployment_metadata"})
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
("code_location_deploy_data", Optional[CodeLocationDeployData]),
|
|
286
|
-
("dagster_library_versions", Optional[Mapping[str, str]]),
|
|
287
|
-
],
|
|
288
|
-
)
|
|
289
|
-
):
|
|
290
|
-
def __new__(
|
|
291
|
-
cls,
|
|
292
|
-
repository_datas,
|
|
293
|
-
container_image,
|
|
294
|
-
executable_path,
|
|
295
|
-
code_location_deploy_data=None,
|
|
296
|
-
dagster_library_versions: Optional[Mapping[str, str]] = None,
|
|
297
|
-
):
|
|
298
|
-
return super(cls, LoadRepositoriesResponse).__new__(
|
|
299
|
-
cls,
|
|
300
|
-
check.list_param(
|
|
301
|
-
repository_datas,
|
|
302
|
-
"repository_datas",
|
|
303
|
-
of_type=DagsterCloudRepositoryData,
|
|
304
|
-
),
|
|
305
|
-
check.opt_str_param(container_image, "container_image"),
|
|
306
|
-
check.opt_str_param(executable_path, "executable_path"),
|
|
307
|
-
check.opt_inst_param(
|
|
308
|
-
code_location_deploy_data, "code_location_deploy_data", CodeLocationDeployData
|
|
309
|
-
),
|
|
310
|
-
check.opt_nullable_mapping_param(dagster_library_versions, "dagster_library_versions"),
|
|
311
|
-
)
|
|
190
|
+
@record
|
|
191
|
+
class LoadRepositoriesResponse:
|
|
192
|
+
repository_datas: Sequence[DagsterCloudRepositoryData]
|
|
193
|
+
container_image: Optional[str]
|
|
194
|
+
executable_path: Optional[str]
|
|
195
|
+
code_location_deploy_data: Optional[CodeLocationDeployData] = None
|
|
196
|
+
dagster_library_versions: Optional[Mapping[str, str]] = None
|
|
312
197
|
|
|
313
198
|
|
|
314
199
|
@whitelist_for_serdes
|
|
315
|
-
|
|
316
|
-
|
|
200
|
+
@record
|
|
201
|
+
class PingLocationArgs:
|
|
202
|
+
location_name: str
|
|
317
203
|
|
|
318
204
|
|
|
319
205
|
@whitelist_for_serdes(storage_field_names={"dagster_run": "pipeline_run"})
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
cls,
|
|
324
|
-
check.inst_param(dagster_run, "dagster_run", DagsterRun),
|
|
325
|
-
)
|
|
206
|
+
@record
|
|
207
|
+
class LaunchRunArgs:
|
|
208
|
+
dagster_run: DagsterRun
|
|
326
209
|
|
|
327
210
|
|
|
328
211
|
@whitelist_for_serdes(storage_field_names={"dagster_run": "pipeline_run"})
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
cls,
|
|
333
|
-
check.inst_param(dagster_run, "dagster_run", DagsterRun),
|
|
334
|
-
)
|
|
212
|
+
@record
|
|
213
|
+
class TerminateRunArgs:
|
|
214
|
+
dagster_run: DagsterRun
|
|
335
215
|
|
|
336
216
|
|
|
337
217
|
@whitelist_for_serdes
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
("is_branch_deployment", bool),
|
|
348
|
-
],
|
|
349
|
-
)
|
|
350
|
-
):
|
|
218
|
+
@record_custom
|
|
219
|
+
class DagsterCloudApiRequest(IHaveNew):
|
|
220
|
+
request_id: str
|
|
221
|
+
request_api: DagsterCloudApi
|
|
222
|
+
request_args: Any
|
|
223
|
+
deployment_name: str
|
|
224
|
+
expire_at: float
|
|
225
|
+
is_branch_deployment: bool
|
|
226
|
+
|
|
351
227
|
def __new__(
|
|
352
228
|
cls,
|
|
353
229
|
request_id: str,
|
|
@@ -357,20 +233,18 @@ class DagsterCloudApiRequest(
|
|
|
357
233
|
expire_at: Optional[float] = None,
|
|
358
234
|
is_branch_deployment: Optional[bool] = None,
|
|
359
235
|
):
|
|
360
|
-
return super(
|
|
236
|
+
return super().__new__(
|
|
361
237
|
cls,
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
request_args,
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
),
|
|
373
|
-
check.opt_bool_param(is_branch_deployment, "is_branch_deployment", default=False),
|
|
238
|
+
request_id=request_id,
|
|
239
|
+
request_api=request_api,
|
|
240
|
+
request_args=request_args,
|
|
241
|
+
deployment_name=deployment_name,
|
|
242
|
+
expire_at=expire_at
|
|
243
|
+
if expire_at is not None
|
|
244
|
+
else (
|
|
245
|
+
get_current_datetime() + timedelta(milliseconds=DEFAULT_EXPIRATION_MILLISECONDS)
|
|
246
|
+
).timestamp(),
|
|
247
|
+
is_branch_deployment=False if is_branch_deployment is None else is_branch_deployment,
|
|
374
248
|
)
|
|
375
249
|
|
|
376
250
|
@property
|
|
@@ -401,54 +275,30 @@ DagsterCloudApiResponseTypesTuple = (
|
|
|
401
275
|
|
|
402
276
|
|
|
403
277
|
@whitelist_for_serdes
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
def __new__(cls, event_log_entries: Sequence[EventLogEntry]):
|
|
408
|
-
return super().__new__(cls, event_log_entries=event_log_entries)
|
|
278
|
+
@record
|
|
279
|
+
class StoreEventBatchRequest:
|
|
280
|
+
event_log_entries: Sequence[EventLogEntry]
|
|
409
281
|
|
|
410
282
|
|
|
411
283
|
@whitelist_for_serdes
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
("request_api", str),
|
|
418
|
-
("response", DagsterCloudApiResponse),
|
|
419
|
-
],
|
|
420
|
-
)
|
|
421
|
-
):
|
|
422
|
-
def __new__(
|
|
423
|
-
cls,
|
|
424
|
-
request_id: str,
|
|
425
|
-
request_api: str,
|
|
426
|
-
response: DagsterCloudApiResponse,
|
|
427
|
-
):
|
|
428
|
-
return super().__new__(
|
|
429
|
-
cls,
|
|
430
|
-
request_id=check.str_param(request_id, "request_id"),
|
|
431
|
-
request_api=check.str_param(request_api, "request_api"),
|
|
432
|
-
response=check.inst_param(response, "response", DagsterCloudApiResponseTypesTuple),
|
|
433
|
-
)
|
|
284
|
+
@record
|
|
285
|
+
class DagsterCloudUploadApiResponse:
|
|
286
|
+
request_id: str
|
|
287
|
+
request_api: str
|
|
288
|
+
response: DagsterCloudApiResponse
|
|
434
289
|
|
|
435
290
|
|
|
436
291
|
@whitelist_for_serdes
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
return super(TimestampedError, cls).__new__(
|
|
448
|
-
cls,
|
|
449
|
-
timestamp=check.opt_float_param(timestamp, "timestamp"),
|
|
450
|
-
error=check.inst_param(error, "error", SerializableErrorInfo),
|
|
451
|
-
)
|
|
292
|
+
@record
|
|
293
|
+
class BatchDagsterCloudUploadApiResponse:
|
|
294
|
+
batch: list[DagsterCloudUploadApiResponse]
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
@whitelist_for_serdes
|
|
298
|
+
@record
|
|
299
|
+
class TimestampedError:
|
|
300
|
+
timestamp: Optional[float]
|
|
301
|
+
error: SerializableErrorInfo
|
|
452
302
|
|
|
453
303
|
|
|
454
304
|
class UserCodeDeploymentType(enum.Enum):
|
|
@@ -479,26 +329,22 @@ class AgentHeartbeatMetadata(TypedDict):
|
|
|
479
329
|
version: NotRequired[str]
|
|
480
330
|
image_tag: NotRequired[str]
|
|
481
331
|
type: NotRequired[str]
|
|
482
|
-
queues: NotRequired[
|
|
332
|
+
queues: NotRequired[list[str]]
|
|
483
333
|
|
|
484
334
|
|
|
485
335
|
@whitelist_for_serdes
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
("agent_queues_config", AgentQueuesConfig),
|
|
499
|
-
],
|
|
500
|
-
)
|
|
501
|
-
):
|
|
336
|
+
@record_custom
|
|
337
|
+
class AgentHeartbeat(IHaveNew):
|
|
338
|
+
timestamp: float
|
|
339
|
+
agent_id: str
|
|
340
|
+
agent_label: Optional[str]
|
|
341
|
+
agent_type: Optional[str]
|
|
342
|
+
errors: Optional[Sequence[TimestampedError]]
|
|
343
|
+
metadata: AgentHeartbeatMetadata
|
|
344
|
+
run_worker_statuses: Optional[CloudRunWorkerStatuses]
|
|
345
|
+
code_server_heartbeats: Sequence[CloudCodeServerHeartbeat]
|
|
346
|
+
agent_queues_config: AgentQueuesConfig
|
|
347
|
+
|
|
502
348
|
def __new__(
|
|
503
349
|
cls,
|
|
504
350
|
timestamp: float,
|
|
@@ -511,30 +357,22 @@ class AgentHeartbeat(
|
|
|
511
357
|
code_server_heartbeats: Optional[Sequence[CloudCodeServerHeartbeat]] = None,
|
|
512
358
|
agent_queues_config: Optional[AgentQueuesConfig] = None,
|
|
513
359
|
):
|
|
514
|
-
return super(
|
|
360
|
+
return super().__new__(
|
|
515
361
|
cls,
|
|
516
|
-
timestamp=
|
|
517
|
-
agent_id=
|
|
518
|
-
agent_label=
|
|
519
|
-
agent_type=
|
|
520
|
-
errors=
|
|
521
|
-
metadata=
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
run_worker_statuses, "run_worker_statuses", CloudRunWorkerStatuses
|
|
526
|
-
),
|
|
527
|
-
code_server_heartbeats=check.opt_list_param(
|
|
528
|
-
code_server_heartbeats, "code_server_heartbeats", of_type=CloudCodeServerHeartbeat
|
|
529
|
-
),
|
|
530
|
-
agent_queues_config=(
|
|
531
|
-
check.opt_inst_param(agent_queues_config, "agent_queues_config", AgentQueuesConfig)
|
|
532
|
-
or AgentQueuesConfig()
|
|
533
|
-
),
|
|
362
|
+
timestamp=timestamp,
|
|
363
|
+
agent_id=agent_id,
|
|
364
|
+
agent_label=agent_label,
|
|
365
|
+
agent_type=agent_type,
|
|
366
|
+
errors=errors,
|
|
367
|
+
metadata=metadata or {},
|
|
368
|
+
run_worker_statuses=run_worker_statuses,
|
|
369
|
+
code_server_heartbeats=code_server_heartbeats or [],
|
|
370
|
+
agent_queues_config=agent_queues_config or AgentQueuesConfig(),
|
|
534
371
|
)
|
|
535
372
|
|
|
536
373
|
def without_messages_and_errors(self) -> "AgentHeartbeat":
|
|
537
|
-
return
|
|
374
|
+
return copy(
|
|
375
|
+
self,
|
|
538
376
|
errors=[],
|
|
539
377
|
run_worker_statuses=self.run_worker_statuses.without_messages_and_errors()
|
|
540
378
|
if self.run_worker_statuses
|
|
@@ -552,3 +390,81 @@ class AgentHeartbeat(
|
|
|
552
390
|
if metrics and keys_not_none(["container_utilization", "request_utilization"], metrics):
|
|
553
391
|
return metrics
|
|
554
392
|
return None
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
class FileFormat:
|
|
396
|
+
JSON = "json"
|
|
397
|
+
GZIPPED_JSON = "json.gz"
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
class SnapshotType:
|
|
401
|
+
ERROR = "error"
|
|
402
|
+
JOB = "job"
|
|
403
|
+
REPOSITORY = "repository"
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
@whitelist_for_serdes
|
|
407
|
+
@record
|
|
408
|
+
class StoredSnapshot:
|
|
409
|
+
sha1: str
|
|
410
|
+
format: str
|
|
411
|
+
uri: str
|
|
412
|
+
decompressed_size: int
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
@whitelist_for_serdes
|
|
416
|
+
@record
|
|
417
|
+
class SnapshotUploadData:
|
|
418
|
+
sha1: str
|
|
419
|
+
format: str
|
|
420
|
+
uri: str
|
|
421
|
+
presigned_put_url: str
|
|
422
|
+
id: str
|
|
423
|
+
type: str
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
@whitelist_for_serdes
|
|
427
|
+
@record
|
|
428
|
+
class CheckSnapshotResult:
|
|
429
|
+
stored_snapshot: Optional[StoredSnapshot]
|
|
430
|
+
upload_data: Optional[SnapshotUploadData]
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
@whitelist_for_serdes
|
|
434
|
+
@record
|
|
435
|
+
class ConfirmUploadResult:
|
|
436
|
+
stored_snapshot: StoredSnapshot
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
@whitelist_for_serdes
|
|
440
|
+
@record
|
|
441
|
+
class DagsterCloudRepositoryManifest:
|
|
442
|
+
name: str
|
|
443
|
+
code_pointer: CodePointer
|
|
444
|
+
stored_snapshot: StoredSnapshot
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
@whitelist_for_serdes
|
|
448
|
+
@record
|
|
449
|
+
class DagsterCloudCodeLocationManifest:
|
|
450
|
+
repositories: Sequence[DagsterCloudRepositoryManifest]
|
|
451
|
+
container_image: Optional[str]
|
|
452
|
+
executable_path: Optional[str]
|
|
453
|
+
dagster_library_versions: Optional[Mapping[str, str]]
|
|
454
|
+
code_location_deploy_data: CodeLocationDeployData
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
@whitelist_for_serdes
|
|
458
|
+
@record
|
|
459
|
+
class DagsterCloudCodeLocationUpdateResult:
|
|
460
|
+
location_name: str
|
|
461
|
+
manifest: Optional[DagsterCloudCodeLocationManifest]
|
|
462
|
+
error_snapshot: Optional[StoredSnapshot]
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
@whitelist_for_serdes
|
|
466
|
+
@record
|
|
467
|
+
class DagsterCloudCodeLocationUpdateResponse:
|
|
468
|
+
updated: bool
|
|
469
|
+
message: str
|
|
470
|
+
missing_job_snapshots: Optional[Sequence[JobSelector]]
|