dagster-cloud 1.11.10__py3-none-any.whl → 1.11.12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dagster_cloud/auth/constants.py +21 -5
- dagster_cloud/instance/__init__.py +53 -34
- dagster_cloud/storage/compute_logs/compute_log_manager.py +3 -3
- dagster_cloud/storage/defs_state/storage.py +63 -5
- dagster_cloud/version.py +1 -1
- {dagster_cloud-1.11.10.dist-info → dagster_cloud-1.11.12.dist-info}/METADATA +7 -7
- {dagster_cloud-1.11.10.dist-info → dagster_cloud-1.11.12.dist-info}/RECORD +9 -10
- dagster_cloud/storage/defs_state/base_storage.py +0 -90
- {dagster_cloud-1.11.10.dist-info → dagster_cloud-1.11.12.dist-info}/WHEEL +0 -0
- {dagster_cloud-1.11.10.dist-info → dagster_cloud-1.11.12.dist-info}/top_level.txt +0 -0
dagster_cloud/auth/constants.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import uuid
|
|
1
2
|
from typing import Optional
|
|
2
3
|
|
|
3
4
|
from dagster._core.errors import DagsterInvariantViolationError
|
|
@@ -19,12 +20,27 @@ def get_organization_public_id_from_api_token(api_token: str) -> Optional[str]:
|
|
|
19
20
|
return split_token[2]
|
|
20
21
|
|
|
21
22
|
|
|
22
|
-
def
|
|
23
|
+
def decode_region_from_uuid(regional_token: str) -> Optional[str]:
|
|
24
|
+
try:
|
|
25
|
+
regional_uuid = uuid.UUID(regional_token)
|
|
26
|
+
except ValueError:
|
|
27
|
+
# if it's not an actual uuid, we can't decode region
|
|
28
|
+
return None
|
|
29
|
+
|
|
30
|
+
# custom uuids contain region subdomains in the first 2 bytes
|
|
31
|
+
if regional_uuid.version != 8 or regional_uuid.variant != uuid.RFC_4122:
|
|
32
|
+
return None
|
|
33
|
+
|
|
34
|
+
uuid_bytes = regional_uuid.bytes
|
|
35
|
+
return uuid_bytes[:2].decode("ascii")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def decode_agent_token(agent_token: str) -> tuple[Optional[str], Optional[str]]:
|
|
23
39
|
split_token = agent_token.split(":")
|
|
24
40
|
|
|
25
41
|
# Legacy agent token format - organization must be specified in dagster.yaml
|
|
26
42
|
if len(split_token) == 1:
|
|
27
|
-
return None
|
|
43
|
+
return None, None
|
|
28
44
|
|
|
29
45
|
token_type, *token = split_token
|
|
30
46
|
|
|
@@ -35,6 +51,6 @@ def get_organization_name_from_agent_token(agent_token: str) -> Optional[str]:
|
|
|
35
51
|
"Generate a new agent token in Dagster Cloud."
|
|
36
52
|
)
|
|
37
53
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
return organization
|
|
54
|
+
# token format: agent:<org>:<uuid>
|
|
55
|
+
organization, uuid_str = token
|
|
56
|
+
return organization, decode_region_from_uuid(uuid_str)
|
|
@@ -34,7 +34,7 @@ from urllib3 import Retry
|
|
|
34
34
|
from dagster_cloud.agent import AgentQueuesConfig
|
|
35
35
|
from dagster_cloud.version import __version__
|
|
36
36
|
|
|
37
|
-
from ..auth.constants import
|
|
37
|
+
from ..auth.constants import decode_agent_token
|
|
38
38
|
from ..opentelemetry.config import opentelemetry_config_schema
|
|
39
39
|
from ..opentelemetry.controller import OpenTelemetryController
|
|
40
40
|
from ..storage.client import dagster_cloud_api_config
|
|
@@ -87,14 +87,18 @@ class DagsterCloudAgentInstance(DagsterCloudInstance):
|
|
|
87
87
|
super().__init__(*args, **kwargs)
|
|
88
88
|
|
|
89
89
|
self._unprocessed_dagster_cloud_api_config = dagster_cloud_api
|
|
90
|
-
self._dagster_cloud_api_config =
|
|
91
|
-
|
|
90
|
+
self._dagster_cloud_api_config = check.not_none(
|
|
91
|
+
self._get_processed_config(
|
|
92
|
+
"dagster_cloud_api",
|
|
93
|
+
dagster_cloud_api,
|
|
94
|
+
dagster_cloud_api_config(),
|
|
95
|
+
)
|
|
92
96
|
)
|
|
93
97
|
|
|
94
98
|
check.invariant(
|
|
95
99
|
not (
|
|
96
|
-
self._dagster_cloud_api_config.get("deployment")
|
|
97
|
-
and self._dagster_cloud_api_config.get("deployments")
|
|
100
|
+
self._dagster_cloud_api_config.get("deployment")
|
|
101
|
+
and self._dagster_cloud_api_config.get("deployments")
|
|
98
102
|
),
|
|
99
103
|
"Cannot set both deployment and deployments in `dagster_cloud_api`",
|
|
100
104
|
)
|
|
@@ -175,16 +179,16 @@ class DagsterCloudAgentInstance(DagsterCloudInstance):
|
|
|
175
179
|
return processed_config.value
|
|
176
180
|
|
|
177
181
|
def _dagster_cloud_api_config_for_deployment(self, deployment_name: Optional[str]):
|
|
178
|
-
new_api_config = dict(copy.deepcopy(self._dagster_cloud_api_config))
|
|
182
|
+
new_api_config = dict(copy.deepcopy(self._dagster_cloud_api_config))
|
|
179
183
|
if deployment_name:
|
|
180
|
-
new_api_config["deployment"] = deployment_name
|
|
184
|
+
new_api_config["deployment"] = deployment_name
|
|
181
185
|
if self.includes_branch_deployments:
|
|
182
|
-
del new_api_config["branch_deployments"]
|
|
183
|
-
new_api_config.pop("deployments", None)
|
|
184
|
-
new_api_config.pop("all_serverless_deployments", None)
|
|
186
|
+
del new_api_config["branch_deployments"]
|
|
187
|
+
new_api_config.pop("deployments", None)
|
|
188
|
+
new_api_config.pop("all_serverless_deployments", None)
|
|
185
189
|
else:
|
|
186
|
-
new_api_config.pop("deployment", None)
|
|
187
|
-
new_api_config.pop("deployments", None)
|
|
190
|
+
new_api_config.pop("deployment", None)
|
|
191
|
+
new_api_config.pop("deployments", None)
|
|
188
192
|
|
|
189
193
|
return new_api_config
|
|
190
194
|
|
|
@@ -238,11 +242,11 @@ class DagsterCloudAgentInstance(DagsterCloudInstance):
|
|
|
238
242
|
return socket_option
|
|
239
243
|
|
|
240
244
|
def _socket_options(self):
|
|
241
|
-
if self._dagster_cloud_api_config.get("socket_options") is None:
|
|
245
|
+
if self._dagster_cloud_api_config.get("socket_options") is None:
|
|
242
246
|
return None
|
|
243
247
|
|
|
244
248
|
translated_socket_options = []
|
|
245
|
-
for socket_option in self._dagster_cloud_api_config["socket_options"]:
|
|
249
|
+
for socket_option in self._dagster_cloud_api_config["socket_options"]:
|
|
246
250
|
check.invariant(
|
|
247
251
|
len(socket_option) == 3, "Each socket option must be a list of three values"
|
|
248
252
|
)
|
|
@@ -284,7 +288,7 @@ class DagsterCloudAgentInstance(DagsterCloudInstance):
|
|
|
284
288
|
adapter_kwargs=dict(
|
|
285
289
|
max_retries=Retry(
|
|
286
290
|
total=self.dagster_cloud_api_retries,
|
|
287
|
-
backoff_factor=self._dagster_cloud_api_config["backoff_factor"],
|
|
291
|
+
backoff_factor=self._dagster_cloud_api_config["backoff_factor"],
|
|
288
292
|
),
|
|
289
293
|
socket_options=self._socket_options(),
|
|
290
294
|
)
|
|
@@ -314,21 +318,26 @@ class DagsterCloudAgentInstance(DagsterCloudInstance):
|
|
|
314
318
|
|
|
315
319
|
@property
|
|
316
320
|
def dagster_cloud_url(self):
|
|
317
|
-
if "url" in self._dagster_cloud_api_config:
|
|
318
|
-
return self._dagster_cloud_api_config["url"]
|
|
321
|
+
if "url" in self._dagster_cloud_api_config:
|
|
322
|
+
return self._dagster_cloud_api_config["url"]
|
|
319
323
|
|
|
320
|
-
organization =
|
|
324
|
+
organization, region = decode_agent_token(self.dagster_cloud_agent_token)
|
|
321
325
|
if not organization:
|
|
322
326
|
raise DagsterInvariantViolationError(
|
|
323
327
|
"Could not derive Dagster Cloud URL from agent token. Create a new agent token or"
|
|
324
328
|
" set the `url` field under `dagster_cloud_api` in your `dagster.yaml`."
|
|
325
329
|
)
|
|
326
330
|
|
|
327
|
-
return
|
|
331
|
+
return (
|
|
332
|
+
f"https://{organization}.agent.{region}.dagster.cloud"
|
|
333
|
+
if region
|
|
334
|
+
else f"https://{organization}.agent.dagster.cloud"
|
|
335
|
+
)
|
|
328
336
|
|
|
329
337
|
@property
|
|
330
338
|
def organization_name(self) -> Optional[str]:
|
|
331
|
-
|
|
339
|
+
organization, _ = decode_agent_token(self.dagster_cloud_agent_token)
|
|
340
|
+
return organization
|
|
332
341
|
|
|
333
342
|
@property
|
|
334
343
|
def deployment_name(self) -> Optional[str]:
|
|
@@ -343,26 +352,31 @@ class DagsterCloudAgentInstance(DagsterCloudInstance):
|
|
|
343
352
|
|
|
344
353
|
@property
|
|
345
354
|
def deployment_names(self) -> list[str]:
|
|
346
|
-
if self._dagster_cloud_api_config.get("deployment"):
|
|
347
|
-
return [self._dagster_cloud_api_config["deployment"]]
|
|
355
|
+
if self._dagster_cloud_api_config.get("deployment"):
|
|
356
|
+
return [self._dagster_cloud_api_config["deployment"]]
|
|
348
357
|
|
|
349
|
-
return self._dagster_cloud_api_config.get("deployments", [])
|
|
358
|
+
return self._dagster_cloud_api_config.get("deployments", [])
|
|
350
359
|
|
|
351
360
|
@property
|
|
352
361
|
def include_all_serverless_deployments(self) -> bool:
|
|
353
|
-
return self._dagster_cloud_api_config.get("all_serverless_deployments")
|
|
362
|
+
return self._dagster_cloud_api_config.get("all_serverless_deployments") or False
|
|
354
363
|
|
|
355
364
|
@property
|
|
356
365
|
def dagit_url(self):
|
|
357
|
-
organization =
|
|
366
|
+
organization, region = decode_agent_token(self.dagster_cloud_agent_token)
|
|
358
367
|
if not organization:
|
|
359
368
|
raise Exception(
|
|
360
369
|
"Could not derive Dagster Cloud URL from agent token to generate a Dagit URL."
|
|
361
370
|
" Generate a new agent token in the Dagit UI."
|
|
362
371
|
)
|
|
363
372
|
|
|
364
|
-
deployment = self._dagster_cloud_api_config.get("deployment")
|
|
365
|
-
|
|
373
|
+
deployment = self._dagster_cloud_api_config.get("deployment")
|
|
374
|
+
base_url = (
|
|
375
|
+
f"https://{organization}.{region}.dagster.cloud/"
|
|
376
|
+
if region
|
|
377
|
+
else f"https://{organization}.dagster.cloud/"
|
|
378
|
+
)
|
|
379
|
+
return base_url + (f"{deployment}/" if deployment else "")
|
|
366
380
|
|
|
367
381
|
@property
|
|
368
382
|
def dagster_cloud_graphql_url(self):
|
|
@@ -412,33 +426,38 @@ class DagsterCloudAgentInstance(DagsterCloudInstance):
|
|
|
412
426
|
return get_agent_headers(self._dagster_cloud_api_config, scope=scope) # pyright: ignore[reportArgumentType]
|
|
413
427
|
|
|
414
428
|
@property
|
|
415
|
-
def dagster_cloud_agent_token(self):
|
|
416
|
-
|
|
429
|
+
def dagster_cloud_agent_token(self) -> str:
|
|
430
|
+
check.invariant(
|
|
431
|
+
self._dagster_cloud_api_config.get("agent_token") is not None,
|
|
432
|
+
"No agent token found in dagster_cloud_api configuration. An agent token is required"
|
|
433
|
+
" for Dagster Cloud authentication.",
|
|
434
|
+
)
|
|
435
|
+
return self._dagster_cloud_api_config["agent_token"]
|
|
417
436
|
|
|
418
437
|
@property
|
|
419
438
|
def dagster_cloud_api_retries(self) -> int:
|
|
420
|
-
return self._dagster_cloud_api_config["retries"]
|
|
439
|
+
return self._dagster_cloud_api_config["retries"]
|
|
421
440
|
|
|
422
441
|
@property
|
|
423
442
|
def dagster_cloud_api_timeout(self) -> int:
|
|
424
|
-
return self._dagster_cloud_api_config["timeout"]
|
|
443
|
+
return self._dagster_cloud_api_config["timeout"]
|
|
425
444
|
|
|
426
445
|
@property
|
|
427
446
|
def dagster_cloud_api_proxies(self) -> Optional[dict[str, str]]:
|
|
428
447
|
# Requests library modifies the proxies key so create a copy
|
|
429
448
|
return (
|
|
430
449
|
self._dagster_cloud_api_config.get("proxies").copy() # pyright: ignore[reportOptionalMemberAccess]
|
|
431
|
-
if self._dagster_cloud_api_config.get("proxies")
|
|
450
|
+
if self._dagster_cloud_api_config.get("proxies")
|
|
432
451
|
else {}
|
|
433
452
|
)
|
|
434
453
|
|
|
435
454
|
@property
|
|
436
455
|
def dagster_cloud_api_agent_label(self) -> Optional[str]:
|
|
437
|
-
return self._dagster_cloud_api_config.get("agent_label")
|
|
456
|
+
return self._dagster_cloud_api_config.get("agent_label")
|
|
438
457
|
|
|
439
458
|
@property
|
|
440
459
|
def includes_branch_deployments(self) -> bool:
|
|
441
|
-
return self._dagster_cloud_api_config.get("branch_deployments", False)
|
|
460
|
+
return self._dagster_cloud_api_config.get("branch_deployments", False)
|
|
442
461
|
|
|
443
462
|
@property
|
|
444
463
|
def instance_uuid(self) -> str:
|
|
@@ -103,6 +103,7 @@ class CloudComputeLogManager(
|
|
|
103
103
|
# for back-compat
|
|
104
104
|
"run_id": log_key[0],
|
|
105
105
|
"key": log_key[-1],
|
|
106
|
+
"method": "PUT",
|
|
106
107
|
}
|
|
107
108
|
if partial:
|
|
108
109
|
params["partial"] = True
|
|
@@ -119,10 +120,9 @@ class CloudComputeLogManager(
|
|
|
119
120
|
if resp_data.get("skip_upload"):
|
|
120
121
|
return
|
|
121
122
|
|
|
122
|
-
self._upload_session.
|
|
123
|
+
self._upload_session.put(
|
|
123
124
|
resp_data["url"],
|
|
124
|
-
data=
|
|
125
|
-
files={"file": data},
|
|
125
|
+
data=data,
|
|
126
126
|
timeout=self._instance.dagster_cloud_api_timeout,
|
|
127
127
|
)
|
|
128
128
|
|
|
@@ -1,18 +1,36 @@
|
|
|
1
|
+
from pathlib import Path
|
|
1
2
|
from typing import TYPE_CHECKING, Any, Optional
|
|
2
3
|
|
|
3
4
|
import dagster._check as check
|
|
5
|
+
from dagster._core.storage.defs_state.base import DefsStateStorage
|
|
4
6
|
from dagster._serdes import ConfigurableClass, ConfigurableClassData
|
|
7
|
+
from dagster_cloud_cli.core.artifacts import download_artifact, upload_artifact
|
|
8
|
+
from dagster_cloud_cli.core.headers.auth import DagsterCloudInstanceScope
|
|
9
|
+
from dagster_shared.serdes import deserialize_value
|
|
10
|
+
from dagster_shared.serdes.objects.models.defs_state_info import DefsStateInfo
|
|
5
11
|
from typing_extensions import Self
|
|
6
12
|
|
|
7
|
-
from dagster_cloud.storage.defs_state.base_storage import BaseGraphQLDefsStateStorage
|
|
8
|
-
|
|
9
13
|
if TYPE_CHECKING:
|
|
10
14
|
from dagster_cloud.instance import DagsterCloudAgentInstance # noqa: F401
|
|
11
15
|
|
|
16
|
+
GET_LATEST_DEFS_STATE_INFO_QUERY = """
|
|
17
|
+
query getLatestDefsStateInfo {
|
|
18
|
+
latestDefsStateInfo
|
|
19
|
+
}
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
SET_LATEST_VERSION_MUTATION = """
|
|
23
|
+
mutation setLatestDefsStateVersion($key: String!, $version: String!) {
|
|
24
|
+
defsState {
|
|
25
|
+
setLatestDefsStateVersion(key: $key, version: $version) {
|
|
26
|
+
ok
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
"""
|
|
31
|
+
|
|
12
32
|
|
|
13
|
-
class GraphQLDefsStateStorage(
|
|
14
|
-
BaseGraphQLDefsStateStorage["DagsterCloudAgentInstance"], ConfigurableClass
|
|
15
|
-
):
|
|
33
|
+
class GraphQLDefsStateStorage(DefsStateStorage["DagsterCloudAgentInstance"], ConfigurableClass):
|
|
16
34
|
def __init__(
|
|
17
35
|
self, inst_data: Optional[ConfigurableClassData] = None, override_graphql_client=None
|
|
18
36
|
):
|
|
@@ -53,3 +71,43 @@ class GraphQLDefsStateStorage(
|
|
|
53
71
|
if self._override_graphql_client
|
|
54
72
|
else self._instance.graphql_client # pyright: ignore[reportAttributeAccessIssue]
|
|
55
73
|
)
|
|
74
|
+
|
|
75
|
+
def _execute_query(self, query, variables=None, idempotent_mutation=False):
|
|
76
|
+
return self.graphql_client.execute(
|
|
77
|
+
query, variable_values=variables, idempotent_mutation=idempotent_mutation
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
def _get_artifact_key(self, key: str, version: str) -> str:
|
|
81
|
+
return f"__state__/{key}/{version}"
|
|
82
|
+
|
|
83
|
+
def download_state_to_path(self, key: str, version: str, path: Path) -> None:
|
|
84
|
+
download_artifact(
|
|
85
|
+
url=self.url,
|
|
86
|
+
scope=DagsterCloudInstanceScope.DEPLOYMENT,
|
|
87
|
+
api_token=self.api_token,
|
|
88
|
+
key=self._get_artifact_key(key, version),
|
|
89
|
+
path=path,
|
|
90
|
+
deployment=self.deployment,
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
def upload_state_from_path(self, key: str, version: str, path: Path) -> None:
|
|
94
|
+
upload_artifact(
|
|
95
|
+
url=self.url,
|
|
96
|
+
scope=DagsterCloudInstanceScope.DEPLOYMENT,
|
|
97
|
+
api_token=self.api_token,
|
|
98
|
+
key=self._get_artifact_key(key, version),
|
|
99
|
+
path=path,
|
|
100
|
+
deployment=self.deployment,
|
|
101
|
+
)
|
|
102
|
+
self.set_latest_version(key, version)
|
|
103
|
+
|
|
104
|
+
def get_latest_defs_state_info(self) -> Optional[DefsStateInfo]:
|
|
105
|
+
res = self._execute_query(GET_LATEST_DEFS_STATE_INFO_QUERY)
|
|
106
|
+
result = res["data"]["latestDefsStateInfo"]
|
|
107
|
+
if result is not None:
|
|
108
|
+
return deserialize_value(result, DefsStateInfo)
|
|
109
|
+
else:
|
|
110
|
+
return None
|
|
111
|
+
|
|
112
|
+
def set_latest_version(self, key: str, version: str) -> None:
|
|
113
|
+
self._execute_query(SET_LATEST_VERSION_MUTATION, variables={"key": key, "version": version})
|
dagster_cloud/version.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = "1.11.
|
|
1
|
+
__version__ = "1.11.12"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dagster-cloud
|
|
3
|
-
Version: 1.11.
|
|
3
|
+
Version: 1.11.12
|
|
4
4
|
Author-email: Elementl <support@elementl.com>
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
Project-URL: Homepage, https://dagster.io/cloud
|
|
@@ -28,9 +28,9 @@ Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
|
|
|
28
28
|
Classifier: Operating System :: OS Independent
|
|
29
29
|
Requires-Python: <3.14,>=3.9
|
|
30
30
|
Description-Content-Type: text/markdown
|
|
31
|
-
Requires-Dist: dagster==1.11.
|
|
32
|
-
Requires-Dist: dagster-shared==1.11.
|
|
33
|
-
Requires-Dist: dagster-cloud-cli==1.11.
|
|
31
|
+
Requires-Dist: dagster==1.11.12
|
|
32
|
+
Requires-Dist: dagster-shared==1.11.12
|
|
33
|
+
Requires-Dist: dagster-cloud-cli==1.11.12
|
|
34
34
|
Requires-Dist: opentelemetry-api<2,>=1.27.0
|
|
35
35
|
Requires-Dist: opentelemetry-sdk<2,>=1.27.0
|
|
36
36
|
Requires-Dist: opentelemetry-exporter-otlp-proto-grpc<2,>=1.27.0
|
|
@@ -65,12 +65,12 @@ Provides-Extra: insights
|
|
|
65
65
|
Requires-Dist: pyarrow; extra == "insights"
|
|
66
66
|
Provides-Extra: docker
|
|
67
67
|
Requires-Dist: docker; extra == "docker"
|
|
68
|
-
Requires-Dist: dagster-docker==0.27.
|
|
68
|
+
Requires-Dist: dagster-docker==0.27.12; extra == "docker"
|
|
69
69
|
Provides-Extra: kubernetes
|
|
70
70
|
Requires-Dist: kubernetes; extra == "kubernetes"
|
|
71
|
-
Requires-Dist: dagster-k8s==0.27.
|
|
71
|
+
Requires-Dist: dagster-k8s==0.27.12; extra == "kubernetes"
|
|
72
72
|
Provides-Extra: ecs
|
|
73
|
-
Requires-Dist: dagster-aws==0.27.
|
|
73
|
+
Requires-Dist: dagster-aws==0.27.12; extra == "ecs"
|
|
74
74
|
Requires-Dist: boto3; extra == "ecs"
|
|
75
75
|
Provides-Extra: sandbox
|
|
76
76
|
Requires-Dist: supervisor; extra == "sandbox"
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
dagster_cloud/__init__.py,sha256=q3UY5eTfCSqZrr35mmn1b1B_AD1VbfR4Alj8cZ49Xg4,316
|
|
2
2
|
dagster_cloud/constants.py,sha256=CPAqXJ99SWGMviksdIA2A9894FEvHChNk8UcP4TluYM,455
|
|
3
3
|
dagster_cloud/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
|
-
dagster_cloud/version.py,sha256
|
|
4
|
+
dagster_cloud/version.py,sha256=9Rb3yC-7xmQLAyvOVo4ZifFK2ok7cTSv1S1wYtretwE,24
|
|
5
5
|
dagster_cloud/agent/__init__.py,sha256=_erVyIrxuHUiyNerwX8vNZcKZN8NAloTEkPq8vPZ3MI,811
|
|
6
6
|
dagster_cloud/agent/dagster_cloud_agent.py,sha256=C72rumzRFFX41L2y7j-pJFrTAnvW60ZiS_AQRTUjExg,57013
|
|
7
7
|
dagster_cloud/agent/queries.py,sha256=iI84GQ1Zxt5ryo6M1ELIaIae-gwUY14QPPMUeiFK97o,1837
|
|
@@ -19,7 +19,7 @@ dagster_cloud/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
|
|
|
19
19
|
dagster_cloud/api/dagster_cloud_api.py,sha256=6ge6WX8SCRKu9wTD0YgTBvCHgPN1_DFrIH_Y0Y9GFuA,14589
|
|
20
20
|
dagster_cloud/artifacts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
21
21
|
dagster_cloud/auth/__init__.py,sha256=rUuAWah3YD-0tatQPFlmLNJpsGQ9ZRj7sUOSTrnf0zQ,248
|
|
22
|
-
dagster_cloud/auth/constants.py,sha256=
|
|
22
|
+
dagster_cloud/auth/constants.py,sha256=zFhKSUBkBKz5_eSTCp5FV8K_HFYJRGHDby-xC7KE4bo,1808
|
|
23
23
|
dagster_cloud/batching/__init__.py,sha256=522PJZX1fmhpA91M32eDIwvYqn2PuqBodIOTcJmsXwc,40
|
|
24
24
|
dagster_cloud/batching/batcher.py,sha256=kt9F-WdMnCJstCJFuHqagG_eHFTWjlfRQdhE-Cc9Hvw,7796
|
|
25
25
|
dagster_cloud/dagster_insights/__init__.py,sha256=_qTVj3INjL-Y2YADy7uElk4HStZUBZoPD32jY5DbwF4,1864
|
|
@@ -46,7 +46,7 @@ dagster_cloud/execution/cloud_run_launcher/process.py,sha256=eYxvzcaJbqYKjHcXMUU
|
|
|
46
46
|
dagster_cloud/execution/monitoring/__init__.py,sha256=2_o2UjeEPvfSpnOb0zky3ZQs-G9VHABAXzjvfMnAh8A,17782
|
|
47
47
|
dagster_cloud/execution/utils/__init__.py,sha256=EfU-tK5-ScZYLF_zmWqUgIrmLsb9DDHWFN-CSg3BSXY,254
|
|
48
48
|
dagster_cloud/execution/utils/process.py,sha256=Qz6Vp4d2siT7z_la7n1zybJLXfRG3hvTu9ePFJA4TA0,883
|
|
49
|
-
dagster_cloud/instance/__init__.py,sha256=
|
|
49
|
+
dagster_cloud/instance/__init__.py,sha256=bZ4ZOUaa4Xg71DmYAHdxxTfSv9BEdjXB70esDWQyX-4,26193
|
|
50
50
|
dagster_cloud/instrumentation/__init__.py,sha256=OKaT4qkwQNmCFdKveVrgB7JIo5sZD1F1rutQKdozK_4,886
|
|
51
51
|
dagster_cloud/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
52
52
|
dagster_cloud/metadata/source_code.py,sha256=yCUfD6V-wR2s4AUrmVpX_Dd7Gln7SYbtpuEpAH71J1I,5952
|
|
@@ -91,11 +91,10 @@ dagster_cloud/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3h
|
|
|
91
91
|
dagster_cloud/storage/client.py,sha256=jsqlQehNImOygYEAq0NoEVlbCn5EbIS0VZJabuVtxL8,2992
|
|
92
92
|
dagster_cloud/storage/tags.py,sha256=YeWSWgpguqvpndFGMM0W7gORn0F8P3CDcanElE7n2Vo,2414
|
|
93
93
|
dagster_cloud/storage/compute_logs/__init__.py,sha256=6O5IqKYrNZ6Yz0cataEa2_8u6kYAIP6PXOoeqh5jvjo,82
|
|
94
|
-
dagster_cloud/storage/compute_logs/compute_log_manager.py,sha256=
|
|
94
|
+
dagster_cloud/storage/compute_logs/compute_log_manager.py,sha256=OBNuGj0PECWmnAfbayFtRixztOcqaK-IGmgOD0AokfY,4857
|
|
95
95
|
dagster_cloud/storage/defs_state/__init__.py,sha256=MCt72laA0K6t7dtZfqxlS9I7MYtFzI6lTR2NN40A9e8,72
|
|
96
|
-
dagster_cloud/storage/defs_state/base_storage.py,sha256=_OtLfplIG5yVCZOjCZbdmEfCKm5yrgBdKXt6Wi8DMZ0,2976
|
|
97
96
|
dagster_cloud/storage/defs_state/queries.py,sha256=UICxqqWH1gCpE_YR7TedxanES2gZ_7dR00y6KR8HWQA,365
|
|
98
|
-
dagster_cloud/storage/defs_state/storage.py,sha256=
|
|
97
|
+
dagster_cloud/storage/defs_state/storage.py,sha256=V-YO63i8DBGCiCj5Ptu9U12jOMBCxQCvaF77YQWJrA8,4006
|
|
99
98
|
dagster_cloud/storage/event_logs/__init__.py,sha256=N45uxmVOvMtOSYJFktLE32sJ6Ju9EFcwMH4b8L2DDFk,70
|
|
100
99
|
dagster_cloud/storage/event_logs/queries.py,sha256=sgkHhHX-0Vr9WukcRjbuGSH8vjC9razSNnwqXBrihX8,19924
|
|
101
100
|
dagster_cloud/storage/event_logs/storage.py,sha256=bu3GOrPu_RkCr29xic78-rRfI4h_ZTMOVZGWQj8jtSc,56438
|
|
@@ -129,7 +128,7 @@ dagster_cloud/workspace/user_code_launcher/__init__.py,sha256=6ib1U47aqB4R9BbzJV
|
|
|
129
128
|
dagster_cloud/workspace/user_code_launcher/process.py,sha256=VZMfBf4OmXNwP0H0ckUMpqD-glqsBgACXS-xO6Pn6nM,14274
|
|
130
129
|
dagster_cloud/workspace/user_code_launcher/user_code_launcher.py,sha256=DqWeZ_w-h4tP2ceLyvOHWeSPM6dUnr7gY_VmuSKsEYA,102911
|
|
131
130
|
dagster_cloud/workspace/user_code_launcher/utils.py,sha256=t8Epee9MrXtRhWL-b_3avXxgMGrjLScUNWtBUUGpMCg,5285
|
|
132
|
-
dagster_cloud-1.11.
|
|
133
|
-
dagster_cloud-1.11.
|
|
134
|
-
dagster_cloud-1.11.
|
|
135
|
-
dagster_cloud-1.11.
|
|
131
|
+
dagster_cloud-1.11.12.dist-info/METADATA,sha256=DP3Nwd6mkUdfyODjBTiBsHAN0oMSYwtRMubwl1IY4p4,6584
|
|
132
|
+
dagster_cloud-1.11.12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
133
|
+
dagster_cloud-1.11.12.dist-info/top_level.txt,sha256=2hMt-U33jyCgnywNrDB9Ih0EpaVmiO6dFkYcJ7Iwx4I,14
|
|
134
|
+
dagster_cloud-1.11.12.dist-info/RECORD,,
|
|
@@ -1,90 +0,0 @@
|
|
|
1
|
-
from abc import ABC, abstractmethod
|
|
2
|
-
from pathlib import Path
|
|
3
|
-
from typing import Any, Generic, Optional
|
|
4
|
-
|
|
5
|
-
from dagster._core.instance.types import T_DagsterInstance
|
|
6
|
-
from dagster._core.storage.defs_state.base import DefsStateStorage
|
|
7
|
-
from dagster_cloud_cli.core.artifacts import download_artifact, upload_artifact
|
|
8
|
-
from dagster_cloud_cli.core.headers.auth import DagsterCloudInstanceScope
|
|
9
|
-
from dagster_shared.serdes import deserialize_value
|
|
10
|
-
from dagster_shared.serdes.objects.models.defs_state_info import DefsStateInfo
|
|
11
|
-
|
|
12
|
-
GET_LATEST_DEFS_STATE_INFO_QUERY = """
|
|
13
|
-
query getLatestDefsStateInfo {
|
|
14
|
-
latestDefsStateInfo
|
|
15
|
-
}
|
|
16
|
-
"""
|
|
17
|
-
|
|
18
|
-
SET_LATEST_VERSION_MUTATION = """
|
|
19
|
-
mutation setLatestDefsStateVersion($key: String!, $version: String!) {
|
|
20
|
-
defsState {
|
|
21
|
-
setLatestDefsStateVersion(key: $key, version: $version) {
|
|
22
|
-
ok
|
|
23
|
-
}
|
|
24
|
-
}
|
|
25
|
-
}
|
|
26
|
-
"""
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
class BaseGraphQLDefsStateStorage(
|
|
30
|
-
DefsStateStorage[T_DagsterInstance], ABC, Generic[T_DagsterInstance]
|
|
31
|
-
):
|
|
32
|
-
"""Base implementation of a DefsStateStorage that uses a GraphQL client to
|
|
33
|
-
interact with the Dagster+ API.
|
|
34
|
-
"""
|
|
35
|
-
|
|
36
|
-
@property
|
|
37
|
-
@abstractmethod
|
|
38
|
-
def url(self) -> str: ...
|
|
39
|
-
|
|
40
|
-
@property
|
|
41
|
-
@abstractmethod
|
|
42
|
-
def api_token(self) -> str: ...
|
|
43
|
-
|
|
44
|
-
@property
|
|
45
|
-
@abstractmethod
|
|
46
|
-
def deployment(self) -> str: ...
|
|
47
|
-
|
|
48
|
-
@property
|
|
49
|
-
@abstractmethod
|
|
50
|
-
def graphql_client(self) -> Any: ...
|
|
51
|
-
|
|
52
|
-
def _execute_query(self, query, variables=None, idempotent_mutation=False):
|
|
53
|
-
return self.graphql_client.execute(
|
|
54
|
-
query, variable_values=variables, idempotent_mutation=idempotent_mutation
|
|
55
|
-
)
|
|
56
|
-
|
|
57
|
-
def _get_artifact_key(self, key: str, version: str) -> str:
|
|
58
|
-
return f"__state__/{key}/{version}"
|
|
59
|
-
|
|
60
|
-
def download_state_to_path(self, key: str, version: str, path: Path) -> None:
|
|
61
|
-
download_artifact(
|
|
62
|
-
url=self.url,
|
|
63
|
-
scope=DagsterCloudInstanceScope.DEPLOYMENT,
|
|
64
|
-
api_token=self.api_token,
|
|
65
|
-
key=self._get_artifact_key(key, version),
|
|
66
|
-
path=path,
|
|
67
|
-
deployment=self.deployment,
|
|
68
|
-
)
|
|
69
|
-
|
|
70
|
-
def upload_state_from_path(self, key: str, version: str, path: Path) -> None:
|
|
71
|
-
upload_artifact(
|
|
72
|
-
url=self.url,
|
|
73
|
-
scope=DagsterCloudInstanceScope.DEPLOYMENT,
|
|
74
|
-
api_token=self.api_token,
|
|
75
|
-
key=self._get_artifact_key(key, version),
|
|
76
|
-
path=path,
|
|
77
|
-
deployment=self.deployment,
|
|
78
|
-
)
|
|
79
|
-
self.set_latest_version(key, version)
|
|
80
|
-
|
|
81
|
-
def get_latest_defs_state_info(self) -> Optional[DefsStateInfo]:
|
|
82
|
-
res = self._execute_query(GET_LATEST_DEFS_STATE_INFO_QUERY)
|
|
83
|
-
result = res["data"]["latestDefsStateInfo"]
|
|
84
|
-
if result is not None:
|
|
85
|
-
return deserialize_value(result, DefsStateInfo)
|
|
86
|
-
else:
|
|
87
|
-
return None
|
|
88
|
-
|
|
89
|
-
def set_latest_version(self, key: str, version: str) -> None:
|
|
90
|
-
self._execute_query(SET_LATEST_VERSION_MUTATION, variables={"key": key, "version": version})
|
|
File without changes
|
|
File without changes
|