dagster-aws 0.26.11__tar.gz → 0.28.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/LICENSE +1 -1
- dagster_aws-0.28.5/PKG-INFO +44 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/athena/resources.py +1 -1
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/ecs/executor.py +8 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/ecs/launcher.py +45 -5
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/ecs/tasks.py +10 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/ecs/utils.py +4 -1
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/pipes/context_injectors.py +6 -5
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/pipes/message_readers.py +2 -2
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/rds/resources.py +1 -1
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/redshift/resources.py +1 -1
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/s3/compute_log_manager.py +13 -15
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/s3/io_manager.py +2 -2
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/s3/resources.py +1 -1
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/s3/sensor.py +2 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/secretsmanager/resources.py +2 -2
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/ssm/resources.py +1 -1
- dagster_aws-0.28.5/dagster_aws/version.py +1 -0
- dagster_aws-0.28.5/dagster_aws.egg-info/PKG-INFO +44 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws.egg-info/requires.txt +2 -1
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/setup.py +4 -3
- dagster-aws-0.26.11/PKG-INFO +0 -20
- dagster-aws-0.26.11/dagster_aws/version.py +0 -1
- dagster-aws-0.26.11/dagster_aws.egg-info/PKG-INFO +0 -20
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/MANIFEST.in +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/README.md +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/__init__.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/_stubs.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/athena/__init__.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/cloudwatch/__init__.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/cloudwatch/loggers.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/ecr/__init__.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/ecr/resources.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/ecs/__init__.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/ecs/container_context.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/ecs/test_utils.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/emr/__init__.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/emr/configs.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/emr/configs_spark.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/emr/emr.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/emr/emr_step_main.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/emr/main.py.template +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/emr/pyspark_step_launcher.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/emr/types.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/emr/utils.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/pipes/__init__.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/pipes/clients/__init__.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/pipes/clients/ecs.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/pipes/clients/emr.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/pipes/clients/emr_containers.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/pipes/clients/emr_serverless.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/pipes/clients/glue.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/pipes/clients/lambda_.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/pipes/clients/utils.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/py.typed +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/rds/__init__.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/redshift/__init__.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/s3/__init__.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/s3/file_manager.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/s3/ops.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/s3/s3_fake_resource.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/s3/utils.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/secretsmanager/__init__.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/secretsmanager/secrets.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/ssm/__init__.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/ssm/parameters.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/utils/__init__.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/utils/mrjob/__init__.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/utils/mrjob/log4j.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/utils/mrjob/retry.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/utils/mrjob/utils.py +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws.egg-info/SOURCES.txt +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws.egg-info/dependency_links.txt +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws.egg-info/not-zip-safe +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws.egg-info/top_level.txt +0 -0
- {dagster-aws-0.26.11 → dagster_aws-0.28.5}/setup.cfg +0 -0
|
@@ -186,7 +186,7 @@
|
|
|
186
186
|
same "printed page" as the copyright notice for easier
|
|
187
187
|
identification within third-party archives.
|
|
188
188
|
|
|
189
|
-
Copyright
|
|
189
|
+
Copyright 2025 Dagster Labs, Inc.
|
|
190
190
|
|
|
191
191
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
192
|
you may not use this file except in compliance with the License.
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dagster-aws
|
|
3
|
+
Version: 0.28.5
|
|
4
|
+
Summary: Package for AWS-specific Dagster framework solid and resource components.
|
|
5
|
+
Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-aws
|
|
6
|
+
Author: Dagster Labs
|
|
7
|
+
Author-email: hello@dagsterlabs.com
|
|
8
|
+
License: Apache-2.0
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
13
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Requires-Python: >=3.10,<3.14
|
|
16
|
+
License-File: LICENSE
|
|
17
|
+
Requires-Dist: boto3
|
|
18
|
+
Requires-Dist: dagster==1.12.5
|
|
19
|
+
Requires-Dist: packaging
|
|
20
|
+
Requires-Dist: requests
|
|
21
|
+
Provides-Extra: redshift
|
|
22
|
+
Requires-Dist: psycopg2-binary; extra == "redshift"
|
|
23
|
+
Provides-Extra: pyspark
|
|
24
|
+
Requires-Dist: dagster-pyspark; extra == "pyspark"
|
|
25
|
+
Provides-Extra: stubs
|
|
26
|
+
Requires-Dist: boto3-stubs-lite[ecs,emr,emr-containers,emr-serverless,glue,logs,s3]; extra == "stubs"
|
|
27
|
+
Provides-Extra: test
|
|
28
|
+
Requires-Dist: botocore!=1.32.1; extra == "test"
|
|
29
|
+
Requires-Dist: moto[emrcontainers,emrserverless,glue,logs,s3,server]<5.0,>=2.2.8; extra == "test"
|
|
30
|
+
Requires-Dist: requests-mock; extra == "test"
|
|
31
|
+
Requires-Dist: xmltodict==0.12.0; extra == "test"
|
|
32
|
+
Requires-Dist: flaky; extra == "test"
|
|
33
|
+
Requires-Dist: pytest-cases; extra == "test"
|
|
34
|
+
Requires-Dist: s3fs; extra == "test"
|
|
35
|
+
Dynamic: author
|
|
36
|
+
Dynamic: author-email
|
|
37
|
+
Dynamic: classifier
|
|
38
|
+
Dynamic: home-page
|
|
39
|
+
Dynamic: license
|
|
40
|
+
Dynamic: license-file
|
|
41
|
+
Dynamic: provides-extra
|
|
42
|
+
Dynamic: requires-dist
|
|
43
|
+
Dynamic: requires-python
|
|
44
|
+
Dynamic: summary
|
|
@@ -254,7 +254,7 @@ class AthenaClientResource(ResourceWithAthenaConfig):
|
|
|
254
254
|
def example_athena_asset(athena: AthenaClientResource):
|
|
255
255
|
return athena.get_client().execute_query("SELECT 1", fetch_results=True)
|
|
256
256
|
|
|
257
|
-
|
|
257
|
+
Definitions(
|
|
258
258
|
assets=[example_athena_asset],
|
|
259
259
|
resources={"athena": AthenaClientResource()}
|
|
260
260
|
)
|
|
@@ -17,6 +17,10 @@ from dagster._core.definitions.executor_definition import multiple_process_execu
|
|
|
17
17
|
from dagster._core.definitions.metadata import MetadataValue
|
|
18
18
|
from dagster._core.events import DagsterEvent, EngineEventData
|
|
19
19
|
from dagster._core.execution.retries import RetryMode, get_retries_config
|
|
20
|
+
from dagster._core.execution.step_dependency_config import (
|
|
21
|
+
StepDependencyConfig,
|
|
22
|
+
get_step_dependency_config_field,
|
|
23
|
+
)
|
|
20
24
|
from dagster._core.execution.tags import get_tag_concurrency_limits_config
|
|
21
25
|
from dagster._core.executor.base import Executor
|
|
22
26
|
from dagster._core.executor.init import InitExecutorContext
|
|
@@ -68,6 +72,7 @@ _ECS_EXECUTOR_CONFIG_SCHEMA = {
|
|
|
68
72
|
),
|
|
69
73
|
),
|
|
70
74
|
"tag_concurrency_limits": get_tag_concurrency_limits_config(),
|
|
75
|
+
"step_dependency_config": get_step_dependency_config_field(),
|
|
71
76
|
}
|
|
72
77
|
|
|
73
78
|
|
|
@@ -134,6 +139,9 @@ def ecs_executor(init_context: InitExecutorContext) -> Executor:
|
|
|
134
139
|
max_concurrent=check.opt_int_elem(exc_cfg, "max_concurrent"),
|
|
135
140
|
tag_concurrency_limits=check.opt_list_elem(exc_cfg, "tag_concurrency_limits"),
|
|
136
141
|
should_verify_step=True,
|
|
142
|
+
step_dependency_config=StepDependencyConfig.from_config(
|
|
143
|
+
exc_cfg.get("step_dependency_config") # type: ignore
|
|
144
|
+
),
|
|
137
145
|
)
|
|
138
146
|
|
|
139
147
|
|
|
@@ -5,7 +5,7 @@ import uuid
|
|
|
5
5
|
import warnings
|
|
6
6
|
from collections import namedtuple
|
|
7
7
|
from collections.abc import Mapping, Sequence
|
|
8
|
-
from typing import Any, Optional
|
|
8
|
+
from typing import TYPE_CHECKING, Any, Optional
|
|
9
9
|
|
|
10
10
|
import boto3
|
|
11
11
|
from botocore.exceptions import ClientError
|
|
@@ -30,11 +30,12 @@ from dagster._core.launcher.base import (
|
|
|
30
30
|
WorkerStatus,
|
|
31
31
|
)
|
|
32
32
|
from dagster._core.storage.dagster_run import DagsterRun
|
|
33
|
-
from dagster._core.storage.tags import RUN_WORKER_ID_TAG
|
|
33
|
+
from dagster._core.storage.tags import HIDDEN_TAG_PREFIX, RUN_WORKER_ID_TAG
|
|
34
34
|
from dagster._grpc.types import ExecuteRunArgs
|
|
35
35
|
from dagster._serdes import ConfigurableClass
|
|
36
36
|
from dagster._serdes.config_class import ConfigurableClassData
|
|
37
37
|
from dagster._utils.backoff import backoff
|
|
38
|
+
from dagster._utils.tags import get_boolean_tag_value
|
|
38
39
|
from typing_extensions import Self
|
|
39
40
|
|
|
40
41
|
from dagster_aws.ecs.container_context import (
|
|
@@ -81,6 +82,9 @@ TAGS_TO_EXCLUDE_FROM_PROPAGATION = {"dagster/op_selection", "dagster/solid_selec
|
|
|
81
82
|
DEFAULT_REGISTER_TASK_DEFINITION_RETRIES = 5
|
|
82
83
|
DEFAULT_RUN_TASK_RETRIES = 5
|
|
83
84
|
|
|
85
|
+
if TYPE_CHECKING:
|
|
86
|
+
from botocore.config import Config
|
|
87
|
+
|
|
84
88
|
|
|
85
89
|
class EcsRunLauncher(RunLauncher[T_DagsterInstance], ConfigurableClass):
|
|
86
90
|
"""RunLauncher that starts a task in ECS for each Dagster job run.
|
|
@@ -119,10 +123,13 @@ class EcsRunLauncher(RunLauncher[T_DagsterInstance], ConfigurableClass):
|
|
|
119
123
|
task_definition_prefix: str = "run",
|
|
120
124
|
):
|
|
121
125
|
self._inst_data = inst_data
|
|
122
|
-
|
|
126
|
+
|
|
127
|
+
boto_client_config = self.get_boto_client_config()
|
|
128
|
+
|
|
129
|
+
self.ecs = boto3.client("ecs", config=boto_client_config)
|
|
123
130
|
self.ec2 = boto3.resource("ec2")
|
|
124
|
-
self.secrets_manager = boto3.client("secretsmanager")
|
|
125
|
-
self.logs = boto3.client("logs")
|
|
131
|
+
self.secrets_manager = boto3.client("secretsmanager", config=boto_client_config)
|
|
132
|
+
self.logs = boto3.client("logs", config=boto_client_config)
|
|
126
133
|
|
|
127
134
|
self._task_definition_prefix = task_definition_prefix
|
|
128
135
|
|
|
@@ -236,6 +243,9 @@ class EcsRunLauncher(RunLauncher[T_DagsterInstance], ConfigurableClass):
|
|
|
236
243
|
self._current_task_metadata = None
|
|
237
244
|
self._current_task = None
|
|
238
245
|
|
|
246
|
+
def get_boto_client_config(self) -> Optional["Config"]:
|
|
247
|
+
return None
|
|
248
|
+
|
|
239
249
|
@property
|
|
240
250
|
def inst_data(self):
|
|
241
251
|
return self._inst_data
|
|
@@ -855,6 +865,28 @@ class EcsRunLauncher(RunLauncher[T_DagsterInstance], ConfigurableClass):
|
|
|
855
865
|
task.get("stoppedReason", "")
|
|
856
866
|
)
|
|
857
867
|
|
|
868
|
+
def _add_eni_id_tags(self, run: DagsterRun, task: dict[str, Any]):
|
|
869
|
+
attachments = task.get("attachments", [])
|
|
870
|
+
eni_ids = {}
|
|
871
|
+
eni_count = 0
|
|
872
|
+
for attachment in attachments:
|
|
873
|
+
if attachment.get("type") == "ElasticNetworkInterface":
|
|
874
|
+
details = {d["name"]: d["value"] for d in attachment.get("details", [])}
|
|
875
|
+
|
|
876
|
+
if "networkInterfaceId" in details:
|
|
877
|
+
if eni_count == 0:
|
|
878
|
+
eni_ids[f"{HIDDEN_TAG_PREFIX}eni_id"] = details["networkInterfaceId"]
|
|
879
|
+
else:
|
|
880
|
+
eni_ids[f"{HIDDEN_TAG_PREFIX}eni_id_{eni_count}"] = details[
|
|
881
|
+
"networkInterfaceId"
|
|
882
|
+
]
|
|
883
|
+
eni_count += 1
|
|
884
|
+
self._instance.add_run_tags(run.run_id, eni_ids)
|
|
885
|
+
if eni_count > 0:
|
|
886
|
+
logging.info(f"Added {eni_count} ENI ID tags for run {run.run_id}: {eni_ids}")
|
|
887
|
+
else:
|
|
888
|
+
logging.warning(f"No ENI IDs found for run {run.run_id}")
|
|
889
|
+
|
|
858
890
|
def check_run_worker_health(self, run: DagsterRun):
|
|
859
891
|
run_worker_id = run.tags.get(RUN_WORKER_ID_TAG)
|
|
860
892
|
|
|
@@ -870,6 +902,14 @@ class EcsRunLauncher(RunLauncher[T_DagsterInstance], ConfigurableClass):
|
|
|
870
902
|
|
|
871
903
|
t = tasks[0]
|
|
872
904
|
|
|
905
|
+
if get_boolean_tag_value(os.getenv("DAGSTER_AWS_ENI_TAGGING_ENABLED")) and not run.tags.get(
|
|
906
|
+
f"{HIDDEN_TAG_PREFIX}eni_id"
|
|
907
|
+
):
|
|
908
|
+
try:
|
|
909
|
+
self._add_eni_id_tags(run, t)
|
|
910
|
+
except Exception:
|
|
911
|
+
logging.exception(f"Error adding ENI ID tags for run {run.run_id}")
|
|
912
|
+
|
|
873
913
|
if t.get("lastStatus") in RUNNING_STATUSES:
|
|
874
914
|
return CheckRunHealthResult(WorkerStatus.RUNNING, run_worker_id=run_worker_id)
|
|
875
915
|
elif t.get("lastStatus") in STOPPED_STATUSES:
|
|
@@ -7,6 +7,9 @@ import requests
|
|
|
7
7
|
from dagster._utils.backoff import backoff
|
|
8
8
|
from dagster._utils.merger import merge_dicts
|
|
9
9
|
|
|
10
|
+
# https://docs.aws.amazon.com/AmazonECS/latest/developerguide/fargate-task-storage.html
|
|
11
|
+
DEFAULT_EPHEMERAL_STORAGE = 20
|
|
12
|
+
|
|
10
13
|
|
|
11
14
|
def _arns_match(arn1: Optional[str], arn2: Optional[str]):
|
|
12
15
|
arn1 = arn1 or ""
|
|
@@ -22,6 +25,12 @@ def _arns_match(arn1: Optional[str], arn2: Optional[str]):
|
|
|
22
25
|
return arn1 == arn2
|
|
23
26
|
|
|
24
27
|
|
|
28
|
+
def _ephemeral_storage_matches(storage1: Optional[int], storage2: Optional[int]):
|
|
29
|
+
if (storage1 or DEFAULT_EPHEMERAL_STORAGE) == (storage2 or DEFAULT_EPHEMERAL_STORAGE):
|
|
30
|
+
return True
|
|
31
|
+
return False
|
|
32
|
+
|
|
33
|
+
|
|
25
34
|
class DagsterEcsTaskDefinitionConfig(
|
|
26
35
|
NamedTuple(
|
|
27
36
|
"_DagsterEcsTaskDefinitionConfig",
|
|
@@ -169,6 +178,7 @@ class DagsterEcsTaskDefinitionConfig(
|
|
|
169
178
|
and self.environment == other.environment
|
|
170
179
|
and self.cpu == other.cpu
|
|
171
180
|
and self.memory == other.memory
|
|
181
|
+
and _ephemeral_storage_matches(self.ephemeral_storage, other.ephemeral_storage)
|
|
172
182
|
and _arns_match(self.execution_role_arn, other.execution_role_arn)
|
|
173
183
|
and _arns_match(self.task_role_arn, other.task_role_arn)
|
|
174
184
|
):
|
|
@@ -3,7 +3,7 @@ import re
|
|
|
3
3
|
from collections.abc import Mapping
|
|
4
4
|
from typing import Any
|
|
5
5
|
|
|
6
|
-
from dagster._core.
|
|
6
|
+
from dagster._core.remote_origin import RemoteJobOrigin
|
|
7
7
|
|
|
8
8
|
from dagster_aws.ecs.tasks import DagsterEcsTaskDefinitionConfig
|
|
9
9
|
|
|
@@ -160,4 +160,7 @@ def is_transient_task_stopped_reason(stopped_reason: str) -> bool:
|
|
|
160
160
|
):
|
|
161
161
|
return True
|
|
162
162
|
|
|
163
|
+
if "The Service Discovery instance could not be registered" in stopped_reason:
|
|
164
|
+
return True
|
|
165
|
+
|
|
163
166
|
return False
|
|
@@ -4,15 +4,15 @@ import random
|
|
|
4
4
|
import string
|
|
5
5
|
from collections.abc import Iterator
|
|
6
6
|
from contextlib import contextmanager
|
|
7
|
-
from typing import TYPE_CHECKING
|
|
7
|
+
from typing import TYPE_CHECKING, Optional
|
|
8
8
|
|
|
9
|
-
import boto3
|
|
10
9
|
import dagster._check as check
|
|
11
10
|
from dagster._core.pipes.client import PipesContextInjector, PipesParams
|
|
12
11
|
from dagster._core.pipes.utils import PipesEnvContextInjector
|
|
13
12
|
|
|
14
13
|
if TYPE_CHECKING:
|
|
15
14
|
from dagster_pipes import PipesContextData
|
|
15
|
+
from mypy_boto3_s3 import S3Client
|
|
16
16
|
|
|
17
17
|
_CONTEXT_FILENAME = "context.json"
|
|
18
18
|
|
|
@@ -22,20 +22,21 @@ class PipesS3ContextInjector(PipesContextInjector):
|
|
|
22
22
|
|
|
23
23
|
Args:
|
|
24
24
|
bucket (str): The S3 bucket to write to.
|
|
25
|
-
client (
|
|
25
|
+
client (S3Client): A boto3 client to use to write to S3.
|
|
26
26
|
key_prefix (Optional[str]): An optional prefix to use for the S3 key. Defaults to a random
|
|
27
27
|
string.
|
|
28
28
|
|
|
29
29
|
"""
|
|
30
30
|
|
|
31
|
-
def __init__(self, *, bucket: str, client:
|
|
31
|
+
def __init__(self, *, bucket: str, client: "S3Client", key_prefix: Optional[str] = None):
|
|
32
32
|
super().__init__()
|
|
33
33
|
self.bucket = check.str_param(bucket, "bucket")
|
|
34
34
|
self.client = client
|
|
35
|
+
self.key_prefix = key_prefix
|
|
35
36
|
|
|
36
37
|
@contextmanager
|
|
37
38
|
def inject_context(self, context: "PipesContextData") -> Iterator[PipesParams]: # pyright: ignore[reportIncompatibleMethodOverride]
|
|
38
|
-
key_prefix = "
|
|
39
|
+
key_prefix = f"{self.key_prefix or ''}{''.join(random.choices(string.ascii_letters, k=30))}"
|
|
39
40
|
key = os.path.join(key_prefix, _CONTEXT_FILENAME)
|
|
40
41
|
self.client.put_object(
|
|
41
42
|
Body=json.dumps(context).encode("utf-8"), Bucket=self.bucket, Key=key
|
|
@@ -4,11 +4,11 @@ import os
|
|
|
4
4
|
import random
|
|
5
5
|
import string
|
|
6
6
|
import sys
|
|
7
|
-
from collections.abc import Generator, Iterator, Sequence
|
|
7
|
+
from collections.abc import Callable, Generator, Iterator, Sequence
|
|
8
8
|
from contextlib import contextmanager
|
|
9
9
|
from datetime import datetime
|
|
10
10
|
from threading import Event, Thread
|
|
11
|
-
from typing import IO, TYPE_CHECKING, Any,
|
|
11
|
+
from typing import IO, TYPE_CHECKING, Any, Optional, cast
|
|
12
12
|
|
|
13
13
|
import boto3
|
|
14
14
|
import dagster._check as check
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import os
|
|
2
2
|
from collections.abc import Iterator, Mapping, Sequence
|
|
3
3
|
from contextlib import contextmanager
|
|
4
|
-
from typing import Any, Optional
|
|
4
|
+
from typing import IO, Any, Optional
|
|
5
5
|
|
|
6
6
|
import boto3
|
|
7
7
|
import dagster_shared.seven as seven
|
|
@@ -14,8 +14,8 @@ from dagster import (
|
|
|
14
14
|
)
|
|
15
15
|
from dagster._config.config_type import Noneable
|
|
16
16
|
from dagster._core.storage.cloud_storage_compute_log_manager import (
|
|
17
|
-
CloudStorageComputeLogManager,
|
|
18
17
|
PollingComputeLogSubscriptionManager,
|
|
18
|
+
TruncatingCloudStorageComputeLogManager,
|
|
19
19
|
)
|
|
20
20
|
from dagster._core.storage.compute_log_manager import CapturedLogContext, ComputeIOType
|
|
21
21
|
from dagster._core.storage.local_compute_log_manager import (
|
|
@@ -23,13 +23,13 @@ from dagster._core.storage.local_compute_log_manager import (
|
|
|
23
23
|
LocalComputeLogManager,
|
|
24
24
|
)
|
|
25
25
|
from dagster._serdes import ConfigurableClass, ConfigurableClassData
|
|
26
|
-
from dagster._utils import ensure_dir
|
|
26
|
+
from dagster._utils import ensure_dir
|
|
27
27
|
from typing_extensions import Self
|
|
28
28
|
|
|
29
29
|
POLLING_INTERVAL = 5
|
|
30
30
|
|
|
31
31
|
|
|
32
|
-
class S3ComputeLogManager(
|
|
32
|
+
class S3ComputeLogManager(TruncatingCloudStorageComputeLogManager, ConfigurableClass):
|
|
33
33
|
"""Logs compute function stdout and stderr to S3.
|
|
34
34
|
|
|
35
35
|
Users should not instantiate this class directly. Instead, use a YAML block in ``dagster.yaml``
|
|
@@ -114,6 +114,7 @@ class S3ComputeLogManager(CloudStorageComputeLogManager, ConfigurableClass):
|
|
|
114
114
|
self._region = self._s3_session.meta.region_name
|
|
115
115
|
else:
|
|
116
116
|
self._region = region
|
|
117
|
+
super().__init__()
|
|
117
118
|
|
|
118
119
|
@property
|
|
119
120
|
def inst_data(self):
|
|
@@ -209,7 +210,7 @@ class S3ComputeLogManager(CloudStorageComputeLogManager, ConfigurableClass):
|
|
|
209
210
|
to_delete = [{"Key": key} for key in s3_keys_to_remove]
|
|
210
211
|
self._s3_session.delete_objects(Bucket=self._s3_bucket, Delete={"Objects": to_delete})
|
|
211
212
|
|
|
212
|
-
def download_url_for_type(self, log_key: Sequence[str], io_type: ComputeIOType):
|
|
213
|
+
def download_url_for_type(self, log_key: Sequence[str], io_type: ComputeIOType):
|
|
213
214
|
if not self.is_capture_complete(log_key):
|
|
214
215
|
return None
|
|
215
216
|
|
|
@@ -234,22 +235,19 @@ class S3ComputeLogManager(CloudStorageComputeLogManager, ConfigurableClass):
|
|
|
234
235
|
return False
|
|
235
236
|
return True
|
|
236
237
|
|
|
237
|
-
def
|
|
238
|
-
self, log_key: Sequence[str], io_type: ComputeIOType, partial=False
|
|
238
|
+
def _upload_file_obj(
|
|
239
|
+
self, data: IO[bytes], log_key: Sequence[str], io_type: ComputeIOType, partial=False
|
|
239
240
|
):
|
|
240
241
|
path = self.local_manager.get_captured_local_path(log_key, IO_TYPE_EXTENSION[io_type])
|
|
241
|
-
ensure_file(path)
|
|
242
|
-
|
|
243
242
|
if (self._skip_empty_files or partial) and os.stat(path).st_size == 0:
|
|
244
243
|
return
|
|
245
244
|
|
|
246
245
|
s3_key = self._s3_key(log_key, io_type, partial=partial)
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
self._s3_session.upload_fileobj(data, self._s3_bucket, s3_key, ExtraArgs=extra_args)
|
|
246
|
+
extra_args = {
|
|
247
|
+
"ContentType": "text/plain",
|
|
248
|
+
**(self._upload_extra_args if self._upload_extra_args else {}),
|
|
249
|
+
}
|
|
250
|
+
self._s3_session.upload_fileobj(data, self._s3_bucket, s3_key, ExtraArgs=extra_args)
|
|
253
251
|
|
|
254
252
|
def download_from_cloud_storage(
|
|
255
253
|
self, log_key: Sequence[str], io_type: ComputeIOType, partial=False
|
|
@@ -112,7 +112,7 @@ class S3PickleIOManager(ConfigurableIOManager):
|
|
|
112
112
|
def asset2(asset1):
|
|
113
113
|
return asset1[:5]
|
|
114
114
|
|
|
115
|
-
|
|
115
|
+
Definitions(
|
|
116
116
|
assets=[asset1, asset2],
|
|
117
117
|
resources={
|
|
118
118
|
"io_manager": S3PickleIOManager(
|
|
@@ -200,7 +200,7 @@ def s3_pickle_io_manager(init_context):
|
|
|
200
200
|
def asset2(asset1):
|
|
201
201
|
return asset1[:5]
|
|
202
202
|
|
|
203
|
-
|
|
203
|
+
Definitions(
|
|
204
204
|
assets=[asset1, asset2],
|
|
205
205
|
resources={
|
|
206
206
|
"io_manager": s3_pickle_io_manager.configured(
|
|
@@ -48,7 +48,7 @@ class SecretsManagerResource(ResourceWithBoto3Configuration):
|
|
|
48
48
|
def example_job():
|
|
49
49
|
example_secretsmanager_op()
|
|
50
50
|
|
|
51
|
-
|
|
51
|
+
Definitions(
|
|
52
52
|
jobs=[example_job],
|
|
53
53
|
resources={
|
|
54
54
|
'secretsmanager': SecretsManagerResource(
|
|
@@ -158,7 +158,7 @@ class SecretsManagerSecretsResource(ResourceWithBoto3Configuration):
|
|
|
158
158
|
example_secretsmanager_secrets_op()
|
|
159
159
|
example_secretsmanager_secrets_op_2()
|
|
160
160
|
|
|
161
|
-
|
|
161
|
+
Definitions(
|
|
162
162
|
jobs=[example_job],
|
|
163
163
|
resources={
|
|
164
164
|
'secrets': SecretsManagerSecretsResource(
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.28.5"
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dagster-aws
|
|
3
|
+
Version: 0.28.5
|
|
4
|
+
Summary: Package for AWS-specific Dagster framework solid and resource components.
|
|
5
|
+
Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-aws
|
|
6
|
+
Author: Dagster Labs
|
|
7
|
+
Author-email: hello@dagsterlabs.com
|
|
8
|
+
License: Apache-2.0
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
13
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Requires-Python: >=3.10,<3.14
|
|
16
|
+
License-File: LICENSE
|
|
17
|
+
Requires-Dist: boto3
|
|
18
|
+
Requires-Dist: dagster==1.12.5
|
|
19
|
+
Requires-Dist: packaging
|
|
20
|
+
Requires-Dist: requests
|
|
21
|
+
Provides-Extra: redshift
|
|
22
|
+
Requires-Dist: psycopg2-binary; extra == "redshift"
|
|
23
|
+
Provides-Extra: pyspark
|
|
24
|
+
Requires-Dist: dagster-pyspark; extra == "pyspark"
|
|
25
|
+
Provides-Extra: stubs
|
|
26
|
+
Requires-Dist: boto3-stubs-lite[ecs,emr,emr-containers,emr-serverless,glue,logs,s3]; extra == "stubs"
|
|
27
|
+
Provides-Extra: test
|
|
28
|
+
Requires-Dist: botocore!=1.32.1; extra == "test"
|
|
29
|
+
Requires-Dist: moto[emrcontainers,emrserverless,glue,logs,s3,server]<5.0,>=2.2.8; extra == "test"
|
|
30
|
+
Requires-Dist: requests-mock; extra == "test"
|
|
31
|
+
Requires-Dist: xmltodict==0.12.0; extra == "test"
|
|
32
|
+
Requires-Dist: flaky; extra == "test"
|
|
33
|
+
Requires-Dist: pytest-cases; extra == "test"
|
|
34
|
+
Requires-Dist: s3fs; extra == "test"
|
|
35
|
+
Dynamic: author
|
|
36
|
+
Dynamic: author-email
|
|
37
|
+
Dynamic: classifier
|
|
38
|
+
Dynamic: home-page
|
|
39
|
+
Dynamic: license
|
|
40
|
+
Dynamic: license-file
|
|
41
|
+
Dynamic: provides-extra
|
|
42
|
+
Dynamic: requires-dist
|
|
43
|
+
Dynamic: requires-python
|
|
44
|
+
Dynamic: summary
|
|
@@ -23,19 +23,19 @@ setup(
|
|
|
23
23
|
description="Package for AWS-specific Dagster framework solid and resource components.",
|
|
24
24
|
url="https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-aws",
|
|
25
25
|
classifiers=[
|
|
26
|
-
"Programming Language :: Python :: 3.9",
|
|
27
26
|
"Programming Language :: Python :: 3.10",
|
|
28
27
|
"Programming Language :: Python :: 3.11",
|
|
29
28
|
"Programming Language :: Python :: 3.12",
|
|
29
|
+
"Programming Language :: Python :: 3.13",
|
|
30
30
|
"License :: OSI Approved :: Apache Software License",
|
|
31
31
|
"Operating System :: OS Independent",
|
|
32
32
|
],
|
|
33
33
|
packages=find_packages(exclude=["dagster_aws_tests*"]),
|
|
34
34
|
include_package_data=True,
|
|
35
|
-
python_requires=">=3.
|
|
35
|
+
python_requires=">=3.10,<3.14",
|
|
36
36
|
install_requires=[
|
|
37
37
|
"boto3",
|
|
38
|
-
"dagster==1.
|
|
38
|
+
"dagster==1.12.5",
|
|
39
39
|
"packaging",
|
|
40
40
|
"requests",
|
|
41
41
|
],
|
|
@@ -52,6 +52,7 @@ setup(
|
|
|
52
52
|
"xmltodict==0.12.0", # pinned until moto>=3.1.9 (https://github.com/spulec/moto/issues/5112)
|
|
53
53
|
"flaky",
|
|
54
54
|
"pytest-cases",
|
|
55
|
+
"s3fs",
|
|
55
56
|
],
|
|
56
57
|
},
|
|
57
58
|
zip_safe=False,
|
dagster-aws-0.26.11/PKG-INFO
DELETED
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.1
|
|
2
|
-
Name: dagster-aws
|
|
3
|
-
Version: 0.26.11
|
|
4
|
-
Summary: Package for AWS-specific Dagster framework solid and resource components.
|
|
5
|
-
Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-aws
|
|
6
|
-
Author: Dagster Labs
|
|
7
|
-
Author-email: hello@dagsterlabs.com
|
|
8
|
-
License: Apache-2.0
|
|
9
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
10
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
11
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
-
Classifier: License :: OSI Approved :: Apache Software License
|
|
14
|
-
Classifier: Operating System :: OS Independent
|
|
15
|
-
Requires-Python: >=3.9,<3.13
|
|
16
|
-
Provides-Extra: redshift
|
|
17
|
-
Provides-Extra: pyspark
|
|
18
|
-
Provides-Extra: stubs
|
|
19
|
-
Provides-Extra: test
|
|
20
|
-
License-File: LICENSE
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.26.11"
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.1
|
|
2
|
-
Name: dagster-aws
|
|
3
|
-
Version: 0.26.11
|
|
4
|
-
Summary: Package for AWS-specific Dagster framework solid and resource components.
|
|
5
|
-
Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-aws
|
|
6
|
-
Author: Dagster Labs
|
|
7
|
-
Author-email: hello@dagsterlabs.com
|
|
8
|
-
License: Apache-2.0
|
|
9
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
10
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
11
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
-
Classifier: License :: OSI Approved :: Apache Software License
|
|
14
|
-
Classifier: Operating System :: OS Independent
|
|
15
|
-
Requires-Python: >=3.9,<3.13
|
|
16
|
-
Provides-Extra: redshift
|
|
17
|
-
Provides-Extra: pyspark
|
|
18
|
-
Provides-Extra: stubs
|
|
19
|
-
Provides-Extra: test
|
|
20
|
-
License-File: LICENSE
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|