dagster-aws 0.28.1__tar.gz → 0.28.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dagster_aws-0.28.1/dagster_aws.egg-info → dagster_aws-0.28.2}/PKG-INFO +3 -4
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/ecs/launcher.py +13 -4
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/pipes/message_readers.py +2 -2
- dagster_aws-0.28.2/dagster_aws/version.py +1 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2/dagster_aws.egg-info}/PKG-INFO +3 -4
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws.egg-info/requires.txt +1 -1
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/setup.py +2 -3
- dagster_aws-0.28.1/dagster_aws/version.py +0 -1
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/LICENSE +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/MANIFEST.in +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/README.md +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/__init__.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/_stubs.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/athena/__init__.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/athena/resources.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/cloudwatch/__init__.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/cloudwatch/loggers.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/ecr/__init__.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/ecr/resources.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/ecs/__init__.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/ecs/container_context.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/ecs/executor.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/ecs/tasks.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/ecs/test_utils.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/ecs/utils.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/emr/__init__.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/emr/configs.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/emr/configs_spark.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/emr/emr.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/emr/emr_step_main.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/emr/main.py.template +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/emr/pyspark_step_launcher.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/emr/types.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/emr/utils.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/pipes/__init__.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/pipes/clients/__init__.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/pipes/clients/ecs.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/pipes/clients/emr.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/pipes/clients/emr_containers.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/pipes/clients/emr_serverless.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/pipes/clients/glue.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/pipes/clients/lambda_.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/pipes/clients/utils.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/pipes/context_injectors.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/py.typed +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/rds/__init__.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/rds/resources.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/redshift/__init__.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/redshift/resources.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/s3/__init__.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/s3/compute_log_manager.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/s3/file_manager.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/s3/io_manager.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/s3/ops.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/s3/resources.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/s3/s3_fake_resource.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/s3/sensor.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/s3/utils.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/secretsmanager/__init__.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/secretsmanager/resources.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/secretsmanager/secrets.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/ssm/__init__.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/ssm/parameters.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/ssm/resources.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/utils/__init__.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/utils/mrjob/__init__.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/utils/mrjob/log4j.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/utils/mrjob/retry.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws/utils/mrjob/utils.py +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws.egg-info/SOURCES.txt +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws.egg-info/dependency_links.txt +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws.egg-info/not-zip-safe +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/dagster_aws.egg-info/top_level.txt +0 -0
- {dagster_aws-0.28.1 → dagster_aws-0.28.2}/setup.cfg +0 -0
|
@@ -1,22 +1,21 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dagster-aws
|
|
3
|
-
Version: 0.28.
|
|
3
|
+
Version: 0.28.2
|
|
4
4
|
Summary: Package for AWS-specific Dagster framework solid and resource components.
|
|
5
5
|
Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-aws
|
|
6
6
|
Author: Dagster Labs
|
|
7
7
|
Author-email: hello@dagsterlabs.com
|
|
8
8
|
License: Apache-2.0
|
|
9
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
10
9
|
Classifier: Programming Language :: Python :: 3.10
|
|
11
10
|
Classifier: Programming Language :: Python :: 3.11
|
|
12
11
|
Classifier: Programming Language :: Python :: 3.12
|
|
13
12
|
Classifier: Programming Language :: Python :: 3.13
|
|
14
13
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
15
14
|
Classifier: Operating System :: OS Independent
|
|
16
|
-
Requires-Python: >=3.
|
|
15
|
+
Requires-Python: >=3.10,<3.14
|
|
17
16
|
License-File: LICENSE
|
|
18
17
|
Requires-Dist: boto3
|
|
19
|
-
Requires-Dist: dagster==1.12.
|
|
18
|
+
Requires-Dist: dagster==1.12.2
|
|
20
19
|
Requires-Dist: packaging
|
|
21
20
|
Requires-Dist: requests
|
|
22
21
|
Provides-Extra: redshift
|
|
@@ -5,7 +5,7 @@ import uuid
|
|
|
5
5
|
import warnings
|
|
6
6
|
from collections import namedtuple
|
|
7
7
|
from collections.abc import Mapping, Sequence
|
|
8
|
-
from typing import Any, Optional
|
|
8
|
+
from typing import TYPE_CHECKING, Any, Optional
|
|
9
9
|
|
|
10
10
|
import boto3
|
|
11
11
|
from botocore.exceptions import ClientError
|
|
@@ -82,6 +82,9 @@ TAGS_TO_EXCLUDE_FROM_PROPAGATION = {"dagster/op_selection", "dagster/solid_selec
|
|
|
82
82
|
DEFAULT_REGISTER_TASK_DEFINITION_RETRIES = 5
|
|
83
83
|
DEFAULT_RUN_TASK_RETRIES = 5
|
|
84
84
|
|
|
85
|
+
if TYPE_CHECKING:
|
|
86
|
+
from botocore.config import Config
|
|
87
|
+
|
|
85
88
|
|
|
86
89
|
class EcsRunLauncher(RunLauncher[T_DagsterInstance], ConfigurableClass):
|
|
87
90
|
"""RunLauncher that starts a task in ECS for each Dagster job run.
|
|
@@ -120,10 +123,13 @@ class EcsRunLauncher(RunLauncher[T_DagsterInstance], ConfigurableClass):
|
|
|
120
123
|
task_definition_prefix: str = "run",
|
|
121
124
|
):
|
|
122
125
|
self._inst_data = inst_data
|
|
123
|
-
|
|
126
|
+
|
|
127
|
+
boto_client_config = self.get_boto_client_config()
|
|
128
|
+
|
|
129
|
+
self.ecs = boto3.client("ecs", config=boto_client_config)
|
|
124
130
|
self.ec2 = boto3.resource("ec2")
|
|
125
|
-
self.secrets_manager = boto3.client("secretsmanager")
|
|
126
|
-
self.logs = boto3.client("logs")
|
|
131
|
+
self.secrets_manager = boto3.client("secretsmanager", config=boto_client_config)
|
|
132
|
+
self.logs = boto3.client("logs", config=boto_client_config)
|
|
127
133
|
|
|
128
134
|
self._task_definition_prefix = task_definition_prefix
|
|
129
135
|
|
|
@@ -237,6 +243,9 @@ class EcsRunLauncher(RunLauncher[T_DagsterInstance], ConfigurableClass):
|
|
|
237
243
|
self._current_task_metadata = None
|
|
238
244
|
self._current_task = None
|
|
239
245
|
|
|
246
|
+
def get_boto_client_config(self) -> Optional["Config"]:
|
|
247
|
+
return None
|
|
248
|
+
|
|
240
249
|
@property
|
|
241
250
|
def inst_data(self):
|
|
242
251
|
return self._inst_data
|
|
@@ -4,11 +4,11 @@ import os
|
|
|
4
4
|
import random
|
|
5
5
|
import string
|
|
6
6
|
import sys
|
|
7
|
-
from collections.abc import Generator, Iterator, Sequence
|
|
7
|
+
from collections.abc import Callable, Generator, Iterator, Sequence
|
|
8
8
|
from contextlib import contextmanager
|
|
9
9
|
from datetime import datetime
|
|
10
10
|
from threading import Event, Thread
|
|
11
|
-
from typing import IO, TYPE_CHECKING, Any,
|
|
11
|
+
from typing import IO, TYPE_CHECKING, Any, Optional, cast
|
|
12
12
|
|
|
13
13
|
import boto3
|
|
14
14
|
import dagster._check as check
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.28.2"
|
|
@@ -1,22 +1,21 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dagster-aws
|
|
3
|
-
Version: 0.28.
|
|
3
|
+
Version: 0.28.2
|
|
4
4
|
Summary: Package for AWS-specific Dagster framework solid and resource components.
|
|
5
5
|
Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-aws
|
|
6
6
|
Author: Dagster Labs
|
|
7
7
|
Author-email: hello@dagsterlabs.com
|
|
8
8
|
License: Apache-2.0
|
|
9
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
10
9
|
Classifier: Programming Language :: Python :: 3.10
|
|
11
10
|
Classifier: Programming Language :: Python :: 3.11
|
|
12
11
|
Classifier: Programming Language :: Python :: 3.12
|
|
13
12
|
Classifier: Programming Language :: Python :: 3.13
|
|
14
13
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
15
14
|
Classifier: Operating System :: OS Independent
|
|
16
|
-
Requires-Python: >=3.
|
|
15
|
+
Requires-Python: >=3.10,<3.14
|
|
17
16
|
License-File: LICENSE
|
|
18
17
|
Requires-Dist: boto3
|
|
19
|
-
Requires-Dist: dagster==1.12.
|
|
18
|
+
Requires-Dist: dagster==1.12.2
|
|
20
19
|
Requires-Dist: packaging
|
|
21
20
|
Requires-Dist: requests
|
|
22
21
|
Provides-Extra: redshift
|
|
@@ -23,7 +23,6 @@ setup(
|
|
|
23
23
|
description="Package for AWS-specific Dagster framework solid and resource components.",
|
|
24
24
|
url="https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-aws",
|
|
25
25
|
classifiers=[
|
|
26
|
-
"Programming Language :: Python :: 3.9",
|
|
27
26
|
"Programming Language :: Python :: 3.10",
|
|
28
27
|
"Programming Language :: Python :: 3.11",
|
|
29
28
|
"Programming Language :: Python :: 3.12",
|
|
@@ -33,10 +32,10 @@ setup(
|
|
|
33
32
|
],
|
|
34
33
|
packages=find_packages(exclude=["dagster_aws_tests*"]),
|
|
35
34
|
include_package_data=True,
|
|
36
|
-
python_requires=">=3.
|
|
35
|
+
python_requires=">=3.10,<3.14",
|
|
37
36
|
install_requires=[
|
|
38
37
|
"boto3",
|
|
39
|
-
"dagster==1.12.
|
|
38
|
+
"dagster==1.12.2",
|
|
40
39
|
"packaging",
|
|
41
40
|
"requests",
|
|
42
41
|
],
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.28.1"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|