dagster-aws 0.28.11__tar.gz → 0.28.13__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dagster_aws-0.28.11/dagster_aws.egg-info → dagster_aws-0.28.13}/PKG-INFO +2 -2
- dagster_aws-0.28.13/dagster_aws/__init__.py +19 -0
- dagster_aws-0.28.13/dagster_aws/components/__init__.py +22 -0
- dagster_aws-0.28.13/dagster_aws/components/athena.py +35 -0
- dagster_aws-0.28.13/dagster_aws/components/credentials.py +98 -0
- dagster_aws-0.28.13/dagster_aws/components/ecr.py +36 -0
- dagster_aws-0.28.13/dagster_aws/components/rds.py +33 -0
- dagster_aws-0.28.13/dagster_aws/components/s3.py +67 -0
- dagster_aws-0.28.13/dagster_aws/components/secretsmanager.py +71 -0
- dagster_aws-0.28.13/dagster_aws/components/ssm.py +79 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/redshift/__init__.py +4 -0
- dagster_aws-0.28.13/dagster_aws/redshift/components.py +73 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/ssm/resources.py +2 -1
- dagster_aws-0.28.13/dagster_aws/version.py +1 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13/dagster_aws.egg-info}/PKG-INFO +2 -2
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws.egg-info/SOURCES.txt +10 -0
- dagster_aws-0.28.13/dagster_aws.egg-info/entry_points.txt +2 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws.egg-info/requires.txt +1 -1
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/setup.py +6 -1
- dagster_aws-0.28.11/dagster_aws/__init__.py +0 -5
- dagster_aws-0.28.11/dagster_aws/version.py +0 -1
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/LICENSE +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/MANIFEST.in +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/README.md +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/_stubs.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/athena/__init__.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/athena/resources.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/cloudwatch/__init__.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/cloudwatch/loggers.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/ecr/__init__.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/ecr/resources.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/ecs/__init__.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/ecs/container_context.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/ecs/executor.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/ecs/launcher.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/ecs/tasks.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/ecs/test_utils.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/ecs/utils.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/emr/__init__.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/emr/configs.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/emr/configs_spark.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/emr/emr.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/emr/emr_step_main.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/emr/main.py.template +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/emr/pyspark_step_launcher.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/emr/types.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/emr/utils.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/pipes/__init__.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/pipes/clients/__init__.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/pipes/clients/ecs.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/pipes/clients/emr.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/pipes/clients/emr_containers.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/pipes/clients/emr_serverless.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/pipes/clients/glue.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/pipes/clients/lambda_.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/pipes/clients/utils.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/pipes/context_injectors.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/pipes/message_readers.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/py.typed +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/rds/__init__.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/rds/resources.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/redshift/resources.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/s3/__init__.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/s3/compute_log_manager.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/s3/file_manager.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/s3/io_manager.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/s3/ops.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/s3/resources.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/s3/s3_fake_resource.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/s3/sensor.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/s3/utils.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/secretsmanager/__init__.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/secretsmanager/resources.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/secretsmanager/secrets.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/ssm/__init__.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/ssm/parameters.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/utils/__init__.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/utils/mrjob/__init__.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/utils/mrjob/log4j.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/utils/mrjob/retry.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws/utils/mrjob/utils.py +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws.egg-info/dependency_links.txt +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws.egg-info/not-zip-safe +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/dagster_aws.egg-info/top_level.txt +0 -0
- {dagster_aws-0.28.11 → dagster_aws-0.28.13}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dagster-aws
|
|
3
|
-
Version: 0.28.
|
|
3
|
+
Version: 0.28.13
|
|
4
4
|
Summary: Package for AWS-specific Dagster framework solid and resource components.
|
|
5
5
|
Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-aws
|
|
6
6
|
Author: Dagster Labs
|
|
@@ -16,7 +16,7 @@ Classifier: Operating System :: OS Independent
|
|
|
16
16
|
Requires-Python: >=3.10,<3.15
|
|
17
17
|
License-File: LICENSE
|
|
18
18
|
Requires-Dist: boto3
|
|
19
|
-
Requires-Dist: dagster==1.12.
|
|
19
|
+
Requires-Dist: dagster==1.12.13
|
|
20
20
|
Requires-Dist: packaging
|
|
21
21
|
Requires-Dist: requests
|
|
22
22
|
Provides-Extra: redshift
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
from dagster_shared.libraries import DagsterLibraryRegistry
|
|
2
|
+
|
|
3
|
+
from dagster_aws.components import (
|
|
4
|
+
AthenaClientResourceComponent as AthenaClientResourceComponent,
|
|
5
|
+
AthenaCredentialsComponent as AthenaCredentialsComponent,
|
|
6
|
+
Boto3CredentialsComponent as Boto3CredentialsComponent,
|
|
7
|
+
ECRPublicResourceComponent as ECRPublicResourceComponent,
|
|
8
|
+
ParameterStoreResourceComponent as ParameterStoreResourceComponent,
|
|
9
|
+
RDSResourceComponent as RDSResourceComponent,
|
|
10
|
+
S3CredentialsComponent as S3CredentialsComponent,
|
|
11
|
+
S3FileManagerResourceComponent as S3FileManagerResourceComponent,
|
|
12
|
+
S3ResourceComponent as S3ResourceComponent,
|
|
13
|
+
SecretsManagerResourceComponent as SecretsManagerResourceComponent,
|
|
14
|
+
SecretsManagerSecretsResourceComponent as SecretsManagerSecretsResourceComponent,
|
|
15
|
+
SSMResourceComponent as SSMResourceComponent,
|
|
16
|
+
)
|
|
17
|
+
from dagster_aws.version import __version__
|
|
18
|
+
|
|
19
|
+
DagsterLibraryRegistry.register("dagster-aws", __version__)
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
from dagster_aws.components.athena import (
|
|
2
|
+
AthenaClientResourceComponent as AthenaClientResourceComponent,
|
|
3
|
+
)
|
|
4
|
+
from dagster_aws.components.credentials import (
|
|
5
|
+
AthenaCredentialsComponent as AthenaCredentialsComponent,
|
|
6
|
+
Boto3CredentialsComponent as Boto3CredentialsComponent,
|
|
7
|
+
S3CredentialsComponent as S3CredentialsComponent,
|
|
8
|
+
)
|
|
9
|
+
from dagster_aws.components.ecr import ECRPublicResourceComponent as ECRPublicResourceComponent
|
|
10
|
+
from dagster_aws.components.rds import RDSResourceComponent as RDSResourceComponent
|
|
11
|
+
from dagster_aws.components.s3 import (
|
|
12
|
+
S3FileManagerResourceComponent as S3FileManagerResourceComponent,
|
|
13
|
+
S3ResourceComponent as S3ResourceComponent,
|
|
14
|
+
)
|
|
15
|
+
from dagster_aws.components.secretsmanager import (
|
|
16
|
+
SecretsManagerResourceComponent as SecretsManagerResourceComponent,
|
|
17
|
+
SecretsManagerSecretsResourceComponent as SecretsManagerSecretsResourceComponent,
|
|
18
|
+
)
|
|
19
|
+
from dagster_aws.components.ssm import (
|
|
20
|
+
ParameterStoreResourceComponent as ParameterStoreResourceComponent,
|
|
21
|
+
SSMResourceComponent as SSMResourceComponent,
|
|
22
|
+
)
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
from functools import cached_property
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
4
|
+
import dagster as dg
|
|
5
|
+
from dagster._annotations import preview, public
|
|
6
|
+
from pydantic import Field
|
|
7
|
+
|
|
8
|
+
from dagster_aws.athena.resources import AthenaClientResource
|
|
9
|
+
from dagster_aws.components.credentials import AthenaCredentialsComponent
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@public
|
|
13
|
+
@preview
|
|
14
|
+
class AthenaClientResourceComponent(dg.Component, dg.Resolvable, dg.Model):
|
|
15
|
+
"""A component that provides an AthenaClientResource for executing queries against Amazon Athena."""
|
|
16
|
+
|
|
17
|
+
credentials: AthenaCredentialsComponent = Field(
|
|
18
|
+
description="Credentials for connecting to Athena."
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
resource_key: Optional[str] = Field(
|
|
22
|
+
default=None,
|
|
23
|
+
description="The key under which the Athena resource will be bound to the definitions.",
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
@cached_property
|
|
27
|
+
def resource(self) -> AthenaClientResource:
|
|
28
|
+
"""Resolves credentials and returns a configured Athena resource."""
|
|
29
|
+
return AthenaClientResource(**self.credentials.render_as_dict())
|
|
30
|
+
|
|
31
|
+
def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
|
|
32
|
+
"""Binds the Athena resource to the specified resource key in the definitions."""
|
|
33
|
+
if self.resource_key is None:
|
|
34
|
+
return dg.Definitions()
|
|
35
|
+
return dg.Definitions(resources={self.resource_key: self.resource})
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
from typing import Optional
|
|
2
|
+
|
|
3
|
+
import dagster as dg
|
|
4
|
+
from dagster._annotations import preview, public
|
|
5
|
+
from pydantic import BaseModel, Field
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class CredentialsRenderMixin:
|
|
9
|
+
"""A mixin to provide shared dictionary rendering logic for AWS credentials."""
|
|
10
|
+
|
|
11
|
+
def render_as_dict(self) -> dict:
|
|
12
|
+
"""Returns the credentials as a dictionary, excluding None values."""
|
|
13
|
+
assert isinstance(self, BaseModel)
|
|
14
|
+
return self.model_dump(exclude_none=True)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@public
|
|
18
|
+
@preview
|
|
19
|
+
class Boto3CredentialsComponent(dg.Component, dg.Resolvable, dg.Model, CredentialsRenderMixin):
|
|
20
|
+
"""Configuration for standard AWS SDK (Boto3) credentials and session settings."""
|
|
21
|
+
|
|
22
|
+
region_name: Optional[str] = Field(
|
|
23
|
+
default=None, description="Specifies a custom region for the Boto3 session"
|
|
24
|
+
)
|
|
25
|
+
endpoint_url: Optional[str] = Field(
|
|
26
|
+
default=None, description="Specifies a custom endpoint for the Boto3 session."
|
|
27
|
+
)
|
|
28
|
+
aws_access_key_id: Optional[str] = Field(
|
|
29
|
+
default=None, description="AWS access key ID to use when creating the boto3 session."
|
|
30
|
+
)
|
|
31
|
+
aws_secret_access_key: Optional[str] = Field(
|
|
32
|
+
default=None, description="AWS secret access key to use when creating the boto3 session."
|
|
33
|
+
)
|
|
34
|
+
aws_session_token: Optional[str] = Field(
|
|
35
|
+
default=None, description="AWS session token to use when creating the boto3 session."
|
|
36
|
+
)
|
|
37
|
+
profile_name: Optional[str] = Field(
|
|
38
|
+
default=None, description="Specifies a profile to connect that session"
|
|
39
|
+
)
|
|
40
|
+
use_ssl: bool = Field(
|
|
41
|
+
default=True, description="Whether or not to use SSL. By default, SSL is used."
|
|
42
|
+
)
|
|
43
|
+
verify: Optional[bool] = Field(
|
|
44
|
+
default=True,
|
|
45
|
+
description="Whether or not to verify SSL certificates. By default SSL certificates are verified.",
|
|
46
|
+
)
|
|
47
|
+
max_attempts: int = Field(
|
|
48
|
+
default=5,
|
|
49
|
+
description=(
|
|
50
|
+
"This provides Boto3's retry handler with a value of maximum retry attempts, where the"
|
|
51
|
+
" initial call counts toward the max_attempts value that you provide"
|
|
52
|
+
),
|
|
53
|
+
)
|
|
54
|
+
retry_mode: Optional[str] = Field(
|
|
55
|
+
default=None, description="Specifies the retry mode to use for the Boto3 session."
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
|
|
59
|
+
return dg.Definitions()
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@public
|
|
63
|
+
@preview
|
|
64
|
+
class S3CredentialsComponent(Boto3CredentialsComponent):
|
|
65
|
+
"""Configuration for S3-specific credentials, including unsigned session support."""
|
|
66
|
+
|
|
67
|
+
use_unsigned_session: Optional[bool] = Field(
|
|
68
|
+
default=None, description="Specifies whether to use an unsigned S3 session."
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@public
|
|
73
|
+
@preview
|
|
74
|
+
class AthenaCredentialsComponent(Boto3CredentialsComponent):
|
|
75
|
+
"""Configuration for Athena-specific authentication and query settings."""
|
|
76
|
+
|
|
77
|
+
workgroup: Optional[str] = Field(
|
|
78
|
+
default="primary",
|
|
79
|
+
description=(
|
|
80
|
+
"The Athena WorkGroup to use."
|
|
81
|
+
" https://docs.aws.amazon.com/athena/latest/ug/manage-queries-control-costs-with-workgroups.html"
|
|
82
|
+
),
|
|
83
|
+
)
|
|
84
|
+
polling_interval: int = Field(
|
|
85
|
+
default=5,
|
|
86
|
+
description=(
|
|
87
|
+
"Time in seconds between checks to see if a query execution is finished. 5 seconds"
|
|
88
|
+
" by default. Must be non-negative."
|
|
89
|
+
),
|
|
90
|
+
)
|
|
91
|
+
max_polls: int = Field(
|
|
92
|
+
default=120,
|
|
93
|
+
description=(
|
|
94
|
+
"Number of times to poll before timing out. 120 attempts by default. When coupled"
|
|
95
|
+
" with the default polling_interval, queries will timeout after 10 minutes (120 * 5"
|
|
96
|
+
" seconds). Must be greater than 0."
|
|
97
|
+
),
|
|
98
|
+
)
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from functools import cached_property
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
4
|
+
import dagster as dg
|
|
5
|
+
from dagster._annotations import preview, public
|
|
6
|
+
from pydantic import Field
|
|
7
|
+
|
|
8
|
+
from dagster_aws.components.credentials import Boto3CredentialsComponent
|
|
9
|
+
from dagster_aws.ecr import ECRPublicResource
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@public
|
|
13
|
+
@preview
|
|
14
|
+
class ECRPublicResourceComponent(dg.Component, dg.Resolvable, dg.Model):
|
|
15
|
+
"""A component that provides an ECRPublicResource for connecting to AWS Public ECR."""
|
|
16
|
+
|
|
17
|
+
credentials: Optional[Boto3CredentialsComponent] = Field(
|
|
18
|
+
default=None,
|
|
19
|
+
description="Optional AWS credentials. If not provided, environment defaults will be used.",
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
resource_key: Optional[str] = Field(
|
|
23
|
+
default=None,
|
|
24
|
+
description="The key under which the ECR Public resource will be bound to the definitions.",
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
@cached_property
|
|
28
|
+
def resource(self) -> ECRPublicResource:
|
|
29
|
+
if self.credentials:
|
|
30
|
+
return ECRPublicResource(**self.credentials.render_as_dict())
|
|
31
|
+
return ECRPublicResource()
|
|
32
|
+
|
|
33
|
+
def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
|
|
34
|
+
if self.resource_key is None:
|
|
35
|
+
return dg.Definitions()
|
|
36
|
+
return dg.Definitions(resources={self.resource_key: self.resource})
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
from functools import cached_property
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
4
|
+
import dagster as dg
|
|
5
|
+
from dagster._annotations import preview, public
|
|
6
|
+
from pydantic import Field
|
|
7
|
+
|
|
8
|
+
from dagster_aws.components.credentials import Boto3CredentialsComponent
|
|
9
|
+
from dagster_aws.rds.resources import RDSResource
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@public
|
|
13
|
+
@preview
|
|
14
|
+
class RDSResourceComponent(dg.Component, dg.Resolvable, dg.Model):
|
|
15
|
+
"""A component that provides an RDSResource for interacting with AWS RDS service."""
|
|
16
|
+
|
|
17
|
+
credentials: Boto3CredentialsComponent = Field(
|
|
18
|
+
description="AWS credentials - inline configuration."
|
|
19
|
+
)
|
|
20
|
+
resource_key: Optional[str] = Field(
|
|
21
|
+
default=None,
|
|
22
|
+
description="The key under which the RDS resource will be bound to the definitions.",
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
@cached_property
|
|
26
|
+
def resource(self) -> RDSResource:
|
|
27
|
+
"""Resolves credentials and returns a configured RDS resource."""
|
|
28
|
+
return RDSResource(**self.credentials.render_as_dict())
|
|
29
|
+
|
|
30
|
+
def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
|
|
31
|
+
if self.resource_key is None:
|
|
32
|
+
return dg.Definitions()
|
|
33
|
+
return dg.Definitions(resources={self.resource_key: self.resource})
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
from functools import cached_property
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
4
|
+
import dagster as dg
|
|
5
|
+
from dagster._annotations import preview, public
|
|
6
|
+
from pydantic import Field
|
|
7
|
+
|
|
8
|
+
from dagster_aws.components.credentials import S3CredentialsComponent
|
|
9
|
+
from dagster_aws.s3.resources import S3FileManagerResource, S3Resource
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@public
|
|
13
|
+
@preview
|
|
14
|
+
class S3ResourceComponent(dg.Component, dg.Resolvable, dg.Model):
|
|
15
|
+
"""A component that provides an S3Resource for interacting with AWS S3."""
|
|
16
|
+
|
|
17
|
+
credentials: S3CredentialsComponent = Field(description="Credentials for connecting to S3.")
|
|
18
|
+
|
|
19
|
+
resource_key: Optional[str] = Field(
|
|
20
|
+
default=None,
|
|
21
|
+
description="The key under which the S3 resource will be bound to the definitions.",
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
@cached_property
|
|
25
|
+
def resource(self) -> S3Resource:
|
|
26
|
+
"""Resolves credentials and returns a configured S3 resource."""
|
|
27
|
+
return S3Resource(**self.credentials.render_as_dict())
|
|
28
|
+
|
|
29
|
+
def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
|
|
30
|
+
if self.resource_key is None:
|
|
31
|
+
return dg.Definitions()
|
|
32
|
+
return dg.Definitions(resources={self.resource_key: self.resource})
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@public
|
|
36
|
+
@preview
|
|
37
|
+
class S3FileManagerResourceComponent(dg.Component, dg.Resolvable, dg.Model):
|
|
38
|
+
"""A component that provides an S3FileManagerResource for file management operations on AWS S3."""
|
|
39
|
+
|
|
40
|
+
s3_bucket: str = Field(description="S3 bucket to use for the file manager.")
|
|
41
|
+
s3_prefix: str = Field(
|
|
42
|
+
default="dagster", description="Prefix to use for the S3 bucket for this file manager."
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
credentials: S3CredentialsComponent = Field(description="Credentials for connecting to S3.")
|
|
46
|
+
|
|
47
|
+
resource_key: Optional[str] = Field(
|
|
48
|
+
default=None,
|
|
49
|
+
description="The key under which the S3FileManager resource will be bound to the definitions.",
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
@cached_property
|
|
53
|
+
def resource(self) -> S3FileManagerResource:
|
|
54
|
+
"""Resolves credentials and returns a configured S3FileManagerResource."""
|
|
55
|
+
return S3FileManagerResource(
|
|
56
|
+
s3_bucket=self.s3_bucket, s3_prefix=self.s3_prefix, **self.credentials.render_as_dict()
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
|
|
60
|
+
if self.resource_key is None:
|
|
61
|
+
return dg.Definitions()
|
|
62
|
+
return dg.Definitions(resources={self.resource_key: self.resource})
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
S3ResourceComponent.model_rebuild()
|
|
66
|
+
|
|
67
|
+
S3FileManagerResourceComponent.model_rebuild()
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
from functools import cached_property
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
4
|
+
import dagster as dg
|
|
5
|
+
from dagster._annotations import preview, public
|
|
6
|
+
from pydantic import Field
|
|
7
|
+
|
|
8
|
+
from dagster_aws.components.credentials import Boto3CredentialsComponent
|
|
9
|
+
from dagster_aws.secretsmanager.resources import (
|
|
10
|
+
SecretsManagerResource,
|
|
11
|
+
SecretsManagerSecretsResource,
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@public
|
|
16
|
+
@preview
|
|
17
|
+
class SecretsManagerResourceComponent(dg.Component, dg.Resolvable, dg.Model):
|
|
18
|
+
"""A component that provides a SecretsManagerResource for interacting with AWS Secrets Manager."""
|
|
19
|
+
|
|
20
|
+
credentials: Boto3CredentialsComponent = Field(
|
|
21
|
+
description="AWS credentials - inline configuration."
|
|
22
|
+
)
|
|
23
|
+
resource_key: Optional[str] = Field(
|
|
24
|
+
default=None,
|
|
25
|
+
description="The key under which the SecretsManager resource will be bound to the definitions.",
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
@cached_property
|
|
29
|
+
def resource(self) -> SecretsManagerResource:
|
|
30
|
+
"""Resolves credentials and returns a configured SecretsManager resource."""
|
|
31
|
+
return SecretsManagerResource(**self.credentials.render_as_dict())
|
|
32
|
+
|
|
33
|
+
def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
|
|
34
|
+
if self.resource_key is None:
|
|
35
|
+
return dg.Definitions()
|
|
36
|
+
return dg.Definitions(resources={self.resource_key: self.resource})
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@public
|
|
40
|
+
@preview
|
|
41
|
+
class SecretsManagerSecretsResourceComponent(dg.Component, dg.Resolvable, dg.Model):
|
|
42
|
+
"""A component that provides a SecretsManagerSecretsResource for fetching secrets from AWS Secrets Manager."""
|
|
43
|
+
|
|
44
|
+
credentials: Boto3CredentialsComponent = Field(
|
|
45
|
+
description="AWS credentials - inline configuration."
|
|
46
|
+
)
|
|
47
|
+
secrets: list[str] = Field(
|
|
48
|
+
default=[], description="An array of AWS Secrets Manager secrets ARNs to fetch."
|
|
49
|
+
)
|
|
50
|
+
secrets_tag: Optional[str] = Field(
|
|
51
|
+
default=None,
|
|
52
|
+
description="AWS Secrets Manager secrets with this tag will be fetched and made available.",
|
|
53
|
+
)
|
|
54
|
+
resource_key: Optional[str] = Field(
|
|
55
|
+
default=None,
|
|
56
|
+
description="The key under which the SecretsManagerSecrets resource will be bound to the definitions.",
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
@cached_property
|
|
60
|
+
def resource(self) -> SecretsManagerSecretsResource:
|
|
61
|
+
creds_data = self.credentials.render_as_dict()
|
|
62
|
+
return SecretsManagerSecretsResource(
|
|
63
|
+
**creds_data,
|
|
64
|
+
secrets=self.secrets,
|
|
65
|
+
secrets_tag=self.secrets_tag,
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
|
|
69
|
+
if self.resource_key is None:
|
|
70
|
+
return dg.Definitions()
|
|
71
|
+
return dg.Definitions(resources={self.resource_key: self.resource})
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
from functools import cached_property
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
4
|
+
import dagster as dg
|
|
5
|
+
from dagster._annotations import preview, public
|
|
6
|
+
from pydantic import Field
|
|
7
|
+
|
|
8
|
+
from dagster_aws.components.credentials import Boto3CredentialsComponent
|
|
9
|
+
from dagster_aws.ssm.resources import ParameterStoreResource, ParameterStoreTag, SSMResource
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@public
|
|
13
|
+
@preview
|
|
14
|
+
class SSMResourceComponent(dg.Component, dg.Resolvable, dg.Model):
|
|
15
|
+
"""A component that provides a SSMResource for interacting with AWS Systems Manager."""
|
|
16
|
+
|
|
17
|
+
credentials: Boto3CredentialsComponent = Field(
|
|
18
|
+
description="AWS credentials - inline configuration."
|
|
19
|
+
)
|
|
20
|
+
resource_key: Optional[str] = Field(
|
|
21
|
+
default=None, description="The key under which the resource will be bound in definitions."
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
@cached_property
|
|
25
|
+
def resource(self) -> SSMResource:
|
|
26
|
+
"""Resolves credentials and returns a configured SSM resource."""
|
|
27
|
+
return SSMResource(**self.credentials.render_as_dict())
|
|
28
|
+
|
|
29
|
+
def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
|
|
30
|
+
if self.resource_key is None:
|
|
31
|
+
return dg.Definitions()
|
|
32
|
+
return dg.Definitions(resources={self.resource_key: self.resource})
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@public
|
|
36
|
+
@preview
|
|
37
|
+
class ParameterStoreResourceComponent(dg.Component, dg.Resolvable, dg.Model):
|
|
38
|
+
"""A component that provides a ParameterStoreResource for fetching parameters from AWS SSM Parameter Store."""
|
|
39
|
+
|
|
40
|
+
credentials: Boto3CredentialsComponent = Field(
|
|
41
|
+
description="AWS credentials - inline configuration."
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
parameters: list[str] = Field(
|
|
45
|
+
default=[],
|
|
46
|
+
description="AWS SSM Parameter store parameters with this tag will be fetched and made available.",
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
parameter_tags: list[ParameterStoreTag] = Field(
|
|
50
|
+
default=[],
|
|
51
|
+
description="AWS SSM Parameter store parameters with this tag will be fetched and made available.",
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
parameter_paths: list[str] = Field(
|
|
55
|
+
default=[], description="List of path prefixes to pull parameters from."
|
|
56
|
+
)
|
|
57
|
+
with_decryption: bool = Field(
|
|
58
|
+
default=False, description="Whether to decrypt secure string parameters."
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
resource_key: Optional[str] = Field(
|
|
62
|
+
default=None,
|
|
63
|
+
description="The key under which the ParameterStore resource will be bound to the definitions.",
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
@cached_property
|
|
67
|
+
def resource(self) -> ParameterStoreResource:
|
|
68
|
+
return ParameterStoreResource(
|
|
69
|
+
**self.credentials.render_as_dict(),
|
|
70
|
+
parameters=self.parameters,
|
|
71
|
+
parameter_paths=self.parameter_paths,
|
|
72
|
+
parameter_tags=self.parameter_tags,
|
|
73
|
+
with_decryption=self.with_decryption,
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
|
|
77
|
+
if self.resource_key is None:
|
|
78
|
+
return dg.Definitions()
|
|
79
|
+
return dg.Definitions(resources={self.resource_key: self.resource})
|
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
from dagster_aws.redshift.components import (
|
|
2
|
+
RedshiftClientResourceComponent as RedshiftClientResourceComponent,
|
|
3
|
+
RedshiftCredentialsComponent as RedshiftCredentialsComponent,
|
|
4
|
+
)
|
|
1
5
|
from dagster_aws.redshift.resources import (
|
|
2
6
|
FakeRedshiftClient as FakeRedshiftClient,
|
|
3
7
|
FakeRedshiftClientResource as FakeRedshiftClientResource,
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
from functools import cached_property
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
4
|
+
import dagster as dg
|
|
5
|
+
from dagster._annotations import preview, public
|
|
6
|
+
from pydantic import BaseModel, Field
|
|
7
|
+
|
|
8
|
+
from dagster_aws.redshift.resources import RedshiftClientResource
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class CredentialsRenderMixin:
|
|
12
|
+
"""A mixin to provide shared dictionary rendering logic for AWS credentials."""
|
|
13
|
+
|
|
14
|
+
def render_as_dict(self) -> dict:
|
|
15
|
+
"""Returns the credentials as a dictionary, excluding None values."""
|
|
16
|
+
assert isinstance(self, BaseModel)
|
|
17
|
+
return self.model_dump(exclude_none=True)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@public
|
|
21
|
+
@preview
|
|
22
|
+
class RedshiftCredentialsComponent(dg.Component, dg.Resolvable, dg.Model, CredentialsRenderMixin):
|
|
23
|
+
"""Credentials and connection configuration for Redshift."""
|
|
24
|
+
|
|
25
|
+
host: Optional[str] = Field(default=None, description="Redshift host")
|
|
26
|
+
port: int = Field(default=5439, description="Redshift port")
|
|
27
|
+
user: Optional[str] = Field(default=None, description="Username for Redshift connection")
|
|
28
|
+
password: Optional[str] = Field(default=None, description="Password for Redshift connection")
|
|
29
|
+
database: Optional[str] = Field(
|
|
30
|
+
default=None,
|
|
31
|
+
description=(
|
|
32
|
+
"Name of the default database to use. After login, you can use USE DATABASE to change"
|
|
33
|
+
" the database."
|
|
34
|
+
),
|
|
35
|
+
)
|
|
36
|
+
autocommit: Optional[bool] = Field(default=None, description="Whether to autocommit queries")
|
|
37
|
+
connect_timeout: int = Field(
|
|
38
|
+
default=5, description="Timeout for connection to Redshift cluster. Defaults to 5 seconds."
|
|
39
|
+
)
|
|
40
|
+
sslmode: Optional[str] = Field(
|
|
41
|
+
default="require",
|
|
42
|
+
description=(
|
|
43
|
+
"SSL mode to use. See the Redshift documentation for reference:"
|
|
44
|
+
" https://docs.aws.amazon.com/redshift/latest/mgmt/connecting-ssl-support.html"
|
|
45
|
+
),
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
|
|
49
|
+
return dg.Definitions()
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@public
|
|
53
|
+
@preview
|
|
54
|
+
class RedshiftClientResourceComponent(dg.Component, dg.Resolvable, dg.Model):
|
|
55
|
+
"""A component that provides a RedshiftClientResource for connecting to and querying Amazon Redshift."""
|
|
56
|
+
|
|
57
|
+
credentials: RedshiftCredentialsComponent = Field(
|
|
58
|
+
description="Redshift credentials - inline configuration."
|
|
59
|
+
)
|
|
60
|
+
resource_key: Optional[str] = Field(
|
|
61
|
+
default=None,
|
|
62
|
+
description="The key under which the Redshift resource will be bound to the definitions.",
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
@cached_property
|
|
66
|
+
def resource(self) -> RedshiftClientResource:
|
|
67
|
+
"""Resolves credentials and returns a configured Redshift resource."""
|
|
68
|
+
return RedshiftClientResource(**self.credentials.render_as_dict())
|
|
69
|
+
|
|
70
|
+
def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
|
|
71
|
+
if self.resource_key is None:
|
|
72
|
+
return dg.Definitions()
|
|
73
|
+
return dg.Definitions(resources={self.resource_key: self.resource})
|
|
@@ -2,6 +2,7 @@ from collections.abc import Generator
|
|
|
2
2
|
from contextlib import contextmanager
|
|
3
3
|
from typing import TYPE_CHECKING, Any, Optional, cast
|
|
4
4
|
|
|
5
|
+
import dagster as dg
|
|
5
6
|
from dagster import (
|
|
6
7
|
Config,
|
|
7
8
|
Field as LegacyDagsterField,
|
|
@@ -136,7 +137,7 @@ def ssm_resource(context) -> "botocore.client.ssm": # pyright: ignore (reportAt
|
|
|
136
137
|
|
|
137
138
|
|
|
138
139
|
@beta
|
|
139
|
-
class ParameterStoreTag(Config):
|
|
140
|
+
class ParameterStoreTag(dg.Resolvable, Config):
|
|
140
141
|
key: str = Field(description="Tag key to search for.")
|
|
141
142
|
values: Optional[list[str]] = Field(default=None, description="List")
|
|
142
143
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.28.13"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dagster-aws
|
|
3
|
-
Version: 0.28.
|
|
3
|
+
Version: 0.28.13
|
|
4
4
|
Summary: Package for AWS-specific Dagster framework solid and resource components.
|
|
5
5
|
Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-aws
|
|
6
6
|
Author: Dagster Labs
|
|
@@ -16,7 +16,7 @@ Classifier: Operating System :: OS Independent
|
|
|
16
16
|
Requires-Python: >=3.10,<3.15
|
|
17
17
|
License-File: LICENSE
|
|
18
18
|
Requires-Dist: boto3
|
|
19
|
-
Requires-Dist: dagster==1.12.
|
|
19
|
+
Requires-Dist: dagster==1.12.13
|
|
20
20
|
Requires-Dist: packaging
|
|
21
21
|
Requires-Dist: requests
|
|
22
22
|
Provides-Extra: redshift
|
|
@@ -10,6 +10,7 @@ dagster_aws/version.py
|
|
|
10
10
|
dagster_aws.egg-info/PKG-INFO
|
|
11
11
|
dagster_aws.egg-info/SOURCES.txt
|
|
12
12
|
dagster_aws.egg-info/dependency_links.txt
|
|
13
|
+
dagster_aws.egg-info/entry_points.txt
|
|
13
14
|
dagster_aws.egg-info/not-zip-safe
|
|
14
15
|
dagster_aws.egg-info/requires.txt
|
|
15
16
|
dagster_aws.egg-info/top_level.txt
|
|
@@ -17,6 +18,14 @@ dagster_aws/athena/__init__.py
|
|
|
17
18
|
dagster_aws/athena/resources.py
|
|
18
19
|
dagster_aws/cloudwatch/__init__.py
|
|
19
20
|
dagster_aws/cloudwatch/loggers.py
|
|
21
|
+
dagster_aws/components/__init__.py
|
|
22
|
+
dagster_aws/components/athena.py
|
|
23
|
+
dagster_aws/components/credentials.py
|
|
24
|
+
dagster_aws/components/ecr.py
|
|
25
|
+
dagster_aws/components/rds.py
|
|
26
|
+
dagster_aws/components/s3.py
|
|
27
|
+
dagster_aws/components/secretsmanager.py
|
|
28
|
+
dagster_aws/components/ssm.py
|
|
20
29
|
dagster_aws/ecr/__init__.py
|
|
21
30
|
dagster_aws/ecr/resources.py
|
|
22
31
|
dagster_aws/ecs/__init__.py
|
|
@@ -49,6 +58,7 @@ dagster_aws/pipes/clients/utils.py
|
|
|
49
58
|
dagster_aws/rds/__init__.py
|
|
50
59
|
dagster_aws/rds/resources.py
|
|
51
60
|
dagster_aws/redshift/__init__.py
|
|
61
|
+
dagster_aws/redshift/components.py
|
|
52
62
|
dagster_aws/redshift/resources.py
|
|
53
63
|
dagster_aws/s3/__init__.py
|
|
54
64
|
dagster_aws/s3/compute_log_manager.py
|
|
@@ -36,7 +36,7 @@ setup(
|
|
|
36
36
|
python_requires=">=3.10,<3.15",
|
|
37
37
|
install_requires=[
|
|
38
38
|
"boto3",
|
|
39
|
-
"dagster==1.12.
|
|
39
|
+
"dagster==1.12.13",
|
|
40
40
|
"packaging",
|
|
41
41
|
"requests",
|
|
42
42
|
],
|
|
@@ -56,5 +56,10 @@ setup(
|
|
|
56
56
|
"s3fs",
|
|
57
57
|
],
|
|
58
58
|
},
|
|
59
|
+
entry_points={
|
|
60
|
+
"dagster_dg_cli.registry_modules": [
|
|
61
|
+
"dagster_aws = dagster_aws.components",
|
|
62
|
+
],
|
|
63
|
+
},
|
|
59
64
|
zip_safe=False,
|
|
60
65
|
)
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.28.11"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|