dagster-aws 0.28.11__tar.gz → 0.28.12__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. {dagster_aws-0.28.11/dagster_aws.egg-info → dagster_aws-0.28.12}/PKG-INFO +2 -2
  2. dagster_aws-0.28.12/dagster_aws/__init__.py +19 -0
  3. dagster_aws-0.28.12/dagster_aws/components/__init__.py +22 -0
  4. dagster_aws-0.28.12/dagster_aws/components/athena.py +35 -0
  5. dagster_aws-0.28.12/dagster_aws/components/credentials.py +98 -0
  6. dagster_aws-0.28.12/dagster_aws/components/ecr.py +36 -0
  7. dagster_aws-0.28.12/dagster_aws/components/rds.py +33 -0
  8. dagster_aws-0.28.12/dagster_aws/components/s3.py +67 -0
  9. dagster_aws-0.28.12/dagster_aws/components/secretsmanager.py +71 -0
  10. dagster_aws-0.28.12/dagster_aws/components/ssm.py +79 -0
  11. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/redshift/__init__.py +4 -0
  12. dagster_aws-0.28.12/dagster_aws/redshift/components.py +73 -0
  13. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/ssm/resources.py +2 -1
  14. dagster_aws-0.28.12/dagster_aws/version.py +1 -0
  15. {dagster_aws-0.28.11 → dagster_aws-0.28.12/dagster_aws.egg-info}/PKG-INFO +2 -2
  16. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws.egg-info/SOURCES.txt +10 -0
  17. dagster_aws-0.28.12/dagster_aws.egg-info/entry_points.txt +2 -0
  18. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws.egg-info/requires.txt +1 -1
  19. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/setup.py +6 -1
  20. dagster_aws-0.28.11/dagster_aws/__init__.py +0 -5
  21. dagster_aws-0.28.11/dagster_aws/version.py +0 -1
  22. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/LICENSE +0 -0
  23. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/MANIFEST.in +0 -0
  24. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/README.md +0 -0
  25. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/_stubs.py +0 -0
  26. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/athena/__init__.py +0 -0
  27. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/athena/resources.py +0 -0
  28. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/cloudwatch/__init__.py +0 -0
  29. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/cloudwatch/loggers.py +0 -0
  30. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/ecr/__init__.py +0 -0
  31. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/ecr/resources.py +0 -0
  32. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/ecs/__init__.py +0 -0
  33. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/ecs/container_context.py +0 -0
  34. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/ecs/executor.py +0 -0
  35. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/ecs/launcher.py +0 -0
  36. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/ecs/tasks.py +0 -0
  37. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/ecs/test_utils.py +0 -0
  38. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/ecs/utils.py +0 -0
  39. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/emr/__init__.py +0 -0
  40. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/emr/configs.py +0 -0
  41. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/emr/configs_spark.py +0 -0
  42. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/emr/emr.py +0 -0
  43. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/emr/emr_step_main.py +0 -0
  44. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/emr/main.py.template +0 -0
  45. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/emr/pyspark_step_launcher.py +0 -0
  46. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/emr/types.py +0 -0
  47. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/emr/utils.py +0 -0
  48. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/pipes/__init__.py +0 -0
  49. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/pipes/clients/__init__.py +0 -0
  50. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/pipes/clients/ecs.py +0 -0
  51. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/pipes/clients/emr.py +0 -0
  52. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/pipes/clients/emr_containers.py +0 -0
  53. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/pipes/clients/emr_serverless.py +0 -0
  54. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/pipes/clients/glue.py +0 -0
  55. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/pipes/clients/lambda_.py +0 -0
  56. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/pipes/clients/utils.py +0 -0
  57. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/pipes/context_injectors.py +0 -0
  58. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/pipes/message_readers.py +0 -0
  59. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/py.typed +0 -0
  60. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/rds/__init__.py +0 -0
  61. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/rds/resources.py +0 -0
  62. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/redshift/resources.py +0 -0
  63. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/s3/__init__.py +0 -0
  64. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/s3/compute_log_manager.py +0 -0
  65. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/s3/file_manager.py +0 -0
  66. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/s3/io_manager.py +0 -0
  67. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/s3/ops.py +0 -0
  68. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/s3/resources.py +0 -0
  69. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/s3/s3_fake_resource.py +0 -0
  70. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/s3/sensor.py +0 -0
  71. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/s3/utils.py +0 -0
  72. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/secretsmanager/__init__.py +0 -0
  73. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/secretsmanager/resources.py +0 -0
  74. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/secretsmanager/secrets.py +0 -0
  75. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/ssm/__init__.py +0 -0
  76. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/ssm/parameters.py +0 -0
  77. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/utils/__init__.py +0 -0
  78. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/utils/mrjob/__init__.py +0 -0
  79. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/utils/mrjob/log4j.py +0 -0
  80. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/utils/mrjob/retry.py +0 -0
  81. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws/utils/mrjob/utils.py +0 -0
  82. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws.egg-info/dependency_links.txt +0 -0
  83. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws.egg-info/not-zip-safe +0 -0
  84. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/dagster_aws.egg-info/top_level.txt +0 -0
  85. {dagster_aws-0.28.11 → dagster_aws-0.28.12}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dagster-aws
3
- Version: 0.28.11
3
+ Version: 0.28.12
4
4
  Summary: Package for AWS-specific Dagster framework solid and resource components.
5
5
  Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-aws
6
6
  Author: Dagster Labs
@@ -16,7 +16,7 @@ Classifier: Operating System :: OS Independent
16
16
  Requires-Python: >=3.10,<3.15
17
17
  License-File: LICENSE
18
18
  Requires-Dist: boto3
19
- Requires-Dist: dagster==1.12.11
19
+ Requires-Dist: dagster==1.12.12
20
20
  Requires-Dist: packaging
21
21
  Requires-Dist: requests
22
22
  Provides-Extra: redshift
@@ -0,0 +1,19 @@
1
+ from dagster_shared.libraries import DagsterLibraryRegistry
2
+
3
+ from dagster_aws.components import (
4
+ AthenaClientResourceComponent as AthenaClientResourceComponent,
5
+ AthenaCredentialsComponent as AthenaCredentialsComponent,
6
+ Boto3CredentialsComponent as Boto3CredentialsComponent,
7
+ ECRPublicResourceComponent as ECRPublicResourceComponent,
8
+ ParameterStoreResourceComponent as ParameterStoreResourceComponent,
9
+ RDSResourceComponent as RDSResourceComponent,
10
+ S3CredentialsComponent as S3CredentialsComponent,
11
+ S3FileManagerResourceComponent as S3FileManagerResourceComponent,
12
+ S3ResourceComponent as S3ResourceComponent,
13
+ SecretsManagerResourceComponent as SecretsManagerResourceComponent,
14
+ SecretsManagerSecretsResourceComponent as SecretsManagerSecretsResourceComponent,
15
+ SSMResourceComponent as SSMResourceComponent,
16
+ )
17
+ from dagster_aws.version import __version__
18
+
19
+ DagsterLibraryRegistry.register("dagster-aws", __version__)
@@ -0,0 +1,22 @@
1
+ from dagster_aws.components.athena import (
2
+ AthenaClientResourceComponent as AthenaClientResourceComponent,
3
+ )
4
+ from dagster_aws.components.credentials import (
5
+ AthenaCredentialsComponent as AthenaCredentialsComponent,
6
+ Boto3CredentialsComponent as Boto3CredentialsComponent,
7
+ S3CredentialsComponent as S3CredentialsComponent,
8
+ )
9
+ from dagster_aws.components.ecr import ECRPublicResourceComponent as ECRPublicResourceComponent
10
+ from dagster_aws.components.rds import RDSResourceComponent as RDSResourceComponent
11
+ from dagster_aws.components.s3 import (
12
+ S3FileManagerResourceComponent as S3FileManagerResourceComponent,
13
+ S3ResourceComponent as S3ResourceComponent,
14
+ )
15
+ from dagster_aws.components.secretsmanager import (
16
+ SecretsManagerResourceComponent as SecretsManagerResourceComponent,
17
+ SecretsManagerSecretsResourceComponent as SecretsManagerSecretsResourceComponent,
18
+ )
19
+ from dagster_aws.components.ssm import (
20
+ ParameterStoreResourceComponent as ParameterStoreResourceComponent,
21
+ SSMResourceComponent as SSMResourceComponent,
22
+ )
@@ -0,0 +1,35 @@
1
+ from functools import cached_property
2
+ from typing import Optional
3
+
4
+ import dagster as dg
5
+ from dagster._annotations import preview, public
6
+ from pydantic import Field
7
+
8
+ from dagster_aws.athena.resources import AthenaClientResource
9
+ from dagster_aws.components.credentials import AthenaCredentialsComponent
10
+
11
+
12
+ @public
13
+ @preview
14
+ class AthenaClientResourceComponent(dg.Component, dg.Resolvable, dg.Model):
15
+ """A component that provides an AthenaClientResource for executing queries against Amazon Athena."""
16
+
17
+ credentials: AthenaCredentialsComponent = Field(
18
+ description="Credentials for connecting to Athena."
19
+ )
20
+
21
+ resource_key: Optional[str] = Field(
22
+ default=None,
23
+ description="The key under which the Athena resource will be bound to the definitions.",
24
+ )
25
+
26
+ @cached_property
27
+ def resource(self) -> AthenaClientResource:
28
+ """Resolves credentials and returns a configured Athena resource."""
29
+ return AthenaClientResource(**self.credentials.render_as_dict())
30
+
31
+ def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
32
+ """Binds the Athena resource to the specified resource key in the definitions."""
33
+ if self.resource_key is None:
34
+ return dg.Definitions()
35
+ return dg.Definitions(resources={self.resource_key: self.resource})
@@ -0,0 +1,98 @@
1
+ from typing import Optional
2
+
3
+ import dagster as dg
4
+ from dagster._annotations import preview, public
5
+ from pydantic import BaseModel, Field
6
+
7
+
8
+ class CredentialsRenderMixin:
9
+ """A mixin to provide shared dictionary rendering logic for AWS credentials."""
10
+
11
+ def render_as_dict(self) -> dict:
12
+ """Returns the credentials as a dictionary, excluding None values."""
13
+ assert isinstance(self, BaseModel)
14
+ return self.model_dump(exclude_none=True)
15
+
16
+
17
+ @public
18
+ @preview
19
+ class Boto3CredentialsComponent(dg.Component, dg.Resolvable, dg.Model, CredentialsRenderMixin):
20
+ """Configuration for standard AWS SDK (Boto3) credentials and session settings."""
21
+
22
+ region_name: Optional[str] = Field(
23
+ default=None, description="Specifies a custom region for the Boto3 session"
24
+ )
25
+ endpoint_url: Optional[str] = Field(
26
+ default=None, description="Specifies a custom endpoint for the Boto3 session."
27
+ )
28
+ aws_access_key_id: Optional[str] = Field(
29
+ default=None, description="AWS access key ID to use when creating the boto3 session."
30
+ )
31
+ aws_secret_access_key: Optional[str] = Field(
32
+ default=None, description="AWS secret access key to use when creating the boto3 session."
33
+ )
34
+ aws_session_token: Optional[str] = Field(
35
+ default=None, description="AWS session token to use when creating the boto3 session."
36
+ )
37
+ profile_name: Optional[str] = Field(
38
+ default=None, description="Specifies a profile to connect that session"
39
+ )
40
+ use_ssl: bool = Field(
41
+ default=True, description="Whether or not to use SSL. By default, SSL is used."
42
+ )
43
+ verify: Optional[bool] = Field(
44
+ default=True,
45
+ description="Whether or not to verify SSL certificates. By default SSL certificates are verified.",
46
+ )
47
+ max_attempts: int = Field(
48
+ default=5,
49
+ description=(
50
+ "This provides Boto3's retry handler with a value of maximum retry attempts, where the"
51
+ " initial call counts toward the max_attempts value that you provide"
52
+ ),
53
+ )
54
+ retry_mode: Optional[str] = Field(
55
+ default=None, description="Specifies the retry mode to use for the Boto3 session."
56
+ )
57
+
58
+ def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
59
+ return dg.Definitions()
60
+
61
+
62
+ @public
63
+ @preview
64
+ class S3CredentialsComponent(Boto3CredentialsComponent):
65
+ """Configuration for S3-specific credentials, including unsigned session support."""
66
+
67
+ use_unsigned_session: Optional[bool] = Field(
68
+ default=None, description="Specifies whether to use an unsigned S3 session."
69
+ )
70
+
71
+
72
+ @public
73
+ @preview
74
+ class AthenaCredentialsComponent(Boto3CredentialsComponent):
75
+ """Configuration for Athena-specific authentication and query settings."""
76
+
77
+ workgroup: Optional[str] = Field(
78
+ default="primary",
79
+ description=(
80
+ "The Athena WorkGroup to use."
81
+ " https://docs.aws.amazon.com/athena/latest/ug/manage-queries-control-costs-with-workgroups.html"
82
+ ),
83
+ )
84
+ polling_interval: int = Field(
85
+ default=5,
86
+ description=(
87
+ "Time in seconds between checks to see if a query execution is finished. 5 seconds"
88
+ " by default. Must be non-negative."
89
+ ),
90
+ )
91
+ max_polls: int = Field(
92
+ default=120,
93
+ description=(
94
+ "Number of times to poll before timing out. 120 attempts by default. When coupled"
95
+ " with the default polling_interval, queries will timeout after 10 minutes (120 * 5"
96
+ " seconds). Must be greater than 0."
97
+ ),
98
+ )
@@ -0,0 +1,36 @@
1
+ from functools import cached_property
2
+ from typing import Optional
3
+
4
+ import dagster as dg
5
+ from dagster._annotations import preview, public
6
+ from pydantic import Field
7
+
8
+ from dagster_aws.components.credentials import Boto3CredentialsComponent
9
+ from dagster_aws.ecr import ECRPublicResource
10
+
11
+
12
+ @public
13
+ @preview
14
+ class ECRPublicResourceComponent(dg.Component, dg.Resolvable, dg.Model):
15
+ """A component that provides an ECRPublicResource for connecting to AWS Public ECR."""
16
+
17
+ credentials: Optional[Boto3CredentialsComponent] = Field(
18
+ default=None,
19
+ description="Optional AWS credentials. If not provided, environment defaults will be used.",
20
+ )
21
+
22
+ resource_key: Optional[str] = Field(
23
+ default=None,
24
+ description="The key under which the ECR Public resource will be bound to the definitions.",
25
+ )
26
+
27
+ @cached_property
28
+ def resource(self) -> ECRPublicResource:
29
+ if self.credentials:
30
+ return ECRPublicResource(**self.credentials.render_as_dict())
31
+ return ECRPublicResource()
32
+
33
+ def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
34
+ if self.resource_key is None:
35
+ return dg.Definitions()
36
+ return dg.Definitions(resources={self.resource_key: self.resource})
@@ -0,0 +1,33 @@
1
+ from functools import cached_property
2
+ from typing import Optional
3
+
4
+ import dagster as dg
5
+ from dagster._annotations import preview, public
6
+ from pydantic import Field
7
+
8
+ from dagster_aws.components.credentials import Boto3CredentialsComponent
9
+ from dagster_aws.rds.resources import RDSResource
10
+
11
+
12
+ @public
13
+ @preview
14
+ class RDSResourceComponent(dg.Component, dg.Resolvable, dg.Model):
15
+ """A component that provides an RDSResource for interacting with AWS RDS service."""
16
+
17
+ credentials: Boto3CredentialsComponent = Field(
18
+ description="AWS credentials - inline configuration."
19
+ )
20
+ resource_key: Optional[str] = Field(
21
+ default=None,
22
+ description="The key under which the RDS resource will be bound to the definitions.",
23
+ )
24
+
25
+ @cached_property
26
+ def resource(self) -> RDSResource:
27
+ """Resolves credentials and returns a configured RDS resource."""
28
+ return RDSResource(**self.credentials.render_as_dict())
29
+
30
+ def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
31
+ if self.resource_key is None:
32
+ return dg.Definitions()
33
+ return dg.Definitions(resources={self.resource_key: self.resource})
@@ -0,0 +1,67 @@
1
+ from functools import cached_property
2
+ from typing import Optional
3
+
4
+ import dagster as dg
5
+ from dagster._annotations import preview, public
6
+ from pydantic import Field
7
+
8
+ from dagster_aws.components.credentials import S3CredentialsComponent
9
+ from dagster_aws.s3.resources import S3FileManagerResource, S3Resource
10
+
11
+
12
+ @public
13
+ @preview
14
+ class S3ResourceComponent(dg.Component, dg.Resolvable, dg.Model):
15
+ """A component that provides an S3Resource for interacting with AWS S3."""
16
+
17
+ credentials: S3CredentialsComponent = Field(description="Credentials for connecting to S3.")
18
+
19
+ resource_key: Optional[str] = Field(
20
+ default=None,
21
+ description="The key under which the S3 resource will be bound to the definitions.",
22
+ )
23
+
24
+ @cached_property
25
+ def resource(self) -> S3Resource:
26
+ """Resolves credentials and returns a configured S3 resource."""
27
+ return S3Resource(**self.credentials.render_as_dict())
28
+
29
+ def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
30
+ if self.resource_key is None:
31
+ return dg.Definitions()
32
+ return dg.Definitions(resources={self.resource_key: self.resource})
33
+
34
+
35
+ @public
36
+ @preview
37
+ class S3FileManagerResourceComponent(dg.Component, dg.Resolvable, dg.Model):
38
+ """A component that provides an S3FileManagerResource for file management operations on AWS S3."""
39
+
40
+ s3_bucket: str = Field(description="S3 bucket to use for the file manager.")
41
+ s3_prefix: str = Field(
42
+ default="dagster", description="Prefix to use for the S3 bucket for this file manager."
43
+ )
44
+
45
+ credentials: S3CredentialsComponent = Field(description="Credentials for connecting to S3.")
46
+
47
+ resource_key: Optional[str] = Field(
48
+ default=None,
49
+ description="The key under which the S3FileManager resource will be bound to the definitions.",
50
+ )
51
+
52
+ @cached_property
53
+ def resource(self) -> S3FileManagerResource:
54
+ """Resolves credentials and returns a configured S3FileManagerResource."""
55
+ return S3FileManagerResource(
56
+ s3_bucket=self.s3_bucket, s3_prefix=self.s3_prefix, **self.credentials.render_as_dict()
57
+ )
58
+
59
+ def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
60
+ if self.resource_key is None:
61
+ return dg.Definitions()
62
+ return dg.Definitions(resources={self.resource_key: self.resource})
63
+
64
+
65
+ S3ResourceComponent.model_rebuild()
66
+
67
+ S3FileManagerResourceComponent.model_rebuild()
@@ -0,0 +1,71 @@
1
+ from functools import cached_property
2
+ from typing import Optional
3
+
4
+ import dagster as dg
5
+ from dagster._annotations import preview, public
6
+ from pydantic import Field
7
+
8
+ from dagster_aws.components.credentials import Boto3CredentialsComponent
9
+ from dagster_aws.secretsmanager.resources import (
10
+ SecretsManagerResource,
11
+ SecretsManagerSecretsResource,
12
+ )
13
+
14
+
15
+ @public
16
+ @preview
17
+ class SecretsManagerResourceComponent(dg.Component, dg.Resolvable, dg.Model):
18
+ """A component that provides a SecretsManagerResource for interacting with AWS Secrets Manager."""
19
+
20
+ credentials: Boto3CredentialsComponent = Field(
21
+ description="AWS credentials - inline configuration."
22
+ )
23
+ resource_key: Optional[str] = Field(
24
+ default=None,
25
+ description="The key under which the SecretsManager resource will be bound to the definitions.",
26
+ )
27
+
28
+ @cached_property
29
+ def resource(self) -> SecretsManagerResource:
30
+ """Resolves credentials and returns a configured SecretsManager resource."""
31
+ return SecretsManagerResource(**self.credentials.render_as_dict())
32
+
33
+ def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
34
+ if self.resource_key is None:
35
+ return dg.Definitions()
36
+ return dg.Definitions(resources={self.resource_key: self.resource})
37
+
38
+
39
+ @public
40
+ @preview
41
+ class SecretsManagerSecretsResourceComponent(dg.Component, dg.Resolvable, dg.Model):
42
+ """A component that provides a SecretsManagerSecretsResource for fetching secrets from AWS Secrets Manager."""
43
+
44
+ credentials: Boto3CredentialsComponent = Field(
45
+ description="AWS credentials - inline configuration."
46
+ )
47
+ secrets: list[str] = Field(
48
+ default=[], description="An array of AWS Secrets Manager secrets ARNs to fetch."
49
+ )
50
+ secrets_tag: Optional[str] = Field(
51
+ default=None,
52
+ description="AWS Secrets Manager secrets with this tag will be fetched and made available.",
53
+ )
54
+ resource_key: Optional[str] = Field(
55
+ default=None,
56
+ description="The key under which the SecretsManagerSecrets resource will be bound to the definitions.",
57
+ )
58
+
59
+ @cached_property
60
+ def resource(self) -> SecretsManagerSecretsResource:
61
+ creds_data = self.credentials.render_as_dict()
62
+ return SecretsManagerSecretsResource(
63
+ **creds_data,
64
+ secrets=self.secrets,
65
+ secrets_tag=self.secrets_tag,
66
+ )
67
+
68
+ def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
69
+ if self.resource_key is None:
70
+ return dg.Definitions()
71
+ return dg.Definitions(resources={self.resource_key: self.resource})
@@ -0,0 +1,79 @@
1
+ from functools import cached_property
2
+ from typing import Optional
3
+
4
+ import dagster as dg
5
+ from dagster._annotations import preview, public
6
+ from pydantic import Field
7
+
8
+ from dagster_aws.components.credentials import Boto3CredentialsComponent
9
+ from dagster_aws.ssm.resources import ParameterStoreResource, ParameterStoreTag, SSMResource
10
+
11
+
12
+ @public
13
+ @preview
14
+ class SSMResourceComponent(dg.Component, dg.Resolvable, dg.Model):
15
+ """A component that provides a SSMResource for interacting with AWS Systems Manager."""
16
+
17
+ credentials: Boto3CredentialsComponent = Field(
18
+ description="AWS credentials - inline configuration."
19
+ )
20
+ resource_key: Optional[str] = Field(
21
+ default=None, description="The key under which the resource will be bound in definitions."
22
+ )
23
+
24
+ @cached_property
25
+ def resource(self) -> SSMResource:
26
+ """Resolves credentials and returns a configured SSM resource."""
27
+ return SSMResource(**self.credentials.render_as_dict())
28
+
29
+ def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
30
+ if self.resource_key is None:
31
+ return dg.Definitions()
32
+ return dg.Definitions(resources={self.resource_key: self.resource})
33
+
34
+
35
+ @public
36
+ @preview
37
+ class ParameterStoreResourceComponent(dg.Component, dg.Resolvable, dg.Model):
38
+ """A component that provides a ParameterStoreResource for fetching parameters from AWS SSM Parameter Store."""
39
+
40
+ credentials: Boto3CredentialsComponent = Field(
41
+ description="AWS credentials - inline configuration."
42
+ )
43
+
44
+ parameters: list[str] = Field(
45
+ default=[],
46
+ description="AWS SSM Parameter store parameters with this tag will be fetched and made available.",
47
+ )
48
+
49
+ parameter_tags: list[ParameterStoreTag] = Field(
50
+ default=[],
51
+ description="AWS SSM Parameter store parameters with this tag will be fetched and made available.",
52
+ )
53
+
54
+ parameter_paths: list[str] = Field(
55
+ default=[], description="List of path prefixes to pull parameters from."
56
+ )
57
+ with_decryption: bool = Field(
58
+ default=False, description="Whether to decrypt secure string parameters."
59
+ )
60
+
61
+ resource_key: Optional[str] = Field(
62
+ default=None,
63
+ description="The key under which the ParameterStore resource will be bound to the definitions.",
64
+ )
65
+
66
+ @cached_property
67
+ def resource(self) -> ParameterStoreResource:
68
+ return ParameterStoreResource(
69
+ **self.credentials.render_as_dict(),
70
+ parameters=self.parameters,
71
+ parameter_paths=self.parameter_paths,
72
+ parameter_tags=self.parameter_tags,
73
+ with_decryption=self.with_decryption,
74
+ )
75
+
76
+ def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
77
+ if self.resource_key is None:
78
+ return dg.Definitions()
79
+ return dg.Definitions(resources={self.resource_key: self.resource})
@@ -1,3 +1,7 @@
1
+ from dagster_aws.redshift.components import (
2
+ RedshiftClientResourceComponent as RedshiftClientResourceComponent,
3
+ RedshiftCredentialsComponent as RedshiftCredentialsComponent,
4
+ )
1
5
  from dagster_aws.redshift.resources import (
2
6
  FakeRedshiftClient as FakeRedshiftClient,
3
7
  FakeRedshiftClientResource as FakeRedshiftClientResource,
@@ -0,0 +1,73 @@
1
+ from functools import cached_property
2
+ from typing import Optional
3
+
4
+ import dagster as dg
5
+ from dagster._annotations import preview, public
6
+ from pydantic import BaseModel, Field
7
+
8
+ from dagster_aws.redshift.resources import RedshiftClientResource
9
+
10
+
11
+ class CredentialsRenderMixin:
12
+ """A mixin to provide shared dictionary rendering logic for AWS credentials."""
13
+
14
+ def render_as_dict(self) -> dict:
15
+ """Returns the credentials as a dictionary, excluding None values."""
16
+ assert isinstance(self, BaseModel)
17
+ return self.model_dump(exclude_none=True)
18
+
19
+
20
+ @public
21
+ @preview
22
+ class RedshiftCredentialsComponent(dg.Component, dg.Resolvable, dg.Model, CredentialsRenderMixin):
23
+ """Credentials and connection configuration for Redshift."""
24
+
25
+ host: Optional[str] = Field(default=None, description="Redshift host")
26
+ port: int = Field(default=5439, description="Redshift port")
27
+ user: Optional[str] = Field(default=None, description="Username for Redshift connection")
28
+ password: Optional[str] = Field(default=None, description="Password for Redshift connection")
29
+ database: Optional[str] = Field(
30
+ default=None,
31
+ description=(
32
+ "Name of the default database to use. After login, you can use USE DATABASE to change"
33
+ " the database."
34
+ ),
35
+ )
36
+ autocommit: Optional[bool] = Field(default=None, description="Whether to autocommit queries")
37
+ connect_timeout: int = Field(
38
+ default=5, description="Timeout for connection to Redshift cluster. Defaults to 5 seconds."
39
+ )
40
+ sslmode: Optional[str] = Field(
41
+ default="require",
42
+ description=(
43
+ "SSL mode to use. See the Redshift documentation for reference:"
44
+ " https://docs.aws.amazon.com/redshift/latest/mgmt/connecting-ssl-support.html"
45
+ ),
46
+ )
47
+
48
+ def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
49
+ return dg.Definitions()
50
+
51
+
52
+ @public
53
+ @preview
54
+ class RedshiftClientResourceComponent(dg.Component, dg.Resolvable, dg.Model):
55
+ """A component that provides a RedshiftClientResource for connecting to and querying Amazon Redshift."""
56
+
57
+ credentials: RedshiftCredentialsComponent = Field(
58
+ description="Redshift credentials - inline configuration."
59
+ )
60
+ resource_key: Optional[str] = Field(
61
+ default=None,
62
+ description="The key under which the Redshift resource will be bound to the definitions.",
63
+ )
64
+
65
+ @cached_property
66
+ def resource(self) -> RedshiftClientResource:
67
+ """Resolves credentials and returns a configured Redshift resource."""
68
+ return RedshiftClientResource(**self.credentials.render_as_dict())
69
+
70
+ def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
71
+ if self.resource_key is None:
72
+ return dg.Definitions()
73
+ return dg.Definitions(resources={self.resource_key: self.resource})
@@ -2,6 +2,7 @@ from collections.abc import Generator
2
2
  from contextlib import contextmanager
3
3
  from typing import TYPE_CHECKING, Any, Optional, cast
4
4
 
5
+ import dagster as dg
5
6
  from dagster import (
6
7
  Config,
7
8
  Field as LegacyDagsterField,
@@ -136,7 +137,7 @@ def ssm_resource(context) -> "botocore.client.ssm": # pyright: ignore (reportAt
136
137
 
137
138
 
138
139
  @beta
139
- class ParameterStoreTag(Config):
140
+ class ParameterStoreTag(dg.Resolvable, Config):
140
141
  key: str = Field(description="Tag key to search for.")
141
142
  values: Optional[list[str]] = Field(default=None, description="List")
142
143
 
@@ -0,0 +1 @@
1
+ __version__ = "0.28.12"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dagster-aws
3
- Version: 0.28.11
3
+ Version: 0.28.12
4
4
  Summary: Package for AWS-specific Dagster framework solid and resource components.
5
5
  Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-aws
6
6
  Author: Dagster Labs
@@ -16,7 +16,7 @@ Classifier: Operating System :: OS Independent
16
16
  Requires-Python: >=3.10,<3.15
17
17
  License-File: LICENSE
18
18
  Requires-Dist: boto3
19
- Requires-Dist: dagster==1.12.11
19
+ Requires-Dist: dagster==1.12.12
20
20
  Requires-Dist: packaging
21
21
  Requires-Dist: requests
22
22
  Provides-Extra: redshift
@@ -10,6 +10,7 @@ dagster_aws/version.py
10
10
  dagster_aws.egg-info/PKG-INFO
11
11
  dagster_aws.egg-info/SOURCES.txt
12
12
  dagster_aws.egg-info/dependency_links.txt
13
+ dagster_aws.egg-info/entry_points.txt
13
14
  dagster_aws.egg-info/not-zip-safe
14
15
  dagster_aws.egg-info/requires.txt
15
16
  dagster_aws.egg-info/top_level.txt
@@ -17,6 +18,14 @@ dagster_aws/athena/__init__.py
17
18
  dagster_aws/athena/resources.py
18
19
  dagster_aws/cloudwatch/__init__.py
19
20
  dagster_aws/cloudwatch/loggers.py
21
+ dagster_aws/components/__init__.py
22
+ dagster_aws/components/athena.py
23
+ dagster_aws/components/credentials.py
24
+ dagster_aws/components/ecr.py
25
+ dagster_aws/components/rds.py
26
+ dagster_aws/components/s3.py
27
+ dagster_aws/components/secretsmanager.py
28
+ dagster_aws/components/ssm.py
20
29
  dagster_aws/ecr/__init__.py
21
30
  dagster_aws/ecr/resources.py
22
31
  dagster_aws/ecs/__init__.py
@@ -49,6 +58,7 @@ dagster_aws/pipes/clients/utils.py
49
58
  dagster_aws/rds/__init__.py
50
59
  dagster_aws/rds/resources.py
51
60
  dagster_aws/redshift/__init__.py
61
+ dagster_aws/redshift/components.py
52
62
  dagster_aws/redshift/resources.py
53
63
  dagster_aws/s3/__init__.py
54
64
  dagster_aws/s3/compute_log_manager.py
@@ -0,0 +1,2 @@
1
+ [dagster_dg_cli.registry_modules]
2
+ dagster_aws = dagster_aws.components
@@ -1,5 +1,5 @@
1
1
  boto3
2
- dagster==1.12.11
2
+ dagster==1.12.12
3
3
  packaging
4
4
  requests
5
5
 
@@ -36,7 +36,7 @@ setup(
36
36
  python_requires=">=3.10,<3.15",
37
37
  install_requires=[
38
38
  "boto3",
39
- "dagster==1.12.11",
39
+ "dagster==1.12.12",
40
40
  "packaging",
41
41
  "requests",
42
42
  ],
@@ -56,5 +56,10 @@ setup(
56
56
  "s3fs",
57
57
  ],
58
58
  },
59
+ entry_points={
60
+ "dagster_dg_cli.registry_modules": [
61
+ "dagster_aws = dagster_aws.components",
62
+ ],
63
+ },
59
64
  zip_safe=False,
60
65
  )
@@ -1,5 +0,0 @@
1
- from dagster_shared.libraries import DagsterLibraryRegistry
2
-
3
- from dagster_aws.version import __version__
4
-
5
- DagsterLibraryRegistry.register("dagster-aws", __version__)
@@ -1 +0,0 @@
1
- __version__ = "0.28.11"
File without changes
File without changes
File without changes
File without changes