dagster-aws 0.20.15__tar.gz → 0.28.5__tar.gz

{dagster-aws-0.20.15 → dagster_aws-0.28.5}/LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2023 Dagster Labs, Inc".
+   Copyright 2025 Dagster Labs, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

{dagster-aws-0.20.15 → dagster_aws-0.28.5}/MANIFEST.in

@@ -4,3 +4,5 @@ recursive-include dagster_aws *.txt
 recursive-include dagster_aws *.template
 include LICENSE
 include dagster_aws/py.typed
+
+exclude ruff.toml

dagster_aws-0.28.5/PKG-INFO

@@ -0,0 +1,44 @@
+Metadata-Version: 2.4
+Name: dagster-aws
+Version: 0.28.5
+Summary: Package for AWS-specific Dagster framework solid and resource components.
+Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-aws
+Author: Dagster Labs
+Author-email: hello@dagsterlabs.com
+License: Apache-2.0
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10,<3.14
+License-File: LICENSE
+Requires-Dist: boto3
+Requires-Dist: dagster==1.12.5
+Requires-Dist: packaging
+Requires-Dist: requests
+Provides-Extra: redshift
+Requires-Dist: psycopg2-binary; extra == "redshift"
+Provides-Extra: pyspark
+Requires-Dist: dagster-pyspark; extra == "pyspark"
+Provides-Extra: stubs
+Requires-Dist: boto3-stubs-lite[ecs,emr,emr-containers,emr-serverless,glue,logs,s3]; extra == "stubs"
+Provides-Extra: test
+Requires-Dist: botocore!=1.32.1; extra == "test"
+Requires-Dist: moto[emrcontainers,emrserverless,glue,logs,s3,server]<5.0,>=2.2.8; extra == "test"
+Requires-Dist: requests-mock; extra == "test"
+Requires-Dist: xmltodict==0.12.0; extra == "test"
+Requires-Dist: flaky; extra == "test"
+Requires-Dist: pytest-cases; extra == "test"
+Requires-Dist: s3fs; extra == "test"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

dagster_aws-0.28.5/README.md

@@ -0,0 +1,4 @@
+# dagster-aws
+
+The docs for `dagster-aws` can be found
+[here](https://docs.dagster.io/api/python-api/libraries/dagster-aws).

dagster_aws-0.28.5/dagster_aws/__init__.py

@@ -0,0 +1,5 @@
+from dagster_shared.libraries import DagsterLibraryRegistry
+
+from dagster_aws.version import __version__
+
+DagsterLibraryRegistry.register("dagster-aws", __version__)

dagster_aws-0.28.5/dagster_aws/_stubs.py

@@ -0,0 +1,50 @@
+# Stubs have been sourced from
+#
+#    https://youtype.github.io/boto3_stubs_docs/mypy_boto3_s3/type_defs/#objecttypedef
+#
+# as to not require any additional dependencies
+
+from datetime import datetime
+from typing import Literal, Optional, TypedDict
+
+ChecksumAlgorithmType = Literal[
+    "CRC32",
+    "CRC32C",
+    "SHA1",
+    "SHA256",
+]
+
+ObjectStorageClassType = Literal[
+    "DEEP_ARCHIVE",
+    "EXPRESS_ONEZONE",
+    "GLACIER",
+    "GLACIER_IR",
+    "INTELLIGENT_TIERING",
+    "ONEZONE_IA",
+    "OUTPOSTS",
+    "REDUCED_REDUNDANCY",
+    "SNOW",
+    "STANDARD",
+    "STANDARD_IA",
+]
+
+
+class OwnerTypeDef(TypedDict):
+    DisplayName: Optional[str]
+    ID: Optional[str]
+
+
+class RestoreStatusTypeDef(TypedDict):
+    IsRestoreInProgress: Optional[bool]
+    RestoreExpiryDate: Optional[datetime]
+
+
+class ObjectTypeDef(TypedDict):
+    Key: str
+    LastModified: datetime
+    ETag: Optional[str]
+    ChecksumAlgorithm: Optional[list[ChecksumAlgorithmType]]
+    Size: Optional[int]
+    StorageClass: Optional[ObjectStorageClassType]
+    Owner: Optional[OwnerTypeDef]
+    RestoreStatus: Optional[RestoreStatusTypeDef]

{dagster-aws-0.20.15 → dagster_aws-0.28.5}/dagster_aws/athena/__init__.py

@@ -1,4 +1,4 @@
-from .resources import (
+from dagster_aws.athena.resources import (
     AthenaClient as AthenaClient,
     AthenaClientResource as AthenaClientResource,
     AthenaError as AthenaError,

{dagster-aws-0.20.15 → dagster_aws-0.28.5}/dagster_aws/athena/resources.py

@@ -82,7 +82,7 @@ class AthenaClient:
             raise AthenaTimeout()
 
         if state != "SUCCEEDED":
-            raise AthenaError(execution["Status"]["StateChangeReason"])
+            raise AthenaError(execution["Status"]["StateChangeReason"])  # pyright: ignore[reportPossiblyUnboundVariable]
 
     def _results(self, execution_id):
         execution = self.client.get_query_execution(QueryExecutionId=execution_id)["QueryExecution"]
@@ -254,7 +254,7 @@ class AthenaClientResource(ResourceWithAthenaConfig):
                 def example_athena_asset(athena: AthenaClientResource):
                     return athena.get_client().execute_query("SELECT 1", fetch_results=True)
 
-                defs = Definitions(
+                Definitions(
                     assets=[example_athena_asset],
                     resources={"athena": AthenaClientResource()}
                 )

dagster_aws-0.28.5/dagster_aws/cloudwatch/__init__.py

@@ -0,0 +1 @@
+from dagster_aws.cloudwatch.loggers import cloudwatch_logger as cloudwatch_logger

{dagster-aws-0.20.15 → dagster_aws-0.28.5}/dagster_aws/cloudwatch/loggers.py

@@ -1,15 +1,17 @@
 import datetime
 import logging
+from typing import Optional
 
 import boto3
 from dagster import (
     Field,
     StringSource,
     _check as check,
-    _seven,
     logger,
 )
+from dagster._annotations import deprecated
 from dagster._core.utils import coerce_valid_log_level
+from dagster_shared import seven
 
 # The maximum batch size is 1,048,576 bytes, and this size is calculated as the sum of all event
 # messages in UTF-8, plus 26 bytes for each log event.
@@ -28,20 +30,29 @@ def millisecond_timestamp(dt):
     return int(microsecond_timestamp / 1000)
 
 
+@deprecated(breaking_version="0.27", additional_warn_text="Use `PipesCloudWatchLogReader` instead.")
 class CloudwatchLogsHandler(logging.Handler):
     def __init__(
         self,
         log_group_name,
         log_stream_name,
         aws_region=None,
-        aws_secret_access_key=None,
-        aws_access_key_id=None,
+        aws_access_key_id: Optional[str] = None,
+        aws_secret_access_key: Optional[str] = None,
+        endpoint_url: Optional[str] = None,
+        use_ssl: bool = True,
+        aws_session_token: Optional[str] = None,
+        verify: Optional[bool] = None,
     ):
         self.client = boto3.client(
             "logs",
             region_name=aws_region,
             aws_access_key_id=aws_access_key_id,
             aws_secret_access_key=aws_secret_access_key,
+            endpoint_url=endpoint_url,
+            use_ssl=use_ssl,
+            aws_session_token=aws_session_token,
+            verify=verify,
         )
         self.log_group_name = check.str_param(log_group_name, "log_group_name")
         # Maybe we should make this optional, and default to the run_id
@@ -53,7 +64,7 @@ class CloudwatchLogsHandler(logging.Handler):
         self.check_log_group()
         self.check_log_stream()
 
-        super(CloudwatchLogsHandler, self).__init__()
+        super().__init__()
 
     def check_log_group(self):
         # Check that log group exists
@@ -111,7 +122,7 @@ class CloudwatchLogsHandler(logging.Handler):
     def log_error(self, record, exc):
         logging.critical("Error while logging!")
         try:
-            logging.error(f"Attempted to log: {_seven.json.dumps(record.__dict__)}")
+            logging.error(f"Attempted to log: {seven.json.dumps(record.__dict__)}")
         except Exception:
             pass
         logging.exception(str(exc))
@@ -123,7 +134,7 @@ class CloudwatchLogsHandler(logging.Handler):
         self._emit(record, retry=True)
 
     def _emit(self, record, retry=False):
-        message = _seven.json.dumps(record.__dict__)
+        message = seven.json.dumps(record.__dict__)
         timestamp = millisecond_timestamp(
             datetime.datetime.strptime(record.dagster_meta["log_timestamp"], "%Y-%m-%dT%H:%M:%S.%f")
         )
@@ -191,6 +202,7 @@ class CloudwatchLogsHandler(logging.Handler):
     },
     description="The default colored console logger.",
 )
+@deprecated(breaking_version="0.27", additional_warn_text="Use `PipesCloudWatchLogReader` instead.")
 def cloudwatch_logger(init_context):
     """This logger provides support for sending Dagster logs to AWS CloudWatch.
 

{dagster-aws-0.20.15 → dagster_aws-0.28.5}/dagster_aws/ecr/__init__.py

@@ -1,4 +1,4 @@
-from .resources import (
+from dagster_aws.ecr.resources import (
     ECRPublicResource as ECRPublicResource,
     FakeECRPublicResource as FakeECRPublicResource,
     ecr_public_resource as ecr_public_resource,

{dagster-aws-0.20.15 → dagster_aws-0.28.5}/dagster_aws/ecr/resources.py

@@ -1,14 +1,34 @@
 import datetime
+from typing import Optional
 
 import boto3
 from botocore.stub import Stubber
 from dagster import ConfigurableResource, resource
+from dagster._annotations import beta
 from dagster._core.definitions.resource_definition import dagster_maintained_resource
 
 
 class ECRPublicClient:
-    def __init__(self):
-        self.client = boto3.client("ecr-public")
+    def __init__(
+        self,
+        region_name: Optional[str] = None,
+        endpoint_url: Optional[str] = None,
+        use_ssl: bool = True,
+        aws_access_key_id: Optional[str] = None,
+        aws_secret_access_key: Optional[str] = None,
+        aws_session_token: Optional[str] = None,
+        verify: Optional[bool] = None,
+    ):
+        self.client = boto3.client(
+            "ecr-public",
+            region_name=region_name,
+            use_ssl=use_ssl,
+            verify=verify,
+            endpoint_url=endpoint_url,
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+            aws_session_token=aws_session_token,
+        )
 
     def get_login_password(self):
         return self.client.get_authorization_token()["authorizationData"]["authorizationToken"]
@@ -34,6 +54,7 @@ class FakeECRPublicClient(ECRPublicClient):
         return result
 
 
+@beta
 class ECRPublicResource(ConfigurableResource):
     """This resource enables connecting to AWS Public and getting a login password from it.
     Similar to the AWS CLI's `aws ecr-public get-login-password` command.
@@ -47,6 +68,7 @@ class ECRPublicResource(ConfigurableResource):
         return ECRPublicClient()
 
 
+@beta
 class FakeECRPublicResource(ConfigurableResource):
     """This resource behaves like ecr_public_resource except it stubs out the real AWS API
     requests and always returns `'token'` as its login password.
@@ -60,6 +82,7 @@ class FakeECRPublicResource(ConfigurableResource):
         return FakeECRPublicClient()
 
 
+@beta
 @dagster_maintained_resource
 @resource(
     description=(
@@ -71,6 +94,7 @@ def ecr_public_resource(context) -> ECRPublicClient:
     return ECRPublicResource.from_resource_context(context).get_client()
 
 
+@beta
 @dagster_maintained_resource
 @resource(
     description=(

dagster_aws-0.28.5/dagster_aws/ecs/__init__.py

@@ -0,0 +1,3 @@
+from dagster_aws.ecs.executor import ecs_executor as ecs_executor
+from dagster_aws.ecs.launcher import EcsRunLauncher as EcsRunLauncher
+from dagster_aws.ecs.tasks import EcsEventualConsistencyTimeout as EcsEventualConsistencyTimeout

{dagster-aws-0.20.15 → dagster_aws-0.28.5}/dagster_aws/ecs/container_context.py

@@ -1,4 +1,5 @@
-from typing import TYPE_CHECKING, Any, Mapping, NamedTuple, Optional, Sequence, cast
+from collections.abc import Mapping, Sequence
+from typing import TYPE_CHECKING, Any, NamedTuple, Optional, cast
 
 from dagster import (
     Array,
@@ -16,10 +17,10 @@ from dagster._core.errors import DagsterInvalidConfigError
 from dagster._core.storage.dagster_run import DagsterRun
 from dagster._core.utils import parse_env_var
 
-from ..secretsmanager import get_tagged_secrets
+from dagster_aws.secretsmanager import get_tagged_secrets
 
 if TYPE_CHECKING:
-    from . import EcsRunLauncher
+    from dagster_aws.ecs import EcsRunLauncher
 
 # Config shared between EcsRunLauncher and EcsContainerContext
 SHARED_ECS_SCHEMA = {
@@ -186,6 +187,12 @@ ECS_CONTAINER_CONTEXT_SCHEMA = {
                     is_required=False,
                     description="The ephemeral storage, in GiB, to use for the launched task.",
                 ),
+                "replica_count": Field(
+                    int,
+                    default_value=1,
+                    is_required=False,
+                    description="The number of code server instances to launch.",
+                ),
             }
         )
     ),
@@ -209,6 +216,11 @@ ECS_CONTAINER_CONTEXT_SCHEMA = {
         is_required=False,
         description="Additional sidecar containers to include in run task definitions.",
     ),
+    "server_health_check": Field(
+        Permissive(),
+        is_required=False,
+        description="Health check to include in code server task definitions.",
+    ),
     **SHARED_TASK_DEFINITION_FIELDS,
     **SHARED_ECS_SCHEMA,
 }
@@ -235,6 +247,7 @@ class EcsContainerContext(
             ("server_ecs_tags", Sequence[Mapping[str, Optional[str]]]),
             ("run_ecs_tags", Sequence[Mapping[str, Optional[str]]]),
             ("repository_credentials", Optional[str]),
+            ("server_health_check", Optional[Mapping[str, Any]]),
         ],
     )
 ):
@@ -259,8 +272,9 @@ class EcsContainerContext(
         server_ecs_tags: Optional[Sequence[Mapping[str, Optional[str]]]] = None,
         run_ecs_tags: Optional[Sequence[Mapping[str, Optional[str]]]] = None,
         repository_credentials: Optional[str] = None,
+        server_health_check: Optional[Mapping[str, Any]] = None,
     ):
-        return super(EcsContainerContext, cls).__new__(
+        return super().__new__(
             cls,
             secrets=check.opt_sequence_param(secrets, "secrets"),
             secrets_tags=check.opt_sequence_param(secrets_tags, "secrets_tags"),
@@ -287,6 +301,7 @@ class EcsContainerContext(
             repository_credentials=check.opt_str_param(
                 repository_credentials, "repository_credentials"
             ),
+            server_health_check=check.opt_mapping_param(server_health_check, "server_health_check"),
         )
 
     def merge(self, other: "EcsContainerContext") -> "EcsContainerContext":
@@ -311,6 +326,7 @@ class EcsContainerContext(
             server_ecs_tags=[*other.server_ecs_tags, *self.server_ecs_tags],
             run_ecs_tags=[*other.run_ecs_tags, *self.run_ecs_tags],
             repository_credentials=other.repository_credentials or self.repository_credentials,
+            server_health_check=other.server_health_check or self.server_health_check,
         )
 
     def get_secrets_dict(self, secrets_manager) -> Mapping[str, str]:
@@ -383,7 +399,7 @@ class EcsContainerContext(
                 run_ecs_container_context,
             )
 
-        processed_context_value = cast(Mapping[str, Any], processed_container_context.value)
+        processed_context_value = cast("Mapping[str, Any]", processed_container_context.value)
 
         return shared_container_context.merge(
             EcsContainerContext(
@@ -404,5 +420,6 @@ class EcsContainerContext(
                 server_ecs_tags=processed_context_value.get("server_ecs_tags"),
                 run_ecs_tags=processed_context_value.get("run_ecs_tags"),
                 repository_credentials=processed_context_value.get("repository_credentials"),
+                server_health_check=processed_context_value.get("server_health_check"),
             )
         )