dagster-aws 0.26.11__tar.gz → 0.28.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/LICENSE +1 -1
  2. dagster_aws-0.28.5/PKG-INFO +44 -0
  3. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/athena/resources.py +1 -1
  4. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/ecs/executor.py +8 -0
  5. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/ecs/launcher.py +45 -5
  6. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/ecs/tasks.py +10 -0
  7. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/ecs/utils.py +4 -1
  8. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/pipes/context_injectors.py +6 -5
  9. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/pipes/message_readers.py +2 -2
  10. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/rds/resources.py +1 -1
  11. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/redshift/resources.py +1 -1
  12. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/s3/compute_log_manager.py +13 -15
  13. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/s3/io_manager.py +2 -2
  14. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/s3/resources.py +1 -1
  15. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/s3/sensor.py +2 -0
  16. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/secretsmanager/resources.py +2 -2
  17. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/ssm/resources.py +1 -1
  18. dagster_aws-0.28.5/dagster_aws/version.py +1 -0
  19. dagster_aws-0.28.5/dagster_aws.egg-info/PKG-INFO +44 -0
  20. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws.egg-info/requires.txt +2 -1
  21. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/setup.py +4 -3
  22. dagster-aws-0.26.11/PKG-INFO +0 -20
  23. dagster-aws-0.26.11/dagster_aws/version.py +0 -1
  24. dagster-aws-0.26.11/dagster_aws.egg-info/PKG-INFO +0 -20
  25. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/MANIFEST.in +0 -0
  26. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/README.md +0 -0
  27. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/__init__.py +0 -0
  28. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/_stubs.py +0 -0
  29. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/athena/__init__.py +0 -0
  30. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/cloudwatch/__init__.py +0 -0
  31. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/cloudwatch/loggers.py +0 -0
  32. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/ecr/__init__.py +0 -0
  33. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/ecr/resources.py +0 -0
  34. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/ecs/__init__.py +0 -0
  35. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/ecs/container_context.py +0 -0
  36. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/ecs/test_utils.py +0 -0
  37. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/emr/__init__.py +0 -0
  38. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/emr/configs.py +0 -0
  39. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/emr/configs_spark.py +0 -0
  40. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/emr/emr.py +0 -0
  41. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/emr/emr_step_main.py +0 -0
  42. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/emr/main.py.template +0 -0
  43. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/emr/pyspark_step_launcher.py +0 -0
  44. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/emr/types.py +0 -0
  45. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/emr/utils.py +0 -0
  46. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/pipes/__init__.py +0 -0
  47. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/pipes/clients/__init__.py +0 -0
  48. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/pipes/clients/ecs.py +0 -0
  49. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/pipes/clients/emr.py +0 -0
  50. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/pipes/clients/emr_containers.py +0 -0
  51. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/pipes/clients/emr_serverless.py +0 -0
  52. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/pipes/clients/glue.py +0 -0
  53. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/pipes/clients/lambda_.py +0 -0
  54. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/pipes/clients/utils.py +0 -0
  55. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/py.typed +0 -0
  56. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/rds/__init__.py +0 -0
  57. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/redshift/__init__.py +0 -0
  58. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/s3/__init__.py +0 -0
  59. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/s3/file_manager.py +0 -0
  60. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/s3/ops.py +0 -0
  61. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/s3/s3_fake_resource.py +0 -0
  62. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/s3/utils.py +0 -0
  63. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/secretsmanager/__init__.py +0 -0
  64. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/secretsmanager/secrets.py +0 -0
  65. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/ssm/__init__.py +0 -0
  66. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/ssm/parameters.py +0 -0
  67. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/utils/__init__.py +0 -0
  68. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/utils/mrjob/__init__.py +0 -0
  69. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/utils/mrjob/log4j.py +0 -0
  70. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/utils/mrjob/retry.py +0 -0
  71. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws/utils/mrjob/utils.py +0 -0
  72. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws.egg-info/SOURCES.txt +0 -0
  73. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws.egg-info/dependency_links.txt +0 -0
  74. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws.egg-info/not-zip-safe +0 -0
  75. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/dagster_aws.egg-info/top_level.txt +0 -0
  76. {dagster-aws-0.26.11 → dagster_aws-0.28.5}/setup.cfg +0 -0
@@ -186,7 +186,7 @@
186
186
  same "printed page" as the copyright notice for easier
187
187
  identification within third-party archives.
188
188
 
189
- Copyright 2023 Dagster Labs, Inc".
189
+ Copyright 2025 Dagster Labs, Inc.
190
190
 
191
191
  Licensed under the Apache License, Version 2.0 (the "License");
192
192
  you may not use this file except in compliance with the License.
@@ -0,0 +1,44 @@
1
+ Metadata-Version: 2.4
2
+ Name: dagster-aws
3
+ Version: 0.28.5
4
+ Summary: Package for AWS-specific Dagster framework solid and resource components.
5
+ Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-aws
6
+ Author: Dagster Labs
7
+ Author-email: hello@dagsterlabs.com
8
+ License: Apache-2.0
9
+ Classifier: Programming Language :: Python :: 3.10
10
+ Classifier: Programming Language :: Python :: 3.11
11
+ Classifier: Programming Language :: Python :: 3.12
12
+ Classifier: Programming Language :: Python :: 3.13
13
+ Classifier: License :: OSI Approved :: Apache Software License
14
+ Classifier: Operating System :: OS Independent
15
+ Requires-Python: >=3.10,<3.14
16
+ License-File: LICENSE
17
+ Requires-Dist: boto3
18
+ Requires-Dist: dagster==1.12.5
19
+ Requires-Dist: packaging
20
+ Requires-Dist: requests
21
+ Provides-Extra: redshift
22
+ Requires-Dist: psycopg2-binary; extra == "redshift"
23
+ Provides-Extra: pyspark
24
+ Requires-Dist: dagster-pyspark; extra == "pyspark"
25
+ Provides-Extra: stubs
26
+ Requires-Dist: boto3-stubs-lite[ecs,emr,emr-containers,emr-serverless,glue,logs,s3]; extra == "stubs"
27
+ Provides-Extra: test
28
+ Requires-Dist: botocore!=1.32.1; extra == "test"
29
+ Requires-Dist: moto[emrcontainers,emrserverless,glue,logs,s3,server]<5.0,>=2.2.8; extra == "test"
30
+ Requires-Dist: requests-mock; extra == "test"
31
+ Requires-Dist: xmltodict==0.12.0; extra == "test"
32
+ Requires-Dist: flaky; extra == "test"
33
+ Requires-Dist: pytest-cases; extra == "test"
34
+ Requires-Dist: s3fs; extra == "test"
35
+ Dynamic: author
36
+ Dynamic: author-email
37
+ Dynamic: classifier
38
+ Dynamic: home-page
39
+ Dynamic: license
40
+ Dynamic: license-file
41
+ Dynamic: provides-extra
42
+ Dynamic: requires-dist
43
+ Dynamic: requires-python
44
+ Dynamic: summary
@@ -254,7 +254,7 @@ class AthenaClientResource(ResourceWithAthenaConfig):
254
254
  def example_athena_asset(athena: AthenaClientResource):
255
255
  return athena.get_client().execute_query("SELECT 1", fetch_results=True)
256
256
 
257
- defs = Definitions(
257
+ Definitions(
258
258
  assets=[example_athena_asset],
259
259
  resources={"athena": AthenaClientResource()}
260
260
  )
@@ -17,6 +17,10 @@ from dagster._core.definitions.executor_definition import multiple_process_execu
17
17
  from dagster._core.definitions.metadata import MetadataValue
18
18
  from dagster._core.events import DagsterEvent, EngineEventData
19
19
  from dagster._core.execution.retries import RetryMode, get_retries_config
20
+ from dagster._core.execution.step_dependency_config import (
21
+ StepDependencyConfig,
22
+ get_step_dependency_config_field,
23
+ )
20
24
  from dagster._core.execution.tags import get_tag_concurrency_limits_config
21
25
  from dagster._core.executor.base import Executor
22
26
  from dagster._core.executor.init import InitExecutorContext
@@ -68,6 +72,7 @@ _ECS_EXECUTOR_CONFIG_SCHEMA = {
68
72
  ),
69
73
  ),
70
74
  "tag_concurrency_limits": get_tag_concurrency_limits_config(),
75
+ "step_dependency_config": get_step_dependency_config_field(),
71
76
  }
72
77
 
73
78
 
@@ -134,6 +139,9 @@ def ecs_executor(init_context: InitExecutorContext) -> Executor:
134
139
  max_concurrent=check.opt_int_elem(exc_cfg, "max_concurrent"),
135
140
  tag_concurrency_limits=check.opt_list_elem(exc_cfg, "tag_concurrency_limits"),
136
141
  should_verify_step=True,
142
+ step_dependency_config=StepDependencyConfig.from_config(
143
+ exc_cfg.get("step_dependency_config") # type: ignore
144
+ ),
137
145
  )
138
146
 
139
147
 
@@ -5,7 +5,7 @@ import uuid
5
5
  import warnings
6
6
  from collections import namedtuple
7
7
  from collections.abc import Mapping, Sequence
8
- from typing import Any, Optional
8
+ from typing import TYPE_CHECKING, Any, Optional
9
9
 
10
10
  import boto3
11
11
  from botocore.exceptions import ClientError
@@ -30,11 +30,12 @@ from dagster._core.launcher.base import (
30
30
  WorkerStatus,
31
31
  )
32
32
  from dagster._core.storage.dagster_run import DagsterRun
33
- from dagster._core.storage.tags import RUN_WORKER_ID_TAG
33
+ from dagster._core.storage.tags import HIDDEN_TAG_PREFIX, RUN_WORKER_ID_TAG
34
34
  from dagster._grpc.types import ExecuteRunArgs
35
35
  from dagster._serdes import ConfigurableClass
36
36
  from dagster._serdes.config_class import ConfigurableClassData
37
37
  from dagster._utils.backoff import backoff
38
+ from dagster._utils.tags import get_boolean_tag_value
38
39
  from typing_extensions import Self
39
40
 
40
41
  from dagster_aws.ecs.container_context import (
@@ -81,6 +82,9 @@ TAGS_TO_EXCLUDE_FROM_PROPAGATION = {"dagster/op_selection", "dagster/solid_selec
81
82
  DEFAULT_REGISTER_TASK_DEFINITION_RETRIES = 5
82
83
  DEFAULT_RUN_TASK_RETRIES = 5
83
84
 
85
+ if TYPE_CHECKING:
86
+ from botocore.config import Config
87
+
84
88
 
85
89
  class EcsRunLauncher(RunLauncher[T_DagsterInstance], ConfigurableClass):
86
90
  """RunLauncher that starts a task in ECS for each Dagster job run.
@@ -119,10 +123,13 @@ class EcsRunLauncher(RunLauncher[T_DagsterInstance], ConfigurableClass):
119
123
  task_definition_prefix: str = "run",
120
124
  ):
121
125
  self._inst_data = inst_data
122
- self.ecs = boto3.client("ecs")
126
+
127
+ boto_client_config = self.get_boto_client_config()
128
+
129
+ self.ecs = boto3.client("ecs", config=boto_client_config)
123
130
  self.ec2 = boto3.resource("ec2")
124
- self.secrets_manager = boto3.client("secretsmanager")
125
- self.logs = boto3.client("logs")
131
+ self.secrets_manager = boto3.client("secretsmanager", config=boto_client_config)
132
+ self.logs = boto3.client("logs", config=boto_client_config)
126
133
 
127
134
  self._task_definition_prefix = task_definition_prefix
128
135
 
@@ -236,6 +243,9 @@ class EcsRunLauncher(RunLauncher[T_DagsterInstance], ConfigurableClass):
236
243
  self._current_task_metadata = None
237
244
  self._current_task = None
238
245
 
246
+ def get_boto_client_config(self) -> Optional["Config"]:
247
+ return None
248
+
239
249
  @property
240
250
  def inst_data(self):
241
251
  return self._inst_data
@@ -855,6 +865,28 @@ class EcsRunLauncher(RunLauncher[T_DagsterInstance], ConfigurableClass):
855
865
  task.get("stoppedReason", "")
856
866
  )
857
867
 
868
+ def _add_eni_id_tags(self, run: DagsterRun, task: dict[str, Any]):
869
+ attachments = task.get("attachments", [])
870
+ eni_ids = {}
871
+ eni_count = 0
872
+ for attachment in attachments:
873
+ if attachment.get("type") == "ElasticNetworkInterface":
874
+ details = {d["name"]: d["value"] for d in attachment.get("details", [])}
875
+
876
+ if "networkInterfaceId" in details:
877
+ if eni_count == 0:
878
+ eni_ids[f"{HIDDEN_TAG_PREFIX}eni_id"] = details["networkInterfaceId"]
879
+ else:
880
+ eni_ids[f"{HIDDEN_TAG_PREFIX}eni_id_{eni_count}"] = details[
881
+ "networkInterfaceId"
882
+ ]
883
+ eni_count += 1
884
+ self._instance.add_run_tags(run.run_id, eni_ids)
885
+ if eni_count > 0:
886
+ logging.info(f"Added {eni_count} ENI ID tags for run {run.run_id}: {eni_ids}")
887
+ else:
888
+ logging.warning(f"No ENI IDs found for run {run.run_id}")
889
+
858
890
  def check_run_worker_health(self, run: DagsterRun):
859
891
  run_worker_id = run.tags.get(RUN_WORKER_ID_TAG)
860
892
 
@@ -870,6 +902,14 @@ class EcsRunLauncher(RunLauncher[T_DagsterInstance], ConfigurableClass):
870
902
 
871
903
  t = tasks[0]
872
904
 
905
+ if get_boolean_tag_value(os.getenv("DAGSTER_AWS_ENI_TAGGING_ENABLED")) and not run.tags.get(
906
+ f"{HIDDEN_TAG_PREFIX}eni_id"
907
+ ):
908
+ try:
909
+ self._add_eni_id_tags(run, t)
910
+ except Exception:
911
+ logging.exception(f"Error adding ENI ID tags for run {run.run_id}")
912
+
873
913
  if t.get("lastStatus") in RUNNING_STATUSES:
874
914
  return CheckRunHealthResult(WorkerStatus.RUNNING, run_worker_id=run_worker_id)
875
915
  elif t.get("lastStatus") in STOPPED_STATUSES:
@@ -7,6 +7,9 @@ import requests
7
7
  from dagster._utils.backoff import backoff
8
8
  from dagster._utils.merger import merge_dicts
9
9
 
10
+ # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/fargate-task-storage.html
11
+ DEFAULT_EPHEMERAL_STORAGE = 20
12
+
10
13
 
11
14
  def _arns_match(arn1: Optional[str], arn2: Optional[str]):
12
15
  arn1 = arn1 or ""
@@ -22,6 +25,12 @@ def _arns_match(arn1: Optional[str], arn2: Optional[str]):
22
25
  return arn1 == arn2
23
26
 
24
27
 
28
+ def _ephemeral_storage_matches(storage1: Optional[int], storage2: Optional[int]):
29
+ if (storage1 or DEFAULT_EPHEMERAL_STORAGE) == (storage2 or DEFAULT_EPHEMERAL_STORAGE):
30
+ return True
31
+ return False
32
+
33
+
25
34
  class DagsterEcsTaskDefinitionConfig(
26
35
  NamedTuple(
27
36
  "_DagsterEcsTaskDefinitionConfig",
@@ -169,6 +178,7 @@ class DagsterEcsTaskDefinitionConfig(
169
178
  and self.environment == other.environment
170
179
  and self.cpu == other.cpu
171
180
  and self.memory == other.memory
181
+ and _ephemeral_storage_matches(self.ephemeral_storage, other.ephemeral_storage)
172
182
  and _arns_match(self.execution_role_arn, other.execution_role_arn)
173
183
  and _arns_match(self.task_role_arn, other.task_role_arn)
174
184
  ):
@@ -3,7 +3,7 @@ import re
3
3
  from collections.abc import Mapping
4
4
  from typing import Any
5
5
 
6
- from dagster._core.remote_representation.origin import RemoteJobOrigin
6
+ from dagster._core.remote_origin import RemoteJobOrigin
7
7
 
8
8
  from dagster_aws.ecs.tasks import DagsterEcsTaskDefinitionConfig
9
9
 
@@ -160,4 +160,7 @@ def is_transient_task_stopped_reason(stopped_reason: str) -> bool:
160
160
  ):
161
161
  return True
162
162
 
163
+ if "The Service Discovery instance could not be registered" in stopped_reason:
164
+ return True
165
+
163
166
  return False
@@ -4,15 +4,15 @@ import random
4
4
  import string
5
5
  from collections.abc import Iterator
6
6
  from contextlib import contextmanager
7
- from typing import TYPE_CHECKING
7
+ from typing import TYPE_CHECKING, Optional
8
8
 
9
- import boto3
10
9
  import dagster._check as check
11
10
  from dagster._core.pipes.client import PipesContextInjector, PipesParams
12
11
  from dagster._core.pipes.utils import PipesEnvContextInjector
13
12
 
14
13
  if TYPE_CHECKING:
15
14
  from dagster_pipes import PipesContextData
15
+ from mypy_boto3_s3 import S3Client
16
16
 
17
17
  _CONTEXT_FILENAME = "context.json"
18
18
 
@@ -22,20 +22,21 @@ class PipesS3ContextInjector(PipesContextInjector):
22
22
 
23
23
  Args:
24
24
  bucket (str): The S3 bucket to write to.
25
- client (boto3.client): A boto3 client to use to write to S3.
25
+ client (S3Client): A boto3 client to use to write to S3.
26
26
  key_prefix (Optional[str]): An optional prefix to use for the S3 key. Defaults to a random
27
27
  string.
28
28
 
29
29
  """
30
30
 
31
- def __init__(self, *, bucket: str, client: boto3.client): # pyright: ignore (reportGeneralTypeIssues)
31
+ def __init__(self, *, bucket: str, client: "S3Client", key_prefix: Optional[str] = None):
32
32
  super().__init__()
33
33
  self.bucket = check.str_param(bucket, "bucket")
34
34
  self.client = client
35
+ self.key_prefix = key_prefix
35
36
 
36
37
  @contextmanager
37
38
  def inject_context(self, context: "PipesContextData") -> Iterator[PipesParams]: # pyright: ignore[reportIncompatibleMethodOverride]
38
- key_prefix = "".join(random.choices(string.ascii_letters, k=30))
39
+ key_prefix = f"{self.key_prefix or ''}{''.join(random.choices(string.ascii_letters, k=30))}"
39
40
  key = os.path.join(key_prefix, _CONTEXT_FILENAME)
40
41
  self.client.put_object(
41
42
  Body=json.dumps(context).encode("utf-8"), Bucket=self.bucket, Key=key
@@ -4,11 +4,11 @@ import os
4
4
  import random
5
5
  import string
6
6
  import sys
7
- from collections.abc import Generator, Iterator, Sequence
7
+ from collections.abc import Callable, Generator, Iterator, Sequence
8
8
  from contextlib import contextmanager
9
9
  from datetime import datetime
10
10
  from threading import Event, Thread
11
- from typing import IO, TYPE_CHECKING, Any, Callable, Optional, cast
11
+ from typing import IO, TYPE_CHECKING, Any, Optional, cast
12
12
 
13
13
  import boto3
14
14
  import dagster._check as check
@@ -40,7 +40,7 @@ class RDSResource(ResourceWithBoto3Configuration):
40
40
  sql="SELECT * from mytable",
41
41
  )
42
42
 
43
- defs = Definitions(
43
+ Definitions(
44
44
  assets=[my_table],
45
45
  resources={
46
46
  "rds_resource": RDSResource(
@@ -301,7 +301,7 @@ class RedshiftClientResource(ConfigurableResource):
301
301
  database='dev',
302
302
  )
303
303
 
304
- defs = Definitions(
304
+ Definitions(
305
305
  assets=[example_redshift_asset],
306
306
  resources={'redshift': redshift_configured},
307
307
  )
@@ -1,7 +1,7 @@
1
1
  import os
2
2
  from collections.abc import Iterator, Mapping, Sequence
3
3
  from contextlib import contextmanager
4
- from typing import Any, Optional
4
+ from typing import IO, Any, Optional
5
5
 
6
6
  import boto3
7
7
  import dagster_shared.seven as seven
@@ -14,8 +14,8 @@ from dagster import (
14
14
  )
15
15
  from dagster._config.config_type import Noneable
16
16
  from dagster._core.storage.cloud_storage_compute_log_manager import (
17
- CloudStorageComputeLogManager,
18
17
  PollingComputeLogSubscriptionManager,
18
+ TruncatingCloudStorageComputeLogManager,
19
19
  )
20
20
  from dagster._core.storage.compute_log_manager import CapturedLogContext, ComputeIOType
21
21
  from dagster._core.storage.local_compute_log_manager import (
@@ -23,13 +23,13 @@ from dagster._core.storage.local_compute_log_manager import (
23
23
  LocalComputeLogManager,
24
24
  )
25
25
  from dagster._serdes import ConfigurableClass, ConfigurableClassData
26
- from dagster._utils import ensure_dir, ensure_file
26
+ from dagster._utils import ensure_dir
27
27
  from typing_extensions import Self
28
28
 
29
29
  POLLING_INTERVAL = 5
30
30
 
31
31
 
32
- class S3ComputeLogManager(CloudStorageComputeLogManager, ConfigurableClass):
32
+ class S3ComputeLogManager(TruncatingCloudStorageComputeLogManager, ConfigurableClass):
33
33
  """Logs compute function stdout and stderr to S3.
34
34
 
35
35
  Users should not instantiate this class directly. Instead, use a YAML block in ``dagster.yaml``
@@ -114,6 +114,7 @@ class S3ComputeLogManager(CloudStorageComputeLogManager, ConfigurableClass):
114
114
  self._region = self._s3_session.meta.region_name
115
115
  else:
116
116
  self._region = region
117
+ super().__init__()
117
118
 
118
119
  @property
119
120
  def inst_data(self):
@@ -209,7 +210,7 @@ class S3ComputeLogManager(CloudStorageComputeLogManager, ConfigurableClass):
209
210
  to_delete = [{"Key": key} for key in s3_keys_to_remove]
210
211
  self._s3_session.delete_objects(Bucket=self._s3_bucket, Delete={"Objects": to_delete})
211
212
 
212
- def download_url_for_type(self, log_key: Sequence[str], io_type: ComputeIOType): # pyright: ignore[reportIncompatibleMethodOverride]
213
+ def download_url_for_type(self, log_key: Sequence[str], io_type: ComputeIOType):
213
214
  if not self.is_capture_complete(log_key):
214
215
  return None
215
216
 
@@ -234,22 +235,19 @@ class S3ComputeLogManager(CloudStorageComputeLogManager, ConfigurableClass):
234
235
  return False
235
236
  return True
236
237
 
237
- def upload_to_cloud_storage(
238
- self, log_key: Sequence[str], io_type: ComputeIOType, partial=False
238
+ def _upload_file_obj(
239
+ self, data: IO[bytes], log_key: Sequence[str], io_type: ComputeIOType, partial=False
239
240
  ):
240
241
  path = self.local_manager.get_captured_local_path(log_key, IO_TYPE_EXTENSION[io_type])
241
- ensure_file(path)
242
-
243
242
  if (self._skip_empty_files or partial) and os.stat(path).st_size == 0:
244
243
  return
245
244
 
246
245
  s3_key = self._s3_key(log_key, io_type, partial=partial)
247
- with open(path, "rb") as data:
248
- extra_args = {
249
- "ContentType": "text/plain",
250
- **(self._upload_extra_args if self._upload_extra_args else {}),
251
- }
252
- self._s3_session.upload_fileobj(data, self._s3_bucket, s3_key, ExtraArgs=extra_args)
246
+ extra_args = {
247
+ "ContentType": "text/plain",
248
+ **(self._upload_extra_args if self._upload_extra_args else {}),
249
+ }
250
+ self._s3_session.upload_fileobj(data, self._s3_bucket, s3_key, ExtraArgs=extra_args)
253
251
 
254
252
  def download_from_cloud_storage(
255
253
  self, log_key: Sequence[str], io_type: ComputeIOType, partial=False
@@ -112,7 +112,7 @@ class S3PickleIOManager(ConfigurableIOManager):
112
112
  def asset2(asset1):
113
113
  return asset1[:5]
114
114
 
115
- defs = Definitions(
115
+ Definitions(
116
116
  assets=[asset1, asset2],
117
117
  resources={
118
118
  "io_manager": S3PickleIOManager(
@@ -200,7 +200,7 @@ def s3_pickle_io_manager(init_context):
200
200
  def asset2(asset1):
201
201
  return asset1[:5]
202
202
 
203
- defs = Definitions(
203
+ Definitions(
204
204
  assets=[asset1, asset2],
205
205
  resources={
206
206
  "io_manager": s3_pickle_io_manager.configured(
@@ -74,7 +74,7 @@ class S3Resource(ResourceWithS3Configuration, IAttachDifferentObjectToOpContext)
74
74
  def example_job():
75
75
  example_s3_op()
76
76
 
77
- defs = Definitions(
77
+ Definitions(
78
78
  jobs=[example_job],
79
79
  resources={'s3': S3Resource(region_name='us-west-1')}
80
80
  )
@@ -65,6 +65,8 @@ def get_objects(
65
65
  for idx, obj in enumerate(sorted_objects):
66
66
  if obj.get("LastModified") > since_last_modified:
67
67
  return sorted_objects[idx:]
68
+ # If no new files are found, return an empty list.
69
+ return []
68
70
 
69
71
  return sorted_objects
70
72
 
@@ -48,7 +48,7 @@ class SecretsManagerResource(ResourceWithBoto3Configuration):
48
48
  def example_job():
49
49
  example_secretsmanager_op()
50
50
 
51
- defs = Definitions(
51
+ Definitions(
52
52
  jobs=[example_job],
53
53
  resources={
54
54
  'secretsmanager': SecretsManagerResource(
@@ -158,7 +158,7 @@ class SecretsManagerSecretsResource(ResourceWithBoto3Configuration):
158
158
  example_secretsmanager_secrets_op()
159
159
  example_secretsmanager_secrets_op_2()
160
160
 
161
- defs = Definitions(
161
+ Definitions(
162
162
  jobs=[example_job],
163
163
  resources={
164
164
  'secrets': SecretsManagerSecretsResource(
@@ -51,7 +51,7 @@ class SSMResource(ResourceWithBoto3Configuration):
51
51
  def example_job():
52
52
  example_ssm_op()
53
53
 
54
- defs = Definitions(
54
+ Definitions(
55
55
  jobs=[example_job],
56
56
  resources={
57
57
  'ssm': SSMResource(
@@ -0,0 +1 @@
1
+ __version__ = "0.28.5"
@@ -0,0 +1,44 @@
1
+ Metadata-Version: 2.4
2
+ Name: dagster-aws
3
+ Version: 0.28.5
4
+ Summary: Package for AWS-specific Dagster framework solid and resource components.
5
+ Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-aws
6
+ Author: Dagster Labs
7
+ Author-email: hello@dagsterlabs.com
8
+ License: Apache-2.0
9
+ Classifier: Programming Language :: Python :: 3.10
10
+ Classifier: Programming Language :: Python :: 3.11
11
+ Classifier: Programming Language :: Python :: 3.12
12
+ Classifier: Programming Language :: Python :: 3.13
13
+ Classifier: License :: OSI Approved :: Apache Software License
14
+ Classifier: Operating System :: OS Independent
15
+ Requires-Python: >=3.10,<3.14
16
+ License-File: LICENSE
17
+ Requires-Dist: boto3
18
+ Requires-Dist: dagster==1.12.5
19
+ Requires-Dist: packaging
20
+ Requires-Dist: requests
21
+ Provides-Extra: redshift
22
+ Requires-Dist: psycopg2-binary; extra == "redshift"
23
+ Provides-Extra: pyspark
24
+ Requires-Dist: dagster-pyspark; extra == "pyspark"
25
+ Provides-Extra: stubs
26
+ Requires-Dist: boto3-stubs-lite[ecs,emr,emr-containers,emr-serverless,glue,logs,s3]; extra == "stubs"
27
+ Provides-Extra: test
28
+ Requires-Dist: botocore!=1.32.1; extra == "test"
29
+ Requires-Dist: moto[emrcontainers,emrserverless,glue,logs,s3,server]<5.0,>=2.2.8; extra == "test"
30
+ Requires-Dist: requests-mock; extra == "test"
31
+ Requires-Dist: xmltodict==0.12.0; extra == "test"
32
+ Requires-Dist: flaky; extra == "test"
33
+ Requires-Dist: pytest-cases; extra == "test"
34
+ Requires-Dist: s3fs; extra == "test"
35
+ Dynamic: author
36
+ Dynamic: author-email
37
+ Dynamic: classifier
38
+ Dynamic: home-page
39
+ Dynamic: license
40
+ Dynamic: license-file
41
+ Dynamic: provides-extra
42
+ Dynamic: requires-dist
43
+ Dynamic: requires-python
44
+ Dynamic: summary
@@ -1,5 +1,5 @@
1
1
  boto3
2
- dagster==1.10.11
2
+ dagster==1.12.5
3
3
  packaging
4
4
  requests
5
5
 
@@ -19,3 +19,4 @@ requests-mock
19
19
  xmltodict==0.12.0
20
20
  flaky
21
21
  pytest-cases
22
+ s3fs
@@ -23,19 +23,19 @@ setup(
23
23
  description="Package for AWS-specific Dagster framework solid and resource components.",
24
24
  url="https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-aws",
25
25
  classifiers=[
26
- "Programming Language :: Python :: 3.9",
27
26
  "Programming Language :: Python :: 3.10",
28
27
  "Programming Language :: Python :: 3.11",
29
28
  "Programming Language :: Python :: 3.12",
29
+ "Programming Language :: Python :: 3.13",
30
30
  "License :: OSI Approved :: Apache Software License",
31
31
  "Operating System :: OS Independent",
32
32
  ],
33
33
  packages=find_packages(exclude=["dagster_aws_tests*"]),
34
34
  include_package_data=True,
35
- python_requires=">=3.9,<3.13",
35
+ python_requires=">=3.10,<3.14",
36
36
  install_requires=[
37
37
  "boto3",
38
- "dagster==1.10.11",
38
+ "dagster==1.12.5",
39
39
  "packaging",
40
40
  "requests",
41
41
  ],
@@ -52,6 +52,7 @@ setup(
52
52
  "xmltodict==0.12.0", # pinned until moto>=3.1.9 (https://github.com/spulec/moto/issues/5112)
53
53
  "flaky",
54
54
  "pytest-cases",
55
+ "s3fs",
55
56
  ],
56
57
  },
57
58
  zip_safe=False,
@@ -1,20 +0,0 @@
1
- Metadata-Version: 2.1
2
- Name: dagster-aws
3
- Version: 0.26.11
4
- Summary: Package for AWS-specific Dagster framework solid and resource components.
5
- Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-aws
6
- Author: Dagster Labs
7
- Author-email: hello@dagsterlabs.com
8
- License: Apache-2.0
9
- Classifier: Programming Language :: Python :: 3.9
10
- Classifier: Programming Language :: Python :: 3.10
11
- Classifier: Programming Language :: Python :: 3.11
12
- Classifier: Programming Language :: Python :: 3.12
13
- Classifier: License :: OSI Approved :: Apache Software License
14
- Classifier: Operating System :: OS Independent
15
- Requires-Python: >=3.9,<3.13
16
- Provides-Extra: redshift
17
- Provides-Extra: pyspark
18
- Provides-Extra: stubs
19
- Provides-Extra: test
20
- License-File: LICENSE
@@ -1 +0,0 @@
1
- __version__ = "0.26.11"
@@ -1,20 +0,0 @@
1
- Metadata-Version: 2.1
2
- Name: dagster-aws
3
- Version: 0.26.11
4
- Summary: Package for AWS-specific Dagster framework solid and resource components.
5
- Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-aws
6
- Author: Dagster Labs
7
- Author-email: hello@dagsterlabs.com
8
- License: Apache-2.0
9
- Classifier: Programming Language :: Python :: 3.9
10
- Classifier: Programming Language :: Python :: 3.10
11
- Classifier: Programming Language :: Python :: 3.11
12
- Classifier: Programming Language :: Python :: 3.12
13
- Classifier: License :: OSI Approved :: Apache Software License
14
- Classifier: Operating System :: OS Independent
15
- Requires-Python: >=3.9,<3.13
16
- Provides-Extra: redshift
17
- Provides-Extra: pyspark
18
- Provides-Extra: stubs
19
- Provides-Extra: test
20
- License-File: LICENSE
File without changes
File without changes
File without changes