dagster-aws 0.28.0__tar.gz → 0.28.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dagster_aws-0.28.0/dagster_aws.egg-info → dagster_aws-0.28.1}/PKG-INFO +2 -2
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/ecs/launcher.py +32 -1
- dagster_aws-0.28.1/dagster_aws/version.py +1 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1/dagster_aws.egg-info}/PKG-INFO +2 -2
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws.egg-info/requires.txt +1 -1
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/setup.py +1 -1
- dagster_aws-0.28.0/dagster_aws/version.py +0 -1
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/LICENSE +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/MANIFEST.in +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/README.md +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/__init__.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/_stubs.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/athena/__init__.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/athena/resources.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/cloudwatch/__init__.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/cloudwatch/loggers.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/ecr/__init__.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/ecr/resources.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/ecs/__init__.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/ecs/container_context.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/ecs/executor.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/ecs/tasks.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/ecs/test_utils.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/ecs/utils.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/emr/__init__.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/emr/configs.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/emr/configs_spark.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/emr/emr.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/emr/emr_step_main.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/emr/main.py.template +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/emr/pyspark_step_launcher.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/emr/types.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/emr/utils.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/pipes/__init__.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/pipes/clients/__init__.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/pipes/clients/ecs.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/pipes/clients/emr.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/pipes/clients/emr_containers.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/pipes/clients/emr_serverless.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/pipes/clients/glue.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/pipes/clients/lambda_.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/pipes/clients/utils.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/pipes/context_injectors.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/pipes/message_readers.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/py.typed +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/rds/__init__.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/rds/resources.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/redshift/__init__.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/redshift/resources.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/s3/__init__.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/s3/compute_log_manager.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/s3/file_manager.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/s3/io_manager.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/s3/ops.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/s3/resources.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/s3/s3_fake_resource.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/s3/sensor.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/s3/utils.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/secretsmanager/__init__.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/secretsmanager/resources.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/secretsmanager/secrets.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/ssm/__init__.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/ssm/parameters.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/ssm/resources.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/utils/__init__.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/utils/mrjob/__init__.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/utils/mrjob/log4j.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/utils/mrjob/retry.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws/utils/mrjob/utils.py +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws.egg-info/SOURCES.txt +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws.egg-info/dependency_links.txt +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws.egg-info/not-zip-safe +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/dagster_aws.egg-info/top_level.txt +0 -0
- {dagster_aws-0.28.0 → dagster_aws-0.28.1}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dagster-aws
|
|
3
|
-
Version: 0.28.
|
|
3
|
+
Version: 0.28.1
|
|
4
4
|
Summary: Package for AWS-specific Dagster framework solid and resource components.
|
|
5
5
|
Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-aws
|
|
6
6
|
Author: Dagster Labs
|
|
@@ -16,7 +16,7 @@ Classifier: Operating System :: OS Independent
|
|
|
16
16
|
Requires-Python: >=3.9,<3.14
|
|
17
17
|
License-File: LICENSE
|
|
18
18
|
Requires-Dist: boto3
|
|
19
|
-
Requires-Dist: dagster==1.12.
|
|
19
|
+
Requires-Dist: dagster==1.12.1
|
|
20
20
|
Requires-Dist: packaging
|
|
21
21
|
Requires-Dist: requests
|
|
22
22
|
Provides-Extra: redshift
|
|
@@ -30,11 +30,12 @@ from dagster._core.launcher.base import (
|
|
|
30
30
|
WorkerStatus,
|
|
31
31
|
)
|
|
32
32
|
from dagster._core.storage.dagster_run import DagsterRun
|
|
33
|
-
from dagster._core.storage.tags import RUN_WORKER_ID_TAG
|
|
33
|
+
from dagster._core.storage.tags import HIDDEN_TAG_PREFIX, RUN_WORKER_ID_TAG
|
|
34
34
|
from dagster._grpc.types import ExecuteRunArgs
|
|
35
35
|
from dagster._serdes import ConfigurableClass
|
|
36
36
|
from dagster._serdes.config_class import ConfigurableClassData
|
|
37
37
|
from dagster._utils.backoff import backoff
|
|
38
|
+
from dagster._utils.tags import get_boolean_tag_value
|
|
38
39
|
from typing_extensions import Self
|
|
39
40
|
|
|
40
41
|
from dagster_aws.ecs.container_context import (
|
|
@@ -855,6 +856,28 @@ class EcsRunLauncher(RunLauncher[T_DagsterInstance], ConfigurableClass):
|
|
|
855
856
|
task.get("stoppedReason", "")
|
|
856
857
|
)
|
|
857
858
|
|
|
859
|
+
def _add_eni_id_tags(self, run: DagsterRun, task: dict[str, Any]):
|
|
860
|
+
attachments = task.get("attachments", [])
|
|
861
|
+
eni_ids = {}
|
|
862
|
+
eni_count = 0
|
|
863
|
+
for attachment in attachments:
|
|
864
|
+
if attachment.get("type") == "ElasticNetworkInterface":
|
|
865
|
+
details = {d["name"]: d["value"] for d in attachment.get("details", [])}
|
|
866
|
+
|
|
867
|
+
if "networkInterfaceId" in details:
|
|
868
|
+
if eni_count == 0:
|
|
869
|
+
eni_ids[f"{HIDDEN_TAG_PREFIX}eni_id"] = details["networkInterfaceId"]
|
|
870
|
+
else:
|
|
871
|
+
eni_ids[f"{HIDDEN_TAG_PREFIX}eni_id_{eni_count}"] = details[
|
|
872
|
+
"networkInterfaceId"
|
|
873
|
+
]
|
|
874
|
+
eni_count += 1
|
|
875
|
+
self._instance.add_run_tags(run.run_id, eni_ids)
|
|
876
|
+
if eni_count > 0:
|
|
877
|
+
logging.info(f"Added {eni_count} ENI ID tags for run {run.run_id}: {eni_ids}")
|
|
878
|
+
else:
|
|
879
|
+
logging.warning(f"No ENI IDs found for run {run.run_id}")
|
|
880
|
+
|
|
858
881
|
def check_run_worker_health(self, run: DagsterRun):
|
|
859
882
|
run_worker_id = run.tags.get(RUN_WORKER_ID_TAG)
|
|
860
883
|
|
|
@@ -870,6 +893,14 @@ class EcsRunLauncher(RunLauncher[T_DagsterInstance], ConfigurableClass):
|
|
|
870
893
|
|
|
871
894
|
t = tasks[0]
|
|
872
895
|
|
|
896
|
+
if get_boolean_tag_value(os.getenv("DAGSTER_AWS_ENI_TAGGING_ENABLED")) and not run.tags.get(
|
|
897
|
+
f"{HIDDEN_TAG_PREFIX}eni_id"
|
|
898
|
+
):
|
|
899
|
+
try:
|
|
900
|
+
self._add_eni_id_tags(run, t)
|
|
901
|
+
except Exception:
|
|
902
|
+
logging.exception(f"Error adding ENI ID tags for run {run.run_id}")
|
|
903
|
+
|
|
873
904
|
if t.get("lastStatus") in RUNNING_STATUSES:
|
|
874
905
|
return CheckRunHealthResult(WorkerStatus.RUNNING, run_worker_id=run_worker_id)
|
|
875
906
|
elif t.get("lastStatus") in STOPPED_STATUSES:
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.28.1"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dagster-aws
|
|
3
|
-
Version: 0.28.
|
|
3
|
+
Version: 0.28.1
|
|
4
4
|
Summary: Package for AWS-specific Dagster framework solid and resource components.
|
|
5
5
|
Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-aws
|
|
6
6
|
Author: Dagster Labs
|
|
@@ -16,7 +16,7 @@ Classifier: Operating System :: OS Independent
|
|
|
16
16
|
Requires-Python: >=3.9,<3.14
|
|
17
17
|
License-File: LICENSE
|
|
18
18
|
Requires-Dist: boto3
|
|
19
|
-
Requires-Dist: dagster==1.12.
|
|
19
|
+
Requires-Dist: dagster==1.12.1
|
|
20
20
|
Requires-Dist: packaging
|
|
21
21
|
Requires-Dist: requests
|
|
22
22
|
Provides-Extra: redshift
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.28.0"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|