dagster-aws 0.27.16__py3-none-any.whl → 0.28.1__py3-none-any.whl

dagster_aws/ecs/launcher.py

@@ -30,11 +30,12 @@ from dagster._core.launcher.base import (
     WorkerStatus,
 )
 from dagster._core.storage.dagster_run import DagsterRun
-from dagster._core.storage.tags import RUN_WORKER_ID_TAG
+from dagster._core.storage.tags import HIDDEN_TAG_PREFIX, RUN_WORKER_ID_TAG
 from dagster._grpc.types import ExecuteRunArgs
 from dagster._serdes import ConfigurableClass
 from dagster._serdes.config_class import ConfigurableClassData
 from dagster._utils.backoff import backoff
+from dagster._utils.tags import get_boolean_tag_value
 from typing_extensions import Self
 
 from dagster_aws.ecs.container_context import (
@@ -855,6 +856,28 @@ class EcsRunLauncher(RunLauncher[T_DagsterInstance], ConfigurableClass):
             task.get("stoppedReason", "")
         )
 
+    def _add_eni_id_tags(self, run: DagsterRun, task: dict[str, Any]):
+        attachments = task.get("attachments", [])
+        eni_ids = {}
+        eni_count = 0
+        for attachment in attachments:
+            if attachment.get("type") == "ElasticNetworkInterface":
+                details = {d["name"]: d["value"] for d in attachment.get("details", [])}
+
+                if "networkInterfaceId" in details:
+                    if eni_count == 0:
+                        eni_ids[f"{HIDDEN_TAG_PREFIX}eni_id"] = details["networkInterfaceId"]
+                    else:
+                        eni_ids[f"{HIDDEN_TAG_PREFIX}eni_id_{eni_count}"] = details[
+                            "networkInterfaceId"
+                        ]
+                    eni_count += 1
+        self._instance.add_run_tags(run.run_id, eni_ids)
+        if eni_count > 0:
+            logging.info(f"Added {eni_count} ENI ID tags for run {run.run_id}: {eni_ids}")
+        else:
+            logging.warning(f"No ENI IDs found for run {run.run_id}")
+
     def check_run_worker_health(self, run: DagsterRun):
         run_worker_id = run.tags.get(RUN_WORKER_ID_TAG)
 
@@ -870,6 +893,14 @@ class EcsRunLauncher(RunLauncher[T_DagsterInstance], ConfigurableClass):
 
         t = tasks[0]
 
+        if get_boolean_tag_value(os.getenv("DAGSTER_AWS_ENI_TAGGING_ENABLED")) and not run.tags.get(
+            f"{HIDDEN_TAG_PREFIX}eni_id"
+        ):
+            try:
+                self._add_eni_id_tags(run, t)
+            except Exception:
+                logging.exception(f"Error adding ENI ID tags for run {run.run_id}")
+
         if t.get("lastStatus") in RUNNING_STATUSES:
             return CheckRunHealthResult(WorkerStatus.RUNNING, run_worker_id=run_worker_id)
         elif t.get("lastStatus") in STOPPED_STATUSES:

dagster_aws/pipes/context_injectors.py

@@ -4,15 +4,15 @@ import random
 import string
 from collections.abc import Iterator
 from contextlib import contextmanager
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 
-import boto3
 import dagster._check as check
 from dagster._core.pipes.client import PipesContextInjector, PipesParams
 from dagster._core.pipes.utils import PipesEnvContextInjector
 
 if TYPE_CHECKING:
     from dagster_pipes import PipesContextData
+    from mypy_boto3_s3 import S3Client
 
 _CONTEXT_FILENAME = "context.json"
 
@@ -22,20 +22,21 @@ class PipesS3ContextInjector(PipesContextInjector):
 
     Args:
         bucket (str): The S3 bucket to write to.
-        client (boto3.client): A boto3 client to use to write to S3.
+        client (S3Client): A boto3 client to use to write to S3.
         key_prefix (Optional[str]): An optional prefix to use for the S3 key. Defaults to a random
             string.
 
     """
 
-    def __init__(self, *, bucket: str, client: boto3.client):  # pyright: ignore (reportGeneralTypeIssues)
+    def __init__(self, *, bucket: str, client: "S3Client", key_prefix: Optional[str] = None):
         super().__init__()
         self.bucket = check.str_param(bucket, "bucket")
         self.client = client
+        self.key_prefix = key_prefix
 
     @contextmanager
     def inject_context(self, context: "PipesContextData") -> Iterator[PipesParams]:  # pyright: ignore[reportIncompatibleMethodOverride]
-        key_prefix = "".join(random.choices(string.ascii_letters, k=30))
+        key_prefix = f"{self.key_prefix or ''}{''.join(random.choices(string.ascii_letters, k=30))}"
         key = os.path.join(key_prefix, _CONTEXT_FILENAME)
         self.client.put_object(
             Body=json.dumps(context).encode("utf-8"), Bucket=self.bucket, Key=key

dagster_aws/version.py

@@ -1 +1 @@
-__version__ = "0.27.16"
+__version__ = "0.28.1"

{dagster_aws-0.27.16.dist-info → dagster_aws-0.28.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster-aws
-Version: 0.27.16
+Version: 0.28.1
 Summary: Package for AWS-specific Dagster framework solid and resource components.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-aws
 Author: Dagster Labs
@@ -16,7 +16,7 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9,<3.14
 License-File: LICENSE
 Requires-Dist: boto3
-Requires-Dist: dagster==1.11.16
+Requires-Dist: dagster==1.12.1
 Requires-Dist: packaging
 Requires-Dist: requests
 Provides-Extra: redshift

{dagster_aws-0.27.16.dist-info → dagster_aws-0.28.1.dist-info}/RECORD

@@ -1,7 +1,7 @@
 dagster_aws/__init__.py,sha256=n5d1NvkQvkVnjWetqKFOLCZUkymTJ0v2WMxLGsbA2Hk,166
 dagster_aws/_stubs.py,sha256=3C6ve6WLtuT2HtCn50hBAMIz-eEAwcZo4fCVnNxGBQw,1112
 dagster_aws/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
-dagster_aws/version.py,sha256=gNIskoVbgqFeY6KiyOzkhfP7GLwnNVmVeil7tBvkL44,24
+dagster_aws/version.py,sha256=ZRQKbgDaGz_yuLk-cUKuk6ZBKCSRKZC8nQd041NRNXk,23
 dagster_aws/athena/__init__.py,sha256=J0-QydhSP3pUQk7CawztpaTm3Q1NFEpbubTM1eULbGo,413
 dagster_aws/athena/resources.py,sha256=h7Md_KQXHvtBHA019WZFzAFTClzEUDItDZNdhR2RrPY,11907
 dagster_aws/cloudwatch/__init__.py,sha256=iiyLEyTfNMThU4_SNM_Ns7IGXKR7wTKIvOUz9v_RQBA,82
@@ -11,7 +11,7 @@ dagster_aws/ecr/resources.py,sha256=3SiatAFl9vvkXFmxZjB6e2L20bM6gsKfaveiyTWSRdM,
 dagster_aws/ecs/__init__.py,sha256=E6SAG_YKJcdUIw57amZf3AqzqVUN-GLjrLCpPxSemMg,233
 dagster_aws/ecs/container_context.py,sha256=vB8C4d04iIhd0Gu9vRdYAfua9ib95N25uohEoev9yIY,18154
 dagster_aws/ecs/executor.py,sha256=Fv9pbtJyE0D1DZ2e2uguYS8JHinZFjvcIfwqguYhYDM,16969
-dagster_aws/ecs/launcher.py,sha256=dqNXv2Qm1NqTESywoV5vdC6X2y99mV7_SX4HOLymv6E,37508
+dagster_aws/ecs/launcher.py,sha256=EWkJdWd2e7yDWfT1whZY1n9FqWzTmvAYIM12R03MuhM,38938
 dagster_aws/ecs/tasks.py,sha256=n5zigCqVhTsFQJS9g09eVHVCerZYJjhN_Hd-EEZJS7c,18443
 dagster_aws/ecs/test_utils.py,sha256=VqzOIws-fLoDxGrOMzE9ZuAbOF4t9AURIwjyM2Jid-E,2041
 dagster_aws/ecs/utils.py,sha256=CGgMjdoprtS9golz-H3yIDkSXPHxvOTmy5LJc43VVSo,5389
@@ -25,7 +25,7 @@ dagster_aws/emr/pyspark_step_launcher.py,sha256=wjhxww4ECfWHs-znbcn56eTgNqdBIJ9I
 dagster_aws/emr/types.py,sha256=-JQH3H2niodJuwiyrGW0HqgkV75bD-SRNeXdXuqYY9I,3698
 dagster_aws/emr/utils.py,sha256=AgxJ8L-uprSyjeJcxFgbMMGLsgqtjcuW11X7mIWi1tI,580
 dagster_aws/pipes/__init__.py,sha256=FEqfChhBYzJRVxqAV1jfaXecRUgxJaLq4xgy8AKjNkI,895
-dagster_aws/pipes/context_injectors.py,sha256=KvBN1CzXz6NOB4RTgDKRwkXbWd_1PPB4j1hCz-wfwNA,2198
+dagster_aws/pipes/context_injectors.py,sha256=DV356jY7k53gB9G--iQjOxx5C7P-ZAxTySbeaMs9gv0,2282
 dagster_aws/pipes/message_readers.py,sha256=kMYdzDPdYxUqzo_Oms5iknA6ezGN-27oHYU6E3-yK1c,14690
 dagster_aws/pipes/clients/__init__.py,sha256=p9jEkLonG7gaI0vS6LA_dlZBSk_YSRfH-svT8eU7Myg,564
 dagster_aws/pipes/clients/ecs.py,sha256=eiEXUbB51cFbnQ5llL6fLjCJ8pFPA0UfuWIghBwnf1g,15482
@@ -59,8 +59,8 @@ dagster_aws/utils/mrjob/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG
 dagster_aws/utils/mrjob/log4j.py,sha256=Gj2D5USX8swfBXlWf3-NiqeFvrs3h1SGJkOPbP3mke8,4392
 dagster_aws/utils/mrjob/retry.py,sha256=o8Y1Sgu8ABNnsm9jMCfagJpXmgpjcwDw6ufQmDV_Dus,4805
 dagster_aws/utils/mrjob/utils.py,sha256=pFKobTkgwPxQoQNlzGzD3jFxwIP-flQOspQxNKGg2dc,3684
-dagster_aws-0.27.16.dist-info/licenses/LICENSE,sha256=4lsMW-RCvfVD4_F57wrmpe3vX1xwUk_OAKKmV_XT7Z0,11348
-dagster_aws-0.27.16.dist-info/METADATA,sha256=yKA0p0MVOOxkzkdHS7qmORAA0hnHnE5hJys43ptglHA,1697
-dagster_aws-0.27.16.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dagster_aws-0.27.16.dist-info/top_level.txt,sha256=-hOwIi00tt_K9eXRhiyjst45X-Ffu5_QCdbKcm8UpDg,12
-dagster_aws-0.27.16.dist-info/RECORD,,
+dagster_aws-0.28.1.dist-info/licenses/LICENSE,sha256=4lsMW-RCvfVD4_F57wrmpe3vX1xwUk_OAKKmV_XT7Z0,11348
+dagster_aws-0.28.1.dist-info/METADATA,sha256=vIv_d9Ts-CJqoyTEgp0XQb4UZLwlbv3E_oLtP1hhE6E,1695
+dagster_aws-0.28.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dagster_aws-0.28.1.dist-info/top_level.txt,sha256=-hOwIi00tt_K9eXRhiyjst45X-Ffu5_QCdbKcm8UpDg,12
+dagster_aws-0.28.1.dist-info/RECORD,,