apache-airflow-providers-amazon 9.1.0rc3__py3-none-any.whl → 9.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/amazon/__init__.py +3 -3
- airflow/providers/amazon/aws/auth_manager/avp/facade.py +2 -1
- airflow/providers/amazon/aws/auth_manager/aws_auth_manager.py +4 -12
- airflow/providers/amazon/aws/executors/batch/batch_executor.py +4 -3
- airflow/providers/amazon/aws/executors/batch/utils.py +3 -3
- airflow/providers/amazon/aws/executors/ecs/ecs_executor.py +2 -1
- airflow/providers/amazon/aws/executors/ecs/utils.py +3 -3
- airflow/providers/amazon/aws/fs/s3.py +2 -2
- airflow/providers/amazon/aws/hooks/appflow.py +15 -5
- airflow/providers/amazon/aws/hooks/athena.py +2 -1
- airflow/providers/amazon/aws/hooks/dms.py +161 -0
- airflow/providers/amazon/aws/hooks/dynamodb.py +2 -1
- airflow/providers/amazon/aws/hooks/eks.py +4 -3
- airflow/providers/amazon/aws/hooks/glue.py +5 -1
- airflow/providers/amazon/aws/hooks/kinesis.py +1 -1
- airflow/providers/amazon/aws/hooks/logs.py +2 -1
- airflow/providers/amazon/aws/hooks/redshift_cluster.py +4 -3
- airflow/providers/amazon/aws/hooks/redshift_data.py +2 -1
- airflow/providers/amazon/aws/hooks/redshift_sql.py +2 -6
- airflow/providers/amazon/aws/hooks/s3.py +9 -3
- airflow/providers/amazon/aws/hooks/sagemaker.py +2 -1
- airflow/providers/amazon/aws/hooks/ses.py +2 -1
- airflow/providers/amazon/aws/notifications/sns.py +1 -1
- airflow/providers/amazon/aws/notifications/sqs.py +1 -1
- airflow/providers/amazon/aws/operators/athena.py +5 -2
- airflow/providers/amazon/aws/operators/base_aws.py +1 -1
- airflow/providers/amazon/aws/operators/batch.py +2 -1
- airflow/providers/amazon/aws/operators/bedrock.py +2 -1
- airflow/providers/amazon/aws/operators/cloud_formation.py +2 -1
- airflow/providers/amazon/aws/operators/comprehend.py +2 -1
- airflow/providers/amazon/aws/operators/datasync.py +2 -1
- airflow/providers/amazon/aws/operators/dms.py +531 -1
- airflow/providers/amazon/aws/operators/ec2.py +2 -1
- airflow/providers/amazon/aws/operators/ecs.py +15 -4
- airflow/providers/amazon/aws/operators/eks.py +8 -5
- airflow/providers/amazon/aws/operators/emr.py +31 -8
- airflow/providers/amazon/aws/operators/eventbridge.py +2 -1
- airflow/providers/amazon/aws/operators/glacier.py +2 -1
- airflow/providers/amazon/aws/operators/glue.py +12 -2
- airflow/providers/amazon/aws/operators/glue_crawler.py +2 -1
- airflow/providers/amazon/aws/operators/glue_databrew.py +2 -1
- airflow/providers/amazon/aws/operators/kinesis_analytics.py +2 -1
- airflow/providers/amazon/aws/operators/lambda_function.py +2 -1
- airflow/providers/amazon/aws/operators/neptune.py +2 -1
- airflow/providers/amazon/aws/operators/quicksight.py +2 -1
- airflow/providers/amazon/aws/operators/rds.py +2 -1
- airflow/providers/amazon/aws/operators/redshift_cluster.py +2 -1
- airflow/providers/amazon/aws/operators/s3.py +7 -1
- airflow/providers/amazon/aws/operators/sagemaker.py +2 -1
- airflow/providers/amazon/aws/operators/sns.py +2 -1
- airflow/providers/amazon/aws/operators/sqs.py +2 -1
- airflow/providers/amazon/aws/operators/step_function.py +2 -1
- airflow/providers/amazon/aws/sensors/athena.py +2 -1
- airflow/providers/amazon/aws/sensors/base_aws.py +1 -1
- airflow/providers/amazon/aws/sensors/batch.py +2 -1
- airflow/providers/amazon/aws/sensors/bedrock.py +2 -1
- airflow/providers/amazon/aws/sensors/cloud_formation.py +2 -1
- airflow/providers/amazon/aws/sensors/comprehend.py +2 -1
- airflow/providers/amazon/aws/sensors/dms.py +2 -1
- airflow/providers/amazon/aws/sensors/dynamodb.py +2 -1
- airflow/providers/amazon/aws/sensors/ec2.py +2 -1
- airflow/providers/amazon/aws/sensors/ecs.py +2 -1
- airflow/providers/amazon/aws/sensors/eks.py +2 -1
- airflow/providers/amazon/aws/sensors/emr.py +2 -1
- airflow/providers/amazon/aws/sensors/glacier.py +2 -1
- airflow/providers/amazon/aws/sensors/glue.py +2 -1
- airflow/providers/amazon/aws/sensors/glue_catalog_partition.py +2 -1
- airflow/providers/amazon/aws/sensors/glue_crawler.py +2 -1
- airflow/providers/amazon/aws/sensors/kinesis_analytics.py +2 -1
- airflow/providers/amazon/aws/sensors/lambda_function.py +2 -1
- airflow/providers/amazon/aws/sensors/opensearch_serverless.py +2 -1
- airflow/providers/amazon/aws/sensors/quicksight.py +2 -1
- airflow/providers/amazon/aws/sensors/rds.py +2 -1
- airflow/providers/amazon/aws/sensors/redshift_cluster.py +2 -1
- airflow/providers/amazon/aws/sensors/s3.py +2 -1
- airflow/providers/amazon/aws/sensors/sagemaker.py +2 -1
- airflow/providers/amazon/aws/sensors/sqs.py +2 -1
- airflow/providers/amazon/aws/sensors/step_function.py +2 -1
- airflow/providers/amazon/aws/transfers/azure_blob_to_s3.py +2 -1
- airflow/providers/amazon/aws/transfers/base.py +1 -1
- airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py +2 -1
- airflow/providers/amazon/aws/transfers/exasol_to_s3.py +2 -1
- airflow/providers/amazon/aws/transfers/ftp_to_s3.py +2 -1
- airflow/providers/amazon/aws/transfers/gcs_to_s3.py +4 -3
- airflow/providers/amazon/aws/transfers/glacier_to_gcs.py +2 -1
- airflow/providers/amazon/aws/transfers/google_api_to_s3.py +4 -8
- airflow/providers/amazon/aws/transfers/hive_to_dynamodb.py +2 -1
- airflow/providers/amazon/aws/transfers/imap_attachment_to_s3.py +2 -1
- airflow/providers/amazon/aws/transfers/local_to_s3.py +2 -1
- airflow/providers/amazon/aws/transfers/mongo_to_s3.py +2 -1
- airflow/providers/amazon/aws/transfers/redshift_to_s3.py +2 -1
- airflow/providers/amazon/aws/transfers/s3_to_dynamodb.py +3 -2
- airflow/providers/amazon/aws/transfers/s3_to_ftp.py +2 -1
- airflow/providers/amazon/aws/transfers/s3_to_redshift.py +2 -1
- airflow/providers/amazon/aws/transfers/s3_to_sftp.py +2 -1
- airflow/providers/amazon/aws/transfers/s3_to_sql.py +2 -1
- airflow/providers/amazon/aws/transfers/salesforce_to_s3.py +2 -1
- airflow/providers/amazon/aws/transfers/sftp_to_s3.py +14 -1
- airflow/providers/amazon/aws/transfers/sql_to_s3.py +2 -1
- airflow/providers/amazon/aws/triggers/athena.py +1 -2
- airflow/providers/amazon/aws/triggers/base.py +2 -1
- airflow/providers/amazon/aws/triggers/dms.py +221 -0
- airflow/providers/amazon/aws/triggers/glue.py +3 -2
- airflow/providers/amazon/aws/triggers/redshift_cluster.py +2 -1
- airflow/providers/amazon/aws/triggers/redshift_data.py +2 -1
- airflow/providers/amazon/aws/triggers/s3.py +2 -1
- airflow/providers/amazon/aws/triggers/sagemaker.py +2 -1
- airflow/providers/amazon/aws/triggers/sqs.py +2 -1
- airflow/providers/amazon/aws/utils/__init__.py +1 -15
- airflow/providers/amazon/aws/utils/task_log_fetcher.py +2 -1
- airflow/providers/amazon/aws/utils/waiter.py +20 -0
- airflow/providers/amazon/aws/waiters/dms.json +88 -0
- airflow/providers/amazon/get_provider_info.py +10 -5
- airflow/providers/amazon/version_compat.py +36 -0
- {apache_airflow_providers_amazon-9.1.0rc3.dist-info → apache_airflow_providers_amazon-9.2.0.dist-info}/METADATA +20 -26
- {apache_airflow_providers_amazon-9.1.0rc3.dist-info → apache_airflow_providers_amazon-9.2.0.dist-info}/RECORD +118 -115
- {apache_airflow_providers_amazon-9.1.0rc3.dist-info → apache_airflow_providers_amazon-9.2.0.dist-info}/WHEEL +1 -1
- {apache_airflow_providers_amazon-9.1.0rc3.dist-info → apache_airflow_providers_amazon-9.2.0.dist-info}/entry_points.txt +0 -0
@@ -20,9 +20,10 @@ from __future__ import annotations
|
|
20
20
|
|
21
21
|
import logging
|
22
22
|
from ast import literal_eval
|
23
|
+
from collections.abc import Sequence
|
23
24
|
from datetime import timedelta
|
24
25
|
from functools import cached_property
|
25
|
-
from typing import TYPE_CHECKING, Any,
|
26
|
+
from typing import TYPE_CHECKING, Any, cast
|
26
27
|
|
27
28
|
from botocore.exceptions import ClientError, WaiterError
|
28
29
|
|
@@ -45,8 +46,10 @@ from airflow.providers.cncf.kubernetes.utils.pod_manager import OnFinishAction
|
|
45
46
|
try:
|
46
47
|
from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator
|
47
48
|
except ImportError:
|
48
|
-
# preserve backward compatibility for older versions of cncf.kubernetes provider
|
49
|
-
from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import
|
49
|
+
# preserve backward compatibility for older versions of cncf.kubernetes provider, remove this when minimum cncf.kubernetes provider is 10.0
|
50
|
+
from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import ( # type: ignore[no-redef]
|
51
|
+
KubernetesPodOperator,
|
52
|
+
)
|
50
53
|
|
51
54
|
if TYPE_CHECKING:
|
52
55
|
from airflow.utils.context import Context
|
@@ -335,7 +338,7 @@ class EksCreateClusterOperator(BaseOperator):
|
|
335
338
|
fargate_pod_execution_role_arn=self.fargate_pod_execution_role_arn,
|
336
339
|
fargate_selectors=self.fargate_selectors,
|
337
340
|
create_fargate_profile_kwargs=self.create_fargate_profile_kwargs,
|
338
|
-
subnets=cast(
|
341
|
+
subnets=cast(list[str], self.resources_vpc_config.get("subnetIds")),
|
339
342
|
)
|
340
343
|
|
341
344
|
def deferrable_create_cluster_next(self, context: Context, event: dict[str, Any] | None = None) -> None:
|
@@ -374,7 +377,7 @@ class EksCreateClusterOperator(BaseOperator):
|
|
374
377
|
fargate_pod_execution_role_arn=self.fargate_pod_execution_role_arn,
|
375
378
|
fargate_selectors=self.fargate_selectors,
|
376
379
|
create_fargate_profile_kwargs=self.create_fargate_profile_kwargs,
|
377
|
-
subnets=cast(
|
380
|
+
subnets=cast(list[str], self.resources_vpc_config.get("subnetIds")),
|
378
381
|
)
|
379
382
|
if self.compute == "fargate":
|
380
383
|
self.defer(
|
@@ -18,13 +18,15 @@
|
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
20
|
import ast
|
21
|
+
import warnings
|
22
|
+
from collections.abc import Sequence
|
21
23
|
from datetime import timedelta
|
22
24
|
from functools import cached_property
|
23
|
-
from typing import TYPE_CHECKING, Any
|
25
|
+
from typing import TYPE_CHECKING, Any
|
24
26
|
from uuid import uuid4
|
25
27
|
|
26
28
|
from airflow.configuration import conf
|
27
|
-
from airflow.exceptions import AirflowException
|
29
|
+
from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
|
28
30
|
from airflow.models import BaseOperator
|
29
31
|
from airflow.providers.amazon.aws.hooks.emr import EmrContainerHook, EmrHook, EmrServerlessHook
|
30
32
|
from airflow.providers.amazon.aws.links.emr import (
|
@@ -49,7 +51,11 @@ from airflow.providers.amazon.aws.triggers.emr import (
|
|
49
51
|
EmrTerminateJobFlowTrigger,
|
50
52
|
)
|
51
53
|
from airflow.providers.amazon.aws.utils import validate_execute_complete_event
|
52
|
-
from airflow.providers.amazon.aws.utils.waiter import
|
54
|
+
from airflow.providers.amazon.aws.utils.waiter import (
|
55
|
+
WAITER_POLICY_NAME_MAPPING,
|
56
|
+
WaitPolicy,
|
57
|
+
waiter,
|
58
|
+
)
|
53
59
|
from airflow.providers.amazon.aws.utils.waiter_with_logging import wait
|
54
60
|
from airflow.utils.helpers import exactly_one, prune_dict
|
55
61
|
from airflow.utils.types import NOTSET, ArgNotSet
|
@@ -636,8 +642,14 @@ class EmrCreateJobFlowOperator(BaseOperator):
|
|
636
642
|
:param job_flow_overrides: boto3 style arguments or reference to an arguments file
|
637
643
|
(must be '.json') to override specific ``emr_conn_id`` extra parameters. (templated)
|
638
644
|
:param region_name: Region named passed to EmrHook
|
639
|
-
:param wait_for_completion:
|
645
|
+
:param wait_for_completion: Deprecated - use `wait_policy` instead.
|
646
|
+
Whether to finish task immediately after creation (False) or wait for jobflow
|
640
647
|
completion (True)
|
648
|
+
(default: None)
|
649
|
+
:param wait_policy: Whether to finish the task immediately after creation (None) or:
|
650
|
+
- wait for the jobflow completion (WaitPolicy.WAIT_FOR_COMPLETION)
|
651
|
+
- wait for the jobflow completion and cluster to terminate (WaitPolicy.WAIT_FOR_STEPS_COMPLETION)
|
652
|
+
(default: None)
|
641
653
|
:param waiter_max_attempts: Maximum number of tries before failing.
|
642
654
|
:param waiter_delay: Number of seconds between polling the state of the notebook.
|
643
655
|
:param deferrable: If True, the operator will wait asynchronously for the crawl to complete.
|
@@ -665,7 +677,8 @@ class EmrCreateJobFlowOperator(BaseOperator):
|
|
665
677
|
emr_conn_id: str | None = "emr_default",
|
666
678
|
job_flow_overrides: str | dict[str, Any] | None = None,
|
667
679
|
region_name: str | None = None,
|
668
|
-
wait_for_completion: bool =
|
680
|
+
wait_for_completion: bool | None = None,
|
681
|
+
wait_policy: WaitPolicy | None = None,
|
669
682
|
waiter_max_attempts: int | None = None,
|
670
683
|
waiter_delay: int | None = None,
|
671
684
|
deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
|
@@ -676,11 +689,20 @@ class EmrCreateJobFlowOperator(BaseOperator):
|
|
676
689
|
self.emr_conn_id = emr_conn_id
|
677
690
|
self.job_flow_overrides = job_flow_overrides or {}
|
678
691
|
self.region_name = region_name
|
679
|
-
self.
|
692
|
+
self.wait_policy = wait_policy
|
680
693
|
self.waiter_max_attempts = waiter_max_attempts or 60
|
681
694
|
self.waiter_delay = waiter_delay or 60
|
682
695
|
self.deferrable = deferrable
|
683
696
|
|
697
|
+
if wait_for_completion is not None:
|
698
|
+
warnings.warn(
|
699
|
+
"`wait_for_completion` parameter is deprecated, please use `wait_policy` instead.",
|
700
|
+
AirflowProviderDeprecationWarning,
|
701
|
+
stacklevel=2,
|
702
|
+
)
|
703
|
+
# preserve previous behaviour
|
704
|
+
self.wait_policy = WaitPolicy.WAIT_FOR_COMPLETION if wait_for_completion else None
|
705
|
+
|
684
706
|
@cached_property
|
685
707
|
def _emr_hook(self) -> EmrHook:
|
686
708
|
"""Create and return an EmrHook."""
|
@@ -733,8 +755,9 @@ class EmrCreateJobFlowOperator(BaseOperator):
|
|
733
755
|
# 60 seconds is added to allow the trigger to exit gracefully (i.e. yield TriggerEvent)
|
734
756
|
timeout=timedelta(seconds=self.waiter_max_attempts * self.waiter_delay + 60),
|
735
757
|
)
|
736
|
-
if self.
|
737
|
-
self.
|
758
|
+
if self.wait_policy:
|
759
|
+
waiter_name = WAITER_POLICY_NAME_MAPPING[self.wait_policy]
|
760
|
+
self._emr_hook.get_waiter(waiter_name).wait(
|
738
761
|
ClusterId=self._job_flow_id,
|
739
762
|
WaiterConfig=prune_dict(
|
740
763
|
{
|
@@ -16,7 +16,8 @@
|
|
16
16
|
# under the License.
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
|
-
from
|
19
|
+
from collections.abc import Sequence
|
20
|
+
from typing import TYPE_CHECKING
|
20
21
|
|
21
22
|
from airflow.exceptions import AirflowException
|
22
23
|
from airflow.providers.amazon.aws.hooks.eventbridge import EventBridgeHook
|
@@ -17,7 +17,8 @@
|
|
17
17
|
# under the License.
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
|
-
from
|
20
|
+
from collections.abc import Sequence
|
21
|
+
from typing import TYPE_CHECKING
|
21
22
|
|
22
23
|
from airflow.providers.amazon.aws.hooks.glacier import GlacierHook
|
23
24
|
from airflow.providers.amazon.aws.operators.base_aws import AwsBaseOperator
|
@@ -19,8 +19,9 @@ from __future__ import annotations
|
|
19
19
|
|
20
20
|
import os
|
21
21
|
import urllib.parse
|
22
|
+
from collections.abc import Sequence
|
22
23
|
from functools import cached_property
|
23
|
-
from typing import TYPE_CHECKING, Any
|
24
|
+
from typing import TYPE_CHECKING, Any
|
24
25
|
|
25
26
|
from botocore.exceptions import ClientError
|
26
27
|
|
@@ -74,6 +75,11 @@ class GlueJobOperator(BaseOperator):
|
|
74
75
|
:param update_config: If True, Operator will update job configuration. (default: False)
|
75
76
|
:param replace_script_file: If True, the script file will be replaced in S3. (default: False)
|
76
77
|
:param stop_job_run_on_kill: If True, Operator will stop the job run when task is killed.
|
78
|
+
:param sleep_before_return: time in seconds to wait before returning final status. This is meaningful in case
|
79
|
+
of limiting concurrency, Glue needs 5-10 seconds to clean up resources.
|
80
|
+
Thus if status is returned immediately it might end up in case of more than 1 concurrent run.
|
81
|
+
It is recommended to set this parameter to 10 when you are using concurrency=1.
|
82
|
+
For more information see: https://repost.aws/questions/QUaKgpLBMPSGWO0iq2Fob_bw/glue-run-concurrent-jobs#ANFpCL2fRnQRqgDFuIU_rpvA
|
77
83
|
"""
|
78
84
|
|
79
85
|
template_fields: Sequence[str] = (
|
@@ -118,6 +124,7 @@ class GlueJobOperator(BaseOperator):
|
|
118
124
|
update_config: bool = False,
|
119
125
|
job_poll_interval: int | float = 6,
|
120
126
|
stop_job_run_on_kill: bool = False,
|
127
|
+
sleep_before_return: int = 0,
|
121
128
|
**kwargs,
|
122
129
|
):
|
123
130
|
super().__init__(**kwargs)
|
@@ -145,6 +152,7 @@ class GlueJobOperator(BaseOperator):
|
|
145
152
|
self.job_poll_interval = job_poll_interval
|
146
153
|
self.stop_job_run_on_kill = stop_job_run_on_kill
|
147
154
|
self._job_run_id: str | None = None
|
155
|
+
self.sleep_before_return: int = sleep_before_return
|
148
156
|
|
149
157
|
@cached_property
|
150
158
|
def glue_job_hook(self) -> GlueJobHook:
|
@@ -220,7 +228,9 @@ class GlueJobOperator(BaseOperator):
|
|
220
228
|
method_name="execute_complete",
|
221
229
|
)
|
222
230
|
elif self.wait_for_completion:
|
223
|
-
glue_job_run = self.glue_job_hook.job_completion(
|
231
|
+
glue_job_run = self.glue_job_hook.job_completion(
|
232
|
+
self.job_name, self._job_run_id, self.verbose, self.sleep_before_return
|
233
|
+
)
|
224
234
|
self.log.info(
|
225
235
|
"AWS Glue Job: %s status: %s. Run Id: %s",
|
226
236
|
self.job_name,
|
@@ -17,7 +17,8 @@
|
|
17
17
|
# under the License.
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
|
-
from
|
20
|
+
from collections.abc import Sequence
|
21
|
+
from typing import TYPE_CHECKING, Any
|
21
22
|
|
22
23
|
from airflow.configuration import conf
|
23
24
|
from airflow.exceptions import AirflowException
|
@@ -17,7 +17,8 @@
|
|
17
17
|
# under the License.
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
|
-
from
|
20
|
+
from collections.abc import Sequence
|
21
|
+
from typing import TYPE_CHECKING, Any
|
21
22
|
|
22
23
|
from airflow.configuration import conf
|
23
24
|
from airflow.exceptions import AirflowException
|
@@ -16,7 +16,8 @@
|
|
16
16
|
# under the License.
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
|
-
from
|
19
|
+
from collections.abc import Sequence
|
20
|
+
from typing import TYPE_CHECKING, Any, ClassVar
|
20
21
|
|
21
22
|
from botocore.exceptions import ClientError
|
22
23
|
|
@@ -18,8 +18,9 @@
|
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
20
|
import json
|
21
|
+
from collections.abc import Sequence
|
21
22
|
from datetime import timedelta
|
22
|
-
from typing import TYPE_CHECKING, Any
|
23
|
+
from typing import TYPE_CHECKING, Any
|
23
24
|
|
24
25
|
from airflow.configuration import conf
|
25
26
|
from airflow.exceptions import AirflowException
|
@@ -16,7 +16,8 @@
|
|
16
16
|
# under the License.
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
|
-
from
|
19
|
+
from collections.abc import Sequence
|
20
|
+
from typing import TYPE_CHECKING
|
20
21
|
|
21
22
|
from airflow.providers.amazon.aws.hooks.quicksight import QuickSightHook
|
22
23
|
from airflow.providers.amazon.aws.operators.base_aws import AwsBaseOperator
|
@@ -18,9 +18,10 @@
|
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
20
|
import json
|
21
|
+
from collections.abc import Sequence
|
21
22
|
from datetime import timedelta
|
22
23
|
from functools import cached_property
|
23
|
-
from typing import TYPE_CHECKING, Any
|
24
|
+
from typing import TYPE_CHECKING, Any
|
24
25
|
|
25
26
|
from airflow.configuration import conf
|
26
27
|
from airflow.exceptions import AirflowException
|
@@ -17,8 +17,9 @@
|
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
19
|
import time
|
20
|
+
from collections.abc import Sequence
|
20
21
|
from datetime import timedelta
|
21
|
-
from typing import TYPE_CHECKING, Any
|
22
|
+
from typing import TYPE_CHECKING, Any
|
22
23
|
|
23
24
|
from airflow.configuration import conf
|
24
25
|
from airflow.exceptions import AirflowException
|
@@ -21,8 +21,9 @@ from __future__ import annotations
|
|
21
21
|
|
22
22
|
import subprocess
|
23
23
|
import sys
|
24
|
+
from collections.abc import Sequence
|
24
25
|
from tempfile import NamedTemporaryFile
|
25
|
-
from typing import TYPE_CHECKING
|
26
|
+
from typing import TYPE_CHECKING
|
26
27
|
|
27
28
|
import pytz
|
28
29
|
from dateutil import parser
|
@@ -282,6 +283,8 @@ class S3CopyObjectOperator(BaseOperator):
|
|
282
283
|
CA cert bundle than the one used by botocore.
|
283
284
|
:param acl_policy: String specifying the canned ACL policy for the file being
|
284
285
|
uploaded to the S3 bucket.
|
286
|
+
:param meta_data_directive: Whether to `COPY` the metadata from the source object or `REPLACE` it with
|
287
|
+
metadata that's provided in the request.
|
285
288
|
"""
|
286
289
|
|
287
290
|
template_fields: Sequence[str] = (
|
@@ -302,6 +305,7 @@ class S3CopyObjectOperator(BaseOperator):
|
|
302
305
|
aws_conn_id: str | None = "aws_default",
|
303
306
|
verify: str | bool | None = None,
|
304
307
|
acl_policy: str | None = None,
|
308
|
+
meta_data_directive: str | None = None,
|
305
309
|
**kwargs,
|
306
310
|
):
|
307
311
|
super().__init__(**kwargs)
|
@@ -314,6 +318,7 @@ class S3CopyObjectOperator(BaseOperator):
|
|
314
318
|
self.aws_conn_id = aws_conn_id
|
315
319
|
self.verify = verify
|
316
320
|
self.acl_policy = acl_policy
|
321
|
+
self.meta_data_directive = meta_data_directive
|
317
322
|
|
318
323
|
def execute(self, context: Context):
|
319
324
|
s3_hook = S3Hook(aws_conn_id=self.aws_conn_id, verify=self.verify)
|
@@ -324,6 +329,7 @@ class S3CopyObjectOperator(BaseOperator):
|
|
324
329
|
self.dest_bucket_name,
|
325
330
|
self.source_version_id,
|
326
331
|
self.acl_policy,
|
332
|
+
self.meta_data_directive,
|
327
333
|
)
|
328
334
|
|
329
335
|
def get_openlineage_facets_on_start(self):
|
@@ -19,8 +19,9 @@ from __future__ import annotations
|
|
19
19
|
import datetime
|
20
20
|
import json
|
21
21
|
import time
|
22
|
+
from collections.abc import Sequence
|
22
23
|
from functools import cached_property
|
23
|
-
from typing import TYPE_CHECKING, Any, Callable, ClassVar
|
24
|
+
from typing import TYPE_CHECKING, Any, Callable, ClassVar
|
24
25
|
|
25
26
|
from botocore.exceptions import ClientError
|
26
27
|
|
@@ -19,7 +19,8 @@
|
|
19
19
|
|
20
20
|
from __future__ import annotations
|
21
21
|
|
22
|
-
from
|
22
|
+
from collections.abc import Sequence
|
23
|
+
from typing import TYPE_CHECKING
|
23
24
|
|
24
25
|
from airflow.providers.amazon.aws.hooks.sns import SnsHook
|
25
26
|
from airflow.providers.amazon.aws.operators.base_aws import AwsBaseOperator
|
@@ -18,7 +18,8 @@
|
|
18
18
|
|
19
19
|
from __future__ import annotations
|
20
20
|
|
21
|
-
from
|
21
|
+
from collections.abc import Sequence
|
22
|
+
from typing import TYPE_CHECKING
|
22
23
|
|
23
24
|
from airflow.providers.amazon.aws.hooks.sqs import SqsHook
|
24
25
|
from airflow.providers.amazon.aws.operators.base_aws import AwsBaseOperator
|
@@ -17,8 +17,9 @@
|
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
19
|
import json
|
20
|
+
from collections.abc import Sequence
|
20
21
|
from datetime import timedelta
|
21
|
-
from typing import TYPE_CHECKING, Any
|
22
|
+
from typing import TYPE_CHECKING, Any
|
22
23
|
|
23
24
|
from airflow.configuration import conf
|
24
25
|
from airflow.exceptions import AirflowException
|
@@ -17,7 +17,8 @@
|
|
17
17
|
# under the License.
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
|
-
from
|
20
|
+
from collections.abc import Sequence
|
21
|
+
from typing import TYPE_CHECKING, Any
|
21
22
|
|
22
23
|
from airflow.providers.amazon.aws.sensors.base_aws import AwsBaseSensor
|
23
24
|
from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
|
@@ -16,9 +16,10 @@
|
|
16
16
|
# under the License.
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
|
+
from collections.abc import Sequence
|
19
20
|
from datetime import timedelta
|
20
21
|
from functools import cached_property
|
21
|
-
from typing import TYPE_CHECKING, Any
|
22
|
+
from typing import TYPE_CHECKING, Any
|
22
23
|
|
23
24
|
from airflow.configuration import conf
|
24
25
|
from airflow.exceptions import AirflowException
|
@@ -18,7 +18,8 @@
|
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
20
|
import abc
|
21
|
-
from
|
21
|
+
from collections.abc import Sequence
|
22
|
+
from typing import TYPE_CHECKING, Any, TypeVar
|
22
23
|
|
23
24
|
from airflow.configuration import conf
|
24
25
|
from airflow.exceptions import AirflowException
|
@@ -19,7 +19,8 @@
|
|
19
19
|
|
20
20
|
from __future__ import annotations
|
21
21
|
|
22
|
-
from
|
22
|
+
from collections.abc import Sequence
|
23
|
+
from typing import TYPE_CHECKING
|
23
24
|
|
24
25
|
from airflow.providers.amazon.aws.sensors.base_aws import AwsBaseSensor
|
25
26
|
from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
|
@@ -17,7 +17,8 @@
|
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
19
|
import abc
|
20
|
-
from
|
20
|
+
from collections.abc import Sequence
|
21
|
+
from typing import TYPE_CHECKING, Any
|
21
22
|
|
22
23
|
from airflow.configuration import conf
|
23
24
|
from airflow.exceptions import AirflowException
|
@@ -17,7 +17,8 @@
|
|
17
17
|
# under the License.
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
|
-
from
|
20
|
+
from collections.abc import Iterable, Sequence
|
21
|
+
from typing import TYPE_CHECKING
|
21
22
|
|
22
23
|
from airflow.exceptions import AirflowException
|
23
24
|
from airflow.providers.amazon.aws.hooks.dms import DmsHook
|
@@ -16,7 +16,8 @@
|
|
16
16
|
# under the License.
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
|
-
from
|
19
|
+
from collections.abc import Iterable, Sequence
|
20
|
+
from typing import TYPE_CHECKING, Any
|
20
21
|
|
21
22
|
from botocore.exceptions import ClientError
|
22
23
|
|
@@ -17,8 +17,9 @@
|
|
17
17
|
# under the License.
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
|
+
from collections.abc import Sequence
|
20
21
|
from functools import cached_property
|
21
|
-
from typing import TYPE_CHECKING, Any
|
22
|
+
from typing import TYPE_CHECKING, Any
|
22
23
|
|
23
24
|
from airflow.configuration import conf
|
24
25
|
from airflow.exceptions import AirflowException
|
@@ -16,8 +16,9 @@
|
|
16
16
|
# under the License.
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
|
+
from collections.abc import Sequence
|
19
20
|
from functools import cached_property
|
20
|
-
from typing import TYPE_CHECKING
|
21
|
+
from typing import TYPE_CHECKING
|
21
22
|
|
22
23
|
from airflow.exceptions import AirflowException
|
23
24
|
from airflow.providers.amazon.aws.hooks.ecs import (
|
@@ -19,8 +19,9 @@
|
|
19
19
|
from __future__ import annotations
|
20
20
|
|
21
21
|
from abc import abstractmethod
|
22
|
+
from collections.abc import Sequence
|
22
23
|
from functools import cached_property
|
23
|
-
from typing import TYPE_CHECKING
|
24
|
+
from typing import TYPE_CHECKING
|
24
25
|
|
25
26
|
from airflow.exceptions import AirflowException
|
26
27
|
from airflow.providers.amazon.aws.hooks.eks import (
|
@@ -17,9 +17,10 @@
|
|
17
17
|
# under the License.
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
|
+
from collections.abc import Iterable, Sequence
|
20
21
|
from datetime import timedelta
|
21
22
|
from functools import cached_property
|
22
|
-
from typing import TYPE_CHECKING, Any
|
23
|
+
from typing import TYPE_CHECKING, Any
|
23
24
|
|
24
25
|
from airflow.configuration import conf
|
25
26
|
from airflow.exceptions import (
|
@@ -17,8 +17,9 @@
|
|
17
17
|
# under the License.
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
|
+
from collections.abc import Sequence
|
20
21
|
from enum import Enum
|
21
|
-
from typing import TYPE_CHECKING, Any
|
22
|
+
from typing import TYPE_CHECKING, Any
|
22
23
|
|
23
24
|
from airflow.exceptions import AirflowException
|
24
25
|
from airflow.providers.amazon.aws.hooks.glacier import GlacierHook
|
@@ -17,8 +17,9 @@
|
|
17
17
|
# under the License.
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
|
+
from collections.abc import Sequence
|
20
21
|
from functools import cached_property
|
21
|
-
from typing import TYPE_CHECKING, Any
|
22
|
+
from typing import TYPE_CHECKING, Any
|
22
23
|
|
23
24
|
from airflow.configuration import conf
|
24
25
|
from airflow.exceptions import AirflowException
|
@@ -17,8 +17,9 @@
|
|
17
17
|
# under the License.
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
|
+
from collections.abc import Sequence
|
20
21
|
from datetime import timedelta
|
21
|
-
from typing import TYPE_CHECKING, Any
|
22
|
+
from typing import TYPE_CHECKING, Any
|
22
23
|
|
23
24
|
from airflow.configuration import conf
|
24
25
|
from airflow.exceptions import AirflowException
|
@@ -17,7 +17,8 @@
|
|
17
17
|
# under the License.
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
|
-
from
|
20
|
+
from collections.abc import Sequence
|
21
|
+
from typing import TYPE_CHECKING
|
21
22
|
|
22
23
|
from airflow.exceptions import AirflowException
|
23
24
|
from airflow.providers.amazon.aws.hooks.glue_crawler import GlueCrawlerHook
|
@@ -16,7 +16,8 @@
|
|
16
16
|
# under the License.
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
|
-
from
|
19
|
+
from collections.abc import Sequence
|
20
|
+
from typing import TYPE_CHECKING, Any
|
20
21
|
|
21
22
|
from airflow.configuration import conf
|
22
23
|
from airflow.exceptions import AirflowException
|
@@ -17,7 +17,8 @@
|
|
17
17
|
# under the License.
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
|
-
from
|
20
|
+
from collections.abc import Sequence
|
21
|
+
from typing import TYPE_CHECKING, Any
|
21
22
|
|
22
23
|
from airflow.exceptions import AirflowException
|
23
24
|
from airflow.providers.amazon.aws.hooks.lambda_function import LambdaHook
|
@@ -16,7 +16,8 @@
|
|
16
16
|
# under the License.
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
|
-
from
|
19
|
+
from collections.abc import Sequence
|
20
|
+
from typing import TYPE_CHECKING, Any
|
20
21
|
|
21
22
|
from airflow.configuration import conf
|
22
23
|
from airflow.exceptions import AirflowException
|
@@ -17,7 +17,8 @@
|
|
17
17
|
# under the License.
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
|
-
from
|
20
|
+
from collections.abc import Sequence
|
21
|
+
from typing import TYPE_CHECKING
|
21
22
|
|
22
23
|
from airflow.exceptions import AirflowException
|
23
24
|
from airflow.providers.amazon.aws.hooks.quicksight import QuickSightHook
|
@@ -16,8 +16,9 @@
|
|
16
16
|
# under the License.
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
|
+
from collections.abc import Sequence
|
19
20
|
from functools import cached_property
|
20
|
-
from typing import TYPE_CHECKING
|
21
|
+
from typing import TYPE_CHECKING
|
21
22
|
|
22
23
|
from airflow.exceptions import AirflowNotFoundException
|
23
24
|
from airflow.providers.amazon.aws.hooks.rds import RdsHook
|
@@ -16,9 +16,10 @@
|
|
16
16
|
# under the License.
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
|
+
from collections.abc import Sequence
|
19
20
|
from datetime import timedelta
|
20
21
|
from functools import cached_property
|
21
|
-
from typing import TYPE_CHECKING, Any
|
22
|
+
from typing import TYPE_CHECKING, Any
|
22
23
|
|
23
24
|
from airflow.configuration import conf
|
24
25
|
from airflow.exceptions import AirflowException
|