apache-airflow-providers-amazon 9.18.0rc2__py3-none-any.whl → 9.18.1rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/amazon/__init__.py +1 -1
- airflow/providers/amazon/aws/auth_manager/avp/facade.py +1 -1
- airflow/providers/amazon/aws/bundles/s3.py +1 -1
- airflow/providers/amazon/aws/exceptions.py +1 -1
- airflow/providers/amazon/aws/executors/aws_lambda/lambda_executor.py +1 -2
- airflow/providers/amazon/aws/executors/batch/batch_executor.py +1 -2
- airflow/providers/amazon/aws/executors/ecs/ecs_executor.py +1 -2
- airflow/providers/amazon/aws/hooks/athena.py +1 -1
- airflow/providers/amazon/aws/hooks/athena_sql.py +1 -1
- airflow/providers/amazon/aws/hooks/base_aws.py +2 -6
- airflow/providers/amazon/aws/hooks/batch_client.py +1 -1
- airflow/providers/amazon/aws/hooks/batch_waiters.py +1 -1
- airflow/providers/amazon/aws/hooks/chime.py +1 -1
- airflow/providers/amazon/aws/hooks/comprehend.py +1 -1
- airflow/providers/amazon/aws/hooks/datasync.py +2 -2
- airflow/providers/amazon/aws/hooks/dynamodb.py +1 -1
- airflow/providers/amazon/aws/hooks/ec2.py +1 -1
- airflow/providers/amazon/aws/hooks/elasticache_replication_group.py +1 -1
- airflow/providers/amazon/aws/hooks/emr.py +1 -1
- airflow/providers/amazon/aws/hooks/glue.py +2 -1
- airflow/providers/amazon/aws/hooks/glue_catalog.py +1 -1
- airflow/providers/amazon/aws/hooks/logs.py +10 -2
- airflow/providers/amazon/aws/hooks/quicksight.py +1 -1
- airflow/providers/amazon/aws/hooks/rds.py +1 -1
- airflow/providers/amazon/aws/hooks/redshift_sql.py +11 -2
- airflow/providers/amazon/aws/hooks/s3.py +2 -1
- airflow/providers/amazon/aws/hooks/sagemaker.py +1 -1
- airflow/providers/amazon/aws/hooks/sagemaker_unified_studio.py +1 -2
- airflow/providers/amazon/aws/links/emr.py +1 -1
- airflow/providers/amazon/aws/log/cloudwatch_task_handler.py +50 -20
- airflow/providers/amazon/aws/operators/appflow.py +1 -1
- airflow/providers/amazon/aws/operators/athena.py +1 -1
- airflow/providers/amazon/aws/operators/batch.py +1 -1
- airflow/providers/amazon/aws/operators/bedrock.py +1 -1
- airflow/providers/amazon/aws/operators/comprehend.py +1 -1
- airflow/providers/amazon/aws/operators/datasync.py +1 -2
- airflow/providers/amazon/aws/operators/dms.py +1 -1
- airflow/providers/amazon/aws/operators/ec2.py +1 -1
- airflow/providers/amazon/aws/operators/ecs.py +1 -1
- airflow/providers/amazon/aws/operators/eks.py +2 -1
- airflow/providers/amazon/aws/operators/emr.py +2 -1
- airflow/providers/amazon/aws/operators/eventbridge.py +1 -1
- airflow/providers/amazon/aws/operators/glue.py +1 -1
- airflow/providers/amazon/aws/operators/glue_crawler.py +1 -1
- airflow/providers/amazon/aws/operators/glue_databrew.py +1 -1
- airflow/providers/amazon/aws/operators/kinesis_analytics.py +1 -1
- airflow/providers/amazon/aws/operators/lambda_function.py +1 -1
- airflow/providers/amazon/aws/operators/mwaa.py +1 -1
- airflow/providers/amazon/aws/operators/neptune.py +1 -1
- airflow/providers/amazon/aws/operators/rds.py +1 -1
- airflow/providers/amazon/aws/operators/redshift_cluster.py +1 -1
- airflow/providers/amazon/aws/operators/redshift_data.py +1 -1
- airflow/providers/amazon/aws/operators/s3.py +1 -1
- airflow/providers/amazon/aws/operators/sagemaker.py +1 -1
- airflow/providers/amazon/aws/operators/sagemaker_unified_studio.py +1 -2
- airflow/providers/amazon/aws/operators/step_function.py +1 -1
- airflow/providers/amazon/aws/sensors/athena.py +1 -1
- airflow/providers/amazon/aws/sensors/batch.py +1 -1
- airflow/providers/amazon/aws/sensors/bedrock.py +1 -1
- airflow/providers/amazon/aws/sensors/comprehend.py +1 -1
- airflow/providers/amazon/aws/sensors/dms.py +1 -1
- airflow/providers/amazon/aws/sensors/ec2.py +1 -1
- airflow/providers/amazon/aws/sensors/ecs.py +1 -1
- airflow/providers/amazon/aws/sensors/eks.py +2 -1
- airflow/providers/amazon/aws/sensors/emr.py +1 -3
- airflow/providers/amazon/aws/sensors/glacier.py +1 -1
- airflow/providers/amazon/aws/sensors/glue.py +1 -1
- airflow/providers/amazon/aws/sensors/glue_catalog_partition.py +1 -1
- airflow/providers/amazon/aws/sensors/glue_crawler.py +1 -1
- airflow/providers/amazon/aws/sensors/kinesis_analytics.py +1 -1
- airflow/providers/amazon/aws/sensors/lambda_function.py +1 -1
- airflow/providers/amazon/aws/sensors/mwaa.py +1 -1
- airflow/providers/amazon/aws/sensors/opensearch_serverless.py +1 -1
- airflow/providers/amazon/aws/sensors/quicksight.py +1 -1
- airflow/providers/amazon/aws/sensors/rds.py +1 -1
- airflow/providers/amazon/aws/sensors/redshift_cluster.py +1 -1
- airflow/providers/amazon/aws/sensors/s3.py +1 -2
- airflow/providers/amazon/aws/sensors/sagemaker.py +1 -1
- airflow/providers/amazon/aws/sensors/sagemaker_unified_studio.py +1 -2
- airflow/providers/amazon/aws/sensors/sqs.py +1 -1
- airflow/providers/amazon/aws/sensors/step_function.py +1 -1
- airflow/providers/amazon/aws/transfers/gcs_to_s3.py +1 -2
- airflow/providers/amazon/aws/transfers/redshift_to_s3.py +1 -2
- airflow/providers/amazon/aws/transfers/s3_to_dynamodb.py +1 -2
- airflow/providers/amazon/aws/transfers/s3_to_redshift.py +1 -2
- airflow/providers/amazon/aws/transfers/s3_to_sql.py +1 -2
- airflow/providers/amazon/aws/transfers/sql_to_s3.py +5 -5
- airflow/providers/amazon/aws/triggers/ecs.py +1 -1
- airflow/providers/amazon/aws/triggers/eks.py +1 -1
- airflow/providers/amazon/aws/triggers/sagemaker.py +1 -1
- airflow/providers/amazon/aws/triggers/sqs.py +1 -1
- airflow/providers/amazon/aws/utils/__init__.py +1 -1
- airflow/providers/amazon/aws/utils/connection_wrapper.py +1 -1
- airflow/providers/amazon/aws/utils/waiter.py +1 -1
- airflow/providers/amazon/aws/utils/waiter_with_logging.py +1 -1
- {apache_airflow_providers_amazon-9.18.0rc2.dist-info → apache_airflow_providers_amazon-9.18.1rc1.dist-info}/METADATA +12 -8
- {apache_airflow_providers_amazon-9.18.0rc2.dist-info → apache_airflow_providers_amazon-9.18.1rc1.dist-info}/RECORD +101 -101
- {apache_airflow_providers_amazon-9.18.0rc2.dist-info → apache_airflow_providers_amazon-9.18.1rc1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_amazon-9.18.0rc2.dist-info → apache_airflow_providers_amazon-9.18.1rc1.dist-info}/entry_points.txt +0 -0
- {apache_airflow_providers_amazon-9.18.0rc2.dist-info → apache_airflow_providers_amazon-9.18.1rc1.dist-info}/licenses/LICENSE +0 -0
- {apache_airflow_providers_amazon-9.18.0rc2.dist-info → apache_airflow_providers_amazon-9.18.1rc1.dist-info}/licenses/NOTICE +0 -0
|
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
|
|
|
29
29
|
|
|
30
30
|
__all__ = ["__version__"]
|
|
31
31
|
|
|
32
|
-
__version__ = "9.18.
|
|
32
|
+
__version__ = "9.18.1"
|
|
33
33
|
|
|
34
34
|
if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
|
|
35
35
|
"2.11.0"
|
|
@@ -23,7 +23,6 @@ from pathlib import Path
|
|
|
23
23
|
from typing import TYPE_CHECKING, TypedDict
|
|
24
24
|
|
|
25
25
|
from airflow.configuration import conf
|
|
26
|
-
from airflow.exceptions import AirflowException
|
|
27
26
|
from airflow.providers.amazon.aws.auth_manager.avp.entities import AvpEntities, get_action_id, get_entity_type
|
|
28
27
|
from airflow.providers.amazon.aws.auth_manager.constants import (
|
|
29
28
|
CONF_AVP_POLICY_STORE_ID_KEY,
|
|
@@ -32,6 +31,7 @@ from airflow.providers.amazon.aws.auth_manager.constants import (
|
|
|
32
31
|
CONF_SECTION_NAME,
|
|
33
32
|
)
|
|
34
33
|
from airflow.providers.amazon.aws.hooks.verified_permissions import VerifiedPermissionsHook
|
|
34
|
+
from airflow.providers.common.compat.sdk import AirflowException
|
|
35
35
|
from airflow.utils.helpers import prune_dict
|
|
36
36
|
from airflow.utils.log.logging_mixin import LoggingMixin
|
|
37
37
|
|
|
@@ -22,9 +22,9 @@ from pathlib import Path
|
|
|
22
22
|
import structlog
|
|
23
23
|
|
|
24
24
|
from airflow.dag_processing.bundles.base import BaseDagBundle
|
|
25
|
-
from airflow.exceptions import AirflowException
|
|
26
25
|
from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
|
|
27
26
|
from airflow.providers.amazon.aws.hooks.s3 import S3Hook
|
|
27
|
+
from airflow.providers.common.compat.sdk import AirflowException
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
class S3DagBundle(BaseDagBundle):
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
# under the License.
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
|
-
from airflow.
|
|
20
|
+
from airflow.providers.common.compat.sdk import AirflowException
|
|
21
21
|
|
|
22
22
|
# Note: Any AirflowException raised is expected to cause the TaskInstance
|
|
23
23
|
# to be marked in an ERROR state
|
|
@@ -26,7 +26,6 @@ from boto3.session import NoCredentialsError
|
|
|
26
26
|
from botocore.utils import ClientError
|
|
27
27
|
|
|
28
28
|
from airflow.configuration import conf
|
|
29
|
-
from airflow.exceptions import AirflowException
|
|
30
29
|
from airflow.executors.base_executor import BaseExecutor
|
|
31
30
|
from airflow.models.taskinstancekey import TaskInstanceKey
|
|
32
31
|
from airflow.providers.amazon.aws.executors.aws_lambda.utils import (
|
|
@@ -42,7 +41,7 @@ from airflow.providers.amazon.aws.executors.utils.exponential_backoff_retry impo
|
|
|
42
41
|
)
|
|
43
42
|
from airflow.providers.amazon.aws.hooks.lambda_function import LambdaHook
|
|
44
43
|
from airflow.providers.amazon.aws.hooks.sqs import SqsHook
|
|
45
|
-
from airflow.
|
|
44
|
+
from airflow.providers.common.compat.sdk import AirflowException, Stats
|
|
46
45
|
|
|
47
46
|
try:
|
|
48
47
|
from airflow.sdk import timezone
|
|
@@ -29,7 +29,6 @@ from typing import TYPE_CHECKING, Any
|
|
|
29
29
|
from botocore.exceptions import ClientError, NoCredentialsError
|
|
30
30
|
|
|
31
31
|
from airflow.configuration import conf
|
|
32
|
-
from airflow.exceptions import AirflowException
|
|
33
32
|
from airflow.executors.base_executor import BaseExecutor
|
|
34
33
|
from airflow.providers.amazon.aws.executors.utils.exponential_backoff_retry import (
|
|
35
34
|
calculate_next_attempt_delay,
|
|
@@ -37,7 +36,7 @@ from airflow.providers.amazon.aws.executors.utils.exponential_backoff_retry impo
|
|
|
37
36
|
)
|
|
38
37
|
from airflow.providers.amazon.aws.hooks.batch_client import BatchClientHook
|
|
39
38
|
from airflow.providers.amazon.version_compat import AIRFLOW_V_3_0_PLUS
|
|
40
|
-
from airflow.
|
|
39
|
+
from airflow.providers.common.compat.sdk import AirflowException, Stats
|
|
41
40
|
|
|
42
41
|
try:
|
|
43
42
|
from airflow.sdk import timezone
|
|
@@ -32,7 +32,6 @@ from typing import TYPE_CHECKING
|
|
|
32
32
|
|
|
33
33
|
from botocore.exceptions import ClientError, NoCredentialsError
|
|
34
34
|
|
|
35
|
-
from airflow.exceptions import AirflowException
|
|
36
35
|
from airflow.executors.base_executor import BaseExecutor
|
|
37
36
|
from airflow.providers.amazon.aws.executors.ecs.boto_schema import BotoDescribeTasksSchema, BotoRunTaskSchema
|
|
38
37
|
from airflow.providers.amazon.aws.executors.ecs.utils import (
|
|
@@ -49,7 +48,7 @@ from airflow.providers.amazon.aws.executors.utils.exponential_backoff_retry impo
|
|
|
49
48
|
)
|
|
50
49
|
from airflow.providers.amazon.aws.hooks.ecs import EcsHook
|
|
51
50
|
from airflow.providers.amazon.version_compat import AIRFLOW_V_3_0_PLUS
|
|
52
|
-
from airflow.
|
|
51
|
+
from airflow.providers.common.compat.sdk import AirflowException, Stats
|
|
53
52
|
|
|
54
53
|
try:
|
|
55
54
|
from airflow.sdk import timezone
|
|
@@ -28,9 +28,9 @@ from __future__ import annotations
|
|
|
28
28
|
from collections.abc import Collection
|
|
29
29
|
from typing import TYPE_CHECKING, Any
|
|
30
30
|
|
|
31
|
-
from airflow.exceptions import AirflowException
|
|
32
31
|
from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
|
|
33
32
|
from airflow.providers.amazon.aws.utils.waiter_with_logging import wait
|
|
33
|
+
from airflow.providers.common.compat.sdk import AirflowException
|
|
34
34
|
|
|
35
35
|
if TYPE_CHECKING:
|
|
36
36
|
from botocore.paginate import PageIterator
|
|
@@ -23,9 +23,9 @@ from typing import TYPE_CHECKING, Any
|
|
|
23
23
|
import pyathena
|
|
24
24
|
from sqlalchemy.engine.url import URL
|
|
25
25
|
|
|
26
|
-
from airflow.exceptions import AirflowException, AirflowNotFoundException
|
|
27
26
|
from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
|
|
28
27
|
from airflow.providers.amazon.aws.utils.connection_wrapper import AwsConnectionWrapper
|
|
28
|
+
from airflow.providers.common.compat.sdk import AirflowException, AirflowNotFoundException
|
|
29
29
|
from airflow.providers.common.sql.hooks.sql import DbApiHook
|
|
30
30
|
|
|
31
31
|
if TYPE_CHECKING:
|
|
@@ -52,15 +52,11 @@ from dateutil.tz import tzlocal
|
|
|
52
52
|
from slugify import slugify
|
|
53
53
|
|
|
54
54
|
from airflow.configuration import conf
|
|
55
|
-
from airflow.exceptions import
|
|
56
|
-
AirflowException,
|
|
57
|
-
AirflowNotFoundException,
|
|
58
|
-
AirflowProviderDeprecationWarning,
|
|
59
|
-
)
|
|
55
|
+
from airflow.exceptions import AirflowProviderDeprecationWarning
|
|
60
56
|
from airflow.providers.amazon.aws.utils.connection_wrapper import AwsConnectionWrapper
|
|
61
57
|
from airflow.providers.amazon.aws.utils.identifiers import generate_uuid
|
|
62
58
|
from airflow.providers.amazon.aws.utils.suppress import return_on_error
|
|
63
|
-
from airflow.providers.common.compat.sdk import BaseHook
|
|
59
|
+
from airflow.providers.common.compat.sdk import AirflowException, AirflowNotFoundException, BaseHook
|
|
64
60
|
from airflow.providers_manager import ProvidersManager
|
|
65
61
|
from airflow.utils.helpers import exactly_one
|
|
66
62
|
from airflow.utils.log.logging_mixin import LoggingMixin
|
|
@@ -37,8 +37,8 @@ import botocore.client
|
|
|
37
37
|
import botocore.exceptions
|
|
38
38
|
import botocore.waiter
|
|
39
39
|
|
|
40
|
-
from airflow.exceptions import AirflowException
|
|
41
40
|
from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
|
|
41
|
+
from airflow.providers.common.compat.sdk import AirflowException
|
|
42
42
|
|
|
43
43
|
if TYPE_CHECKING:
|
|
44
44
|
from airflow.providers.amazon.aws.utils.task_log_fetcher import AwsTaskLogFetcher
|
|
@@ -36,8 +36,8 @@ from typing import TYPE_CHECKING, Any
|
|
|
36
36
|
import botocore.exceptions
|
|
37
37
|
import botocore.waiter
|
|
38
38
|
|
|
39
|
-
from airflow.exceptions import AirflowException
|
|
40
39
|
from airflow.providers.amazon.aws.hooks.batch_client import BatchClientHook
|
|
40
|
+
from airflow.providers.common.compat.sdk import AirflowException
|
|
41
41
|
|
|
42
42
|
if TYPE_CHECKING:
|
|
43
43
|
from airflow.providers.amazon.aws.utils.task_log_fetcher import AwsTaskLogFetcher
|
|
@@ -25,7 +25,7 @@ import re
|
|
|
25
25
|
from functools import cached_property
|
|
26
26
|
from typing import Any
|
|
27
27
|
|
|
28
|
-
from airflow.
|
|
28
|
+
from airflow.providers.common.compat.sdk import AirflowException
|
|
29
29
|
from airflow.providers.http.hooks.http import HttpHook
|
|
30
30
|
|
|
31
31
|
|
|
@@ -16,8 +16,8 @@
|
|
|
16
16
|
# under the License.
|
|
17
17
|
from __future__ import annotations
|
|
18
18
|
|
|
19
|
-
from airflow.exceptions import AirflowException
|
|
20
19
|
from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
|
|
20
|
+
from airflow.providers.common.compat.sdk import AirflowException
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
class ComprehendHook(AwsBaseHook):
|
|
@@ -21,9 +21,9 @@ from __future__ import annotations
|
|
|
21
21
|
import time
|
|
22
22
|
from urllib.parse import urlsplit
|
|
23
23
|
|
|
24
|
-
from airflow.exceptions import AirflowBadRequest
|
|
24
|
+
from airflow.exceptions import AirflowBadRequest
|
|
25
25
|
from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
|
|
26
|
-
from airflow.providers.common.compat.sdk import AirflowTaskTimeout
|
|
26
|
+
from airflow.providers.common.compat.sdk import AirflowException, AirflowTaskTimeout
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
class DataSyncHook(AwsBaseHook):
|
|
@@ -25,8 +25,8 @@ from typing import TYPE_CHECKING
|
|
|
25
25
|
|
|
26
26
|
from botocore.exceptions import ClientError
|
|
27
27
|
|
|
28
|
-
from airflow.exceptions import AirflowException
|
|
29
28
|
from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
|
|
29
|
+
from airflow.providers.common.compat.sdk import AirflowException
|
|
30
30
|
|
|
31
31
|
if TYPE_CHECKING:
|
|
32
32
|
from botocore.client import BaseClient
|
|
@@ -22,8 +22,8 @@ import time
|
|
|
22
22
|
from collections.abc import Callable
|
|
23
23
|
from typing import ParamSpec, TypeVar
|
|
24
24
|
|
|
25
|
-
from airflow.exceptions import AirflowException
|
|
26
25
|
from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
|
|
26
|
+
from airflow.providers.common.compat.sdk import AirflowException
|
|
27
27
|
|
|
28
28
|
PS = ParamSpec("PS")
|
|
29
29
|
RT = TypeVar("RT")
|
|
@@ -19,8 +19,8 @@ from __future__ import annotations
|
|
|
19
19
|
|
|
20
20
|
import time
|
|
21
21
|
|
|
22
|
-
from airflow.exceptions import AirflowException
|
|
23
22
|
from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
|
|
23
|
+
from airflow.providers.common.compat.sdk import AirflowException
|
|
24
24
|
|
|
25
25
|
|
|
26
26
|
class ElastiCacheReplicationGroupHook(AwsBaseHook):
|
|
@@ -26,9 +26,9 @@ import tenacity
|
|
|
26
26
|
from botocore.exceptions import ClientError
|
|
27
27
|
from tenacity import retry_if_exception, stop_after_attempt, wait_fixed
|
|
28
28
|
|
|
29
|
-
from airflow.exceptions import AirflowException, AirflowNotFoundException
|
|
30
29
|
from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
|
|
31
30
|
from airflow.providers.amazon.aws.utils.waiter_with_logging import wait
|
|
31
|
+
from airflow.providers.common.compat.sdk import AirflowException, AirflowNotFoundException
|
|
32
32
|
|
|
33
33
|
|
|
34
34
|
class EmrHook(AwsBaseHook):
|
|
@@ -33,9 +33,10 @@ from tenacity import (
|
|
|
33
33
|
wait_exponential,
|
|
34
34
|
)
|
|
35
35
|
|
|
36
|
-
from airflow.exceptions import
|
|
36
|
+
from airflow.exceptions import AirflowProviderDeprecationWarning
|
|
37
37
|
from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
|
|
38
38
|
from airflow.providers.amazon.aws.hooks.logs import AwsLogsHook
|
|
39
|
+
from airflow.providers.common.compat.sdk import AirflowException
|
|
39
40
|
|
|
40
41
|
DEFAULT_LOG_SUFFIX = "output"
|
|
41
42
|
ERROR_LOG_SUFFIX = "error"
|
|
@@ -23,8 +23,8 @@ from typing import Any
|
|
|
23
23
|
|
|
24
24
|
from botocore.exceptions import ClientError
|
|
25
25
|
|
|
26
|
-
from airflow.exceptions import AirflowException
|
|
27
26
|
from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
|
|
27
|
+
from airflow.providers.common.compat.sdk import AirflowException
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
class GlueCatalogHook(AwsBaseHook):
|
|
@@ -19,7 +19,7 @@ from __future__ import annotations
|
|
|
19
19
|
|
|
20
20
|
import asyncio
|
|
21
21
|
from collections.abc import AsyncGenerator, Generator
|
|
22
|
-
from typing import Any
|
|
22
|
+
from typing import Any, TypedDict
|
|
23
23
|
|
|
24
24
|
from botocore.exceptions import ClientError
|
|
25
25
|
|
|
@@ -35,6 +35,14 @@ from airflow.utils.helpers import prune_dict
|
|
|
35
35
|
NUM_CONSECUTIVE_EMPTY_RESPONSE_EXIT_THRESHOLD = 3
|
|
36
36
|
|
|
37
37
|
|
|
38
|
+
class CloudWatchLogEvent(TypedDict):
|
|
39
|
+
"""TypedDict for CloudWatch Log Event."""
|
|
40
|
+
|
|
41
|
+
timestamp: int
|
|
42
|
+
message: str
|
|
43
|
+
ingestionTime: int
|
|
44
|
+
|
|
45
|
+
|
|
38
46
|
class AwsLogsHook(AwsBaseHook):
|
|
39
47
|
"""
|
|
40
48
|
Interact with Amazon CloudWatch Logs.
|
|
@@ -67,7 +75,7 @@ class AwsLogsHook(AwsBaseHook):
|
|
|
67
75
|
start_from_head: bool | None = None,
|
|
68
76
|
continuation_token: ContinuationToken | None = None,
|
|
69
77
|
end_time: int | None = None,
|
|
70
|
-
) -> Generator:
|
|
78
|
+
) -> Generator[CloudWatchLogEvent, None, None]:
|
|
71
79
|
"""
|
|
72
80
|
Return a generator for log items in a single stream; yields all items available at the current moment.
|
|
73
81
|
|
|
@@ -21,8 +21,8 @@ import time
|
|
|
21
21
|
|
|
22
22
|
from botocore.exceptions import ClientError
|
|
23
23
|
|
|
24
|
-
from airflow.exceptions import AirflowException
|
|
25
24
|
from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
|
|
25
|
+
from airflow.providers.common.compat.sdk import AirflowException
|
|
26
26
|
|
|
27
27
|
|
|
28
28
|
class QuickSightHook(AwsBaseHook):
|
|
@@ -23,9 +23,9 @@ import time
|
|
|
23
23
|
from collections.abc import Callable
|
|
24
24
|
from typing import TYPE_CHECKING
|
|
25
25
|
|
|
26
|
-
from airflow.exceptions import AirflowException, AirflowNotFoundException
|
|
27
26
|
from airflow.providers.amazon.aws.hooks.base_aws import AwsGenericHook
|
|
28
27
|
from airflow.providers.amazon.aws.utils.waiter_with_logging import wait
|
|
28
|
+
from airflow.providers.common.compat.sdk import AirflowException, AirflowNotFoundException
|
|
29
29
|
|
|
30
30
|
if TYPE_CHECKING:
|
|
31
31
|
from mypy_boto3_rds import RDSClient # noqa: F401
|
|
@@ -20,12 +20,13 @@ from functools import cached_property
|
|
|
20
20
|
from typing import TYPE_CHECKING
|
|
21
21
|
|
|
22
22
|
import redshift_connector
|
|
23
|
-
|
|
23
|
+
import tenacity
|
|
24
|
+
from redshift_connector import Connection as RedshiftConnection, InterfaceError, OperationalError
|
|
24
25
|
from sqlalchemy import create_engine
|
|
25
26
|
from sqlalchemy.engine.url import URL
|
|
26
27
|
|
|
27
|
-
from airflow.exceptions import AirflowException
|
|
28
28
|
from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
|
|
29
|
+
from airflow.providers.common.compat.sdk import AirflowException
|
|
29
30
|
from airflow.providers.common.sql.hooks.sql import DbApiHook
|
|
30
31
|
|
|
31
32
|
if TYPE_CHECKING:
|
|
@@ -206,6 +207,14 @@ class RedshiftSQLHook(DbApiHook):
|
|
|
206
207
|
pk_columns = [row[0] for row in self.get_records(sql, (schema, table))]
|
|
207
208
|
return pk_columns or None
|
|
208
209
|
|
|
210
|
+
@tenacity.retry(
|
|
211
|
+
stop=tenacity.stop_after_attempt(5),
|
|
212
|
+
wait=tenacity.wait_exponential(max=20),
|
|
213
|
+
# OperationalError is thrown when the connection times out
|
|
214
|
+
# InterfaceError is thrown when the connection is refused
|
|
215
|
+
retry=tenacity.retry_if_exception_type((OperationalError, InterfaceError)),
|
|
216
|
+
reraise=True,
|
|
217
|
+
)
|
|
209
218
|
def get_conn(self) -> RedshiftConnection:
|
|
210
219
|
"""Get a ``redshift_connector.Connection`` object."""
|
|
211
220
|
conn_params = self._get_conn_params()
|
|
@@ -56,11 +56,12 @@ from asgiref.sync import sync_to_async
|
|
|
56
56
|
from boto3.s3.transfer import S3Transfer, TransferConfig
|
|
57
57
|
from botocore.exceptions import ClientError
|
|
58
58
|
|
|
59
|
-
from airflow.exceptions import
|
|
59
|
+
from airflow.exceptions import AirflowProviderDeprecationWarning
|
|
60
60
|
from airflow.providers.amazon.aws.exceptions import S3HookUriParseFailure
|
|
61
61
|
from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
|
|
62
62
|
from airflow.providers.amazon.aws.utils.tags import format_tags
|
|
63
63
|
from airflow.providers.common.compat.lineage.hook import get_hook_lineage_collector
|
|
64
|
+
from airflow.providers.common.compat.sdk import AirflowException, AirflowNotFoundException
|
|
64
65
|
from airflow.utils.helpers import chunks
|
|
65
66
|
|
|
66
67
|
logger = logging.getLogger(__name__)
|
|
@@ -31,11 +31,11 @@ from typing import Any, cast
|
|
|
31
31
|
from asgiref.sync import sync_to_async
|
|
32
32
|
from botocore.exceptions import ClientError
|
|
33
33
|
|
|
34
|
-
from airflow.exceptions import AirflowException
|
|
35
34
|
from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
|
|
36
35
|
from airflow.providers.amazon.aws.hooks.logs import AwsLogsHook
|
|
37
36
|
from airflow.providers.amazon.aws.hooks.s3 import S3Hook
|
|
38
37
|
from airflow.providers.amazon.aws.utils.tags import format_tags
|
|
38
|
+
from airflow.providers.common.compat.sdk import AirflowException
|
|
39
39
|
|
|
40
40
|
try:
|
|
41
41
|
from airflow.sdk import timezone
|
|
@@ -24,9 +24,8 @@ import time
|
|
|
24
24
|
from sagemaker_studio import ClientConfig
|
|
25
25
|
from sagemaker_studio.sagemaker_studio_api import SageMakerStudioAPI
|
|
26
26
|
|
|
27
|
-
from airflow.exceptions import AirflowException
|
|
28
27
|
from airflow.providers.amazon.aws.utils.sagemaker_unified_studio import is_local_runner
|
|
29
|
-
from airflow.providers.common.compat.sdk import BaseHook
|
|
28
|
+
from airflow.providers.common.compat.sdk import AirflowException, BaseHook
|
|
30
29
|
|
|
31
30
|
|
|
32
31
|
class SageMakerNotebookHook(BaseHook):
|
|
@@ -19,10 +19,10 @@ from __future__ import annotations
|
|
|
19
19
|
from typing import TYPE_CHECKING, Any
|
|
20
20
|
from urllib.parse import ParseResult, quote_plus, urlparse
|
|
21
21
|
|
|
22
|
-
from airflow.exceptions import AirflowException
|
|
23
22
|
from airflow.providers.amazon.aws.hooks.emr import EmrServerlessHook
|
|
24
23
|
from airflow.providers.amazon.aws.hooks.s3 import S3Hook
|
|
25
24
|
from airflow.providers.amazon.aws.links.base_aws import BASE_AWS_CONSOLE_LINK, BaseAwsLink
|
|
25
|
+
from airflow.providers.common.compat.sdk import AirflowException
|
|
26
26
|
from airflow.utils.helpers import exactly_one
|
|
27
27
|
|
|
28
28
|
if TYPE_CHECKING:
|
|
@@ -22,6 +22,7 @@ import copy
|
|
|
22
22
|
import json
|
|
23
23
|
import logging
|
|
24
24
|
import os
|
|
25
|
+
from collections.abc import Generator
|
|
25
26
|
from datetime import date, datetime, timedelta, timezone
|
|
26
27
|
from functools import cached_property
|
|
27
28
|
from pathlib import Path
|
|
@@ -40,8 +41,15 @@ if TYPE_CHECKING:
|
|
|
40
41
|
import structlog.typing
|
|
41
42
|
|
|
42
43
|
from airflow.models.taskinstance import TaskInstance
|
|
44
|
+
from airflow.providers.amazon.aws.hooks.logs import CloudWatchLogEvent
|
|
43
45
|
from airflow.sdk.types import RuntimeTaskInstanceProtocol as RuntimeTI
|
|
44
|
-
from airflow.utils.log.file_task_handler import
|
|
46
|
+
from airflow.utils.log.file_task_handler import (
|
|
47
|
+
LogMessages,
|
|
48
|
+
LogResponse,
|
|
49
|
+
LogSourceInfo,
|
|
50
|
+
RawLogStream,
|
|
51
|
+
StreamingLogResponse,
|
|
52
|
+
)
|
|
45
53
|
|
|
46
54
|
|
|
47
55
|
def json_serialize_legacy(value: Any) -> str | None:
|
|
@@ -163,20 +171,31 @@ class CloudWatchRemoteLogIO(LoggingMixin): # noqa: D101
|
|
|
163
171
|
self.close()
|
|
164
172
|
return
|
|
165
173
|
|
|
166
|
-
def read(self, relative_path, ti: RuntimeTI) ->
|
|
167
|
-
logs
|
|
174
|
+
def read(self, relative_path: str, ti: RuntimeTI) -> LogResponse:
|
|
175
|
+
messages, logs = self.stream(relative_path, ti)
|
|
176
|
+
str_logs: list[str] = [f"{msg}\n" for group in logs for msg in group]
|
|
177
|
+
|
|
178
|
+
return messages, str_logs
|
|
179
|
+
|
|
180
|
+
def stream(self, relative_path: str, ti: RuntimeTI) -> StreamingLogResponse:
|
|
181
|
+
logs: list[RawLogStream] = []
|
|
168
182
|
messages = [
|
|
169
183
|
f"Reading remote log from Cloudwatch log_group: {self.log_group} log_stream: {relative_path}"
|
|
170
184
|
]
|
|
171
185
|
try:
|
|
172
|
-
|
|
186
|
+
gen: RawLogStream = (
|
|
187
|
+
self._parse_log_event_as_dumped_json(event)
|
|
188
|
+
for event in self.get_cloudwatch_logs(relative_path, ti)
|
|
189
|
+
)
|
|
190
|
+
logs = [gen]
|
|
173
191
|
except Exception as e:
|
|
174
|
-
logs = None
|
|
175
192
|
messages.append(str(e))
|
|
176
193
|
|
|
177
194
|
return messages, logs
|
|
178
195
|
|
|
179
|
-
def get_cloudwatch_logs(
|
|
196
|
+
def get_cloudwatch_logs(
|
|
197
|
+
self, stream_name: str, task_instance: RuntimeTI
|
|
198
|
+
) -> Generator[CloudWatchLogEvent, None, None]:
|
|
180
199
|
"""
|
|
181
200
|
Return all logs from the given log stream.
|
|
182
201
|
|
|
@@ -192,29 +211,22 @@ class CloudWatchRemoteLogIO(LoggingMixin): # noqa: D101
|
|
|
192
211
|
if (end_date := getattr(task_instance, "end_date", None)) is None
|
|
193
212
|
else datetime_to_epoch_utc_ms(end_date + timedelta(seconds=30))
|
|
194
213
|
)
|
|
195
|
-
|
|
214
|
+
return self.hook.get_log_events(
|
|
196
215
|
log_group=self.log_group,
|
|
197
216
|
log_stream_name=stream_name,
|
|
198
217
|
end_time=end_time,
|
|
199
218
|
)
|
|
200
|
-
return "\n".join(self._event_to_str(event) for event in events)
|
|
201
219
|
|
|
202
|
-
def
|
|
220
|
+
def _parse_log_event_as_dumped_json(self, event: CloudWatchLogEvent) -> str:
|
|
203
221
|
event_dt = datetime.fromtimestamp(event["timestamp"] / 1000.0, tz=timezone.utc).isoformat()
|
|
204
|
-
|
|
222
|
+
event_msg = event["message"]
|
|
205
223
|
try:
|
|
206
|
-
message = json.loads(
|
|
224
|
+
message = json.loads(event_msg)
|
|
207
225
|
message["timestamp"] = event_dt
|
|
208
|
-
return message
|
|
209
226
|
except Exception:
|
|
210
|
-
|
|
227
|
+
message = {"timestamp": event_dt, "event": event_msg}
|
|
211
228
|
|
|
212
|
-
|
|
213
|
-
event_dt = datetime.fromtimestamp(event["timestamp"] / 1000.0, tz=timezone.utc)
|
|
214
|
-
# Format a datetime object to a string in Zulu time without milliseconds.
|
|
215
|
-
formatted_event_dt = event_dt.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
216
|
-
message = event["message"]
|
|
217
|
-
return f"[{formatted_event_dt}] {message}"
|
|
229
|
+
return json.dumps(message)
|
|
218
230
|
|
|
219
231
|
|
|
220
232
|
class CloudwatchTaskHandler(FileTaskHandler, LoggingMixin):
|
|
@@ -291,4 +303,22 @@ class CloudwatchTaskHandler(FileTaskHandler, LoggingMixin):
|
|
|
291
303
|
) -> tuple[LogSourceInfo, LogMessages]:
|
|
292
304
|
stream_name = self._render_filename(task_instance, try_number)
|
|
293
305
|
messages, logs = self.io.read(stream_name, task_instance)
|
|
294
|
-
|
|
306
|
+
|
|
307
|
+
messages = [
|
|
308
|
+
f"Reading remote log from Cloudwatch log_group: {self.io.log_group} log_stream: {stream_name}"
|
|
309
|
+
]
|
|
310
|
+
try:
|
|
311
|
+
events = self.io.get_cloudwatch_logs(stream_name, task_instance)
|
|
312
|
+
logs = ["\n".join(self._event_to_str(event) for event in events)]
|
|
313
|
+
except Exception as e:
|
|
314
|
+
logs = []
|
|
315
|
+
messages.append(str(e))
|
|
316
|
+
|
|
317
|
+
return messages, logs
|
|
318
|
+
|
|
319
|
+
def _event_to_str(self, event: CloudWatchLogEvent) -> str:
|
|
320
|
+
event_dt = datetime.fromtimestamp(event["timestamp"] / 1000.0, tz=timezone.utc)
|
|
321
|
+
# Format a datetime object to a string in Zulu time without milliseconds.
|
|
322
|
+
formatted_event_dt = event_dt.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
323
|
+
message = event["message"]
|
|
324
|
+
return f"[{formatted_event_dt}] {message}"
|
|
@@ -20,11 +20,11 @@ import time
|
|
|
20
20
|
from datetime import datetime, timedelta
|
|
21
21
|
from typing import TYPE_CHECKING, cast
|
|
22
22
|
|
|
23
|
-
from airflow.exceptions import AirflowException
|
|
24
23
|
from airflow.providers.amazon.aws.hooks.appflow import AppflowHook
|
|
25
24
|
from airflow.providers.amazon.aws.operators.base_aws import AwsBaseOperator
|
|
26
25
|
from airflow.providers.amazon.aws.utils import datetime_to_epoch_ms
|
|
27
26
|
from airflow.providers.amazon.aws.utils.mixins import AwsBaseHookMixin, AwsHookParams, aws_template_fields
|
|
27
|
+
from airflow.providers.common.compat.sdk import AirflowException
|
|
28
28
|
from airflow.providers.common.compat.standard.operators import ShortCircuitOperator
|
|
29
29
|
|
|
30
30
|
if TYPE_CHECKING:
|
|
@@ -22,13 +22,13 @@ from typing import TYPE_CHECKING, Any
|
|
|
22
22
|
from urllib.parse import urlparse
|
|
23
23
|
|
|
24
24
|
from airflow.configuration import conf
|
|
25
|
-
from airflow.exceptions import AirflowException
|
|
26
25
|
from airflow.providers.amazon.aws.hooks.athena import AthenaHook
|
|
27
26
|
from airflow.providers.amazon.aws.links.athena import AthenaQueryResultsLink
|
|
28
27
|
from airflow.providers.amazon.aws.operators.base_aws import AwsBaseOperator
|
|
29
28
|
from airflow.providers.amazon.aws.triggers.athena import AthenaTrigger
|
|
30
29
|
from airflow.providers.amazon.aws.utils import validate_execute_complete_event
|
|
31
30
|
from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
|
|
31
|
+
from airflow.providers.common.compat.sdk import AirflowException
|
|
32
32
|
|
|
33
33
|
if TYPE_CHECKING:
|
|
34
34
|
from airflow.providers.common.compat.openlineage.facet import BaseFacet, Dataset, DatasetFacet
|
|
@@ -31,7 +31,6 @@ from datetime import timedelta
|
|
|
31
31
|
from typing import TYPE_CHECKING, Any
|
|
32
32
|
|
|
33
33
|
from airflow.configuration import conf
|
|
34
|
-
from airflow.exceptions import AirflowException
|
|
35
34
|
from airflow.providers.amazon.aws.hooks.batch_client import BatchClientHook
|
|
36
35
|
from airflow.providers.amazon.aws.links.batch import (
|
|
37
36
|
BatchJobDefinitionLink,
|
|
@@ -47,6 +46,7 @@ from airflow.providers.amazon.aws.triggers.batch import (
|
|
|
47
46
|
from airflow.providers.amazon.aws.utils import trim_none_values, validate_execute_complete_event
|
|
48
47
|
from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
|
|
49
48
|
from airflow.providers.amazon.aws.utils.task_log_fetcher import AwsTaskLogFetcher
|
|
49
|
+
from airflow.providers.common.compat.sdk import AirflowException
|
|
50
50
|
|
|
51
51
|
if TYPE_CHECKING:
|
|
52
52
|
from airflow.utils.context import Context
|
|
@@ -24,7 +24,6 @@ from typing import TYPE_CHECKING, Any
|
|
|
24
24
|
from botocore.exceptions import ClientError
|
|
25
25
|
|
|
26
26
|
from airflow.configuration import conf
|
|
27
|
-
from airflow.exceptions import AirflowException
|
|
28
27
|
from airflow.providers.amazon.aws.hooks.bedrock import (
|
|
29
28
|
BedrockAgentHook,
|
|
30
29
|
BedrockAgentRuntimeHook,
|
|
@@ -41,6 +40,7 @@ from airflow.providers.amazon.aws.triggers.bedrock import (
|
|
|
41
40
|
)
|
|
42
41
|
from airflow.providers.amazon.aws.utils import validate_execute_complete_event
|
|
43
42
|
from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
|
|
43
|
+
from airflow.providers.common.compat.sdk import AirflowException
|
|
44
44
|
from airflow.utils.helpers import prune_dict
|
|
45
45
|
from airflow.utils.timezone import utcnow
|
|
46
46
|
|
|
@@ -21,7 +21,6 @@ from functools import cached_property
|
|
|
21
21
|
from typing import TYPE_CHECKING, Any, ClassVar
|
|
22
22
|
|
|
23
23
|
from airflow.configuration import conf
|
|
24
|
-
from airflow.exceptions import AirflowException
|
|
25
24
|
from airflow.providers.amazon.aws.hooks.comprehend import ComprehendHook
|
|
26
25
|
from airflow.providers.amazon.aws.links.comprehend import (
|
|
27
26
|
ComprehendDocumentClassifierLink,
|
|
@@ -34,6 +33,7 @@ from airflow.providers.amazon.aws.triggers.comprehend import (
|
|
|
34
33
|
)
|
|
35
34
|
from airflow.providers.amazon.aws.utils import validate_execute_complete_event
|
|
36
35
|
from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
|
|
36
|
+
from airflow.providers.common.compat.sdk import AirflowException
|
|
37
37
|
from airflow.utils.timezone import utcnow
|
|
38
38
|
|
|
39
39
|
if TYPE_CHECKING:
|
|
@@ -23,12 +23,11 @@ import random
|
|
|
23
23
|
from collections.abc import Sequence
|
|
24
24
|
from typing import TYPE_CHECKING, Any
|
|
25
25
|
|
|
26
|
-
from airflow.exceptions import AirflowException
|
|
27
26
|
from airflow.providers.amazon.aws.hooks.datasync import DataSyncHook
|
|
28
27
|
from airflow.providers.amazon.aws.links.datasync import DataSyncTaskExecutionLink, DataSyncTaskLink
|
|
29
28
|
from airflow.providers.amazon.aws.operators.base_aws import AwsBaseOperator
|
|
30
29
|
from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
|
|
31
|
-
from airflow.providers.common.compat.sdk import AirflowTaskTimeout
|
|
30
|
+
from airflow.providers.common.compat.sdk import AirflowException, AirflowTaskTimeout
|
|
32
31
|
|
|
33
32
|
if TYPE_CHECKING:
|
|
34
33
|
from airflow.utils.context import Context
|
|
@@ -22,7 +22,6 @@ from datetime import datetime
|
|
|
22
22
|
from typing import TYPE_CHECKING, Any, ClassVar
|
|
23
23
|
|
|
24
24
|
from airflow.configuration import conf
|
|
25
|
-
from airflow.exceptions import AirflowException
|
|
26
25
|
from airflow.providers.amazon.aws.hooks.dms import DmsHook
|
|
27
26
|
from airflow.providers.amazon.aws.operators.base_aws import AwsBaseOperator
|
|
28
27
|
from airflow.providers.amazon.aws.triggers.dms import (
|
|
@@ -33,6 +32,7 @@ from airflow.providers.amazon.aws.triggers.dms import (
|
|
|
33
32
|
DmsReplicationTerminalStatusTrigger,
|
|
34
33
|
)
|
|
35
34
|
from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
|
|
35
|
+
from airflow.providers.common.compat.sdk import AirflowException
|
|
36
36
|
from airflow.utils.context import Context
|
|
37
37
|
|
|
38
38
|
if TYPE_CHECKING:
|