apache-airflow-providers-amazon 9.18.0rc2__py3-none-any.whl → 9.19.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. airflow/providers/amazon/__init__.py +1 -1
  2. airflow/providers/amazon/aws/auth_manager/avp/facade.py +1 -1
  3. airflow/providers/amazon/aws/bundles/s3.py +1 -1
  4. airflow/providers/amazon/aws/exceptions.py +1 -1
  5. airflow/providers/amazon/aws/executors/aws_lambda/lambda_executor.py +1 -2
  6. airflow/providers/amazon/aws/executors/batch/batch_executor.py +1 -2
  7. airflow/providers/amazon/aws/executors/ecs/ecs_executor.py +1 -2
  8. airflow/providers/amazon/aws/hooks/athena.py +1 -1
  9. airflow/providers/amazon/aws/hooks/athena_sql.py +1 -1
  10. airflow/providers/amazon/aws/hooks/base_aws.py +2 -6
  11. airflow/providers/amazon/aws/hooks/batch_client.py +1 -1
  12. airflow/providers/amazon/aws/hooks/batch_waiters.py +1 -1
  13. airflow/providers/amazon/aws/hooks/chime.py +1 -1
  14. airflow/providers/amazon/aws/hooks/comprehend.py +1 -1
  15. airflow/providers/amazon/aws/hooks/datasync.py +2 -2
  16. airflow/providers/amazon/aws/hooks/dynamodb.py +1 -1
  17. airflow/providers/amazon/aws/hooks/ec2.py +1 -1
  18. airflow/providers/amazon/aws/hooks/elasticache_replication_group.py +1 -1
  19. airflow/providers/amazon/aws/hooks/emr.py +1 -1
  20. airflow/providers/amazon/aws/hooks/glue.py +2 -1
  21. airflow/providers/amazon/aws/hooks/glue_catalog.py +1 -1
  22. airflow/providers/amazon/aws/hooks/logs.py +10 -2
  23. airflow/providers/amazon/aws/hooks/quicksight.py +1 -1
  24. airflow/providers/amazon/aws/hooks/rds.py +1 -1
  25. airflow/providers/amazon/aws/hooks/redshift_sql.py +11 -2
  26. airflow/providers/amazon/aws/hooks/s3.py +2 -1
  27. airflow/providers/amazon/aws/hooks/sagemaker.py +1 -1
  28. airflow/providers/amazon/aws/hooks/sagemaker_unified_studio.py +2 -3
  29. airflow/providers/amazon/aws/links/base_aws.py +1 -1
  30. airflow/providers/amazon/aws/links/emr.py +1 -1
  31. airflow/providers/amazon/aws/log/cloudwatch_task_handler.py +50 -20
  32. airflow/providers/amazon/aws/notifications/chime.py +1 -1
  33. airflow/providers/amazon/aws/operators/appflow.py +2 -2
  34. airflow/providers/amazon/aws/operators/athena.py +2 -2
  35. airflow/providers/amazon/aws/operators/batch.py +2 -2
  36. airflow/providers/amazon/aws/operators/bedrock.py +3 -4
  37. airflow/providers/amazon/aws/operators/cloud_formation.py +1 -1
  38. airflow/providers/amazon/aws/operators/comprehend.py +3 -4
  39. airflow/providers/amazon/aws/operators/datasync.py +2 -3
  40. airflow/providers/amazon/aws/operators/dms.py +2 -6
  41. airflow/providers/amazon/aws/operators/ec2.py +2 -2
  42. airflow/providers/amazon/aws/operators/ecs.py +2 -2
  43. airflow/providers/amazon/aws/operators/eks.py +3 -2
  44. airflow/providers/amazon/aws/operators/emr.py +3 -2
  45. airflow/providers/amazon/aws/operators/eventbridge.py +2 -2
  46. airflow/providers/amazon/aws/operators/glacier.py +1 -1
  47. airflow/providers/amazon/aws/operators/glue.py +2 -2
  48. airflow/providers/amazon/aws/operators/glue_crawler.py +3 -4
  49. airflow/providers/amazon/aws/operators/glue_databrew.py +2 -2
  50. airflow/providers/amazon/aws/operators/kinesis_analytics.py +2 -2
  51. airflow/providers/amazon/aws/operators/lambda_function.py +2 -2
  52. airflow/providers/amazon/aws/operators/mwaa.py +2 -2
  53. airflow/providers/amazon/aws/operators/neptune.py +2 -2
  54. airflow/providers/amazon/aws/operators/quicksight.py +1 -1
  55. airflow/providers/amazon/aws/operators/rds.py +2 -2
  56. airflow/providers/amazon/aws/operators/redshift_cluster.py +2 -2
  57. airflow/providers/amazon/aws/operators/redshift_data.py +2 -2
  58. airflow/providers/amazon/aws/operators/s3.py +2 -2
  59. airflow/providers/amazon/aws/operators/sagemaker.py +2 -2
  60. airflow/providers/amazon/aws/operators/sagemaker_unified_studio.py +2 -3
  61. airflow/providers/amazon/aws/operators/sns.py +1 -1
  62. airflow/providers/amazon/aws/operators/sqs.py +1 -1
  63. airflow/providers/amazon/aws/operators/ssm.py +1 -1
  64. airflow/providers/amazon/aws/operators/step_function.py +2 -2
  65. airflow/providers/amazon/aws/secrets/secrets_manager.py +2 -1
  66. airflow/providers/amazon/aws/secrets/systems_manager.py +2 -1
  67. airflow/providers/amazon/aws/sensors/athena.py +3 -4
  68. airflow/providers/amazon/aws/sensors/batch.py +2 -2
  69. airflow/providers/amazon/aws/sensors/bedrock.py +2 -2
  70. airflow/providers/amazon/aws/sensors/cloud_formation.py +2 -3
  71. airflow/providers/amazon/aws/sensors/comprehend.py +2 -2
  72. airflow/providers/amazon/aws/sensors/dms.py +2 -2
  73. airflow/providers/amazon/aws/sensors/dynamodb.py +1 -1
  74. airflow/providers/amazon/aws/sensors/ec2.py +2 -2
  75. airflow/providers/amazon/aws/sensors/ecs.py +2 -2
  76. airflow/providers/amazon/aws/sensors/eks.py +3 -2
  77. airflow/providers/amazon/aws/sensors/emr.py +2 -4
  78. airflow/providers/amazon/aws/sensors/glacier.py +2 -2
  79. airflow/providers/amazon/aws/sensors/glue.py +2 -2
  80. airflow/providers/amazon/aws/sensors/glue_catalog_partition.py +2 -2
  81. airflow/providers/amazon/aws/sensors/glue_crawler.py +2 -2
  82. airflow/providers/amazon/aws/sensors/kinesis_analytics.py +2 -2
  83. airflow/providers/amazon/aws/sensors/lambda_function.py +2 -2
  84. airflow/providers/amazon/aws/sensors/mwaa.py +2 -2
  85. airflow/providers/amazon/aws/sensors/opensearch_serverless.py +2 -2
  86. airflow/providers/amazon/aws/sensors/quicksight.py +2 -2
  87. airflow/providers/amazon/aws/sensors/rds.py +2 -2
  88. airflow/providers/amazon/aws/sensors/redshift_cluster.py +2 -2
  89. airflow/providers/amazon/aws/sensors/s3.py +5 -7
  90. airflow/providers/amazon/aws/sensors/sagemaker.py +2 -2
  91. airflow/providers/amazon/aws/sensors/sagemaker_unified_studio.py +2 -3
  92. airflow/providers/amazon/aws/sensors/sqs.py +2 -2
  93. airflow/providers/amazon/aws/sensors/ssm.py +1 -1
  94. airflow/providers/amazon/aws/sensors/step_function.py +2 -2
  95. airflow/providers/amazon/aws/transfers/azure_blob_to_s3.py +1 -1
  96. airflow/providers/amazon/aws/transfers/exasol_to_s3.py +1 -1
  97. airflow/providers/amazon/aws/transfers/ftp_to_s3.py +1 -1
  98. airflow/providers/amazon/aws/transfers/gcs_to_s3.py +2 -3
  99. airflow/providers/amazon/aws/transfers/glacier_to_gcs.py +1 -1
  100. airflow/providers/amazon/aws/transfers/google_api_to_s3.py +1 -1
  101. airflow/providers/amazon/aws/transfers/hive_to_dynamodb.py +1 -1
  102. airflow/providers/amazon/aws/transfers/http_to_s3.py +1 -1
  103. airflow/providers/amazon/aws/transfers/imap_attachment_to_s3.py +1 -1
  104. airflow/providers/amazon/aws/transfers/local_to_s3.py +1 -1
  105. airflow/providers/amazon/aws/transfers/mongo_to_s3.py +1 -1
  106. airflow/providers/amazon/aws/transfers/redshift_to_s3.py +2 -3
  107. airflow/providers/amazon/aws/transfers/s3_to_dynamodb.py +2 -3
  108. airflow/providers/amazon/aws/transfers/s3_to_ftp.py +1 -1
  109. airflow/providers/amazon/aws/transfers/s3_to_redshift.py +2 -3
  110. airflow/providers/amazon/aws/transfers/s3_to_sftp.py +1 -1
  111. airflow/providers/amazon/aws/transfers/s3_to_sql.py +2 -3
  112. airflow/providers/amazon/aws/transfers/salesforce_to_s3.py +1 -1
  113. airflow/providers/amazon/aws/transfers/sftp_to_s3.py +1 -1
  114. airflow/providers/amazon/aws/transfers/sql_to_s3.py +6 -6
  115. airflow/providers/amazon/aws/triggers/ecs.py +1 -1
  116. airflow/providers/amazon/aws/triggers/eks.py +1 -1
  117. airflow/providers/amazon/aws/triggers/sagemaker.py +1 -1
  118. airflow/providers/amazon/aws/triggers/sqs.py +1 -1
  119. airflow/providers/amazon/aws/utils/__init__.py +1 -1
  120. airflow/providers/amazon/aws/utils/connection_wrapper.py +1 -1
  121. airflow/providers/amazon/aws/utils/waiter.py +1 -1
  122. airflow/providers/amazon/aws/utils/waiter_with_logging.py +1 -1
  123. {apache_airflow_providers_amazon-9.18.0rc2.dist-info → apache_airflow_providers_amazon-9.19.0.dist-info}/METADATA +18 -14
  124. {apache_airflow_providers_amazon-9.18.0rc2.dist-info → apache_airflow_providers_amazon-9.19.0.dist-info}/RECORD +128 -128
  125. {apache_airflow_providers_amazon-9.18.0rc2.dist-info → apache_airflow_providers_amazon-9.19.0.dist-info}/WHEEL +0 -0
  126. {apache_airflow_providers_amazon-9.18.0rc2.dist-info → apache_airflow_providers_amazon-9.19.0.dist-info}/entry_points.txt +0 -0
  127. {apache_airflow_providers_amazon-9.18.0rc2.dist-info → apache_airflow_providers_amazon-9.19.0.dist-info}/licenses/LICENSE +0 -0
  128. {apache_airflow_providers_amazon-9.18.0rc2.dist-info → apache_airflow_providers_amazon-9.19.0.dist-info}/licenses/NOTICE +0 -0
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
29
29
 
30
30
  __all__ = ["__version__"]
31
31
 
32
- __version__ = "9.18.0"
32
+ __version__ = "9.19.0"
33
33
 
34
34
  if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
35
35
  "2.11.0"
@@ -23,7 +23,6 @@ from pathlib import Path
23
23
  from typing import TYPE_CHECKING, TypedDict
24
24
 
25
25
  from airflow.configuration import conf
26
- from airflow.exceptions import AirflowException
27
26
  from airflow.providers.amazon.aws.auth_manager.avp.entities import AvpEntities, get_action_id, get_entity_type
28
27
  from airflow.providers.amazon.aws.auth_manager.constants import (
29
28
  CONF_AVP_POLICY_STORE_ID_KEY,
@@ -32,6 +31,7 @@ from airflow.providers.amazon.aws.auth_manager.constants import (
32
31
  CONF_SECTION_NAME,
33
32
  )
34
33
  from airflow.providers.amazon.aws.hooks.verified_permissions import VerifiedPermissionsHook
34
+ from airflow.providers.common.compat.sdk import AirflowException
35
35
  from airflow.utils.helpers import prune_dict
36
36
  from airflow.utils.log.logging_mixin import LoggingMixin
37
37
 
@@ -22,9 +22,9 @@ from pathlib import Path
22
22
  import structlog
23
23
 
24
24
  from airflow.dag_processing.bundles.base import BaseDagBundle
25
- from airflow.exceptions import AirflowException
26
25
  from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
27
26
  from airflow.providers.amazon.aws.hooks.s3 import S3Hook
27
+ from airflow.providers.common.compat.sdk import AirflowException
28
28
 
29
29
 
30
30
  class S3DagBundle(BaseDagBundle):
@@ -17,7 +17,7 @@
17
17
  # under the License.
18
18
  from __future__ import annotations
19
19
 
20
- from airflow.exceptions import AirflowException
20
+ from airflow.providers.common.compat.sdk import AirflowException
21
21
 
22
22
  # Note: Any AirflowException raised is expected to cause the TaskInstance
23
23
  # to be marked in an ERROR state
@@ -26,7 +26,6 @@ from boto3.session import NoCredentialsError
26
26
  from botocore.utils import ClientError
27
27
 
28
28
  from airflow.configuration import conf
29
- from airflow.exceptions import AirflowException
30
29
  from airflow.executors.base_executor import BaseExecutor
31
30
  from airflow.models.taskinstancekey import TaskInstanceKey
32
31
  from airflow.providers.amazon.aws.executors.aws_lambda.utils import (
@@ -42,7 +41,7 @@ from airflow.providers.amazon.aws.executors.utils.exponential_backoff_retry impo
42
41
  )
43
42
  from airflow.providers.amazon.aws.hooks.lambda_function import LambdaHook
44
43
  from airflow.providers.amazon.aws.hooks.sqs import SqsHook
45
- from airflow.stats import Stats
44
+ from airflow.providers.common.compat.sdk import AirflowException, Stats
46
45
 
47
46
  try:
48
47
  from airflow.sdk import timezone
@@ -29,7 +29,6 @@ from typing import TYPE_CHECKING, Any
29
29
  from botocore.exceptions import ClientError, NoCredentialsError
30
30
 
31
31
  from airflow.configuration import conf
32
- from airflow.exceptions import AirflowException
33
32
  from airflow.executors.base_executor import BaseExecutor
34
33
  from airflow.providers.amazon.aws.executors.utils.exponential_backoff_retry import (
35
34
  calculate_next_attempt_delay,
@@ -37,7 +36,7 @@ from airflow.providers.amazon.aws.executors.utils.exponential_backoff_retry impo
37
36
  )
38
37
  from airflow.providers.amazon.aws.hooks.batch_client import BatchClientHook
39
38
  from airflow.providers.amazon.version_compat import AIRFLOW_V_3_0_PLUS
40
- from airflow.stats import Stats
39
+ from airflow.providers.common.compat.sdk import AirflowException, Stats
41
40
 
42
41
  try:
43
42
  from airflow.sdk import timezone
@@ -32,7 +32,6 @@ from typing import TYPE_CHECKING
32
32
 
33
33
  from botocore.exceptions import ClientError, NoCredentialsError
34
34
 
35
- from airflow.exceptions import AirflowException
36
35
  from airflow.executors.base_executor import BaseExecutor
37
36
  from airflow.providers.amazon.aws.executors.ecs.boto_schema import BotoDescribeTasksSchema, BotoRunTaskSchema
38
37
  from airflow.providers.amazon.aws.executors.ecs.utils import (
@@ -49,7 +48,7 @@ from airflow.providers.amazon.aws.executors.utils.exponential_backoff_retry impo
49
48
  )
50
49
  from airflow.providers.amazon.aws.hooks.ecs import EcsHook
51
50
  from airflow.providers.amazon.version_compat import AIRFLOW_V_3_0_PLUS
52
- from airflow.stats import Stats
51
+ from airflow.providers.common.compat.sdk import AirflowException, Stats
53
52
 
54
53
  try:
55
54
  from airflow.sdk import timezone
@@ -28,9 +28,9 @@ from __future__ import annotations
28
28
  from collections.abc import Collection
29
29
  from typing import TYPE_CHECKING, Any
30
30
 
31
- from airflow.exceptions import AirflowException
32
31
  from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
33
32
  from airflow.providers.amazon.aws.utils.waiter_with_logging import wait
33
+ from airflow.providers.common.compat.sdk import AirflowException
34
34
 
35
35
  if TYPE_CHECKING:
36
36
  from botocore.paginate import PageIterator
@@ -23,9 +23,9 @@ from typing import TYPE_CHECKING, Any
23
23
  import pyathena
24
24
  from sqlalchemy.engine.url import URL
25
25
 
26
- from airflow.exceptions import AirflowException, AirflowNotFoundException
27
26
  from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
28
27
  from airflow.providers.amazon.aws.utils.connection_wrapper import AwsConnectionWrapper
28
+ from airflow.providers.common.compat.sdk import AirflowException, AirflowNotFoundException
29
29
  from airflow.providers.common.sql.hooks.sql import DbApiHook
30
30
 
31
31
  if TYPE_CHECKING:
@@ -52,15 +52,11 @@ from dateutil.tz import tzlocal
52
52
  from slugify import slugify
53
53
 
54
54
  from airflow.configuration import conf
55
- from airflow.exceptions import (
56
- AirflowException,
57
- AirflowNotFoundException,
58
- AirflowProviderDeprecationWarning,
59
- )
55
+ from airflow.exceptions import AirflowProviderDeprecationWarning
60
56
  from airflow.providers.amazon.aws.utils.connection_wrapper import AwsConnectionWrapper
61
57
  from airflow.providers.amazon.aws.utils.identifiers import generate_uuid
62
58
  from airflow.providers.amazon.aws.utils.suppress import return_on_error
63
- from airflow.providers.common.compat.sdk import BaseHook
59
+ from airflow.providers.common.compat.sdk import AirflowException, AirflowNotFoundException, BaseHook
64
60
  from airflow.providers_manager import ProvidersManager
65
61
  from airflow.utils.helpers import exactly_one
66
62
  from airflow.utils.log.logging_mixin import LoggingMixin
@@ -37,8 +37,8 @@ import botocore.client
37
37
  import botocore.exceptions
38
38
  import botocore.waiter
39
39
 
40
- from airflow.exceptions import AirflowException
41
40
  from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
41
+ from airflow.providers.common.compat.sdk import AirflowException
42
42
 
43
43
  if TYPE_CHECKING:
44
44
  from airflow.providers.amazon.aws.utils.task_log_fetcher import AwsTaskLogFetcher
@@ -36,8 +36,8 @@ from typing import TYPE_CHECKING, Any
36
36
  import botocore.exceptions
37
37
  import botocore.waiter
38
38
 
39
- from airflow.exceptions import AirflowException
40
39
  from airflow.providers.amazon.aws.hooks.batch_client import BatchClientHook
40
+ from airflow.providers.common.compat.sdk import AirflowException
41
41
 
42
42
  if TYPE_CHECKING:
43
43
  from airflow.providers.amazon.aws.utils.task_log_fetcher import AwsTaskLogFetcher
@@ -25,7 +25,7 @@ import re
25
25
  from functools import cached_property
26
26
  from typing import Any
27
27
 
28
- from airflow.exceptions import AirflowException
28
+ from airflow.providers.common.compat.sdk import AirflowException
29
29
  from airflow.providers.http.hooks.http import HttpHook
30
30
 
31
31
 
@@ -16,8 +16,8 @@
16
16
  # under the License.
17
17
  from __future__ import annotations
18
18
 
19
- from airflow.exceptions import AirflowException
20
19
  from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
20
+ from airflow.providers.common.compat.sdk import AirflowException
21
21
 
22
22
 
23
23
  class ComprehendHook(AwsBaseHook):
@@ -21,9 +21,9 @@ from __future__ import annotations
21
21
  import time
22
22
  from urllib.parse import urlsplit
23
23
 
24
- from airflow.exceptions import AirflowBadRequest, AirflowException
24
+ from airflow.exceptions import AirflowBadRequest
25
25
  from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
26
- from airflow.providers.common.compat.sdk import AirflowTaskTimeout
26
+ from airflow.providers.common.compat.sdk import AirflowException, AirflowTaskTimeout
27
27
 
28
28
 
29
29
  class DataSyncHook(AwsBaseHook):
@@ -25,8 +25,8 @@ from typing import TYPE_CHECKING
25
25
 
26
26
  from botocore.exceptions import ClientError
27
27
 
28
- from airflow.exceptions import AirflowException
29
28
  from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
29
+ from airflow.providers.common.compat.sdk import AirflowException
30
30
 
31
31
  if TYPE_CHECKING:
32
32
  from botocore.client import BaseClient
@@ -22,8 +22,8 @@ import time
22
22
  from collections.abc import Callable
23
23
  from typing import ParamSpec, TypeVar
24
24
 
25
- from airflow.exceptions import AirflowException
26
25
  from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
26
+ from airflow.providers.common.compat.sdk import AirflowException
27
27
 
28
28
  PS = ParamSpec("PS")
29
29
  RT = TypeVar("RT")
@@ -19,8 +19,8 @@ from __future__ import annotations
19
19
 
20
20
  import time
21
21
 
22
- from airflow.exceptions import AirflowException
23
22
  from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
23
+ from airflow.providers.common.compat.sdk import AirflowException
24
24
 
25
25
 
26
26
  class ElastiCacheReplicationGroupHook(AwsBaseHook):
@@ -26,9 +26,9 @@ import tenacity
26
26
  from botocore.exceptions import ClientError
27
27
  from tenacity import retry_if_exception, stop_after_attempt, wait_fixed
28
28
 
29
- from airflow.exceptions import AirflowException, AirflowNotFoundException
30
29
  from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
31
30
  from airflow.providers.amazon.aws.utils.waiter_with_logging import wait
31
+ from airflow.providers.common.compat.sdk import AirflowException, AirflowNotFoundException
32
32
 
33
33
 
34
34
  class EmrHook(AwsBaseHook):
@@ -33,9 +33,10 @@ from tenacity import (
33
33
  wait_exponential,
34
34
  )
35
35
 
36
- from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
36
+ from airflow.exceptions import AirflowProviderDeprecationWarning
37
37
  from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
38
38
  from airflow.providers.amazon.aws.hooks.logs import AwsLogsHook
39
+ from airflow.providers.common.compat.sdk import AirflowException
39
40
 
40
41
  DEFAULT_LOG_SUFFIX = "output"
41
42
  ERROR_LOG_SUFFIX = "error"
@@ -23,8 +23,8 @@ from typing import Any
23
23
 
24
24
  from botocore.exceptions import ClientError
25
25
 
26
- from airflow.exceptions import AirflowException
27
26
  from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
27
+ from airflow.providers.common.compat.sdk import AirflowException
28
28
 
29
29
 
30
30
  class GlueCatalogHook(AwsBaseHook):
@@ -19,7 +19,7 @@ from __future__ import annotations
19
19
 
20
20
  import asyncio
21
21
  from collections.abc import AsyncGenerator, Generator
22
- from typing import Any
22
+ from typing import Any, TypedDict
23
23
 
24
24
  from botocore.exceptions import ClientError
25
25
 
@@ -35,6 +35,14 @@ from airflow.utils.helpers import prune_dict
35
35
  NUM_CONSECUTIVE_EMPTY_RESPONSE_EXIT_THRESHOLD = 3
36
36
 
37
37
 
38
+ class CloudWatchLogEvent(TypedDict):
39
+ """TypedDict for CloudWatch Log Event."""
40
+
41
+ timestamp: int
42
+ message: str
43
+ ingestionTime: int
44
+
45
+
38
46
  class AwsLogsHook(AwsBaseHook):
39
47
  """
40
48
  Interact with Amazon CloudWatch Logs.
@@ -67,7 +75,7 @@ class AwsLogsHook(AwsBaseHook):
67
75
  start_from_head: bool | None = None,
68
76
  continuation_token: ContinuationToken | None = None,
69
77
  end_time: int | None = None,
70
- ) -> Generator:
78
+ ) -> Generator[CloudWatchLogEvent, None, None]:
71
79
  """
72
80
  Return a generator for log items in a single stream; yields all items available at the current moment.
73
81
 
@@ -21,8 +21,8 @@ import time
21
21
 
22
22
  from botocore.exceptions import ClientError
23
23
 
24
- from airflow.exceptions import AirflowException
25
24
  from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
25
+ from airflow.providers.common.compat.sdk import AirflowException
26
26
 
27
27
 
28
28
  class QuickSightHook(AwsBaseHook):
@@ -23,9 +23,9 @@ import time
23
23
  from collections.abc import Callable
24
24
  from typing import TYPE_CHECKING
25
25
 
26
- from airflow.exceptions import AirflowException, AirflowNotFoundException
27
26
  from airflow.providers.amazon.aws.hooks.base_aws import AwsGenericHook
28
27
  from airflow.providers.amazon.aws.utils.waiter_with_logging import wait
28
+ from airflow.providers.common.compat.sdk import AirflowException, AirflowNotFoundException
29
29
 
30
30
  if TYPE_CHECKING:
31
31
  from mypy_boto3_rds import RDSClient # noqa: F401
@@ -20,12 +20,13 @@ from functools import cached_property
20
20
  from typing import TYPE_CHECKING
21
21
 
22
22
  import redshift_connector
23
- from redshift_connector import Connection as RedshiftConnection
23
+ import tenacity
24
+ from redshift_connector import Connection as RedshiftConnection, InterfaceError, OperationalError
24
25
  from sqlalchemy import create_engine
25
26
  from sqlalchemy.engine.url import URL
26
27
 
27
- from airflow.exceptions import AirflowException
28
28
  from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
29
+ from airflow.providers.common.compat.sdk import AirflowException
29
30
  from airflow.providers.common.sql.hooks.sql import DbApiHook
30
31
 
31
32
  if TYPE_CHECKING:
@@ -206,6 +207,14 @@ class RedshiftSQLHook(DbApiHook):
206
207
  pk_columns = [row[0] for row in self.get_records(sql, (schema, table))]
207
208
  return pk_columns or None
208
209
 
210
+ @tenacity.retry(
211
+ stop=tenacity.stop_after_attempt(5),
212
+ wait=tenacity.wait_exponential(max=20),
213
+ # OperationalError is thrown when the connection times out
214
+ # InterfaceError is thrown when the connection is refused
215
+ retry=tenacity.retry_if_exception_type((OperationalError, InterfaceError)),
216
+ reraise=True,
217
+ )
209
218
  def get_conn(self) -> RedshiftConnection:
210
219
  """Get a ``redshift_connector.Connection`` object."""
211
220
  conn_params = self._get_conn_params()
@@ -56,11 +56,12 @@ from asgiref.sync import sync_to_async
56
56
  from boto3.s3.transfer import S3Transfer, TransferConfig
57
57
  from botocore.exceptions import ClientError
58
58
 
59
- from airflow.exceptions import AirflowException, AirflowNotFoundException, AirflowProviderDeprecationWarning
59
+ from airflow.exceptions import AirflowProviderDeprecationWarning
60
60
  from airflow.providers.amazon.aws.exceptions import S3HookUriParseFailure
61
61
  from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
62
62
  from airflow.providers.amazon.aws.utils.tags import format_tags
63
63
  from airflow.providers.common.compat.lineage.hook import get_hook_lineage_collector
64
+ from airflow.providers.common.compat.sdk import AirflowException, AirflowNotFoundException
64
65
  from airflow.utils.helpers import chunks
65
66
 
66
67
  logger = logging.getLogger(__name__)
@@ -31,11 +31,11 @@ from typing import Any, cast
31
31
  from asgiref.sync import sync_to_async
32
32
  from botocore.exceptions import ClientError
33
33
 
34
- from airflow.exceptions import AirflowException
35
34
  from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
36
35
  from airflow.providers.amazon.aws.hooks.logs import AwsLogsHook
37
36
  from airflow.providers.amazon.aws.hooks.s3 import S3Hook
38
37
  from airflow.providers.amazon.aws.utils.tags import format_tags
38
+ from airflow.providers.common.compat.sdk import AirflowException
39
39
 
40
40
  try:
41
41
  from airflow.sdk import timezone
@@ -24,9 +24,8 @@ import time
24
24
  from sagemaker_studio import ClientConfig
25
25
  from sagemaker_studio.sagemaker_studio_api import SageMakerStudioAPI
26
26
 
27
- from airflow.exceptions import AirflowException
28
27
  from airflow.providers.amazon.aws.utils.sagemaker_unified_studio import is_local_runner
29
- from airflow.providers.common.compat.sdk import BaseHook
28
+ from airflow.providers.common.compat.sdk import AirflowException, BaseHook
30
29
 
31
30
 
32
31
  class SageMakerNotebookHook(BaseHook):
@@ -124,7 +123,7 @@ class SageMakerNotebookHook(BaseHook):
124
123
  if self.compute:
125
124
  start_execution_params["compute"] = self.compute
126
125
  else:
127
- start_execution_params["compute"] = {"instance_type": "ml.m4.xlarge"}
126
+ start_execution_params["compute"] = {"instance_type": "ml.m6i.xlarge"}
128
127
 
129
128
  print(start_execution_params)
130
129
  return self._sagemaker_studio.execution_client.start_execution(**start_execution_params)
@@ -25,7 +25,7 @@ from airflow.providers.common.compat.sdk import BaseOperatorLink, XCom
25
25
  if TYPE_CHECKING:
26
26
  from airflow.models import BaseOperator
27
27
  from airflow.models.taskinstancekey import TaskInstanceKey
28
- from airflow.utils.context import Context
28
+ from airflow.sdk import Context
29
29
 
30
30
 
31
31
  BASE_AWS_CONSOLE_LINK = "https://console.{aws_domain}"
@@ -19,10 +19,10 @@ from __future__ import annotations
19
19
  from typing import TYPE_CHECKING, Any
20
20
  from urllib.parse import ParseResult, quote_plus, urlparse
21
21
 
22
- from airflow.exceptions import AirflowException
23
22
  from airflow.providers.amazon.aws.hooks.emr import EmrServerlessHook
24
23
  from airflow.providers.amazon.aws.hooks.s3 import S3Hook
25
24
  from airflow.providers.amazon.aws.links.base_aws import BASE_AWS_CONSOLE_LINK, BaseAwsLink
25
+ from airflow.providers.common.compat.sdk import AirflowException
26
26
  from airflow.utils.helpers import exactly_one
27
27
 
28
28
  if TYPE_CHECKING:
@@ -22,6 +22,7 @@ import copy
22
22
  import json
23
23
  import logging
24
24
  import os
25
+ from collections.abc import Generator
25
26
  from datetime import date, datetime, timedelta, timezone
26
27
  from functools import cached_property
27
28
  from pathlib import Path
@@ -40,8 +41,15 @@ if TYPE_CHECKING:
40
41
  import structlog.typing
41
42
 
42
43
  from airflow.models.taskinstance import TaskInstance
44
+ from airflow.providers.amazon.aws.hooks.logs import CloudWatchLogEvent
43
45
  from airflow.sdk.types import RuntimeTaskInstanceProtocol as RuntimeTI
44
- from airflow.utils.log.file_task_handler import LogMessages, LogSourceInfo
46
+ from airflow.utils.log.file_task_handler import (
47
+ LogMessages,
48
+ LogResponse,
49
+ LogSourceInfo,
50
+ RawLogStream,
51
+ StreamingLogResponse,
52
+ )
45
53
 
46
54
 
47
55
  def json_serialize_legacy(value: Any) -> str | None:
@@ -163,20 +171,31 @@ class CloudWatchRemoteLogIO(LoggingMixin): # noqa: D101
163
171
  self.close()
164
172
  return
165
173
 
166
- def read(self, relative_path, ti: RuntimeTI) -> tuple[LogSourceInfo, LogMessages | None]:
167
- logs: LogMessages | None = []
174
+ def read(self, relative_path: str, ti: RuntimeTI) -> LogResponse:
175
+ messages, logs = self.stream(relative_path, ti)
176
+ str_logs: list[str] = [f"{msg}\n" for group in logs for msg in group]
177
+
178
+ return messages, str_logs
179
+
180
+ def stream(self, relative_path: str, ti: RuntimeTI) -> StreamingLogResponse:
181
+ logs: list[RawLogStream] = []
168
182
  messages = [
169
183
  f"Reading remote log from Cloudwatch log_group: {self.log_group} log_stream: {relative_path}"
170
184
  ]
171
185
  try:
172
- logs = [self.get_cloudwatch_logs(relative_path, ti)]
186
+ gen: RawLogStream = (
187
+ self._parse_log_event_as_dumped_json(event)
188
+ for event in self.get_cloudwatch_logs(relative_path, ti)
189
+ )
190
+ logs = [gen]
173
191
  except Exception as e:
174
- logs = None
175
192
  messages.append(str(e))
176
193
 
177
194
  return messages, logs
178
195
 
179
- def get_cloudwatch_logs(self, stream_name: str, task_instance: RuntimeTI):
196
+ def get_cloudwatch_logs(
197
+ self, stream_name: str, task_instance: RuntimeTI
198
+ ) -> Generator[CloudWatchLogEvent, None, None]:
180
199
  """
181
200
  Return all logs from the given log stream.
182
201
 
@@ -192,29 +211,22 @@ class CloudWatchRemoteLogIO(LoggingMixin): # noqa: D101
192
211
  if (end_date := getattr(task_instance, "end_date", None)) is None
193
212
  else datetime_to_epoch_utc_ms(end_date + timedelta(seconds=30))
194
213
  )
195
- events = self.hook.get_log_events(
214
+ return self.hook.get_log_events(
196
215
  log_group=self.log_group,
197
216
  log_stream_name=stream_name,
198
217
  end_time=end_time,
199
218
  )
200
- return "\n".join(self._event_to_str(event) for event in events)
201
219
 
202
- def _event_to_dict(self, event: dict) -> dict:
220
+ def _parse_log_event_as_dumped_json(self, event: CloudWatchLogEvent) -> str:
203
221
  event_dt = datetime.fromtimestamp(event["timestamp"] / 1000.0, tz=timezone.utc).isoformat()
204
- message = event["message"]
222
+ event_msg = event["message"]
205
223
  try:
206
- message = json.loads(message)
224
+ message = json.loads(event_msg)
207
225
  message["timestamp"] = event_dt
208
- return message
209
226
  except Exception:
210
- return {"timestamp": event_dt, "event": message}
227
+ message = {"timestamp": event_dt, "event": event_msg}
211
228
 
212
- def _event_to_str(self, event: dict) -> str:
213
- event_dt = datetime.fromtimestamp(event["timestamp"] / 1000.0, tz=timezone.utc)
214
- # Format a datetime object to a string in Zulu time without milliseconds.
215
- formatted_event_dt = event_dt.strftime("%Y-%m-%dT%H:%M:%SZ")
216
- message = event["message"]
217
- return f"[{formatted_event_dt}] {message}"
229
+ return json.dumps(message)
218
230
 
219
231
 
220
232
  class CloudwatchTaskHandler(FileTaskHandler, LoggingMixin):
@@ -291,4 +303,22 @@ class CloudwatchTaskHandler(FileTaskHandler, LoggingMixin):
291
303
  ) -> tuple[LogSourceInfo, LogMessages]:
292
304
  stream_name = self._render_filename(task_instance, try_number)
293
305
  messages, logs = self.io.read(stream_name, task_instance)
294
- return messages, logs or []
306
+
307
+ messages = [
308
+ f"Reading remote log from Cloudwatch log_group: {self.io.log_group} log_stream: {stream_name}"
309
+ ]
310
+ try:
311
+ events = self.io.get_cloudwatch_logs(stream_name, task_instance)
312
+ logs = ["\n".join(self._event_to_str(event) for event in events)]
313
+ except Exception as e:
314
+ logs = []
315
+ messages.append(str(e))
316
+
317
+ return messages, logs
318
+
319
+ def _event_to_str(self, event: CloudWatchLogEvent) -> str:
320
+ event_dt = datetime.fromtimestamp(event["timestamp"] / 1000.0, tz=timezone.utc)
321
+ # Format a datetime object to a string in Zulu time without milliseconds.
322
+ formatted_event_dt = event_dt.strftime("%Y-%m-%dT%H:%M:%SZ")
323
+ message = event["message"]
324
+ return f"[{formatted_event_dt}] {message}"
@@ -24,7 +24,7 @@ from airflow.providers.amazon.aws.hooks.chime import ChimeWebhookHook
24
24
  from airflow.providers.common.compat.notifier import BaseNotifier
25
25
 
26
26
  if TYPE_CHECKING:
27
- from airflow.utils.context import Context
27
+ from airflow.sdk import Context
28
28
 
29
29
 
30
30
  class ChimeNotifier(BaseNotifier):
@@ -20,11 +20,11 @@ import time
20
20
  from datetime import datetime, timedelta
21
21
  from typing import TYPE_CHECKING, cast
22
22
 
23
- from airflow.exceptions import AirflowException
24
23
  from airflow.providers.amazon.aws.hooks.appflow import AppflowHook
25
24
  from airflow.providers.amazon.aws.operators.base_aws import AwsBaseOperator
26
25
  from airflow.providers.amazon.aws.utils import datetime_to_epoch_ms
27
26
  from airflow.providers.amazon.aws.utils.mixins import AwsBaseHookMixin, AwsHookParams, aws_template_fields
27
+ from airflow.providers.common.compat.sdk import AirflowException
28
28
  from airflow.providers.common.compat.standard.operators import ShortCircuitOperator
29
29
 
30
30
  if TYPE_CHECKING:
@@ -34,7 +34,7 @@ if TYPE_CHECKING:
34
34
  TaskTypeDef,
35
35
  )
36
36
 
37
- from airflow.utils.context import Context
37
+ from airflow.sdk import Context
38
38
 
39
39
  SUPPORTED_SOURCES = {"salesforce", "zendesk"}
40
40
  MANDATORY_FILTER_DATE_MSG = "The filter_date argument is mandatory for {entity}!"
@@ -22,18 +22,18 @@ from typing import TYPE_CHECKING, Any
22
22
  from urllib.parse import urlparse
23
23
 
24
24
  from airflow.configuration import conf
25
- from airflow.exceptions import AirflowException
26
25
  from airflow.providers.amazon.aws.hooks.athena import AthenaHook
27
26
  from airflow.providers.amazon.aws.links.athena import AthenaQueryResultsLink
28
27
  from airflow.providers.amazon.aws.operators.base_aws import AwsBaseOperator
29
28
  from airflow.providers.amazon.aws.triggers.athena import AthenaTrigger
30
29
  from airflow.providers.amazon.aws.utils import validate_execute_complete_event
31
30
  from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
31
+ from airflow.providers.common.compat.sdk import AirflowException
32
32
 
33
33
  if TYPE_CHECKING:
34
34
  from airflow.providers.common.compat.openlineage.facet import BaseFacet, Dataset, DatasetFacet
35
35
  from airflow.providers.openlineage.extractors.base import OperatorLineage
36
- from airflow.utils.context import Context
36
+ from airflow.sdk import Context
37
37
 
38
38
 
39
39
  class AthenaOperator(AwsBaseOperator[AthenaHook]):
@@ -31,7 +31,6 @@ from datetime import timedelta
31
31
  from typing import TYPE_CHECKING, Any
32
32
 
33
33
  from airflow.configuration import conf
34
- from airflow.exceptions import AirflowException
35
34
  from airflow.providers.amazon.aws.hooks.batch_client import BatchClientHook
36
35
  from airflow.providers.amazon.aws.links.batch import (
37
36
  BatchJobDefinitionLink,
@@ -47,9 +46,10 @@ from airflow.providers.amazon.aws.triggers.batch import (
47
46
  from airflow.providers.amazon.aws.utils import trim_none_values, validate_execute_complete_event
48
47
  from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
49
48
  from airflow.providers.amazon.aws.utils.task_log_fetcher import AwsTaskLogFetcher
49
+ from airflow.providers.common.compat.sdk import AirflowException
50
50
 
51
51
  if TYPE_CHECKING:
52
- from airflow.utils.context import Context
52
+ from airflow.sdk import Context
53
53
 
54
54
 
55
55
  class BatchOperator(AwsBaseOperator[BatchClientHook]):