apache-airflow-providers-google 10.7.0rc1__py3-none-any.whl → 10.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. airflow/providers/google/__init__.py +1 -1
  2. airflow/providers/google/ads/hooks/ads.py +10 -6
  3. airflow/providers/google/cloud/_internal_client/secret_manager_client.py +4 -1
  4. airflow/providers/google/cloud/example_dags/example_cloud_sql_query.py +31 -34
  5. airflow/providers/google/cloud/hooks/automl.py +11 -9
  6. airflow/providers/google/cloud/hooks/bigquery.py +30 -36
  7. airflow/providers/google/cloud/hooks/bigquery_dts.py +5 -3
  8. airflow/providers/google/cloud/hooks/bigtable.py +11 -8
  9. airflow/providers/google/cloud/hooks/cloud_batch.py +5 -3
  10. airflow/providers/google/cloud/hooks/cloud_build.py +6 -4
  11. airflow/providers/google/cloud/hooks/cloud_composer.py +14 -10
  12. airflow/providers/google/cloud/hooks/cloud_memorystore.py +5 -3
  13. airflow/providers/google/cloud/hooks/cloud_run.py +5 -3
  14. airflow/providers/google/cloud/hooks/cloud_sql.py +11 -14
  15. airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +8 -6
  16. airflow/providers/google/cloud/hooks/compute.py +5 -3
  17. airflow/providers/google/cloud/hooks/compute_ssh.py +1 -1
  18. airflow/providers/google/cloud/hooks/datacatalog.py +5 -3
  19. airflow/providers/google/cloud/hooks/dataflow.py +8 -11
  20. airflow/providers/google/cloud/hooks/dataform.py +4 -2
  21. airflow/providers/google/cloud/hooks/datafusion.py +24 -6
  22. airflow/providers/google/cloud/hooks/dataplex.py +75 -6
  23. airflow/providers/google/cloud/hooks/dataproc.py +9 -7
  24. airflow/providers/google/cloud/hooks/dataproc_metastore.py +8 -6
  25. airflow/providers/google/cloud/hooks/dlp.py +139 -137
  26. airflow/providers/google/cloud/hooks/gcs.py +15 -20
  27. airflow/providers/google/cloud/hooks/kms.py +4 -2
  28. airflow/providers/google/cloud/hooks/kubernetes_engine.py +34 -34
  29. airflow/providers/google/cloud/hooks/looker.py +4 -1
  30. airflow/providers/google/cloud/hooks/mlengine.py +8 -6
  31. airflow/providers/google/cloud/hooks/natural_language.py +4 -2
  32. airflow/providers/google/cloud/hooks/os_login.py +9 -7
  33. airflow/providers/google/cloud/hooks/pubsub.py +13 -11
  34. airflow/providers/google/cloud/hooks/spanner.py +7 -5
  35. airflow/providers/google/cloud/hooks/speech_to_text.py +4 -2
  36. airflow/providers/google/cloud/hooks/stackdriver.py +6 -5
  37. airflow/providers/google/cloud/hooks/tasks.py +5 -3
  38. airflow/providers/google/cloud/hooks/text_to_speech.py +4 -2
  39. airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +7 -5
  40. airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +6 -4
  41. airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +11 -9
  42. airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +12 -10
  43. airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +8 -6
  44. airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +6 -4
  45. airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +7 -5
  46. airflow/providers/google/cloud/hooks/video_intelligence.py +5 -3
  47. airflow/providers/google/cloud/hooks/vision.py +5 -3
  48. airflow/providers/google/cloud/hooks/workflows.py +8 -6
  49. airflow/providers/google/cloud/links/bigquery.py +1 -1
  50. airflow/providers/google/cloud/links/bigquery_dts.py +1 -1
  51. airflow/providers/google/cloud/links/cloud_functions.py +1 -1
  52. airflow/providers/google/cloud/links/cloud_memorystore.py +1 -1
  53. airflow/providers/google/cloud/links/cloud_sql.py +1 -1
  54. airflow/providers/google/cloud/links/cloud_tasks.py +1 -1
  55. airflow/providers/google/cloud/links/compute.py +1 -1
  56. airflow/providers/google/cloud/links/datacatalog.py +1 -1
  57. airflow/providers/google/cloud/links/dataflow.py +1 -1
  58. airflow/providers/google/cloud/links/dataform.py +1 -1
  59. airflow/providers/google/cloud/links/pubsub.py +1 -1
  60. airflow/providers/google/cloud/links/spanner.py +1 -1
  61. airflow/providers/google/cloud/links/stackdriver.py +1 -1
  62. airflow/providers/google/cloud/links/workflows.py +2 -2
  63. airflow/providers/google/cloud/log/gcs_task_handler.py +5 -7
  64. airflow/providers/google/cloud/log/stackdriver_task_handler.py +8 -4
  65. airflow/providers/google/cloud/operators/automl.py +2 -1
  66. airflow/providers/google/cloud/operators/bigquery.py +6 -2
  67. airflow/providers/google/cloud/operators/bigquery_dts.py +2 -1
  68. airflow/providers/google/cloud/operators/bigtable.py +5 -3
  69. airflow/providers/google/cloud/operators/cloud_batch.py +6 -3
  70. airflow/providers/google/cloud/operators/cloud_build.py +2 -1
  71. airflow/providers/google/cloud/operators/cloud_composer.py +3 -2
  72. airflow/providers/google/cloud/operators/cloud_memorystore.py +3 -2
  73. airflow/providers/google/cloud/operators/cloud_run.py +3 -2
  74. airflow/providers/google/cloud/operators/cloud_sql.py +157 -152
  75. airflow/providers/google/cloud/operators/compute.py +59 -61
  76. airflow/providers/google/cloud/operators/datacatalog.py +3 -2
  77. airflow/providers/google/cloud/operators/dataflow.py +3 -1
  78. airflow/providers/google/cloud/operators/dataform.py +2 -1
  79. airflow/providers/google/cloud/operators/datafusion.py +1 -1
  80. airflow/providers/google/cloud/operators/dataplex.py +110 -8
  81. airflow/providers/google/cloud/operators/dataproc.py +39 -18
  82. airflow/providers/google/cloud/operators/dataproc_metastore.py +2 -1
  83. airflow/providers/google/cloud/operators/dlp.py +3 -2
  84. airflow/providers/google/cloud/operators/functions.py +46 -46
  85. airflow/providers/google/cloud/operators/gcs.py +4 -6
  86. airflow/providers/google/cloud/operators/kubernetes_engine.py +2 -1
  87. airflow/providers/google/cloud/operators/natural_language.py +3 -2
  88. airflow/providers/google/cloud/operators/pubsub.py +2 -1
  89. airflow/providers/google/cloud/operators/speech_to_text.py +3 -2
  90. airflow/providers/google/cloud/operators/stackdriver.py +2 -1
  91. airflow/providers/google/cloud/operators/tasks.py +3 -2
  92. airflow/providers/google/cloud/operators/text_to_speech.py +3 -2
  93. airflow/providers/google/cloud/operators/translate_speech.py +2 -1
  94. airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +2 -1
  95. airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +3 -2
  96. airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +2 -1
  97. airflow/providers/google/cloud/operators/vertex_ai/dataset.py +3 -2
  98. airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +4 -4
  99. airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +3 -2
  100. airflow/providers/google/cloud/operators/vertex_ai/model_service.py +2 -1
  101. airflow/providers/google/cloud/operators/video_intelligence.py +2 -1
  102. airflow/providers/google/cloud/operators/vision.py +3 -2
  103. airflow/providers/google/cloud/operators/workflows.py +7 -5
  104. airflow/providers/google/cloud/secrets/secret_manager.py +2 -2
  105. airflow/providers/google/cloud/sensors/bigquery_dts.py +2 -1
  106. airflow/providers/google/cloud/sensors/dataplex.py +2 -1
  107. airflow/providers/google/cloud/sensors/dataproc_metastore.py +2 -2
  108. airflow/providers/google/cloud/sensors/gcs.py +2 -1
  109. airflow/providers/google/cloud/sensors/workflows.py +2 -1
  110. airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +24 -10
  111. airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +2 -1
  112. airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +2 -1
  113. airflow/providers/google/cloud/transfers/bigquery_to_mysql.py +1 -4
  114. airflow/providers/google/cloud/transfers/bigquery_to_postgres.py +1 -4
  115. airflow/providers/google/cloud/transfers/bigquery_to_sql.py +1 -1
  116. airflow/providers/google/cloud/transfers/calendar_to_gcs.py +4 -2
  117. airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +1 -3
  118. airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +2 -2
  119. airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +2 -1
  120. airflow/providers/google/cloud/transfers/presto_to_gcs.py +5 -4
  121. airflow/providers/google/cloud/transfers/sql_to_gcs.py +1 -1
  122. airflow/providers/google/cloud/transfers/trino_to_gcs.py +5 -4
  123. airflow/providers/google/cloud/triggers/bigquery.py +30 -36
  124. airflow/providers/google/cloud/triggers/bigquery_dts.py +9 -10
  125. airflow/providers/google/cloud/triggers/cloud_batch.py +6 -8
  126. airflow/providers/google/cloud/triggers/cloud_build.py +5 -6
  127. airflow/providers/google/cloud/triggers/cloud_run.py +4 -3
  128. airflow/providers/google/cloud/triggers/cloud_sql.py +10 -10
  129. airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +1 -1
  130. airflow/providers/google/cloud/triggers/dataflow.py +5 -6
  131. airflow/providers/google/cloud/triggers/datafusion.py +5 -6
  132. airflow/providers/google/cloud/triggers/dataplex.py +110 -0
  133. airflow/providers/google/cloud/triggers/dataproc.py +18 -20
  134. airflow/providers/google/cloud/triggers/kubernetes_engine.py +14 -13
  135. airflow/providers/google/cloud/triggers/mlengine.py +5 -5
  136. airflow/providers/google/cloud/triggers/pubsub.py +2 -2
  137. airflow/providers/google/cloud/utils/bigquery_get_data.py +6 -3
  138. airflow/providers/google/cloud/utils/credentials_provider.py +1 -1
  139. airflow/providers/google/cloud/utils/field_validator.py +13 -13
  140. airflow/providers/google/cloud/utils/mlengine_operator_utils.py +5 -3
  141. airflow/providers/google/cloud/utils/mlengine_prediction_summary.py +1 -1
  142. airflow/providers/google/common/hooks/base_google.py +10 -2
  143. airflow/providers/google/common/links/storage.py +1 -1
  144. airflow/providers/google/common/utils/id_token_credentials.py +4 -1
  145. airflow/providers/google/get_provider_info.py +5 -0
  146. airflow/providers/google/marketing_platform/hooks/campaign_manager.py +4 -2
  147. airflow/providers/google/marketing_platform/sensors/display_video.py +6 -3
  148. airflow/providers/google/suite/hooks/calendar.py +4 -2
  149. {apache_airflow_providers_google-10.7.0rc1.dist-info → apache_airflow_providers_google-10.8.0.dist-info}/METADATA +8 -8
  150. {apache_airflow_providers_google-10.7.0rc1.dist-info → apache_airflow_providers_google-10.8.0.dist-info}/RECORD +155 -173
  151. {apache_airflow_providers_google-10.7.0rc1.dist-info → apache_airflow_providers_google-10.8.0.dist-info}/WHEEL +1 -1
  152. airflow/providers/google/ads/_vendor/__init__.py +0 -16
  153. airflow/providers/google/ads/_vendor/googleads/__init__.py +0 -16
  154. airflow/providers/google/ads/_vendor/googleads/interceptors/__init__.py +0 -16
  155. airflow/providers/google/ads/_vendor/googleads/v12/__init__.py +0 -16
  156. airflow/providers/google/ads/_vendor/googleads/v12/common/__init__.py +0 -16
  157. airflow/providers/google/ads/_vendor/googleads/v12/common/types/__init__.py +0 -16
  158. airflow/providers/google/ads/_vendor/googleads/v12/enums/__init__.py +0 -16
  159. airflow/providers/google/ads/_vendor/googleads/v12/enums/types/__init__.py +0 -16
  160. airflow/providers/google/ads/_vendor/googleads/v12/errors/__init__.py +0 -16
  161. airflow/providers/google/ads/_vendor/googleads/v12/errors/types/__init__.py +0 -16
  162. airflow/providers/google/ads/_vendor/googleads/v12/resources/__init__.py +0 -16
  163. airflow/providers/google/ads/_vendor/googleads/v12/resources/types/__init__.py +0 -16
  164. airflow/providers/google/ads/_vendor/googleads/v12/services/__init__.py +0 -16
  165. airflow/providers/google/ads/_vendor/googleads/v12/services/services/__init__.py +0 -16
  166. airflow/providers/google/ads/_vendor/googleads/v12/services/services/customer_service/__init__.py +0 -16
  167. airflow/providers/google/ads/_vendor/googleads/v12/services/services/customer_service/transports/__init__.py +0 -16
  168. airflow/providers/google/ads/_vendor/googleads/v12/services/services/google_ads_service/__init__.py +0 -16
  169. airflow/providers/google/ads/_vendor/googleads/v12/services/services/google_ads_service/transports/__init__.py +0 -16
  170. airflow/providers/google/ads/_vendor/googleads/v12/services/types/__init__.py +0 -16
  171. {apache_airflow_providers_google-10.7.0rc1.dist-info → apache_airflow_providers_google-10.8.0.dist-info}/LICENSE +0 -0
  172. {apache_airflow_providers_google-10.7.0rc1.dist-info → apache_airflow_providers_google-10.8.0.dist-info}/NOTICE +0 -0
  173. {apache_airflow_providers_google-10.7.0rc1.dist-info → apache_airflow_providers_google-10.8.0.dist-info}/entry_points.txt +0 -0
  174. {apache_airflow_providers_google-10.7.0rc1.dist-info → apache_airflow_providers_google-10.8.0.dist-info}/top_level.txt +0 -0
@@ -22,7 +22,6 @@ from typing import TYPE_CHECKING, Sequence
22
22
 
23
23
  from google.api_core.exceptions import NotFound
24
24
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
25
- from google.api_core.retry import Retry
26
25
  from google.cloud.aiplatform import datasets
27
26
  from google.cloud.aiplatform.models import Model
28
27
  from google.cloud.aiplatform_v1.types.training_pipeline import TrainingPipeline
@@ -36,6 +35,8 @@ from airflow.providers.google.cloud.links.vertex_ai import (
36
35
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
37
36
 
38
37
  if TYPE_CHECKING:
38
+ from google.api_core.retry import Retry
39
+
39
40
  from airflow.utils.context import Context
40
41
 
41
42
 
@@ -30,8 +30,6 @@ from typing import TYPE_CHECKING, Sequence
30
30
 
31
31
  from google.api_core.exceptions import NotFound
32
32
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
33
- from google.api_core.retry import Retry
34
- from google.cloud.aiplatform import Model, explain
35
33
  from google.cloud.aiplatform_v1.types import BatchPredictionJob
36
34
 
37
35
  from airflow.providers.google.cloud.hooks.vertex_ai.batch_prediction_job import BatchPredictionJobHook
@@ -42,6 +40,9 @@ from airflow.providers.google.cloud.links.vertex_ai import (
42
40
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
43
41
 
44
42
  if TYPE_CHECKING:
43
+ from google.api_core.retry import Retry
44
+ from google.cloud.aiplatform import Model, explain
45
+
45
46
  from airflow.utils.context import Context
46
47
 
47
48
 
@@ -22,7 +22,6 @@ from typing import TYPE_CHECKING, Sequence
22
22
 
23
23
  from google.api_core.exceptions import NotFound
24
24
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
25
- from google.api_core.retry import Retry
26
25
  from google.cloud.aiplatform.models import Model
27
26
  from google.cloud.aiplatform_v1.types.dataset import Dataset
28
27
  from google.cloud.aiplatform_v1.types.training_pipeline import TrainingPipeline
@@ -36,6 +35,8 @@ from airflow.providers.google.cloud.links.vertex_ai import (
36
35
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
37
36
 
38
37
  if TYPE_CHECKING:
38
+ from google.api_core.retry import Retry
39
+
39
40
  from airflow.utils.context import Context
40
41
 
41
42
 
@@ -22,15 +22,16 @@ from typing import TYPE_CHECKING, Sequence
22
22
 
23
23
  from google.api_core.exceptions import NotFound
24
24
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
25
- from google.api_core.retry import Retry
26
25
  from google.cloud.aiplatform_v1.types import Dataset, ExportDataConfig, ImportDataConfig
27
- from google.protobuf.field_mask_pb2 import FieldMask
28
26
 
29
27
  from airflow.providers.google.cloud.hooks.vertex_ai.dataset import DatasetHook
30
28
  from airflow.providers.google.cloud.links.vertex_ai import VertexAIDatasetLink, VertexAIDatasetListLink
31
29
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
32
30
 
33
31
  if TYPE_CHECKING:
32
+ from google.api_core.retry import Retry
33
+ from google.protobuf.field_mask_pb2 import FieldMask
34
+
34
35
  from airflow.utils.context import Context
35
36
 
36
37
 
@@ -32,9 +32,7 @@ from typing import TYPE_CHECKING, Sequence
32
32
 
33
33
  from google.api_core.exceptions import NotFound
34
34
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
35
- from google.api_core.retry import Retry
36
35
  from google.cloud.aiplatform_v1.types import DeployedModel, Endpoint, endpoint_service
37
- from google.protobuf.field_mask_pb2 import FieldMask
38
36
 
39
37
  from airflow.providers.google.cloud.hooks.vertex_ai.endpoint_service import EndpointServiceHook
40
38
  from airflow.providers.google.cloud.links.vertex_ai import (
@@ -45,6 +43,9 @@ from airflow.providers.google.cloud.links.vertex_ai import (
45
43
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
46
44
 
47
45
  if TYPE_CHECKING:
46
+ from google.api_core.retry import Retry
47
+ from google.protobuf.field_mask_pb2 import FieldMask
48
+
48
49
  from airflow.utils.context import Context
49
50
 
50
51
 
@@ -462,8 +463,7 @@ class UndeployModelOperator(GoogleCloudBaseOperator):
462
463
 
463
464
  :param project_id: Required. The ID of the Google Cloud project that the service belongs to.
464
465
  :param region: Required. The ID of the Google Cloud region that the service belongs to.
465
- :param endpoint_id: Required. The name of the Endpoint resource from which to undeploy a Model. Format:
466
- ``projects/{project}/locations/{location}/endpoints/{endpoint}``
466
+ :param endpoint_id: Required. The ID of the Endpoint resource from which to undeploy a Model.
467
467
  :param deployed_model_id: Required. The ID of the DeployedModel to be undeployed from the Endpoint.
468
468
  :param traffic_split: If this field is provided, then the Endpoint's
469
469
  [traffic_split][google.cloud.aiplatform.v1.Endpoint.traffic_split] will be overwritten with it. If
@@ -31,8 +31,6 @@ from typing import TYPE_CHECKING, Sequence
31
31
 
32
32
  from google.api_core.exceptions import NotFound
33
33
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
34
- from google.api_core.retry import Retry
35
- from google.cloud.aiplatform import gapic, hyperparameter_tuning
36
34
  from google.cloud.aiplatform_v1.types import HyperparameterTuningJob
37
35
 
38
36
  from airflow.providers.google.cloud.hooks.vertex_ai.hyperparameter_tuning_job import (
@@ -45,6 +43,9 @@ from airflow.providers.google.cloud.links.vertex_ai import (
45
43
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
46
44
 
47
45
  if TYPE_CHECKING:
46
+ from google.api_core.retry import Retry
47
+ from google.cloud.aiplatform import gapic, hyperparameter_tuning
48
+
48
49
  from airflow.utils.context import Context
49
50
 
50
51
 
@@ -28,7 +28,6 @@ from typing import TYPE_CHECKING, Sequence
28
28
 
29
29
  from google.api_core.exceptions import NotFound
30
30
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
31
- from google.api_core.retry import Retry
32
31
  from google.cloud.aiplatform_v1.types import Model, model_service
33
32
 
34
33
  from airflow.providers.google.cloud.hooks.vertex_ai.model_service import ModelServiceHook
@@ -40,6 +39,8 @@ from airflow.providers.google.cloud.links.vertex_ai import (
40
39
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
41
40
 
42
41
  if TYPE_CHECKING:
42
+ from google.api_core.retry import Retry
43
+
43
44
  from airflow.utils.context import Context
44
45
 
45
46
 
@@ -21,7 +21,6 @@ from __future__ import annotations
21
21
  from typing import TYPE_CHECKING, Sequence
22
22
 
23
23
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
24
- from google.api_core.retry import Retry
25
24
  from google.cloud.videointelligence_v1 import Feature, VideoContext
26
25
  from google.protobuf.json_format import MessageToDict
27
26
 
@@ -29,6 +28,8 @@ from airflow.providers.google.cloud.hooks.video_intelligence import CloudVideoIn
29
28
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
30
29
 
31
30
  if TYPE_CHECKING:
31
+ from google.api_core.retry import Retry
32
+
32
33
  from airflow.utils.context import Context
33
34
 
34
35
 
@@ -23,7 +23,6 @@ from typing import TYPE_CHECKING, Any, Sequence, Tuple
23
23
 
24
24
  from google.api_core.exceptions import AlreadyExists
25
25
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
26
- from google.api_core.retry import Retry
27
26
  from google.cloud.vision_v1 import (
28
27
  AnnotateImageRequest,
29
28
  Image,
@@ -31,12 +30,14 @@ from google.cloud.vision_v1 import (
31
30
  ProductSet,
32
31
  ReferenceImage,
33
32
  )
34
- from google.protobuf.field_mask_pb2 import FieldMask # type: ignore
35
33
 
36
34
  from airflow.providers.google.cloud.hooks.vision import CloudVisionHook
37
35
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
38
36
 
39
37
  if TYPE_CHECKING:
38
+ from google.api_core.retry import Retry
39
+ from google.protobuf.field_mask_pb2 import FieldMask
40
+
40
41
  from airflow.utils.context import Context
41
42
 
42
43
 
@@ -24,10 +24,8 @@ from typing import TYPE_CHECKING, Sequence
24
24
 
25
25
  from google.api_core.exceptions import AlreadyExists
26
26
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
27
- from google.api_core.retry import Retry
28
27
  from google.cloud.workflows.executions_v1beta import Execution
29
28
  from google.cloud.workflows_v1beta import Workflow
30
- from google.protobuf.field_mask_pb2 import FieldMask
31
29
 
32
30
  from airflow.providers.google.cloud.hooks.workflows import WorkflowsHook
33
31
  from airflow.providers.google.cloud.links.workflows import (
@@ -38,6 +36,9 @@ from airflow.providers.google.cloud.links.workflows import (
38
36
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
39
37
 
40
38
  if TYPE_CHECKING:
39
+ from google.api_core.retry import Retry
40
+ from google.protobuf.field_mask_pb2 import FieldMask
41
+
41
42
  from airflow.utils.context import Context
42
43
  try:
43
44
  from airflow.utils.hashlib_wrapper import md5
@@ -51,7 +52,7 @@ class WorkflowsCreateWorkflowOperator(GoogleCloudBaseOperator):
51
52
  Creates a new workflow.
52
53
 
53
54
  If a workflow with the specified name already exists in the specified
54
- project and location, the long running operation will return
55
+ project and location, the long-running operation will return
55
56
  [ALREADY_EXISTS][google.rpc.Code.ALREADY_EXISTS] error.
56
57
 
57
58
  .. seealso::
@@ -605,7 +606,8 @@ class WorkflowsListExecutionsOperator(GoogleCloudBaseOperator):
605
606
 
606
607
  :param workflow_id: Required. The ID of the workflow to be created.
607
608
  :param start_date_filter: If passed only executions older that this date will be returned.
608
- By default operators return executions from last 60 minutes
609
+ By default, operators return executions from last 60 minutes.
610
+ Note that datetime object must specify a time zone, e.g. ``datetime.timezone.utc``.
609
611
  :param project_id: Required. The ID of the Google Cloud project the cluster belongs to.
610
612
  :param location: Required. The GCP region in which to handle the request.
611
613
  :param retry: A retry object used to retry requests. If ``None`` is specified, requests will not be
@@ -669,7 +671,7 @@ class WorkflowsListExecutionsOperator(GoogleCloudBaseOperator):
669
671
  return [
670
672
  Execution.to_dict(e)
671
673
  for e in execution_iter
672
- if e.start_time.ToDatetime(tzinfo=datetime.timezone.utc) > self.start_date_filter
674
+ if e.start_time > self.start_date_filter # type: ignore
673
675
  ]
674
676
 
675
677
 
@@ -36,8 +36,8 @@ SECRET_ID_PATTERN = r"^[a-zA-Z0-9-_]*$"
36
36
 
37
37
 
38
38
  def _parse_version(val):
39
- val = re.sub(r"(\d+\.\d+\.\d+).*", lambda x: x.group(1), val)
40
- return tuple(int(x) for x in val.split("."))
39
+ match = re.search(r"(\d+)\.(\d+)\.(\d+)", val)
40
+ return tuple(int(x) for x in match.groups())
41
41
 
42
42
 
43
43
  class CloudSecretManagerBackend(BaseSecretsBackend, LoggingMixin):
@@ -21,7 +21,6 @@ from __future__ import annotations
21
21
  from typing import TYPE_CHECKING, Sequence
22
22
 
23
23
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
24
- from google.api_core.retry import Retry
25
24
  from google.cloud.bigquery_datatransfer_v1 import TransferState
26
25
 
27
26
  from airflow.exceptions import AirflowException
@@ -29,6 +28,8 @@ from airflow.providers.google.cloud.hooks.bigquery_dts import BiqQueryDataTransf
29
28
  from airflow.sensors.base import BaseSensorOperator
30
29
 
31
30
  if TYPE_CHECKING:
31
+ from google.api_core.retry import Retry
32
+
32
33
  from airflow.utils.context import Context
33
34
 
34
35
 
@@ -21,10 +21,11 @@ import time
21
21
  from typing import TYPE_CHECKING, Sequence
22
22
 
23
23
  if TYPE_CHECKING:
24
+ from google.api_core.retry import Retry
25
+
24
26
  from airflow.utils.context import Context
25
27
  from google.api_core.exceptions import GoogleAPICallError
26
28
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
27
- from google.api_core.retry import Retry
28
29
  from google.cloud.dataplex_v1.types import DataScanJob
29
30
 
30
31
  from airflow.exceptions import AirflowException
@@ -19,14 +19,14 @@ from __future__ import annotations
19
19
 
20
20
  from typing import TYPE_CHECKING, Sequence
21
21
 
22
- from google.api_core.operation import Operation
23
-
24
22
  from airflow import AirflowException
25
23
  from airflow.providers.google.cloud.hooks.dataproc_metastore import DataprocMetastoreHook
26
24
  from airflow.providers.google.cloud.hooks.gcs import parse_json_from_gcs
27
25
  from airflow.sensors.base import BaseSensorOperator
28
26
 
29
27
  if TYPE_CHECKING:
28
+ from google.api_core.operation import Operation
29
+
30
30
  from airflow.utils.context import Context
31
31
 
32
32
 
@@ -24,7 +24,6 @@ import warnings
24
24
  from datetime import datetime, timedelta
25
25
  from typing import TYPE_CHECKING, Any, Callable, Sequence
26
26
 
27
- from google.api_core.retry import Retry
28
27
  from google.cloud.storage.retry import DEFAULT_RETRY
29
28
 
30
29
  from airflow.configuration import conf
@@ -39,6 +38,8 @@ from airflow.providers.google.cloud.triggers.gcs import (
39
38
  from airflow.sensors.base import BaseSensorOperator, poke_mode_only
40
39
 
41
40
  if TYPE_CHECKING:
41
+ from google.api_core.retry import Retry
42
+
42
43
  from airflow.utils.context import Context
43
44
 
44
45
 
@@ -19,7 +19,6 @@ from __future__ import annotations
19
19
  from typing import TYPE_CHECKING, Sequence
20
20
 
21
21
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
22
- from google.api_core.retry import Retry
23
22
  from google.cloud.workflows.executions_v1beta import Execution
24
23
 
25
24
  from airflow.exceptions import AirflowException
@@ -27,6 +26,8 @@ from airflow.providers.google.cloud.hooks.workflows import WorkflowsHook
27
26
  from airflow.sensors.base import BaseSensorOperator
28
27
 
29
28
  if TYPE_CHECKING:
29
+ from google.api_core.retry import Retry
30
+
30
31
  from airflow.utils.context import Context
31
32
 
32
33
 
@@ -17,10 +17,12 @@
17
17
  # under the License.
18
18
  from __future__ import annotations
19
19
 
20
+ import warnings
20
21
  from tempfile import NamedTemporaryFile
21
22
  from typing import TYPE_CHECKING, Sequence
22
23
 
23
24
  from airflow import AirflowException
25
+ from airflow.exceptions import AirflowProviderDeprecationWarning
24
26
  from airflow.models import BaseOperator
25
27
  from airflow.providers.google.cloud.hooks.gcs import GCSHook, _parse_gcs_url, gcs_object_is_directory
26
28
  from airflow.providers.microsoft.azure.hooks.fileshare import AzureFileShareHook
@@ -73,6 +75,7 @@ class AzureFileShareToGCSOperator(BaseOperator):
73
75
  share_name: str,
74
76
  dest_gcs: str,
75
77
  directory_name: str | None = None,
78
+ directory_path: str | None = None,
76
79
  prefix: str = "",
77
80
  azure_fileshare_conn_id: str = "azure_fileshare_default",
78
81
  gcp_conn_id: str = "google_cloud_default",
@@ -84,7 +87,15 @@ class AzureFileShareToGCSOperator(BaseOperator):
84
87
  super().__init__(**kwargs)
85
88
 
86
89
  self.share_name = share_name
90
+ self.directory_path = directory_path
87
91
  self.directory_name = directory_name
92
+ if self.directory_path is None:
93
+ self.directory_path = directory_name
94
+ warnings.warn(
95
+ "Use 'directory_path' instead of 'directory_name'.",
96
+ AirflowProviderDeprecationWarning,
97
+ stacklevel=2,
98
+ )
88
99
  self.prefix = prefix
89
100
  self.azure_fileshare_conn_id = azure_fileshare_conn_id
90
101
  self.gcp_conn_id = gcp_conn_id
@@ -106,10 +117,12 @@ class AzureFileShareToGCSOperator(BaseOperator):
106
117
 
107
118
  def execute(self, context: Context):
108
119
  self._check_inputs()
109
- azure_fileshare_hook = AzureFileShareHook(self.azure_fileshare_conn_id)
110
- files = azure_fileshare_hook.list_files(
111
- share_name=self.share_name, directory_name=self.directory_name
120
+ azure_fileshare_hook = AzureFileShareHook(
121
+ share_name=self.share_name,
122
+ azure_fileshare_conn_id=self.azure_fileshare_conn_id,
123
+ directory_path=self.directory_path,
112
124
  )
125
+ files = azure_fileshare_hook.list_files()
113
126
 
114
127
  gcs_hook = GCSHook(
115
128
  gcp_conn_id=self.gcp_conn_id,
@@ -141,16 +154,17 @@ class AzureFileShareToGCSOperator(BaseOperator):
141
154
 
142
155
  if files:
143
156
  self.log.info("%s files are going to be synced.", len(files))
144
- if self.directory_name is None:
157
+ if self.directory_path is None:
145
158
  raise RuntimeError("The directory_name must be set!.")
146
159
  for file in files:
160
+ azure_fileshare_hook = AzureFileShareHook(
161
+ share_name=self.share_name,
162
+ azure_fileshare_conn_id=self.azure_fileshare_conn_id,
163
+ directory_path=self.directory_path,
164
+ file_path=file,
165
+ )
147
166
  with NamedTemporaryFile() as temp_file:
148
- azure_fileshare_hook.get_file_to_stream(
149
- stream=temp_file,
150
- share_name=self.share_name,
151
- directory_name=self.directory_name,
152
- file_name=file,
153
- )
167
+ azure_fileshare_hook.get_file_to_stream(stream=temp_file)
154
168
  temp_file.flush()
155
169
 
156
170
  # There will always be a '/' before file because it is
@@ -21,7 +21,6 @@ from __future__ import annotations
21
21
  from typing import TYPE_CHECKING, Any, Sequence
22
22
 
23
23
  from google.api_core.exceptions import Conflict
24
- from google.api_core.retry import Retry
25
24
  from google.cloud.bigquery import DEFAULT_RETRY, UnknownJob
26
25
 
27
26
  from airflow import AirflowException
@@ -32,6 +31,8 @@ from airflow.providers.google.cloud.links.bigquery import BigQueryTableLink
32
31
  from airflow.providers.google.cloud.triggers.bigquery import BigQueryInsertJobTrigger
33
32
 
34
33
  if TYPE_CHECKING:
34
+ from google.api_core.retry import Retry
35
+
35
36
  from airflow.utils.context import Context
36
37
 
37
38
 
@@ -45,7 +45,8 @@ class BigQueryToMsSqlOperator(BigQueryToSqlBaseOperator):
45
45
  :param mssql_conn_id: reference to a specific mssql hook
46
46
  """
47
47
 
48
- template_fields: Sequence[str] = tuple(BigQueryToSqlBaseOperator.template_fields) + (
48
+ template_fields: Sequence[str] = (
49
+ *BigQueryToSqlBaseOperator.template_fields,
49
50
  "source_project_dataset_table",
50
51
  )
51
52
  operator_extra_links = (BigQueryTableLink(),)
@@ -40,10 +40,7 @@ class BigQueryToMySqlOperator(BigQueryToSqlBaseOperator):
40
40
  :param mysql_conn_id: Reference to :ref:`mysql connection id <howto/connection:mysql>`.
41
41
  """
42
42
 
43
- template_fields: Sequence[str] = tuple(BigQueryToSqlBaseOperator.template_fields) + (
44
- "dataset_id",
45
- "table_id",
46
- )
43
+ template_fields: Sequence[str] = (*BigQueryToSqlBaseOperator.template_fields, "dataset_id", "table_id")
47
44
 
48
45
  def __init__(
49
46
  self,
@@ -36,10 +36,7 @@ class BigQueryToPostgresOperator(BigQueryToSqlBaseOperator):
36
36
  :param postgres_conn_id: Reference to :ref:`postgres connection id <howto/connection:postgres>`.
37
37
  """
38
38
 
39
- template_fields: Sequence[str] = tuple(BigQueryToSqlBaseOperator.template_fields) + (
40
- "dataset_id",
41
- "table_id",
42
- )
39
+ template_fields: Sequence[str] = (*BigQueryToSqlBaseOperator.template_fields, "dataset_id", "table_id")
43
40
 
44
41
  def __init__(
45
42
  self,
@@ -22,11 +22,11 @@ import abc
22
22
  from typing import TYPE_CHECKING, Sequence
23
23
 
24
24
  from airflow.models import BaseOperator
25
- from airflow.providers.common.sql.hooks.sql import DbApiHook
26
25
  from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook
27
26
  from airflow.providers.google.cloud.utils.bigquery_get_data import bigquery_get_data
28
27
 
29
28
  if TYPE_CHECKING:
29
+ from airflow.providers.common.sql.hooks.sql import DbApiHook
30
30
  from airflow.utils.context import Context
31
31
 
32
32
 
@@ -17,14 +17,16 @@
17
17
  from __future__ import annotations
18
18
 
19
19
  import json
20
- from datetime import datetime
21
20
  from tempfile import NamedTemporaryFile
22
- from typing import Any, Sequence
21
+ from typing import TYPE_CHECKING, Any, Sequence
23
22
 
24
23
  from airflow.models import BaseOperator
25
24
  from airflow.providers.google.cloud.hooks.gcs import GCSHook
26
25
  from airflow.providers.google.suite.hooks.calendar import GoogleCalendarHook
27
26
 
27
+ if TYPE_CHECKING:
28
+ from datetime import datetime
29
+
28
30
 
29
31
  class GoogleCalendarToGCSOperator(BaseOperator):
30
32
  """
@@ -258,9 +258,7 @@ class CassandraToGCSOperator(BaseOperator):
258
258
 
259
259
  def convert_value(self, value: Any | None) -> Any | None:
260
260
  """Convert value to BQ type."""
261
- if not value:
262
- return value
263
- elif isinstance(value, (str, int, float, bool, dict)):
261
+ if not value or isinstance(value, (str, int, float, bool, dict)):
264
262
  return value
265
263
  elif isinstance(value, bytes):
266
264
  return b64encode(value).decode("ascii")
@@ -23,14 +23,14 @@ import tempfile
23
23
  from enum import Enum
24
24
  from typing import TYPE_CHECKING, Any, Sequence
25
25
 
26
- from facebook_business.adobjects.adsinsights import AdsInsights
27
-
28
26
  from airflow.exceptions import AirflowException
29
27
  from airflow.models import BaseOperator
30
28
  from airflow.providers.facebook.ads.hooks.ads import FacebookAdsReportingHook
31
29
  from airflow.providers.google.cloud.hooks.gcs import GCSHook
32
30
 
33
31
  if TYPE_CHECKING:
32
+ from facebook_business.adobjects.adsinsights import AdsInsights
33
+
34
34
  from airflow.utils.context import Context
35
35
 
36
36
 
@@ -22,7 +22,6 @@ import json
22
22
  from typing import TYPE_CHECKING, Any, Sequence
23
23
 
24
24
  from google.api_core.exceptions import BadRequest, Conflict
25
- from google.api_core.retry import Retry
26
25
  from google.cloud.bigquery import (
27
26
  DEFAULT_RETRY,
28
27
  CopyJob,
@@ -44,6 +43,8 @@ from airflow.providers.google.cloud.links.bigquery import BigQueryTableLink
44
43
  from airflow.providers.google.cloud.triggers.bigquery import BigQueryInsertJobTrigger
45
44
 
46
45
  if TYPE_CHECKING:
46
+ from google.api_core.retry import Retry
47
+
47
48
  from airflow.utils.context import Context
48
49
 
49
50
  ALLOWED_FORMATS = [
@@ -17,14 +17,15 @@
17
17
  # under the License.
18
18
  from __future__ import annotations
19
19
 
20
- from typing import Any
21
-
22
- from prestodb.client import PrestoResult
23
- from prestodb.dbapi import Cursor as PrestoCursor
20
+ from typing import TYPE_CHECKING, Any
24
21
 
25
22
  from airflow.providers.google.cloud.transfers.sql_to_gcs import BaseSQLToGCSOperator
26
23
  from airflow.providers.presto.hooks.presto import PrestoHook
27
24
 
25
+ if TYPE_CHECKING:
26
+ from prestodb.client import PrestoResult
27
+ from prestodb.dbapi import Cursor as PrestoCursor
28
+
28
29
 
29
30
  class _PrestoToGCSPrestoCursorAdapter:
30
31
  """
@@ -235,7 +235,7 @@ class BaseSQLToGCSOperator(BaseOperator):
235
235
  names in GCS, and values are file handles to local files that
236
236
  contain the data for the GCS objects.
237
237
  """
238
- org_schema = list(map(lambda schema_tuple: schema_tuple[0], cursor.description))
238
+ org_schema = [schema_tuple[0] for schema_tuple in cursor.description]
239
239
  schema = [column for column in org_schema if column not in self.exclude_columns]
240
240
 
241
241
  col_type_dict = self._get_col_type_dict()
@@ -17,14 +17,15 @@
17
17
  # under the License.
18
18
  from __future__ import annotations
19
19
 
20
- from typing import Any
21
-
22
- from trino.client import TrinoResult
23
- from trino.dbapi import Cursor as TrinoCursor
20
+ from typing import TYPE_CHECKING, Any
24
21
 
25
22
  from airflow.providers.google.cloud.transfers.sql_to_gcs import BaseSQLToGCSOperator
26
23
  from airflow.providers.trino.hooks.trino import TrinoHook
27
24
 
25
+ if TYPE_CHECKING:
26
+ from trino.client import TrinoResult
27
+ from trino.dbapi import Cursor as TrinoCursor
28
+
28
29
 
29
30
  class _TrinoToGCSTrinoCursorAdapter:
30
31
  """