apache-airflow-providers-google 14.0.0__py3-none-any.whl → 14.1.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. airflow/providers/google/3rd-party-licenses/LICENSES.txt +14 -0
  2. airflow/providers/google/3rd-party-licenses/NOTICE +15 -0
  3. airflow/providers/google/__init__.py +1 -1
  4. airflow/providers/google/_vendor/__init__.py +0 -0
  5. airflow/providers/google/_vendor/json_merge_patch.py +91 -0
  6. airflow/providers/google/ads/hooks/ads.py +12 -11
  7. airflow/providers/google/cloud/_internal_client/secret_manager_client.py +3 -2
  8. airflow/providers/google/cloud/hooks/alloy_db.py +2 -3
  9. airflow/providers/google/cloud/hooks/automl.py +7 -13
  10. airflow/providers/google/cloud/hooks/bigquery.py +21 -22
  11. airflow/providers/google/cloud/hooks/bigquery_dts.py +8 -8
  12. airflow/providers/google/cloud/hooks/bigtable.py +3 -2
  13. airflow/providers/google/cloud/hooks/cloud_batch.py +4 -3
  14. airflow/providers/google/cloud/hooks/cloud_build.py +7 -13
  15. airflow/providers/google/cloud/hooks/cloud_composer.py +7 -12
  16. airflow/providers/google/cloud/hooks/cloud_memorystore.py +4 -3
  17. airflow/providers/google/cloud/hooks/cloud_run.py +4 -3
  18. airflow/providers/google/cloud/hooks/cloud_sql.py +1 -1
  19. airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +8 -9
  20. airflow/providers/google/cloud/hooks/compute.py +3 -3
  21. airflow/providers/google/cloud/hooks/datacatalog.py +4 -3
  22. airflow/providers/google/cloud/hooks/dataflow.py +12 -12
  23. airflow/providers/google/cloud/hooks/dataform.py +3 -2
  24. airflow/providers/google/cloud/hooks/datafusion.py +2 -2
  25. airflow/providers/google/cloud/hooks/dataplex.py +11 -10
  26. airflow/providers/google/cloud/hooks/dataproc.py +5 -4
  27. airflow/providers/google/cloud/hooks/dataproc_metastore.py +4 -3
  28. airflow/providers/google/cloud/hooks/dlp.py +4 -3
  29. airflow/providers/google/cloud/hooks/gcs.py +19 -12
  30. airflow/providers/google/cloud/hooks/kms.py +3 -2
  31. airflow/providers/google/cloud/hooks/kubernetes_engine.py +20 -14
  32. airflow/providers/google/cloud/hooks/life_sciences.py +1 -1
  33. airflow/providers/google/cloud/hooks/managed_kafka.py +227 -3
  34. airflow/providers/google/cloud/hooks/natural_language.py +3 -2
  35. airflow/providers/google/cloud/hooks/os_login.py +3 -2
  36. airflow/providers/google/cloud/hooks/pubsub.py +6 -6
  37. airflow/providers/google/cloud/hooks/secret_manager.py +3 -2
  38. airflow/providers/google/cloud/hooks/spanner.py +2 -2
  39. airflow/providers/google/cloud/hooks/speech_to_text.py +3 -2
  40. airflow/providers/google/cloud/hooks/stackdriver.py +4 -4
  41. airflow/providers/google/cloud/hooks/tasks.py +4 -3
  42. airflow/providers/google/cloud/hooks/text_to_speech.py +3 -2
  43. airflow/providers/google/cloud/hooks/translate.py +6 -15
  44. airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +6 -12
  45. airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +6 -12
  46. airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +7 -13
  47. airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +5 -12
  48. airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +4 -11
  49. airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +4 -3
  50. airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +6 -12
  51. airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +6 -11
  52. airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +6 -12
  53. airflow/providers/google/cloud/hooks/vertex_ai/prediction_service.py +3 -2
  54. airflow/providers/google/cloud/hooks/video_intelligence.py +3 -2
  55. airflow/providers/google/cloud/hooks/vision.py +4 -3
  56. airflow/providers/google/cloud/hooks/workflows.py +3 -2
  57. airflow/providers/google/cloud/links/base.py +7 -1
  58. airflow/providers/google/cloud/links/datafusion.py +8 -2
  59. airflow/providers/google/cloud/links/dataproc.py +8 -1
  60. airflow/providers/google/cloud/links/kubernetes_engine.py +2 -1
  61. airflow/providers/google/cloud/links/managed_kafka.py +30 -0
  62. airflow/providers/google/cloud/log/gcs_task_handler.py +15 -7
  63. airflow/providers/google/cloud/log/stackdriver_task_handler.py +8 -6
  64. airflow/providers/google/cloud/openlineage/utils.py +4 -2
  65. airflow/providers/google/cloud/operators/alloy_db.py +6 -5
  66. airflow/providers/google/cloud/operators/automl.py +11 -9
  67. airflow/providers/google/cloud/operators/bigquery.py +8 -6
  68. airflow/providers/google/cloud/operators/bigquery_dts.py +10 -8
  69. airflow/providers/google/cloud/operators/bigtable.py +3 -1
  70. airflow/providers/google/cloud/operators/cloud_base.py +2 -1
  71. airflow/providers/google/cloud/operators/cloud_batch.py +4 -2
  72. airflow/providers/google/cloud/operators/cloud_build.py +5 -3
  73. airflow/providers/google/cloud/operators/cloud_composer.py +7 -5
  74. airflow/providers/google/cloud/operators/cloud_memorystore.py +6 -4
  75. airflow/providers/google/cloud/operators/cloud_run.py +5 -3
  76. airflow/providers/google/cloud/operators/compute.py +5 -4
  77. airflow/providers/google/cloud/operators/datacatalog.py +11 -9
  78. airflow/providers/google/cloud/operators/dataform.py +5 -3
  79. airflow/providers/google/cloud/operators/datafusion.py +1 -1
  80. airflow/providers/google/cloud/operators/dataplex.py +20 -18
  81. airflow/providers/google/cloud/operators/dataproc.py +34 -5
  82. airflow/providers/google/cloud/operators/dataproc_metastore.py +18 -7
  83. airflow/providers/google/cloud/operators/dlp.py +19 -17
  84. airflow/providers/google/cloud/operators/gcs.py +5 -4
  85. airflow/providers/google/cloud/operators/kubernetes_engine.py +54 -5
  86. airflow/providers/google/cloud/operators/managed_kafka.py +271 -4
  87. airflow/providers/google/cloud/operators/natural_language.py +5 -3
  88. airflow/providers/google/cloud/operators/pubsub.py +10 -8
  89. airflow/providers/google/cloud/operators/speech_to_text.py +5 -3
  90. airflow/providers/google/cloud/operators/stackdriver.py +5 -3
  91. airflow/providers/google/cloud/operators/tasks.py +6 -4
  92. airflow/providers/google/cloud/operators/text_to_speech.py +4 -2
  93. airflow/providers/google/cloud/operators/translate.py +5 -3
  94. airflow/providers/google/cloud/operators/translate_speech.py +4 -2
  95. airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +8 -6
  96. airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +6 -4
  97. airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +8 -6
  98. airflow/providers/google/cloud/operators/vertex_ai/dataset.py +6 -4
  99. airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +6 -4
  100. airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +6 -4
  101. airflow/providers/google/cloud/operators/vertex_ai/model_service.py +6 -4
  102. airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +6 -4
  103. airflow/providers/google/cloud/operators/video_intelligence.py +5 -3
  104. airflow/providers/google/cloud/operators/vision.py +6 -4
  105. airflow/providers/google/cloud/operators/workflows.py +7 -5
  106. airflow/providers/google/cloud/secrets/secret_manager.py +2 -1
  107. airflow/providers/google/cloud/sensors/bigquery_dts.py +5 -3
  108. airflow/providers/google/cloud/sensors/bigtable.py +3 -2
  109. airflow/providers/google/cloud/sensors/cloud_composer.py +1 -1
  110. airflow/providers/google/cloud/sensors/dataplex.py +6 -4
  111. airflow/providers/google/cloud/sensors/dataproc.py +3 -2
  112. airflow/providers/google/cloud/sensors/dataproc_metastore.py +2 -1
  113. airflow/providers/google/cloud/sensors/gcs.py +4 -2
  114. airflow/providers/google/cloud/sensors/pubsub.py +3 -2
  115. airflow/providers/google/cloud/sensors/workflows.py +5 -3
  116. airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +5 -3
  117. airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +12 -10
  118. airflow/providers/google/cloud/triggers/bigquery_dts.py +2 -1
  119. airflow/providers/google/cloud/triggers/cloud_batch.py +2 -1
  120. airflow/providers/google/cloud/triggers/cloud_build.py +2 -1
  121. airflow/providers/google/cloud/triggers/cloud_composer.py +1 -1
  122. airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +3 -2
  123. airflow/providers/google/cloud/triggers/dataflow.py +3 -2
  124. airflow/providers/google/cloud/triggers/dataplex.py +2 -1
  125. airflow/providers/google/cloud/triggers/dataproc.py +3 -2
  126. airflow/providers/google/cloud/triggers/kubernetes_engine.py +1 -1
  127. airflow/providers/google/cloud/triggers/pubsub.py +2 -1
  128. airflow/providers/google/cloud/triggers/vertex_ai.py +8 -7
  129. airflow/providers/google/cloud/utils/credentials_provider.py +4 -3
  130. airflow/providers/google/cloud/utils/external_token_supplier.py +0 -1
  131. airflow/providers/google/cloud/{example_dags/__init__.py → utils/validators.py} +27 -0
  132. airflow/providers/google/common/auth_backend/google_openid.py +14 -5
  133. airflow/providers/google/common/consts.py +2 -1
  134. airflow/providers/google/common/hooks/base_google.py +7 -8
  135. airflow/providers/google/common/hooks/operation_helpers.py +79 -0
  136. airflow/providers/google/get_provider_info.py +11 -6
  137. airflow/providers/google/marketing_platform/hooks/analytics_admin.py +3 -2
  138. airflow/providers/google/marketing_platform/hooks/search_ads.py +1 -1
  139. airflow/providers/google/marketing_platform/links/analytics_admin.py +10 -2
  140. airflow/providers/google/marketing_platform/operators/analytics_admin.py +7 -5
  141. {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-14.1.0rc1.dist-info}/METADATA +35 -38
  142. {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-14.1.0rc1.dist-info}/RECORD +144 -140
  143. {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-14.1.0rc1.dist-info}/WHEEL +1 -1
  144. airflow/providers/google/cloud/example_dags/example_cloud_task.py +0 -54
  145. {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-14.1.0rc1.dist-info}/entry_points.txt +0 -0
@@ -23,6 +23,10 @@ from __future__ import annotations
23
23
  from collections.abc import Sequence
24
24
  from typing import TYPE_CHECKING, Any
25
25
 
26
+ from google.api_core.exceptions import NotFound
27
+ from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
28
+ from google.cloud.aiplatform_v1 import types
29
+
26
30
  from airflow.configuration import conf
27
31
  from airflow.exceptions import AirflowException
28
32
  from airflow.providers.google.cloud.hooks.vertex_ai.hyperparameter_tuning_job import (
@@ -34,15 +38,13 @@ from airflow.providers.google.cloud.links.vertex_ai import (
34
38
  )
35
39
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
36
40
  from airflow.providers.google.cloud.triggers.vertex_ai import CreateHyperparameterTuningJobTrigger
37
- from google.api_core.exceptions import NotFound
38
- from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
39
- from google.cloud.aiplatform_v1 import types
40
41
 
41
42
  if TYPE_CHECKING:
42
- from airflow.utils.context import Context
43
43
  from google.api_core.retry import Retry
44
44
  from google.cloud.aiplatform import HyperparameterTuningJob, gapic, hyperparameter_tuning
45
45
 
46
+ from airflow.utils.context import Context
47
+
46
48
 
47
49
  class CreateHyperparameterTuningJobOperator(GoogleCloudBaseOperator):
48
50
  """
@@ -22,6 +22,10 @@ from __future__ import annotations
22
22
  from collections.abc import Sequence
23
23
  from typing import TYPE_CHECKING
24
24
 
25
+ from google.api_core.exceptions import NotFound
26
+ from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
27
+ from google.cloud.aiplatform_v1.types import Model, model_service
28
+
25
29
  from airflow.providers.google.cloud.hooks.vertex_ai.model_service import ModelServiceHook
26
30
  from airflow.providers.google.cloud.links.vertex_ai import (
27
31
  VertexAIModelExportLink,
@@ -29,14 +33,12 @@ from airflow.providers.google.cloud.links.vertex_ai import (
29
33
  VertexAIModelListLink,
30
34
  )
31
35
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
32
- from google.api_core.exceptions import NotFound
33
- from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
34
- from google.cloud.aiplatform_v1.types import Model, model_service
35
36
 
36
37
  if TYPE_CHECKING:
37
- from airflow.utils.context import Context
38
38
  from google.api_core.retry import Retry
39
39
 
40
+ from airflow.utils.context import Context
41
+
40
42
 
41
43
  class DeleteModelOperator(GoogleCloudBaseOperator):
42
44
  """
@@ -23,6 +23,10 @@ from collections.abc import Sequence
23
23
  from functools import cached_property
24
24
  from typing import TYPE_CHECKING, Any
25
25
 
26
+ from google.api_core.exceptions import NotFound
27
+ from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
28
+ from google.cloud.aiplatform_v1 import types
29
+
26
30
  from airflow.configuration import conf
27
31
  from airflow.exceptions import AirflowException
28
32
  from airflow.providers.google.cloud.hooks.vertex_ai.pipeline_job import PipelineJobHook
@@ -32,16 +36,14 @@ from airflow.providers.google.cloud.links.vertex_ai import (
32
36
  )
33
37
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
34
38
  from airflow.providers.google.cloud.triggers.vertex_ai import RunPipelineJobTrigger
35
- from google.api_core.exceptions import NotFound
36
- from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
37
- from google.cloud.aiplatform_v1 import types
38
39
 
39
40
  if TYPE_CHECKING:
40
- from airflow.utils.context import Context
41
41
  from google.api_core.retry import Retry
42
42
  from google.cloud.aiplatform import PipelineJob
43
43
  from google.cloud.aiplatform.metadata import experiment_resources
44
44
 
45
+ from airflow.utils.context import Context
46
+
45
47
 
46
48
  class RunPipelineJobOperator(GoogleCloudBaseOperator):
47
49
  """
@@ -22,16 +22,18 @@ from __future__ import annotations
22
22
  from collections.abc import Sequence
23
23
  from typing import TYPE_CHECKING
24
24
 
25
- from airflow.providers.google.cloud.hooks.video_intelligence import CloudVideoIntelligenceHook
26
- from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
27
25
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
28
26
  from google.cloud.videointelligence_v1 import Feature, VideoContext
29
27
  from google.protobuf.json_format import MessageToDict
30
28
 
29
+ from airflow.providers.google.cloud.hooks.video_intelligence import CloudVideoIntelligenceHook
30
+ from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
31
+
31
32
  if TYPE_CHECKING:
32
- from airflow.utils.context import Context
33
33
  from google.api_core.retry import Retry
34
34
 
35
+ from airflow.utils.context import Context
36
+
35
37
 
36
38
  class CloudVideoIntelligenceDetectVideoLabelsOperator(GoogleCloudBaseOperator):
37
39
  """
@@ -23,9 +23,6 @@ from collections.abc import Sequence
23
23
  from copy import deepcopy
24
24
  from typing import TYPE_CHECKING, Any
25
25
 
26
- from airflow.providers.google.cloud.hooks.vision import CloudVisionHook
27
- from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
28
- from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
29
26
  from google.api_core.exceptions import AlreadyExists
30
27
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
31
28
  from google.cloud.vision_v1 import (
@@ -36,11 +33,16 @@ from google.cloud.vision_v1 import (
36
33
  ReferenceImage,
37
34
  )
38
35
 
36
+ from airflow.providers.google.cloud.hooks.vision import CloudVisionHook
37
+ from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
38
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
39
+
39
40
  if TYPE_CHECKING:
40
- from airflow.utils.context import Context
41
41
  from google.api_core.retry import Retry
42
42
  from google.protobuf.field_mask_pb2 import FieldMask
43
43
 
44
+ from airflow.utils.context import Context
45
+
44
46
 
45
47
  MetaData = Sequence[tuple[str, str]]
46
48
 
@@ -23,6 +23,11 @@ import uuid
23
23
  from collections.abc import Sequence
24
24
  from typing import TYPE_CHECKING
25
25
 
26
+ from google.api_core.exceptions import AlreadyExists
27
+ from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
28
+ from google.cloud.workflows.executions_v1beta import Execution
29
+ from google.cloud.workflows_v1beta import Workflow
30
+
26
31
  from airflow.providers.google.cloud.hooks.workflows import WorkflowsHook
27
32
  from airflow.providers.google.cloud.links.workflows import (
28
33
  WorkflowsExecutionLink,
@@ -31,16 +36,13 @@ from airflow.providers.google.cloud.links.workflows import (
31
36
  )
32
37
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
33
38
  from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
34
- from google.api_core.exceptions import AlreadyExists
35
- from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
36
- from google.cloud.workflows.executions_v1beta import Execution
37
- from google.cloud.workflows_v1beta import Workflow
38
39
 
39
40
  if TYPE_CHECKING:
40
- from airflow.utils.context import Context
41
41
  from google.api_core.retry import Retry
42
42
  from google.protobuf.field_mask_pb2 import FieldMask
43
43
 
44
+ from airflow.utils.context import Context
45
+
44
46
  from airflow.utils.hashlib_wrapper import md5
45
47
 
46
48
 
@@ -21,6 +21,8 @@ from __future__ import annotations
21
21
  import logging
22
22
  from collections.abc import Sequence
23
23
 
24
+ from google.auth.exceptions import DefaultCredentialsError
25
+
24
26
  from airflow.exceptions import AirflowException
25
27
  from airflow.providers.google.cloud._internal_client.secret_manager_client import _SecretManagerClient
26
28
  from airflow.providers.google.cloud.utils.credentials_provider import (
@@ -30,7 +32,6 @@ from airflow.providers.google.cloud.utils.credentials_provider import (
30
32
  from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
31
33
  from airflow.secrets import BaseSecretsBackend
32
34
  from airflow.utils.log.logging_mixin import LoggingMixin
33
- from google.auth.exceptions import DefaultCredentialsError
34
35
 
35
36
  log = logging.getLogger(__name__)
36
37
 
@@ -22,17 +22,19 @@ from __future__ import annotations
22
22
  from collections.abc import Sequence
23
23
  from typing import TYPE_CHECKING
24
24
 
25
+ from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
26
+ from google.cloud.bigquery_datatransfer_v1 import TransferState
27
+
25
28
  from airflow.exceptions import AirflowException
26
29
  from airflow.providers.google.cloud.hooks.bigquery_dts import BiqQueryDataTransferServiceHook
27
30
  from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
28
31
  from airflow.sensors.base import BaseSensorOperator
29
- from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
30
- from google.cloud.bigquery_datatransfer_v1 import TransferState
31
32
 
32
33
  if TYPE_CHECKING:
33
- from airflow.utils.context import Context
34
34
  from google.api_core.retry import Retry
35
35
 
36
+ from airflow.utils.context import Context
37
+
36
38
 
37
39
  class BigQueryDataTransferServiceTransferRunSensor(BaseSensorOperator):
38
40
  """
@@ -23,13 +23,14 @@ from collections.abc import Sequence
23
23
  from typing import TYPE_CHECKING
24
24
 
25
25
  import google.api_core.exceptions
26
+ from google.cloud.bigtable import enums
27
+ from google.cloud.bigtable.table import ClusterState
28
+
26
29
  from airflow.providers.google.cloud.hooks.bigtable import BigtableHook
27
30
  from airflow.providers.google.cloud.links.bigtable import BigtableTablesLink
28
31
  from airflow.providers.google.cloud.operators.bigtable import BigtableValidationMixin
29
32
  from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
30
33
  from airflow.sensors.base import BaseSensorOperator
31
- from google.cloud.bigtable import enums
32
- from google.cloud.bigtable.table import ClusterState
33
34
 
34
35
  if TYPE_CHECKING:
35
36
  from airflow.utils.context import Context
@@ -26,6 +26,7 @@ from functools import cached_property
26
26
  from typing import TYPE_CHECKING
27
27
 
28
28
  from dateutil import parser
29
+ from google.cloud.orchestration.airflow.service_v1.types import Environment, ExecuteAirflowCommandResponse
29
30
 
30
31
  from airflow.configuration import conf
31
32
  from airflow.exceptions import AirflowException
@@ -34,7 +35,6 @@ from airflow.providers.google.cloud.triggers.cloud_composer import CloudComposer
34
35
  from airflow.providers.google.common.consts import GOOGLE_DEFAULT_DEFERRABLE_METHOD_NAME
35
36
  from airflow.sensors.base import BaseSensorOperator
36
37
  from airflow.utils.state import TaskInstanceState
37
- from google.cloud.orchestration.airflow.service_v1.types import Environment, ExecuteAirflowCommandResponse
38
38
 
39
39
  if TYPE_CHECKING:
40
40
  from airflow.utils.context import Context
@@ -23,9 +23,14 @@ from collections.abc import Sequence
23
23
  from typing import TYPE_CHECKING
24
24
 
25
25
  if TYPE_CHECKING:
26
- from airflow.utils.context import Context
27
26
  from google.api_core.retry import Retry
28
27
 
28
+ from airflow.utils.context import Context
29
+
30
+ from google.api_core.exceptions import GoogleAPICallError
31
+ from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
32
+ from google.cloud.dataplex_v1.types import DataScanJob
33
+
29
34
  from airflow.exceptions import AirflowException
30
35
  from airflow.providers.google.cloud.hooks.dataplex import (
31
36
  AirflowDataQualityScanException,
@@ -33,9 +38,6 @@ from airflow.providers.google.cloud.hooks.dataplex import (
33
38
  DataplexHook,
34
39
  )
35
40
  from airflow.sensors.base import BaseSensorOperator
36
- from google.api_core.exceptions import GoogleAPICallError
37
- from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
38
- from google.cloud.dataplex_v1.types import DataScanJob
39
41
 
40
42
 
41
43
  class TaskState:
@@ -23,12 +23,13 @@ import time
23
23
  from collections.abc import Sequence
24
24
  from typing import TYPE_CHECKING
25
25
 
26
+ from google.api_core.exceptions import ServerError
27
+ from google.cloud.dataproc_v1.types import Batch, JobStatus
28
+
26
29
  from airflow.exceptions import AirflowException
27
30
  from airflow.providers.google.cloud.hooks.dataproc import DataprocHook
28
31
  from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
29
32
  from airflow.sensors.base import BaseSensorOperator
30
- from google.api_core.exceptions import ServerError
31
- from google.cloud.dataproc_v1.types import Batch, JobStatus
32
33
 
33
34
  if TYPE_CHECKING:
34
35
  from airflow.utils.context import Context
@@ -26,9 +26,10 @@ from airflow.providers.google.cloud.hooks.gcs import parse_json_from_gcs
26
26
  from airflow.sensors.base import BaseSensorOperator
27
27
 
28
28
  if TYPE_CHECKING:
29
- from airflow.utils.context import Context
30
29
  from google.api_core.operation import Operation
31
30
 
31
+ from airflow.utils.context import Context
32
+
32
33
 
33
34
  class MetastoreHivePartitionSensor(BaseSensorOperator):
34
35
  """
@@ -25,6 +25,8 @@ from collections.abc import Sequence
25
25
  from datetime import datetime, timedelta
26
26
  from typing import TYPE_CHECKING, Any, Callable
27
27
 
28
+ from google.cloud.storage.retry import DEFAULT_RETRY
29
+
28
30
  from airflow.configuration import conf
29
31
  from airflow.exceptions import AirflowException
30
32
  from airflow.providers.google.cloud.hooks.gcs import GCSHook
@@ -35,12 +37,12 @@ from airflow.providers.google.cloud.triggers.gcs import (
35
37
  GCSUploadSessionTrigger,
36
38
  )
37
39
  from airflow.sensors.base import BaseSensorOperator, poke_mode_only
38
- from google.cloud.storage.retry import DEFAULT_RETRY
39
40
 
40
41
  if TYPE_CHECKING:
41
- from airflow.utils.context import Context
42
42
  from google.api_core.retry import Retry
43
43
 
44
+ from airflow.utils.context import Context
45
+
44
46
 
45
47
  class GCSObjectExistenceSensor(BaseSensorOperator):
46
48
  """
@@ -23,13 +23,14 @@ from collections.abc import Sequence
23
23
  from datetime import timedelta
24
24
  from typing import TYPE_CHECKING, Any, Callable
25
25
 
26
+ from google.cloud import pubsub_v1
27
+ from google.cloud.pubsub_v1.types import ReceivedMessage
28
+
26
29
  from airflow.configuration import conf
27
30
  from airflow.exceptions import AirflowException
28
31
  from airflow.providers.google.cloud.hooks.pubsub import PubSubHook
29
32
  from airflow.providers.google.cloud.triggers.pubsub import PubsubPullTrigger
30
33
  from airflow.sensors.base import BaseSensorOperator
31
- from google.cloud import pubsub_v1
32
- from google.cloud.pubsub_v1.types import ReceivedMessage
33
34
 
34
35
  if TYPE_CHECKING:
35
36
  from airflow.utils.context import Context
@@ -19,17 +19,19 @@ from __future__ import annotations
19
19
  from collections.abc import Sequence
20
20
  from typing import TYPE_CHECKING
21
21
 
22
+ from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
23
+ from google.cloud.workflows.executions_v1beta import Execution
24
+
22
25
  from airflow.exceptions import AirflowException
23
26
  from airflow.providers.google.cloud.hooks.workflows import WorkflowsHook
24
27
  from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
25
28
  from airflow.sensors.base import BaseSensorOperator
26
- from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
27
- from google.cloud.workflows.executions_v1beta import Execution
28
29
 
29
30
  if TYPE_CHECKING:
30
- from airflow.utils.context import Context
31
31
  from google.api_core.retry import Retry
32
32
 
33
+ from airflow.utils.context import Context
34
+
33
35
 
34
36
  class WorkflowExecutionSensor(BaseSensorOperator):
35
37
  """
@@ -22,6 +22,9 @@ from __future__ import annotations
22
22
  from collections.abc import Sequence
23
23
  from typing import TYPE_CHECKING, Any
24
24
 
25
+ from google.api_core.exceptions import Conflict
26
+ from google.cloud.bigquery import DEFAULT_RETRY, UnknownJob
27
+
25
28
  from airflow.configuration import conf
26
29
  from airflow.exceptions import AirflowException
27
30
  from airflow.models import BaseOperator
@@ -30,13 +33,12 @@ from airflow.providers.google.cloud.links.bigquery import BigQueryTableLink
30
33
  from airflow.providers.google.cloud.triggers.bigquery import BigQueryInsertJobTrigger
31
34
  from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
32
35
  from airflow.utils.helpers import merge_dicts
33
- from google.api_core.exceptions import Conflict
34
- from google.cloud.bigquery import DEFAULT_RETRY, UnknownJob
35
36
 
36
37
  if TYPE_CHECKING:
37
- from airflow.utils.context import Context
38
38
  from google.api_core.retry import Retry
39
39
 
40
+ from airflow.utils.context import Context
41
+
40
42
 
41
43
  class BigQueryToGCSOperator(BaseOperator):
42
44
  """
@@ -23,15 +23,6 @@ import json
23
23
  from collections.abc import Sequence
24
24
  from typing import TYPE_CHECKING, Any
25
25
 
26
- from airflow.configuration import conf
27
- from airflow.exceptions import AirflowException
28
- from airflow.models import BaseOperator
29
- from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook, BigQueryJob
30
- from airflow.providers.google.cloud.hooks.gcs import GCSHook
31
- from airflow.providers.google.cloud.links.bigquery import BigQueryTableLink
32
- from airflow.providers.google.cloud.triggers.bigquery import BigQueryInsertJobTrigger
33
- from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
34
- from airflow.utils.helpers import merge_dicts
35
26
  from google.api_core.exceptions import BadRequest, Conflict
36
27
  from google.cloud.bigquery import (
37
28
  DEFAULT_RETRY,
@@ -45,10 +36,21 @@ from google.cloud.bigquery import (
45
36
  )
46
37
  from google.cloud.bigquery.table import EncryptionConfiguration, Table, TableReference
47
38
 
39
+ from airflow.configuration import conf
40
+ from airflow.exceptions import AirflowException
41
+ from airflow.models import BaseOperator
42
+ from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook, BigQueryJob
43
+ from airflow.providers.google.cloud.hooks.gcs import GCSHook
44
+ from airflow.providers.google.cloud.links.bigquery import BigQueryTableLink
45
+ from airflow.providers.google.cloud.triggers.bigquery import BigQueryInsertJobTrigger
46
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
47
+ from airflow.utils.helpers import merge_dicts
48
+
48
49
  if TYPE_CHECKING:
49
- from airflow.utils.context import Context
50
50
  from google.api_core.retry import Retry
51
51
 
52
+ from airflow.utils.context import Context
53
+
52
54
  ALLOWED_FORMATS = [
53
55
  "CSV",
54
56
  "NEWLINE_DELIMITED_JSON",
@@ -21,9 +21,10 @@ import asyncio
21
21
  from collections.abc import AsyncIterator, Sequence
22
22
  from typing import Any
23
23
 
24
+ from google.cloud.bigquery_datatransfer_v1 import TransferRun, TransferState
25
+
24
26
  from airflow.providers.google.cloud.hooks.bigquery_dts import AsyncBiqQueryDataTransferServiceHook
25
27
  from airflow.triggers.base import BaseTrigger, TriggerEvent
26
- from google.cloud.bigquery_datatransfer_v1 import TransferRun, TransferState
27
28
 
28
29
 
29
30
  class BigQueryDataTransferRunTrigger(BaseTrigger):
@@ -20,9 +20,10 @@ import asyncio
20
20
  from collections.abc import AsyncIterator, Sequence
21
21
  from typing import Any
22
22
 
23
+ from google.cloud.batch_v1 import Job, JobStatus
24
+
23
25
  from airflow.providers.google.cloud.hooks.cloud_batch import CloudBatchAsyncHook
24
26
  from airflow.triggers.base import BaseTrigger, TriggerEvent
25
- from google.cloud.batch_v1 import Job, JobStatus
26
27
 
27
28
  DEFAULT_BATCH_LOCATION = "us-central1"
28
29
 
@@ -20,9 +20,10 @@ import asyncio
20
20
  from collections.abc import AsyncIterator, Sequence
21
21
  from typing import Any
22
22
 
23
+ from google.cloud.devtools.cloudbuild_v1.types import Build
24
+
23
25
  from airflow.providers.google.cloud.hooks.cloud_build import CloudBuildAsyncHook
24
26
  from airflow.triggers.base import BaseTrigger, TriggerEvent
25
- from google.cloud.devtools.cloudbuild_v1.types import Build
26
27
 
27
28
 
28
29
  class CloudBuildCreateBuildTrigger(BaseTrigger):
@@ -25,11 +25,11 @@ from datetime import datetime
25
25
  from typing import Any
26
26
 
27
27
  from dateutil import parser
28
+ from google.cloud.orchestration.airflow.service_v1.types import ExecuteAirflowCommandResponse
28
29
 
29
30
  from airflow.exceptions import AirflowException
30
31
  from airflow.providers.google.cloud.hooks.cloud_composer import CloudComposerAsyncHook
31
32
  from airflow.triggers.base import BaseTrigger, TriggerEvent
32
- from google.cloud.orchestration.airflow.service_v1.types import ExecuteAirflowCommandResponse
33
33
 
34
34
 
35
35
  class CloudComposerExecutionTrigger(BaseTrigger):
@@ -21,6 +21,9 @@ import asyncio
21
21
  from collections.abc import AsyncIterator, Iterable, Sequence
22
22
  from typing import Any
23
23
 
24
+ from google.api_core.exceptions import GoogleAPIError
25
+ from google.cloud.storage_transfer_v1.types import TransferOperation
26
+
24
27
  from airflow.exceptions import AirflowException
25
28
  from airflow.providers.google.cloud.hooks.cloud_storage_transfer_service import (
26
29
  CloudDataTransferServiceAsyncHook,
@@ -28,8 +31,6 @@ from airflow.providers.google.cloud.hooks.cloud_storage_transfer_service import
28
31
  )
29
32
  from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
30
33
  from airflow.triggers.base import BaseTrigger, TriggerEvent
31
- from google.api_core.exceptions import GoogleAPIError
32
- from google.cloud.storage_transfer_v1.types import TransferOperation
33
34
 
34
35
 
35
36
  class CloudStorageTransferServiceCreateJobsTrigger(BaseTrigger):
@@ -22,8 +22,6 @@ from collections.abc import Sequence
22
22
  from functools import cached_property
23
23
  from typing import TYPE_CHECKING, Any
24
24
 
25
- from airflow.providers.google.cloud.hooks.dataflow import AsyncDataflowHook, DataflowJobStatus
26
- from airflow.triggers.base import BaseTrigger, TriggerEvent
27
25
  from google.cloud.dataflow_v1beta3 import JobState
28
26
  from google.cloud.dataflow_v1beta3.types import (
29
27
  AutoscalingEvent,
@@ -34,6 +32,9 @@ from google.cloud.dataflow_v1beta3.types import (
34
32
  MetricUpdate,
35
33
  )
36
34
 
35
+ from airflow.providers.google.cloud.hooks.dataflow import AsyncDataflowHook, DataflowJobStatus
36
+ from airflow.triggers.base import BaseTrigger, TriggerEvent
37
+
37
38
  if TYPE_CHECKING:
38
39
  from google.cloud.dataflow_v1beta3.services.messages_v1_beta3.pagers import ListJobMessagesAsyncPager
39
40
 
@@ -22,9 +22,10 @@ from __future__ import annotations
22
22
  import asyncio
23
23
  from collections.abc import AsyncIterator, Sequence
24
24
 
25
+ from google.cloud.dataplex_v1.types import DataScanJob
26
+
25
27
  from airflow.providers.google.cloud.hooks.dataplex import DataplexAsyncHook
26
28
  from airflow.triggers.base import BaseTrigger, TriggerEvent
27
- from google.cloud.dataplex_v1.types import DataScanJob
28
29
 
29
30
 
30
31
  class DataplexDataQualityJobTrigger(BaseTrigger):
@@ -25,6 +25,9 @@ import time
25
25
  from collections.abc import AsyncIterator, Sequence
26
26
  from typing import TYPE_CHECKING, Any
27
27
 
28
+ from google.api_core.exceptions import NotFound
29
+ from google.cloud.dataproc_v1 import Batch, Cluster, ClusterStatus, JobStatus
30
+
28
31
  from airflow.exceptions import AirflowException
29
32
  from airflow.models.taskinstance import TaskInstance
30
33
  from airflow.providers.google.cloud.hooks.dataproc import DataprocAsyncHook, DataprocHook
@@ -33,8 +36,6 @@ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
33
36
  from airflow.triggers.base import BaseTrigger, TriggerEvent
34
37
  from airflow.utils.session import provide_session
35
38
  from airflow.utils.state import TaskInstanceState
36
- from google.api_core.exceptions import NotFound
37
- from google.cloud.dataproc_v1 import Batch, Cluster, ClusterStatus, JobStatus
38
39
 
39
40
  if TYPE_CHECKING:
40
41
  from sqlalchemy.orm.session import Session
@@ -23,6 +23,7 @@ from collections.abc import AsyncIterator, Sequence
23
23
  from functools import cached_property
24
24
  from typing import TYPE_CHECKING, Any
25
25
 
26
+ from google.cloud.container_v1.types import Operation
26
27
  from packaging.version import parse as parse_version
27
28
 
28
29
  from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
@@ -36,7 +37,6 @@ from airflow.providers.google.cloud.hooks.kubernetes_engine import (
36
37
  )
37
38
  from airflow.providers_manager import ProvidersManager
38
39
  from airflow.triggers.base import BaseTrigger, TriggerEvent
39
- from google.cloud.container_v1.types import Operation
40
40
 
41
41
  if TYPE_CHECKING:
42
42
  from datetime import datetime
@@ -23,9 +23,10 @@ from collections.abc import AsyncIterator, Sequence
23
23
  from functools import cached_property
24
24
  from typing import Any
25
25
 
26
+ from google.cloud.pubsub_v1.types import ReceivedMessage
27
+
26
28
  from airflow.providers.google.cloud.hooks.pubsub import PubSubAsyncHook
27
29
  from airflow.triggers.base import BaseTrigger, TriggerEvent
28
- from google.cloud.pubsub_v1.types import ReceivedMessage
29
30
 
30
31
 
31
32
  class PubsubPullTrigger(BaseTrigger):
@@ -20,6 +20,14 @@ from collections.abc import AsyncIterator, Sequence
20
20
  from functools import cached_property
21
21
  from typing import TYPE_CHECKING, Any
22
22
 
23
+ from google.cloud.aiplatform_v1 import (
24
+ BatchPredictionJob,
25
+ HyperparameterTuningJob,
26
+ JobState,
27
+ PipelineState,
28
+ types,
29
+ )
30
+
23
31
  from airflow.exceptions import AirflowException
24
32
  from airflow.providers.google.cloud.hooks.vertex_ai.batch_prediction_job import BatchPredictionJobAsyncHook
25
33
  from airflow.providers.google.cloud.hooks.vertex_ai.custom_job import CustomJobAsyncHook
@@ -28,13 +36,6 @@ from airflow.providers.google.cloud.hooks.vertex_ai.hyperparameter_tuning_job im
28
36
  )
29
37
  from airflow.providers.google.cloud.hooks.vertex_ai.pipeline_job import PipelineJobAsyncHook
30
38
  from airflow.triggers.base import BaseTrigger, TriggerEvent
31
- from google.cloud.aiplatform_v1 import (
32
- BatchPredictionJob,
33
- HyperparameterTuningJob,
34
- JobState,
35
- PipelineState,
36
- types,
37
- )
38
39
 
39
40
  if TYPE_CHECKING:
40
41
  from proto import Message
@@ -29,6 +29,10 @@ from urllib.parse import urlencode
29
29
 
30
30
  import google.auth
31
31
  import google.oauth2.service_account
32
+ from google.auth import impersonated_credentials # type: ignore[attr-defined]
33
+ from google.auth.credentials import AnonymousCredentials, Credentials
34
+ from google.auth.environment_vars import CREDENTIALS, LEGACY_PROJECT, PROJECT
35
+
32
36
  from airflow.exceptions import AirflowException
33
37
  from airflow.providers.google.cloud._internal_client.secret_manager_client import _SecretManagerClient
34
38
  from airflow.providers.google.cloud.utils.external_token_supplier import (
@@ -36,9 +40,6 @@ from airflow.providers.google.cloud.utils.external_token_supplier import (
36
40
  )
37
41
  from airflow.utils.log.logging_mixin import LoggingMixin
38
42
  from airflow.utils.process_utils import patch_environ
39
- from google.auth import impersonated_credentials # type: ignore[attr-defined]
40
- from google.auth.credentials import AnonymousCredentials, Credentials
41
- from google.auth.environment_vars import CREDENTIALS, LEGACY_PROJECT, PROJECT
42
43
 
43
44
  log = logging.getLogger(__name__)
44
45
 
@@ -22,7 +22,6 @@ from functools import wraps
22
22
  from typing import TYPE_CHECKING, Any
23
23
 
24
24
  import requests
25
-
26
25
  from google.auth.exceptions import RefreshError
27
26
  from google.auth.identity_pool import SubjectTokenSupplier
28
27