apache-airflow-providers-google 14.0.0__py3-none-any.whl → 14.1.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/3rd-party-licenses/LICENSES.txt +14 -0
- airflow/providers/google/3rd-party-licenses/NOTICE +15 -0
- airflow/providers/google/__init__.py +1 -1
- airflow/providers/google/_vendor/__init__.py +0 -0
- airflow/providers/google/_vendor/json_merge_patch.py +91 -0
- airflow/providers/google/ads/hooks/ads.py +12 -11
- airflow/providers/google/cloud/_internal_client/secret_manager_client.py +3 -2
- airflow/providers/google/cloud/hooks/alloy_db.py +2 -3
- airflow/providers/google/cloud/hooks/automl.py +7 -13
- airflow/providers/google/cloud/hooks/bigquery.py +21 -22
- airflow/providers/google/cloud/hooks/bigquery_dts.py +8 -8
- airflow/providers/google/cloud/hooks/bigtable.py +3 -2
- airflow/providers/google/cloud/hooks/cloud_batch.py +4 -3
- airflow/providers/google/cloud/hooks/cloud_build.py +7 -13
- airflow/providers/google/cloud/hooks/cloud_composer.py +7 -12
- airflow/providers/google/cloud/hooks/cloud_memorystore.py +4 -3
- airflow/providers/google/cloud/hooks/cloud_run.py +4 -3
- airflow/providers/google/cloud/hooks/cloud_sql.py +1 -1
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +8 -9
- airflow/providers/google/cloud/hooks/compute.py +3 -3
- airflow/providers/google/cloud/hooks/datacatalog.py +4 -3
- airflow/providers/google/cloud/hooks/dataflow.py +12 -12
- airflow/providers/google/cloud/hooks/dataform.py +3 -2
- airflow/providers/google/cloud/hooks/datafusion.py +2 -2
- airflow/providers/google/cloud/hooks/dataplex.py +11 -10
- airflow/providers/google/cloud/hooks/dataproc.py +5 -4
- airflow/providers/google/cloud/hooks/dataproc_metastore.py +4 -3
- airflow/providers/google/cloud/hooks/dlp.py +4 -3
- airflow/providers/google/cloud/hooks/gcs.py +19 -12
- airflow/providers/google/cloud/hooks/kms.py +3 -2
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +20 -14
- airflow/providers/google/cloud/hooks/life_sciences.py +1 -1
- airflow/providers/google/cloud/hooks/managed_kafka.py +227 -3
- airflow/providers/google/cloud/hooks/natural_language.py +3 -2
- airflow/providers/google/cloud/hooks/os_login.py +3 -2
- airflow/providers/google/cloud/hooks/pubsub.py +6 -6
- airflow/providers/google/cloud/hooks/secret_manager.py +3 -2
- airflow/providers/google/cloud/hooks/spanner.py +2 -2
- airflow/providers/google/cloud/hooks/speech_to_text.py +3 -2
- airflow/providers/google/cloud/hooks/stackdriver.py +4 -4
- airflow/providers/google/cloud/hooks/tasks.py +4 -3
- airflow/providers/google/cloud/hooks/text_to_speech.py +3 -2
- airflow/providers/google/cloud/hooks/translate.py +6 -15
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +6 -12
- airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +6 -12
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +7 -13
- airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +5 -12
- airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +4 -11
- airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +4 -3
- airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +6 -12
- airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +6 -11
- airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +6 -12
- airflow/providers/google/cloud/hooks/vertex_ai/prediction_service.py +3 -2
- airflow/providers/google/cloud/hooks/video_intelligence.py +3 -2
- airflow/providers/google/cloud/hooks/vision.py +4 -3
- airflow/providers/google/cloud/hooks/workflows.py +3 -2
- airflow/providers/google/cloud/links/base.py +7 -1
- airflow/providers/google/cloud/links/datafusion.py +8 -2
- airflow/providers/google/cloud/links/dataproc.py +8 -1
- airflow/providers/google/cloud/links/kubernetes_engine.py +2 -1
- airflow/providers/google/cloud/links/managed_kafka.py +30 -0
- airflow/providers/google/cloud/log/gcs_task_handler.py +15 -7
- airflow/providers/google/cloud/log/stackdriver_task_handler.py +8 -6
- airflow/providers/google/cloud/openlineage/utils.py +4 -2
- airflow/providers/google/cloud/operators/alloy_db.py +6 -5
- airflow/providers/google/cloud/operators/automl.py +11 -9
- airflow/providers/google/cloud/operators/bigquery.py +8 -6
- airflow/providers/google/cloud/operators/bigquery_dts.py +10 -8
- airflow/providers/google/cloud/operators/bigtable.py +3 -1
- airflow/providers/google/cloud/operators/cloud_base.py +2 -1
- airflow/providers/google/cloud/operators/cloud_batch.py +4 -2
- airflow/providers/google/cloud/operators/cloud_build.py +5 -3
- airflow/providers/google/cloud/operators/cloud_composer.py +7 -5
- airflow/providers/google/cloud/operators/cloud_memorystore.py +6 -4
- airflow/providers/google/cloud/operators/cloud_run.py +5 -3
- airflow/providers/google/cloud/operators/compute.py +5 -4
- airflow/providers/google/cloud/operators/datacatalog.py +11 -9
- airflow/providers/google/cloud/operators/dataform.py +5 -3
- airflow/providers/google/cloud/operators/datafusion.py +1 -1
- airflow/providers/google/cloud/operators/dataplex.py +20 -18
- airflow/providers/google/cloud/operators/dataproc.py +34 -5
- airflow/providers/google/cloud/operators/dataproc_metastore.py +18 -7
- airflow/providers/google/cloud/operators/dlp.py +19 -17
- airflow/providers/google/cloud/operators/gcs.py +5 -4
- airflow/providers/google/cloud/operators/kubernetes_engine.py +54 -5
- airflow/providers/google/cloud/operators/managed_kafka.py +271 -4
- airflow/providers/google/cloud/operators/natural_language.py +5 -3
- airflow/providers/google/cloud/operators/pubsub.py +10 -8
- airflow/providers/google/cloud/operators/speech_to_text.py +5 -3
- airflow/providers/google/cloud/operators/stackdriver.py +5 -3
- airflow/providers/google/cloud/operators/tasks.py +6 -4
- airflow/providers/google/cloud/operators/text_to_speech.py +4 -2
- airflow/providers/google/cloud/operators/translate.py +5 -3
- airflow/providers/google/cloud/operators/translate_speech.py +4 -2
- airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +8 -6
- airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +6 -4
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +8 -6
- airflow/providers/google/cloud/operators/vertex_ai/dataset.py +6 -4
- airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +6 -4
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +6 -4
- airflow/providers/google/cloud/operators/vertex_ai/model_service.py +6 -4
- airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +6 -4
- airflow/providers/google/cloud/operators/video_intelligence.py +5 -3
- airflow/providers/google/cloud/operators/vision.py +6 -4
- airflow/providers/google/cloud/operators/workflows.py +7 -5
- airflow/providers/google/cloud/secrets/secret_manager.py +2 -1
- airflow/providers/google/cloud/sensors/bigquery_dts.py +5 -3
- airflow/providers/google/cloud/sensors/bigtable.py +3 -2
- airflow/providers/google/cloud/sensors/cloud_composer.py +1 -1
- airflow/providers/google/cloud/sensors/dataplex.py +6 -4
- airflow/providers/google/cloud/sensors/dataproc.py +3 -2
- airflow/providers/google/cloud/sensors/dataproc_metastore.py +2 -1
- airflow/providers/google/cloud/sensors/gcs.py +4 -2
- airflow/providers/google/cloud/sensors/pubsub.py +3 -2
- airflow/providers/google/cloud/sensors/workflows.py +5 -3
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +5 -3
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +12 -10
- airflow/providers/google/cloud/triggers/bigquery_dts.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_batch.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_build.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_composer.py +1 -1
- airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +3 -2
- airflow/providers/google/cloud/triggers/dataflow.py +3 -2
- airflow/providers/google/cloud/triggers/dataplex.py +2 -1
- airflow/providers/google/cloud/triggers/dataproc.py +3 -2
- airflow/providers/google/cloud/triggers/kubernetes_engine.py +1 -1
- airflow/providers/google/cloud/triggers/pubsub.py +2 -1
- airflow/providers/google/cloud/triggers/vertex_ai.py +8 -7
- airflow/providers/google/cloud/utils/credentials_provider.py +4 -3
- airflow/providers/google/cloud/utils/external_token_supplier.py +0 -1
- airflow/providers/google/cloud/{example_dags/__init__.py → utils/validators.py} +27 -0
- airflow/providers/google/common/auth_backend/google_openid.py +14 -5
- airflow/providers/google/common/consts.py +2 -1
- airflow/providers/google/common/hooks/base_google.py +7 -8
- airflow/providers/google/common/hooks/operation_helpers.py +79 -0
- airflow/providers/google/get_provider_info.py +11 -6
- airflow/providers/google/marketing_platform/hooks/analytics_admin.py +3 -2
- airflow/providers/google/marketing_platform/hooks/search_ads.py +1 -1
- airflow/providers/google/marketing_platform/links/analytics_admin.py +10 -2
- airflow/providers/google/marketing_platform/operators/analytics_admin.py +7 -5
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-14.1.0rc1.dist-info}/METADATA +35 -38
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-14.1.0rc1.dist-info}/RECORD +144 -140
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-14.1.0rc1.dist-info}/WHEEL +1 -1
- airflow/providers/google/cloud/example_dags/example_cloud_task.py +0 -54
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-14.1.0rc1.dist-info}/entry_points.txt +0 -0
@@ -23,6 +23,10 @@ from __future__ import annotations
|
|
23
23
|
from collections.abc import Sequence
|
24
24
|
from typing import TYPE_CHECKING, Any
|
25
25
|
|
26
|
+
from google.api_core.exceptions import NotFound
|
27
|
+
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
28
|
+
from google.cloud.aiplatform_v1 import types
|
29
|
+
|
26
30
|
from airflow.configuration import conf
|
27
31
|
from airflow.exceptions import AirflowException
|
28
32
|
from airflow.providers.google.cloud.hooks.vertex_ai.hyperparameter_tuning_job import (
|
@@ -34,15 +38,13 @@ from airflow.providers.google.cloud.links.vertex_ai import (
|
|
34
38
|
)
|
35
39
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
36
40
|
from airflow.providers.google.cloud.triggers.vertex_ai import CreateHyperparameterTuningJobTrigger
|
37
|
-
from google.api_core.exceptions import NotFound
|
38
|
-
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
39
|
-
from google.cloud.aiplatform_v1 import types
|
40
41
|
|
41
42
|
if TYPE_CHECKING:
|
42
|
-
from airflow.utils.context import Context
|
43
43
|
from google.api_core.retry import Retry
|
44
44
|
from google.cloud.aiplatform import HyperparameterTuningJob, gapic, hyperparameter_tuning
|
45
45
|
|
46
|
+
from airflow.utils.context import Context
|
47
|
+
|
46
48
|
|
47
49
|
class CreateHyperparameterTuningJobOperator(GoogleCloudBaseOperator):
|
48
50
|
"""
|
@@ -22,6 +22,10 @@ from __future__ import annotations
|
|
22
22
|
from collections.abc import Sequence
|
23
23
|
from typing import TYPE_CHECKING
|
24
24
|
|
25
|
+
from google.api_core.exceptions import NotFound
|
26
|
+
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
27
|
+
from google.cloud.aiplatform_v1.types import Model, model_service
|
28
|
+
|
25
29
|
from airflow.providers.google.cloud.hooks.vertex_ai.model_service import ModelServiceHook
|
26
30
|
from airflow.providers.google.cloud.links.vertex_ai import (
|
27
31
|
VertexAIModelExportLink,
|
@@ -29,14 +33,12 @@ from airflow.providers.google.cloud.links.vertex_ai import (
|
|
29
33
|
VertexAIModelListLink,
|
30
34
|
)
|
31
35
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
32
|
-
from google.api_core.exceptions import NotFound
|
33
|
-
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
34
|
-
from google.cloud.aiplatform_v1.types import Model, model_service
|
35
36
|
|
36
37
|
if TYPE_CHECKING:
|
37
|
-
from airflow.utils.context import Context
|
38
38
|
from google.api_core.retry import Retry
|
39
39
|
|
40
|
+
from airflow.utils.context import Context
|
41
|
+
|
40
42
|
|
41
43
|
class DeleteModelOperator(GoogleCloudBaseOperator):
|
42
44
|
"""
|
@@ -23,6 +23,10 @@ from collections.abc import Sequence
|
|
23
23
|
from functools import cached_property
|
24
24
|
from typing import TYPE_CHECKING, Any
|
25
25
|
|
26
|
+
from google.api_core.exceptions import NotFound
|
27
|
+
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
28
|
+
from google.cloud.aiplatform_v1 import types
|
29
|
+
|
26
30
|
from airflow.configuration import conf
|
27
31
|
from airflow.exceptions import AirflowException
|
28
32
|
from airflow.providers.google.cloud.hooks.vertex_ai.pipeline_job import PipelineJobHook
|
@@ -32,16 +36,14 @@ from airflow.providers.google.cloud.links.vertex_ai import (
|
|
32
36
|
)
|
33
37
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
34
38
|
from airflow.providers.google.cloud.triggers.vertex_ai import RunPipelineJobTrigger
|
35
|
-
from google.api_core.exceptions import NotFound
|
36
|
-
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
37
|
-
from google.cloud.aiplatform_v1 import types
|
38
39
|
|
39
40
|
if TYPE_CHECKING:
|
40
|
-
from airflow.utils.context import Context
|
41
41
|
from google.api_core.retry import Retry
|
42
42
|
from google.cloud.aiplatform import PipelineJob
|
43
43
|
from google.cloud.aiplatform.metadata import experiment_resources
|
44
44
|
|
45
|
+
from airflow.utils.context import Context
|
46
|
+
|
45
47
|
|
46
48
|
class RunPipelineJobOperator(GoogleCloudBaseOperator):
|
47
49
|
"""
|
@@ -22,16 +22,18 @@ from __future__ import annotations
|
|
22
22
|
from collections.abc import Sequence
|
23
23
|
from typing import TYPE_CHECKING
|
24
24
|
|
25
|
-
from airflow.providers.google.cloud.hooks.video_intelligence import CloudVideoIntelligenceHook
|
26
|
-
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
27
25
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
28
26
|
from google.cloud.videointelligence_v1 import Feature, VideoContext
|
29
27
|
from google.protobuf.json_format import MessageToDict
|
30
28
|
|
29
|
+
from airflow.providers.google.cloud.hooks.video_intelligence import CloudVideoIntelligenceHook
|
30
|
+
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
31
|
+
|
31
32
|
if TYPE_CHECKING:
|
32
|
-
from airflow.utils.context import Context
|
33
33
|
from google.api_core.retry import Retry
|
34
34
|
|
35
|
+
from airflow.utils.context import Context
|
36
|
+
|
35
37
|
|
36
38
|
class CloudVideoIntelligenceDetectVideoLabelsOperator(GoogleCloudBaseOperator):
|
37
39
|
"""
|
@@ -23,9 +23,6 @@ from collections.abc import Sequence
|
|
23
23
|
from copy import deepcopy
|
24
24
|
from typing import TYPE_CHECKING, Any
|
25
25
|
|
26
|
-
from airflow.providers.google.cloud.hooks.vision import CloudVisionHook
|
27
|
-
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
28
|
-
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
29
26
|
from google.api_core.exceptions import AlreadyExists
|
30
27
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
31
28
|
from google.cloud.vision_v1 import (
|
@@ -36,11 +33,16 @@ from google.cloud.vision_v1 import (
|
|
36
33
|
ReferenceImage,
|
37
34
|
)
|
38
35
|
|
36
|
+
from airflow.providers.google.cloud.hooks.vision import CloudVisionHook
|
37
|
+
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
38
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
39
|
+
|
39
40
|
if TYPE_CHECKING:
|
40
|
-
from airflow.utils.context import Context
|
41
41
|
from google.api_core.retry import Retry
|
42
42
|
from google.protobuf.field_mask_pb2 import FieldMask
|
43
43
|
|
44
|
+
from airflow.utils.context import Context
|
45
|
+
|
44
46
|
|
45
47
|
MetaData = Sequence[tuple[str, str]]
|
46
48
|
|
@@ -23,6 +23,11 @@ import uuid
|
|
23
23
|
from collections.abc import Sequence
|
24
24
|
from typing import TYPE_CHECKING
|
25
25
|
|
26
|
+
from google.api_core.exceptions import AlreadyExists
|
27
|
+
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
28
|
+
from google.cloud.workflows.executions_v1beta import Execution
|
29
|
+
from google.cloud.workflows_v1beta import Workflow
|
30
|
+
|
26
31
|
from airflow.providers.google.cloud.hooks.workflows import WorkflowsHook
|
27
32
|
from airflow.providers.google.cloud.links.workflows import (
|
28
33
|
WorkflowsExecutionLink,
|
@@ -31,16 +36,13 @@ from airflow.providers.google.cloud.links.workflows import (
|
|
31
36
|
)
|
32
37
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
33
38
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
34
|
-
from google.api_core.exceptions import AlreadyExists
|
35
|
-
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
36
|
-
from google.cloud.workflows.executions_v1beta import Execution
|
37
|
-
from google.cloud.workflows_v1beta import Workflow
|
38
39
|
|
39
40
|
if TYPE_CHECKING:
|
40
|
-
from airflow.utils.context import Context
|
41
41
|
from google.api_core.retry import Retry
|
42
42
|
from google.protobuf.field_mask_pb2 import FieldMask
|
43
43
|
|
44
|
+
from airflow.utils.context import Context
|
45
|
+
|
44
46
|
from airflow.utils.hashlib_wrapper import md5
|
45
47
|
|
46
48
|
|
@@ -21,6 +21,8 @@ from __future__ import annotations
|
|
21
21
|
import logging
|
22
22
|
from collections.abc import Sequence
|
23
23
|
|
24
|
+
from google.auth.exceptions import DefaultCredentialsError
|
25
|
+
|
24
26
|
from airflow.exceptions import AirflowException
|
25
27
|
from airflow.providers.google.cloud._internal_client.secret_manager_client import _SecretManagerClient
|
26
28
|
from airflow.providers.google.cloud.utils.credentials_provider import (
|
@@ -30,7 +32,6 @@ from airflow.providers.google.cloud.utils.credentials_provider import (
|
|
30
32
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
31
33
|
from airflow.secrets import BaseSecretsBackend
|
32
34
|
from airflow.utils.log.logging_mixin import LoggingMixin
|
33
|
-
from google.auth.exceptions import DefaultCredentialsError
|
34
35
|
|
35
36
|
log = logging.getLogger(__name__)
|
36
37
|
|
@@ -22,17 +22,19 @@ from __future__ import annotations
|
|
22
22
|
from collections.abc import Sequence
|
23
23
|
from typing import TYPE_CHECKING
|
24
24
|
|
25
|
+
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
26
|
+
from google.cloud.bigquery_datatransfer_v1 import TransferState
|
27
|
+
|
25
28
|
from airflow.exceptions import AirflowException
|
26
29
|
from airflow.providers.google.cloud.hooks.bigquery_dts import BiqQueryDataTransferServiceHook
|
27
30
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
28
31
|
from airflow.sensors.base import BaseSensorOperator
|
29
|
-
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
30
|
-
from google.cloud.bigquery_datatransfer_v1 import TransferState
|
31
32
|
|
32
33
|
if TYPE_CHECKING:
|
33
|
-
from airflow.utils.context import Context
|
34
34
|
from google.api_core.retry import Retry
|
35
35
|
|
36
|
+
from airflow.utils.context import Context
|
37
|
+
|
36
38
|
|
37
39
|
class BigQueryDataTransferServiceTransferRunSensor(BaseSensorOperator):
|
38
40
|
"""
|
@@ -23,13 +23,14 @@ from collections.abc import Sequence
|
|
23
23
|
from typing import TYPE_CHECKING
|
24
24
|
|
25
25
|
import google.api_core.exceptions
|
26
|
+
from google.cloud.bigtable import enums
|
27
|
+
from google.cloud.bigtable.table import ClusterState
|
28
|
+
|
26
29
|
from airflow.providers.google.cloud.hooks.bigtable import BigtableHook
|
27
30
|
from airflow.providers.google.cloud.links.bigtable import BigtableTablesLink
|
28
31
|
from airflow.providers.google.cloud.operators.bigtable import BigtableValidationMixin
|
29
32
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
30
33
|
from airflow.sensors.base import BaseSensorOperator
|
31
|
-
from google.cloud.bigtable import enums
|
32
|
-
from google.cloud.bigtable.table import ClusterState
|
33
34
|
|
34
35
|
if TYPE_CHECKING:
|
35
36
|
from airflow.utils.context import Context
|
@@ -26,6 +26,7 @@ from functools import cached_property
|
|
26
26
|
from typing import TYPE_CHECKING
|
27
27
|
|
28
28
|
from dateutil import parser
|
29
|
+
from google.cloud.orchestration.airflow.service_v1.types import Environment, ExecuteAirflowCommandResponse
|
29
30
|
|
30
31
|
from airflow.configuration import conf
|
31
32
|
from airflow.exceptions import AirflowException
|
@@ -34,7 +35,6 @@ from airflow.providers.google.cloud.triggers.cloud_composer import CloudComposer
|
|
34
35
|
from airflow.providers.google.common.consts import GOOGLE_DEFAULT_DEFERRABLE_METHOD_NAME
|
35
36
|
from airflow.sensors.base import BaseSensorOperator
|
36
37
|
from airflow.utils.state import TaskInstanceState
|
37
|
-
from google.cloud.orchestration.airflow.service_v1.types import Environment, ExecuteAirflowCommandResponse
|
38
38
|
|
39
39
|
if TYPE_CHECKING:
|
40
40
|
from airflow.utils.context import Context
|
@@ -23,9 +23,14 @@ from collections.abc import Sequence
|
|
23
23
|
from typing import TYPE_CHECKING
|
24
24
|
|
25
25
|
if TYPE_CHECKING:
|
26
|
-
from airflow.utils.context import Context
|
27
26
|
from google.api_core.retry import Retry
|
28
27
|
|
28
|
+
from airflow.utils.context import Context
|
29
|
+
|
30
|
+
from google.api_core.exceptions import GoogleAPICallError
|
31
|
+
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
32
|
+
from google.cloud.dataplex_v1.types import DataScanJob
|
33
|
+
|
29
34
|
from airflow.exceptions import AirflowException
|
30
35
|
from airflow.providers.google.cloud.hooks.dataplex import (
|
31
36
|
AirflowDataQualityScanException,
|
@@ -33,9 +38,6 @@ from airflow.providers.google.cloud.hooks.dataplex import (
|
|
33
38
|
DataplexHook,
|
34
39
|
)
|
35
40
|
from airflow.sensors.base import BaseSensorOperator
|
36
|
-
from google.api_core.exceptions import GoogleAPICallError
|
37
|
-
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
38
|
-
from google.cloud.dataplex_v1.types import DataScanJob
|
39
41
|
|
40
42
|
|
41
43
|
class TaskState:
|
@@ -23,12 +23,13 @@ import time
|
|
23
23
|
from collections.abc import Sequence
|
24
24
|
from typing import TYPE_CHECKING
|
25
25
|
|
26
|
+
from google.api_core.exceptions import ServerError
|
27
|
+
from google.cloud.dataproc_v1.types import Batch, JobStatus
|
28
|
+
|
26
29
|
from airflow.exceptions import AirflowException
|
27
30
|
from airflow.providers.google.cloud.hooks.dataproc import DataprocHook
|
28
31
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
29
32
|
from airflow.sensors.base import BaseSensorOperator
|
30
|
-
from google.api_core.exceptions import ServerError
|
31
|
-
from google.cloud.dataproc_v1.types import Batch, JobStatus
|
32
33
|
|
33
34
|
if TYPE_CHECKING:
|
34
35
|
from airflow.utils.context import Context
|
@@ -26,9 +26,10 @@ from airflow.providers.google.cloud.hooks.gcs import parse_json_from_gcs
|
|
26
26
|
from airflow.sensors.base import BaseSensorOperator
|
27
27
|
|
28
28
|
if TYPE_CHECKING:
|
29
|
-
from airflow.utils.context import Context
|
30
29
|
from google.api_core.operation import Operation
|
31
30
|
|
31
|
+
from airflow.utils.context import Context
|
32
|
+
|
32
33
|
|
33
34
|
class MetastoreHivePartitionSensor(BaseSensorOperator):
|
34
35
|
"""
|
@@ -25,6 +25,8 @@ from collections.abc import Sequence
|
|
25
25
|
from datetime import datetime, timedelta
|
26
26
|
from typing import TYPE_CHECKING, Any, Callable
|
27
27
|
|
28
|
+
from google.cloud.storage.retry import DEFAULT_RETRY
|
29
|
+
|
28
30
|
from airflow.configuration import conf
|
29
31
|
from airflow.exceptions import AirflowException
|
30
32
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
@@ -35,12 +37,12 @@ from airflow.providers.google.cloud.triggers.gcs import (
|
|
35
37
|
GCSUploadSessionTrigger,
|
36
38
|
)
|
37
39
|
from airflow.sensors.base import BaseSensorOperator, poke_mode_only
|
38
|
-
from google.cloud.storage.retry import DEFAULT_RETRY
|
39
40
|
|
40
41
|
if TYPE_CHECKING:
|
41
|
-
from airflow.utils.context import Context
|
42
42
|
from google.api_core.retry import Retry
|
43
43
|
|
44
|
+
from airflow.utils.context import Context
|
45
|
+
|
44
46
|
|
45
47
|
class GCSObjectExistenceSensor(BaseSensorOperator):
|
46
48
|
"""
|
@@ -23,13 +23,14 @@ from collections.abc import Sequence
|
|
23
23
|
from datetime import timedelta
|
24
24
|
from typing import TYPE_CHECKING, Any, Callable
|
25
25
|
|
26
|
+
from google.cloud import pubsub_v1
|
27
|
+
from google.cloud.pubsub_v1.types import ReceivedMessage
|
28
|
+
|
26
29
|
from airflow.configuration import conf
|
27
30
|
from airflow.exceptions import AirflowException
|
28
31
|
from airflow.providers.google.cloud.hooks.pubsub import PubSubHook
|
29
32
|
from airflow.providers.google.cloud.triggers.pubsub import PubsubPullTrigger
|
30
33
|
from airflow.sensors.base import BaseSensorOperator
|
31
|
-
from google.cloud import pubsub_v1
|
32
|
-
from google.cloud.pubsub_v1.types import ReceivedMessage
|
33
34
|
|
34
35
|
if TYPE_CHECKING:
|
35
36
|
from airflow.utils.context import Context
|
@@ -19,17 +19,19 @@ from __future__ import annotations
|
|
19
19
|
from collections.abc import Sequence
|
20
20
|
from typing import TYPE_CHECKING
|
21
21
|
|
22
|
+
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
23
|
+
from google.cloud.workflows.executions_v1beta import Execution
|
24
|
+
|
22
25
|
from airflow.exceptions import AirflowException
|
23
26
|
from airflow.providers.google.cloud.hooks.workflows import WorkflowsHook
|
24
27
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
25
28
|
from airflow.sensors.base import BaseSensorOperator
|
26
|
-
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
27
|
-
from google.cloud.workflows.executions_v1beta import Execution
|
28
29
|
|
29
30
|
if TYPE_CHECKING:
|
30
|
-
from airflow.utils.context import Context
|
31
31
|
from google.api_core.retry import Retry
|
32
32
|
|
33
|
+
from airflow.utils.context import Context
|
34
|
+
|
33
35
|
|
34
36
|
class WorkflowExecutionSensor(BaseSensorOperator):
|
35
37
|
"""
|
@@ -22,6 +22,9 @@ from __future__ import annotations
|
|
22
22
|
from collections.abc import Sequence
|
23
23
|
from typing import TYPE_CHECKING, Any
|
24
24
|
|
25
|
+
from google.api_core.exceptions import Conflict
|
26
|
+
from google.cloud.bigquery import DEFAULT_RETRY, UnknownJob
|
27
|
+
|
25
28
|
from airflow.configuration import conf
|
26
29
|
from airflow.exceptions import AirflowException
|
27
30
|
from airflow.models import BaseOperator
|
@@ -30,13 +33,12 @@ from airflow.providers.google.cloud.links.bigquery import BigQueryTableLink
|
|
30
33
|
from airflow.providers.google.cloud.triggers.bigquery import BigQueryInsertJobTrigger
|
31
34
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
32
35
|
from airflow.utils.helpers import merge_dicts
|
33
|
-
from google.api_core.exceptions import Conflict
|
34
|
-
from google.cloud.bigquery import DEFAULT_RETRY, UnknownJob
|
35
36
|
|
36
37
|
if TYPE_CHECKING:
|
37
|
-
from airflow.utils.context import Context
|
38
38
|
from google.api_core.retry import Retry
|
39
39
|
|
40
|
+
from airflow.utils.context import Context
|
41
|
+
|
40
42
|
|
41
43
|
class BigQueryToGCSOperator(BaseOperator):
|
42
44
|
"""
|
@@ -23,15 +23,6 @@ import json
|
|
23
23
|
from collections.abc import Sequence
|
24
24
|
from typing import TYPE_CHECKING, Any
|
25
25
|
|
26
|
-
from airflow.configuration import conf
|
27
|
-
from airflow.exceptions import AirflowException
|
28
|
-
from airflow.models import BaseOperator
|
29
|
-
from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook, BigQueryJob
|
30
|
-
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
31
|
-
from airflow.providers.google.cloud.links.bigquery import BigQueryTableLink
|
32
|
-
from airflow.providers.google.cloud.triggers.bigquery import BigQueryInsertJobTrigger
|
33
|
-
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
34
|
-
from airflow.utils.helpers import merge_dicts
|
35
26
|
from google.api_core.exceptions import BadRequest, Conflict
|
36
27
|
from google.cloud.bigquery import (
|
37
28
|
DEFAULT_RETRY,
|
@@ -45,10 +36,21 @@ from google.cloud.bigquery import (
|
|
45
36
|
)
|
46
37
|
from google.cloud.bigquery.table import EncryptionConfiguration, Table, TableReference
|
47
38
|
|
39
|
+
from airflow.configuration import conf
|
40
|
+
from airflow.exceptions import AirflowException
|
41
|
+
from airflow.models import BaseOperator
|
42
|
+
from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook, BigQueryJob
|
43
|
+
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
44
|
+
from airflow.providers.google.cloud.links.bigquery import BigQueryTableLink
|
45
|
+
from airflow.providers.google.cloud.triggers.bigquery import BigQueryInsertJobTrigger
|
46
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
47
|
+
from airflow.utils.helpers import merge_dicts
|
48
|
+
|
48
49
|
if TYPE_CHECKING:
|
49
|
-
from airflow.utils.context import Context
|
50
50
|
from google.api_core.retry import Retry
|
51
51
|
|
52
|
+
from airflow.utils.context import Context
|
53
|
+
|
52
54
|
ALLOWED_FORMATS = [
|
53
55
|
"CSV",
|
54
56
|
"NEWLINE_DELIMITED_JSON",
|
@@ -21,9 +21,10 @@ import asyncio
|
|
21
21
|
from collections.abc import AsyncIterator, Sequence
|
22
22
|
from typing import Any
|
23
23
|
|
24
|
+
from google.cloud.bigquery_datatransfer_v1 import TransferRun, TransferState
|
25
|
+
|
24
26
|
from airflow.providers.google.cloud.hooks.bigquery_dts import AsyncBiqQueryDataTransferServiceHook
|
25
27
|
from airflow.triggers.base import BaseTrigger, TriggerEvent
|
26
|
-
from google.cloud.bigquery_datatransfer_v1 import TransferRun, TransferState
|
27
28
|
|
28
29
|
|
29
30
|
class BigQueryDataTransferRunTrigger(BaseTrigger):
|
@@ -20,9 +20,10 @@ import asyncio
|
|
20
20
|
from collections.abc import AsyncIterator, Sequence
|
21
21
|
from typing import Any
|
22
22
|
|
23
|
+
from google.cloud.batch_v1 import Job, JobStatus
|
24
|
+
|
23
25
|
from airflow.providers.google.cloud.hooks.cloud_batch import CloudBatchAsyncHook
|
24
26
|
from airflow.triggers.base import BaseTrigger, TriggerEvent
|
25
|
-
from google.cloud.batch_v1 import Job, JobStatus
|
26
27
|
|
27
28
|
DEFAULT_BATCH_LOCATION = "us-central1"
|
28
29
|
|
@@ -20,9 +20,10 @@ import asyncio
|
|
20
20
|
from collections.abc import AsyncIterator, Sequence
|
21
21
|
from typing import Any
|
22
22
|
|
23
|
+
from google.cloud.devtools.cloudbuild_v1.types import Build
|
24
|
+
|
23
25
|
from airflow.providers.google.cloud.hooks.cloud_build import CloudBuildAsyncHook
|
24
26
|
from airflow.triggers.base import BaseTrigger, TriggerEvent
|
25
|
-
from google.cloud.devtools.cloudbuild_v1.types import Build
|
26
27
|
|
27
28
|
|
28
29
|
class CloudBuildCreateBuildTrigger(BaseTrigger):
|
@@ -25,11 +25,11 @@ from datetime import datetime
|
|
25
25
|
from typing import Any
|
26
26
|
|
27
27
|
from dateutil import parser
|
28
|
+
from google.cloud.orchestration.airflow.service_v1.types import ExecuteAirflowCommandResponse
|
28
29
|
|
29
30
|
from airflow.exceptions import AirflowException
|
30
31
|
from airflow.providers.google.cloud.hooks.cloud_composer import CloudComposerAsyncHook
|
31
32
|
from airflow.triggers.base import BaseTrigger, TriggerEvent
|
32
|
-
from google.cloud.orchestration.airflow.service_v1.types import ExecuteAirflowCommandResponse
|
33
33
|
|
34
34
|
|
35
35
|
class CloudComposerExecutionTrigger(BaseTrigger):
|
@@ -21,6 +21,9 @@ import asyncio
|
|
21
21
|
from collections.abc import AsyncIterator, Iterable, Sequence
|
22
22
|
from typing import Any
|
23
23
|
|
24
|
+
from google.api_core.exceptions import GoogleAPIError
|
25
|
+
from google.cloud.storage_transfer_v1.types import TransferOperation
|
26
|
+
|
24
27
|
from airflow.exceptions import AirflowException
|
25
28
|
from airflow.providers.google.cloud.hooks.cloud_storage_transfer_service import (
|
26
29
|
CloudDataTransferServiceAsyncHook,
|
@@ -28,8 +31,6 @@ from airflow.providers.google.cloud.hooks.cloud_storage_transfer_service import
|
|
28
31
|
)
|
29
32
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
30
33
|
from airflow.triggers.base import BaseTrigger, TriggerEvent
|
31
|
-
from google.api_core.exceptions import GoogleAPIError
|
32
|
-
from google.cloud.storage_transfer_v1.types import TransferOperation
|
33
34
|
|
34
35
|
|
35
36
|
class CloudStorageTransferServiceCreateJobsTrigger(BaseTrigger):
|
@@ -22,8 +22,6 @@ from collections.abc import Sequence
|
|
22
22
|
from functools import cached_property
|
23
23
|
from typing import TYPE_CHECKING, Any
|
24
24
|
|
25
|
-
from airflow.providers.google.cloud.hooks.dataflow import AsyncDataflowHook, DataflowJobStatus
|
26
|
-
from airflow.triggers.base import BaseTrigger, TriggerEvent
|
27
25
|
from google.cloud.dataflow_v1beta3 import JobState
|
28
26
|
from google.cloud.dataflow_v1beta3.types import (
|
29
27
|
AutoscalingEvent,
|
@@ -34,6 +32,9 @@ from google.cloud.dataflow_v1beta3.types import (
|
|
34
32
|
MetricUpdate,
|
35
33
|
)
|
36
34
|
|
35
|
+
from airflow.providers.google.cloud.hooks.dataflow import AsyncDataflowHook, DataflowJobStatus
|
36
|
+
from airflow.triggers.base import BaseTrigger, TriggerEvent
|
37
|
+
|
37
38
|
if TYPE_CHECKING:
|
38
39
|
from google.cloud.dataflow_v1beta3.services.messages_v1_beta3.pagers import ListJobMessagesAsyncPager
|
39
40
|
|
@@ -22,9 +22,10 @@ from __future__ import annotations
|
|
22
22
|
import asyncio
|
23
23
|
from collections.abc import AsyncIterator, Sequence
|
24
24
|
|
25
|
+
from google.cloud.dataplex_v1.types import DataScanJob
|
26
|
+
|
25
27
|
from airflow.providers.google.cloud.hooks.dataplex import DataplexAsyncHook
|
26
28
|
from airflow.triggers.base import BaseTrigger, TriggerEvent
|
27
|
-
from google.cloud.dataplex_v1.types import DataScanJob
|
28
29
|
|
29
30
|
|
30
31
|
class DataplexDataQualityJobTrigger(BaseTrigger):
|
@@ -25,6 +25,9 @@ import time
|
|
25
25
|
from collections.abc import AsyncIterator, Sequence
|
26
26
|
from typing import TYPE_CHECKING, Any
|
27
27
|
|
28
|
+
from google.api_core.exceptions import NotFound
|
29
|
+
from google.cloud.dataproc_v1 import Batch, Cluster, ClusterStatus, JobStatus
|
30
|
+
|
28
31
|
from airflow.exceptions import AirflowException
|
29
32
|
from airflow.models.taskinstance import TaskInstance
|
30
33
|
from airflow.providers.google.cloud.hooks.dataproc import DataprocAsyncHook, DataprocHook
|
@@ -33,8 +36,6 @@ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
|
33
36
|
from airflow.triggers.base import BaseTrigger, TriggerEvent
|
34
37
|
from airflow.utils.session import provide_session
|
35
38
|
from airflow.utils.state import TaskInstanceState
|
36
|
-
from google.api_core.exceptions import NotFound
|
37
|
-
from google.cloud.dataproc_v1 import Batch, Cluster, ClusterStatus, JobStatus
|
38
39
|
|
39
40
|
if TYPE_CHECKING:
|
40
41
|
from sqlalchemy.orm.session import Session
|
@@ -23,6 +23,7 @@ from collections.abc import AsyncIterator, Sequence
|
|
23
23
|
from functools import cached_property
|
24
24
|
from typing import TYPE_CHECKING, Any
|
25
25
|
|
26
|
+
from google.cloud.container_v1.types import Operation
|
26
27
|
from packaging.version import parse as parse_version
|
27
28
|
|
28
29
|
from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
|
@@ -36,7 +37,6 @@ from airflow.providers.google.cloud.hooks.kubernetes_engine import (
|
|
36
37
|
)
|
37
38
|
from airflow.providers_manager import ProvidersManager
|
38
39
|
from airflow.triggers.base import BaseTrigger, TriggerEvent
|
39
|
-
from google.cloud.container_v1.types import Operation
|
40
40
|
|
41
41
|
if TYPE_CHECKING:
|
42
42
|
from datetime import datetime
|
@@ -23,9 +23,10 @@ from collections.abc import AsyncIterator, Sequence
|
|
23
23
|
from functools import cached_property
|
24
24
|
from typing import Any
|
25
25
|
|
26
|
+
from google.cloud.pubsub_v1.types import ReceivedMessage
|
27
|
+
|
26
28
|
from airflow.providers.google.cloud.hooks.pubsub import PubSubAsyncHook
|
27
29
|
from airflow.triggers.base import BaseTrigger, TriggerEvent
|
28
|
-
from google.cloud.pubsub_v1.types import ReceivedMessage
|
29
30
|
|
30
31
|
|
31
32
|
class PubsubPullTrigger(BaseTrigger):
|
@@ -20,6 +20,14 @@ from collections.abc import AsyncIterator, Sequence
|
|
20
20
|
from functools import cached_property
|
21
21
|
from typing import TYPE_CHECKING, Any
|
22
22
|
|
23
|
+
from google.cloud.aiplatform_v1 import (
|
24
|
+
BatchPredictionJob,
|
25
|
+
HyperparameterTuningJob,
|
26
|
+
JobState,
|
27
|
+
PipelineState,
|
28
|
+
types,
|
29
|
+
)
|
30
|
+
|
23
31
|
from airflow.exceptions import AirflowException
|
24
32
|
from airflow.providers.google.cloud.hooks.vertex_ai.batch_prediction_job import BatchPredictionJobAsyncHook
|
25
33
|
from airflow.providers.google.cloud.hooks.vertex_ai.custom_job import CustomJobAsyncHook
|
@@ -28,13 +36,6 @@ from airflow.providers.google.cloud.hooks.vertex_ai.hyperparameter_tuning_job im
|
|
28
36
|
)
|
29
37
|
from airflow.providers.google.cloud.hooks.vertex_ai.pipeline_job import PipelineJobAsyncHook
|
30
38
|
from airflow.triggers.base import BaseTrigger, TriggerEvent
|
31
|
-
from google.cloud.aiplatform_v1 import (
|
32
|
-
BatchPredictionJob,
|
33
|
-
HyperparameterTuningJob,
|
34
|
-
JobState,
|
35
|
-
PipelineState,
|
36
|
-
types,
|
37
|
-
)
|
38
39
|
|
39
40
|
if TYPE_CHECKING:
|
40
41
|
from proto import Message
|
@@ -29,6 +29,10 @@ from urllib.parse import urlencode
|
|
29
29
|
|
30
30
|
import google.auth
|
31
31
|
import google.oauth2.service_account
|
32
|
+
from google.auth import impersonated_credentials # type: ignore[attr-defined]
|
33
|
+
from google.auth.credentials import AnonymousCredentials, Credentials
|
34
|
+
from google.auth.environment_vars import CREDENTIALS, LEGACY_PROJECT, PROJECT
|
35
|
+
|
32
36
|
from airflow.exceptions import AirflowException
|
33
37
|
from airflow.providers.google.cloud._internal_client.secret_manager_client import _SecretManagerClient
|
34
38
|
from airflow.providers.google.cloud.utils.external_token_supplier import (
|
@@ -36,9 +40,6 @@ from airflow.providers.google.cloud.utils.external_token_supplier import (
|
|
36
40
|
)
|
37
41
|
from airflow.utils.log.logging_mixin import LoggingMixin
|
38
42
|
from airflow.utils.process_utils import patch_environ
|
39
|
-
from google.auth import impersonated_credentials # type: ignore[attr-defined]
|
40
|
-
from google.auth.credentials import AnonymousCredentials, Credentials
|
41
|
-
from google.auth.environment_vars import CREDENTIALS, LEGACY_PROJECT, PROJECT
|
42
43
|
|
43
44
|
log = logging.getLogger(__name__)
|
44
45
|
|