apache-airflow-providers-google 10.7.0__py3-none-any.whl → 10.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/__init__.py +1 -1
- airflow/providers/google/ads/hooks/ads.py +10 -6
- airflow/providers/google/cloud/_internal_client/secret_manager_client.py +4 -1
- airflow/providers/google/cloud/example_dags/example_cloud_sql_query.py +31 -34
- airflow/providers/google/cloud/hooks/automl.py +11 -9
- airflow/providers/google/cloud/hooks/bigquery.py +30 -36
- airflow/providers/google/cloud/hooks/bigquery_dts.py +5 -3
- airflow/providers/google/cloud/hooks/bigtable.py +11 -8
- airflow/providers/google/cloud/hooks/cloud_batch.py +5 -3
- airflow/providers/google/cloud/hooks/cloud_build.py +6 -4
- airflow/providers/google/cloud/hooks/cloud_composer.py +14 -10
- airflow/providers/google/cloud/hooks/cloud_memorystore.py +5 -3
- airflow/providers/google/cloud/hooks/cloud_run.py +5 -3
- airflow/providers/google/cloud/hooks/cloud_sql.py +11 -14
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +8 -6
- airflow/providers/google/cloud/hooks/compute.py +5 -3
- airflow/providers/google/cloud/hooks/compute_ssh.py +1 -1
- airflow/providers/google/cloud/hooks/datacatalog.py +5 -3
- airflow/providers/google/cloud/hooks/dataflow.py +8 -11
- airflow/providers/google/cloud/hooks/dataform.py +4 -2
- airflow/providers/google/cloud/hooks/datafusion.py +24 -6
- airflow/providers/google/cloud/hooks/dataplex.py +75 -6
- airflow/providers/google/cloud/hooks/dataproc.py +9 -7
- airflow/providers/google/cloud/hooks/dataproc_metastore.py +8 -6
- airflow/providers/google/cloud/hooks/dlp.py +139 -137
- airflow/providers/google/cloud/hooks/gcs.py +15 -20
- airflow/providers/google/cloud/hooks/kms.py +4 -2
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +34 -34
- airflow/providers/google/cloud/hooks/looker.py +4 -1
- airflow/providers/google/cloud/hooks/mlengine.py +8 -6
- airflow/providers/google/cloud/hooks/natural_language.py +4 -2
- airflow/providers/google/cloud/hooks/os_login.py +9 -7
- airflow/providers/google/cloud/hooks/pubsub.py +13 -11
- airflow/providers/google/cloud/hooks/spanner.py +7 -5
- airflow/providers/google/cloud/hooks/speech_to_text.py +4 -2
- airflow/providers/google/cloud/hooks/stackdriver.py +6 -5
- airflow/providers/google/cloud/hooks/tasks.py +5 -3
- airflow/providers/google/cloud/hooks/text_to_speech.py +4 -2
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +7 -5
- airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +6 -4
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +11 -9
- airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +12 -10
- airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +8 -6
- airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +6 -4
- airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +7 -5
- airflow/providers/google/cloud/hooks/video_intelligence.py +5 -3
- airflow/providers/google/cloud/hooks/vision.py +5 -3
- airflow/providers/google/cloud/hooks/workflows.py +8 -6
- airflow/providers/google/cloud/links/bigquery.py +1 -1
- airflow/providers/google/cloud/links/bigquery_dts.py +1 -1
- airflow/providers/google/cloud/links/cloud_functions.py +1 -1
- airflow/providers/google/cloud/links/cloud_memorystore.py +1 -1
- airflow/providers/google/cloud/links/cloud_sql.py +1 -1
- airflow/providers/google/cloud/links/cloud_tasks.py +1 -1
- airflow/providers/google/cloud/links/compute.py +1 -1
- airflow/providers/google/cloud/links/datacatalog.py +1 -1
- airflow/providers/google/cloud/links/dataflow.py +1 -1
- airflow/providers/google/cloud/links/dataform.py +1 -1
- airflow/providers/google/cloud/links/pubsub.py +1 -1
- airflow/providers/google/cloud/links/spanner.py +1 -1
- airflow/providers/google/cloud/links/stackdriver.py +1 -1
- airflow/providers/google/cloud/links/workflows.py +2 -2
- airflow/providers/google/cloud/log/gcs_task_handler.py +5 -7
- airflow/providers/google/cloud/log/stackdriver_task_handler.py +8 -4
- airflow/providers/google/cloud/operators/automl.py +2 -1
- airflow/providers/google/cloud/operators/bigquery.py +6 -2
- airflow/providers/google/cloud/operators/bigquery_dts.py +2 -1
- airflow/providers/google/cloud/operators/bigtable.py +5 -3
- airflow/providers/google/cloud/operators/cloud_batch.py +6 -3
- airflow/providers/google/cloud/operators/cloud_build.py +2 -1
- airflow/providers/google/cloud/operators/cloud_composer.py +3 -2
- airflow/providers/google/cloud/operators/cloud_memorystore.py +3 -2
- airflow/providers/google/cloud/operators/cloud_run.py +3 -2
- airflow/providers/google/cloud/operators/cloud_sql.py +157 -152
- airflow/providers/google/cloud/operators/compute.py +59 -61
- airflow/providers/google/cloud/operators/datacatalog.py +3 -2
- airflow/providers/google/cloud/operators/dataflow.py +3 -1
- airflow/providers/google/cloud/operators/dataform.py +2 -1
- airflow/providers/google/cloud/operators/datafusion.py +1 -1
- airflow/providers/google/cloud/operators/dataplex.py +110 -8
- airflow/providers/google/cloud/operators/dataproc.py +39 -18
- airflow/providers/google/cloud/operators/dataproc_metastore.py +2 -1
- airflow/providers/google/cloud/operators/dlp.py +3 -2
- airflow/providers/google/cloud/operators/functions.py +46 -46
- airflow/providers/google/cloud/operators/gcs.py +4 -6
- airflow/providers/google/cloud/operators/kubernetes_engine.py +2 -1
- airflow/providers/google/cloud/operators/natural_language.py +3 -2
- airflow/providers/google/cloud/operators/pubsub.py +2 -1
- airflow/providers/google/cloud/operators/speech_to_text.py +3 -2
- airflow/providers/google/cloud/operators/stackdriver.py +2 -1
- airflow/providers/google/cloud/operators/tasks.py +3 -2
- airflow/providers/google/cloud/operators/text_to_speech.py +3 -2
- airflow/providers/google/cloud/operators/translate_speech.py +2 -1
- airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +2 -1
- airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +3 -2
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +2 -1
- airflow/providers/google/cloud/operators/vertex_ai/dataset.py +3 -2
- airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +4 -4
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +3 -2
- airflow/providers/google/cloud/operators/vertex_ai/model_service.py +2 -1
- airflow/providers/google/cloud/operators/video_intelligence.py +2 -1
- airflow/providers/google/cloud/operators/vision.py +3 -2
- airflow/providers/google/cloud/operators/workflows.py +7 -5
- airflow/providers/google/cloud/secrets/secret_manager.py +2 -2
- airflow/providers/google/cloud/sensors/bigquery_dts.py +2 -1
- airflow/providers/google/cloud/sensors/dataplex.py +2 -1
- airflow/providers/google/cloud/sensors/dataproc_metastore.py +2 -2
- airflow/providers/google/cloud/sensors/gcs.py +2 -1
- airflow/providers/google/cloud/sensors/workflows.py +2 -1
- airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +24 -10
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +2 -1
- airflow/providers/google/cloud/transfers/bigquery_to_mysql.py +1 -4
- airflow/providers/google/cloud/transfers/bigquery_to_postgres.py +1 -4
- airflow/providers/google/cloud/transfers/bigquery_to_sql.py +1 -1
- airflow/providers/google/cloud/transfers/calendar_to_gcs.py +4 -2
- airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +1 -3
- airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +2 -1
- airflow/providers/google/cloud/transfers/presto_to_gcs.py +5 -4
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/trino_to_gcs.py +5 -4
- airflow/providers/google/cloud/triggers/bigquery.py +30 -36
- airflow/providers/google/cloud/triggers/bigquery_dts.py +9 -10
- airflow/providers/google/cloud/triggers/cloud_batch.py +6 -8
- airflow/providers/google/cloud/triggers/cloud_build.py +5 -6
- airflow/providers/google/cloud/triggers/cloud_run.py +4 -3
- airflow/providers/google/cloud/triggers/cloud_sql.py +10 -10
- airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +1 -1
- airflow/providers/google/cloud/triggers/dataflow.py +5 -6
- airflow/providers/google/cloud/triggers/datafusion.py +5 -6
- airflow/providers/google/cloud/triggers/dataplex.py +110 -0
- airflow/providers/google/cloud/triggers/dataproc.py +18 -20
- airflow/providers/google/cloud/triggers/kubernetes_engine.py +14 -13
- airflow/providers/google/cloud/triggers/mlengine.py +5 -5
- airflow/providers/google/cloud/triggers/pubsub.py +2 -2
- airflow/providers/google/cloud/utils/bigquery_get_data.py +6 -3
- airflow/providers/google/cloud/utils/credentials_provider.py +1 -1
- airflow/providers/google/cloud/utils/field_validator.py +13 -13
- airflow/providers/google/cloud/utils/mlengine_operator_utils.py +5 -3
- airflow/providers/google/cloud/utils/mlengine_prediction_summary.py +1 -1
- airflow/providers/google/common/hooks/base_google.py +10 -2
- airflow/providers/google/common/links/storage.py +1 -1
- airflow/providers/google/common/utils/id_token_credentials.py +4 -1
- airflow/providers/google/get_provider_info.py +5 -0
- airflow/providers/google/marketing_platform/hooks/campaign_manager.py +4 -2
- airflow/providers/google/marketing_platform/sensors/display_video.py +6 -3
- airflow/providers/google/suite/hooks/calendar.py +4 -2
- {apache_airflow_providers_google-10.7.0.dist-info → apache_airflow_providers_google-10.8.0.dist-info}/METADATA +6 -6
- {apache_airflow_providers_google-10.7.0.dist-info → apache_airflow_providers_google-10.8.0.dist-info}/RECORD +155 -173
- {apache_airflow_providers_google-10.7.0.dist-info → apache_airflow_providers_google-10.8.0.dist-info}/WHEEL +1 -1
- airflow/providers/google/ads/_vendor/__init__.py +0 -16
- airflow/providers/google/ads/_vendor/googleads/__init__.py +0 -16
- airflow/providers/google/ads/_vendor/googleads/interceptors/__init__.py +0 -16
- airflow/providers/google/ads/_vendor/googleads/v12/__init__.py +0 -16
- airflow/providers/google/ads/_vendor/googleads/v12/common/__init__.py +0 -16
- airflow/providers/google/ads/_vendor/googleads/v12/common/types/__init__.py +0 -16
- airflow/providers/google/ads/_vendor/googleads/v12/enums/__init__.py +0 -16
- airflow/providers/google/ads/_vendor/googleads/v12/enums/types/__init__.py +0 -16
- airflow/providers/google/ads/_vendor/googleads/v12/errors/__init__.py +0 -16
- airflow/providers/google/ads/_vendor/googleads/v12/errors/types/__init__.py +0 -16
- airflow/providers/google/ads/_vendor/googleads/v12/resources/__init__.py +0 -16
- airflow/providers/google/ads/_vendor/googleads/v12/resources/types/__init__.py +0 -16
- airflow/providers/google/ads/_vendor/googleads/v12/services/__init__.py +0 -16
- airflow/providers/google/ads/_vendor/googleads/v12/services/services/__init__.py +0 -16
- airflow/providers/google/ads/_vendor/googleads/v12/services/services/customer_service/__init__.py +0 -16
- airflow/providers/google/ads/_vendor/googleads/v12/services/services/customer_service/transports/__init__.py +0 -16
- airflow/providers/google/ads/_vendor/googleads/v12/services/services/google_ads_service/__init__.py +0 -16
- airflow/providers/google/ads/_vendor/googleads/v12/services/services/google_ads_service/transports/__init__.py +0 -16
- airflow/providers/google/ads/_vendor/googleads/v12/services/types/__init__.py +0 -16
- {apache_airflow_providers_google-10.7.0.dist-info → apache_airflow_providers_google-10.8.0.dist-info}/LICENSE +0 -0
- {apache_airflow_providers_google-10.7.0.dist-info → apache_airflow_providers_google-10.8.0.dist-info}/NOTICE +0 -0
- {apache_airflow_providers_google-10.7.0.dist-info → apache_airflow_providers_google-10.8.0.dist-info}/entry_points.txt +0 -0
- {apache_airflow_providers_google-10.7.0.dist-info → apache_airflow_providers_google-10.8.0.dist-info}/top_level.txt +0 -0
@@ -22,7 +22,6 @@ from typing import TYPE_CHECKING, Sequence
|
|
22
22
|
|
23
23
|
from google.api_core.exceptions import NotFound
|
24
24
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
25
|
-
from google.api_core.retry import Retry
|
26
25
|
from google.cloud.aiplatform import datasets
|
27
26
|
from google.cloud.aiplatform.models import Model
|
28
27
|
from google.cloud.aiplatform_v1.types.training_pipeline import TrainingPipeline
|
@@ -36,6 +35,8 @@ from airflow.providers.google.cloud.links.vertex_ai import (
|
|
36
35
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
37
36
|
|
38
37
|
if TYPE_CHECKING:
|
38
|
+
from google.api_core.retry import Retry
|
39
|
+
|
39
40
|
from airflow.utils.context import Context
|
40
41
|
|
41
42
|
|
@@ -30,8 +30,6 @@ from typing import TYPE_CHECKING, Sequence
|
|
30
30
|
|
31
31
|
from google.api_core.exceptions import NotFound
|
32
32
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
33
|
-
from google.api_core.retry import Retry
|
34
|
-
from google.cloud.aiplatform import Model, explain
|
35
33
|
from google.cloud.aiplatform_v1.types import BatchPredictionJob
|
36
34
|
|
37
35
|
from airflow.providers.google.cloud.hooks.vertex_ai.batch_prediction_job import BatchPredictionJobHook
|
@@ -42,6 +40,9 @@ from airflow.providers.google.cloud.links.vertex_ai import (
|
|
42
40
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
43
41
|
|
44
42
|
if TYPE_CHECKING:
|
43
|
+
from google.api_core.retry import Retry
|
44
|
+
from google.cloud.aiplatform import Model, explain
|
45
|
+
|
45
46
|
from airflow.utils.context import Context
|
46
47
|
|
47
48
|
|
@@ -22,7 +22,6 @@ from typing import TYPE_CHECKING, Sequence
|
|
22
22
|
|
23
23
|
from google.api_core.exceptions import NotFound
|
24
24
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
25
|
-
from google.api_core.retry import Retry
|
26
25
|
from google.cloud.aiplatform.models import Model
|
27
26
|
from google.cloud.aiplatform_v1.types.dataset import Dataset
|
28
27
|
from google.cloud.aiplatform_v1.types.training_pipeline import TrainingPipeline
|
@@ -36,6 +35,8 @@ from airflow.providers.google.cloud.links.vertex_ai import (
|
|
36
35
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
37
36
|
|
38
37
|
if TYPE_CHECKING:
|
38
|
+
from google.api_core.retry import Retry
|
39
|
+
|
39
40
|
from airflow.utils.context import Context
|
40
41
|
|
41
42
|
|
@@ -22,15 +22,16 @@ from typing import TYPE_CHECKING, Sequence
|
|
22
22
|
|
23
23
|
from google.api_core.exceptions import NotFound
|
24
24
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
25
|
-
from google.api_core.retry import Retry
|
26
25
|
from google.cloud.aiplatform_v1.types import Dataset, ExportDataConfig, ImportDataConfig
|
27
|
-
from google.protobuf.field_mask_pb2 import FieldMask
|
28
26
|
|
29
27
|
from airflow.providers.google.cloud.hooks.vertex_ai.dataset import DatasetHook
|
30
28
|
from airflow.providers.google.cloud.links.vertex_ai import VertexAIDatasetLink, VertexAIDatasetListLink
|
31
29
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
32
30
|
|
33
31
|
if TYPE_CHECKING:
|
32
|
+
from google.api_core.retry import Retry
|
33
|
+
from google.protobuf.field_mask_pb2 import FieldMask
|
34
|
+
|
34
35
|
from airflow.utils.context import Context
|
35
36
|
|
36
37
|
|
@@ -32,9 +32,7 @@ from typing import TYPE_CHECKING, Sequence
|
|
32
32
|
|
33
33
|
from google.api_core.exceptions import NotFound
|
34
34
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
35
|
-
from google.api_core.retry import Retry
|
36
35
|
from google.cloud.aiplatform_v1.types import DeployedModel, Endpoint, endpoint_service
|
37
|
-
from google.protobuf.field_mask_pb2 import FieldMask
|
38
36
|
|
39
37
|
from airflow.providers.google.cloud.hooks.vertex_ai.endpoint_service import EndpointServiceHook
|
40
38
|
from airflow.providers.google.cloud.links.vertex_ai import (
|
@@ -45,6 +43,9 @@ from airflow.providers.google.cloud.links.vertex_ai import (
|
|
45
43
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
46
44
|
|
47
45
|
if TYPE_CHECKING:
|
46
|
+
from google.api_core.retry import Retry
|
47
|
+
from google.protobuf.field_mask_pb2 import FieldMask
|
48
|
+
|
48
49
|
from airflow.utils.context import Context
|
49
50
|
|
50
51
|
|
@@ -462,8 +463,7 @@ class UndeployModelOperator(GoogleCloudBaseOperator):
|
|
462
463
|
|
463
464
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
464
465
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
465
|
-
:param endpoint_id: Required. The
|
466
|
-
``projects/{project}/locations/{location}/endpoints/{endpoint}``
|
466
|
+
:param endpoint_id: Required. The ID of the Endpoint resource from which to undeploy a Model.
|
467
467
|
:param deployed_model_id: Required. The ID of the DeployedModel to be undeployed from the Endpoint.
|
468
468
|
:param traffic_split: If this field is provided, then the Endpoint's
|
469
469
|
[traffic_split][google.cloud.aiplatform.v1.Endpoint.traffic_split] will be overwritten with it. If
|
@@ -31,8 +31,6 @@ from typing import TYPE_CHECKING, Sequence
|
|
31
31
|
|
32
32
|
from google.api_core.exceptions import NotFound
|
33
33
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
34
|
-
from google.api_core.retry import Retry
|
35
|
-
from google.cloud.aiplatform import gapic, hyperparameter_tuning
|
36
34
|
from google.cloud.aiplatform_v1.types import HyperparameterTuningJob
|
37
35
|
|
38
36
|
from airflow.providers.google.cloud.hooks.vertex_ai.hyperparameter_tuning_job import (
|
@@ -45,6 +43,9 @@ from airflow.providers.google.cloud.links.vertex_ai import (
|
|
45
43
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
46
44
|
|
47
45
|
if TYPE_CHECKING:
|
46
|
+
from google.api_core.retry import Retry
|
47
|
+
from google.cloud.aiplatform import gapic, hyperparameter_tuning
|
48
|
+
|
48
49
|
from airflow.utils.context import Context
|
49
50
|
|
50
51
|
|
@@ -28,7 +28,6 @@ from typing import TYPE_CHECKING, Sequence
|
|
28
28
|
|
29
29
|
from google.api_core.exceptions import NotFound
|
30
30
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
31
|
-
from google.api_core.retry import Retry
|
32
31
|
from google.cloud.aiplatform_v1.types import Model, model_service
|
33
32
|
|
34
33
|
from airflow.providers.google.cloud.hooks.vertex_ai.model_service import ModelServiceHook
|
@@ -40,6 +39,8 @@ from airflow.providers.google.cloud.links.vertex_ai import (
|
|
40
39
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
41
40
|
|
42
41
|
if TYPE_CHECKING:
|
42
|
+
from google.api_core.retry import Retry
|
43
|
+
|
43
44
|
from airflow.utils.context import Context
|
44
45
|
|
45
46
|
|
@@ -21,7 +21,6 @@ from __future__ import annotations
|
|
21
21
|
from typing import TYPE_CHECKING, Sequence
|
22
22
|
|
23
23
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
24
|
-
from google.api_core.retry import Retry
|
25
24
|
from google.cloud.videointelligence_v1 import Feature, VideoContext
|
26
25
|
from google.protobuf.json_format import MessageToDict
|
27
26
|
|
@@ -29,6 +28,8 @@ from airflow.providers.google.cloud.hooks.video_intelligence import CloudVideoIn
|
|
29
28
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
30
29
|
|
31
30
|
if TYPE_CHECKING:
|
31
|
+
from google.api_core.retry import Retry
|
32
|
+
|
32
33
|
from airflow.utils.context import Context
|
33
34
|
|
34
35
|
|
@@ -23,7 +23,6 @@ from typing import TYPE_CHECKING, Any, Sequence, Tuple
|
|
23
23
|
|
24
24
|
from google.api_core.exceptions import AlreadyExists
|
25
25
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
26
|
-
from google.api_core.retry import Retry
|
27
26
|
from google.cloud.vision_v1 import (
|
28
27
|
AnnotateImageRequest,
|
29
28
|
Image,
|
@@ -31,12 +30,14 @@ from google.cloud.vision_v1 import (
|
|
31
30
|
ProductSet,
|
32
31
|
ReferenceImage,
|
33
32
|
)
|
34
|
-
from google.protobuf.field_mask_pb2 import FieldMask # type: ignore
|
35
33
|
|
36
34
|
from airflow.providers.google.cloud.hooks.vision import CloudVisionHook
|
37
35
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
38
36
|
|
39
37
|
if TYPE_CHECKING:
|
38
|
+
from google.api_core.retry import Retry
|
39
|
+
from google.protobuf.field_mask_pb2 import FieldMask
|
40
|
+
|
40
41
|
from airflow.utils.context import Context
|
41
42
|
|
42
43
|
|
@@ -24,10 +24,8 @@ from typing import TYPE_CHECKING, Sequence
|
|
24
24
|
|
25
25
|
from google.api_core.exceptions import AlreadyExists
|
26
26
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
27
|
-
from google.api_core.retry import Retry
|
28
27
|
from google.cloud.workflows.executions_v1beta import Execution
|
29
28
|
from google.cloud.workflows_v1beta import Workflow
|
30
|
-
from google.protobuf.field_mask_pb2 import FieldMask
|
31
29
|
|
32
30
|
from airflow.providers.google.cloud.hooks.workflows import WorkflowsHook
|
33
31
|
from airflow.providers.google.cloud.links.workflows import (
|
@@ -38,6 +36,9 @@ from airflow.providers.google.cloud.links.workflows import (
|
|
38
36
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
39
37
|
|
40
38
|
if TYPE_CHECKING:
|
39
|
+
from google.api_core.retry import Retry
|
40
|
+
from google.protobuf.field_mask_pb2 import FieldMask
|
41
|
+
|
41
42
|
from airflow.utils.context import Context
|
42
43
|
try:
|
43
44
|
from airflow.utils.hashlib_wrapper import md5
|
@@ -51,7 +52,7 @@ class WorkflowsCreateWorkflowOperator(GoogleCloudBaseOperator):
|
|
51
52
|
Creates a new workflow.
|
52
53
|
|
53
54
|
If a workflow with the specified name already exists in the specified
|
54
|
-
project and location, the long
|
55
|
+
project and location, the long-running operation will return
|
55
56
|
[ALREADY_EXISTS][google.rpc.Code.ALREADY_EXISTS] error.
|
56
57
|
|
57
58
|
.. seealso::
|
@@ -605,7 +606,8 @@ class WorkflowsListExecutionsOperator(GoogleCloudBaseOperator):
|
|
605
606
|
|
606
607
|
:param workflow_id: Required. The ID of the workflow to be created.
|
607
608
|
:param start_date_filter: If passed only executions older that this date will be returned.
|
608
|
-
By default operators return executions from last 60 minutes
|
609
|
+
By default, operators return executions from last 60 minutes.
|
610
|
+
Note that datetime object must specify a time zone, e.g. ``datetime.timezone.utc``.
|
609
611
|
:param project_id: Required. The ID of the Google Cloud project the cluster belongs to.
|
610
612
|
:param location: Required. The GCP region in which to handle the request.
|
611
613
|
:param retry: A retry object used to retry requests. If ``None`` is specified, requests will not be
|
@@ -669,7 +671,7 @@ class WorkflowsListExecutionsOperator(GoogleCloudBaseOperator):
|
|
669
671
|
return [
|
670
672
|
Execution.to_dict(e)
|
671
673
|
for e in execution_iter
|
672
|
-
if e.start_time
|
674
|
+
if e.start_time > self.start_date_filter # type: ignore
|
673
675
|
]
|
674
676
|
|
675
677
|
|
@@ -36,8 +36,8 @@ SECRET_ID_PATTERN = r"^[a-zA-Z0-9-_]*$"
|
|
36
36
|
|
37
37
|
|
38
38
|
def _parse_version(val):
|
39
|
-
|
40
|
-
return tuple(int(x) for x in
|
39
|
+
match = re.search(r"(\d+)\.(\d+)\.(\d+)", val)
|
40
|
+
return tuple(int(x) for x in match.groups())
|
41
41
|
|
42
42
|
|
43
43
|
class CloudSecretManagerBackend(BaseSecretsBackend, LoggingMixin):
|
@@ -21,7 +21,6 @@ from __future__ import annotations
|
|
21
21
|
from typing import TYPE_CHECKING, Sequence
|
22
22
|
|
23
23
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
24
|
-
from google.api_core.retry import Retry
|
25
24
|
from google.cloud.bigquery_datatransfer_v1 import TransferState
|
26
25
|
|
27
26
|
from airflow.exceptions import AirflowException
|
@@ -29,6 +28,8 @@ from airflow.providers.google.cloud.hooks.bigquery_dts import BiqQueryDataTransf
|
|
29
28
|
from airflow.sensors.base import BaseSensorOperator
|
30
29
|
|
31
30
|
if TYPE_CHECKING:
|
31
|
+
from google.api_core.retry import Retry
|
32
|
+
|
32
33
|
from airflow.utils.context import Context
|
33
34
|
|
34
35
|
|
@@ -21,10 +21,11 @@ import time
|
|
21
21
|
from typing import TYPE_CHECKING, Sequence
|
22
22
|
|
23
23
|
if TYPE_CHECKING:
|
24
|
+
from google.api_core.retry import Retry
|
25
|
+
|
24
26
|
from airflow.utils.context import Context
|
25
27
|
from google.api_core.exceptions import GoogleAPICallError
|
26
28
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
27
|
-
from google.api_core.retry import Retry
|
28
29
|
from google.cloud.dataplex_v1.types import DataScanJob
|
29
30
|
|
30
31
|
from airflow.exceptions import AirflowException
|
@@ -19,14 +19,14 @@ from __future__ import annotations
|
|
19
19
|
|
20
20
|
from typing import TYPE_CHECKING, Sequence
|
21
21
|
|
22
|
-
from google.api_core.operation import Operation
|
23
|
-
|
24
22
|
from airflow import AirflowException
|
25
23
|
from airflow.providers.google.cloud.hooks.dataproc_metastore import DataprocMetastoreHook
|
26
24
|
from airflow.providers.google.cloud.hooks.gcs import parse_json_from_gcs
|
27
25
|
from airflow.sensors.base import BaseSensorOperator
|
28
26
|
|
29
27
|
if TYPE_CHECKING:
|
28
|
+
from google.api_core.operation import Operation
|
29
|
+
|
30
30
|
from airflow.utils.context import Context
|
31
31
|
|
32
32
|
|
@@ -24,7 +24,6 @@ import warnings
|
|
24
24
|
from datetime import datetime, timedelta
|
25
25
|
from typing import TYPE_CHECKING, Any, Callable, Sequence
|
26
26
|
|
27
|
-
from google.api_core.retry import Retry
|
28
27
|
from google.cloud.storage.retry import DEFAULT_RETRY
|
29
28
|
|
30
29
|
from airflow.configuration import conf
|
@@ -39,6 +38,8 @@ from airflow.providers.google.cloud.triggers.gcs import (
|
|
39
38
|
from airflow.sensors.base import BaseSensorOperator, poke_mode_only
|
40
39
|
|
41
40
|
if TYPE_CHECKING:
|
41
|
+
from google.api_core.retry import Retry
|
42
|
+
|
42
43
|
from airflow.utils.context import Context
|
43
44
|
|
44
45
|
|
@@ -19,7 +19,6 @@ from __future__ import annotations
|
|
19
19
|
from typing import TYPE_CHECKING, Sequence
|
20
20
|
|
21
21
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
22
|
-
from google.api_core.retry import Retry
|
23
22
|
from google.cloud.workflows.executions_v1beta import Execution
|
24
23
|
|
25
24
|
from airflow.exceptions import AirflowException
|
@@ -27,6 +26,8 @@ from airflow.providers.google.cloud.hooks.workflows import WorkflowsHook
|
|
27
26
|
from airflow.sensors.base import BaseSensorOperator
|
28
27
|
|
29
28
|
if TYPE_CHECKING:
|
29
|
+
from google.api_core.retry import Retry
|
30
|
+
|
30
31
|
from airflow.utils.context import Context
|
31
32
|
|
32
33
|
|
@@ -17,10 +17,12 @@
|
|
17
17
|
# under the License.
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
|
+
import warnings
|
20
21
|
from tempfile import NamedTemporaryFile
|
21
22
|
from typing import TYPE_CHECKING, Sequence
|
22
23
|
|
23
24
|
from airflow import AirflowException
|
25
|
+
from airflow.exceptions import AirflowProviderDeprecationWarning
|
24
26
|
from airflow.models import BaseOperator
|
25
27
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook, _parse_gcs_url, gcs_object_is_directory
|
26
28
|
from airflow.providers.microsoft.azure.hooks.fileshare import AzureFileShareHook
|
@@ -73,6 +75,7 @@ class AzureFileShareToGCSOperator(BaseOperator):
|
|
73
75
|
share_name: str,
|
74
76
|
dest_gcs: str,
|
75
77
|
directory_name: str | None = None,
|
78
|
+
directory_path: str | None = None,
|
76
79
|
prefix: str = "",
|
77
80
|
azure_fileshare_conn_id: str = "azure_fileshare_default",
|
78
81
|
gcp_conn_id: str = "google_cloud_default",
|
@@ -84,7 +87,15 @@ class AzureFileShareToGCSOperator(BaseOperator):
|
|
84
87
|
super().__init__(**kwargs)
|
85
88
|
|
86
89
|
self.share_name = share_name
|
90
|
+
self.directory_path = directory_path
|
87
91
|
self.directory_name = directory_name
|
92
|
+
if self.directory_path is None:
|
93
|
+
self.directory_path = directory_name
|
94
|
+
warnings.warn(
|
95
|
+
"Use 'directory_path' instead of 'directory_name'.",
|
96
|
+
AirflowProviderDeprecationWarning,
|
97
|
+
stacklevel=2,
|
98
|
+
)
|
88
99
|
self.prefix = prefix
|
89
100
|
self.azure_fileshare_conn_id = azure_fileshare_conn_id
|
90
101
|
self.gcp_conn_id = gcp_conn_id
|
@@ -106,10 +117,12 @@ class AzureFileShareToGCSOperator(BaseOperator):
|
|
106
117
|
|
107
118
|
def execute(self, context: Context):
|
108
119
|
self._check_inputs()
|
109
|
-
azure_fileshare_hook = AzureFileShareHook(
|
110
|
-
|
111
|
-
|
120
|
+
azure_fileshare_hook = AzureFileShareHook(
|
121
|
+
share_name=self.share_name,
|
122
|
+
azure_fileshare_conn_id=self.azure_fileshare_conn_id,
|
123
|
+
directory_path=self.directory_path,
|
112
124
|
)
|
125
|
+
files = azure_fileshare_hook.list_files()
|
113
126
|
|
114
127
|
gcs_hook = GCSHook(
|
115
128
|
gcp_conn_id=self.gcp_conn_id,
|
@@ -141,16 +154,17 @@ class AzureFileShareToGCSOperator(BaseOperator):
|
|
141
154
|
|
142
155
|
if files:
|
143
156
|
self.log.info("%s files are going to be synced.", len(files))
|
144
|
-
if self.
|
157
|
+
if self.directory_path is None:
|
145
158
|
raise RuntimeError("The directory_name must be set!.")
|
146
159
|
for file in files:
|
160
|
+
azure_fileshare_hook = AzureFileShareHook(
|
161
|
+
share_name=self.share_name,
|
162
|
+
azure_fileshare_conn_id=self.azure_fileshare_conn_id,
|
163
|
+
directory_path=self.directory_path,
|
164
|
+
file_path=file,
|
165
|
+
)
|
147
166
|
with NamedTemporaryFile() as temp_file:
|
148
|
-
azure_fileshare_hook.get_file_to_stream(
|
149
|
-
stream=temp_file,
|
150
|
-
share_name=self.share_name,
|
151
|
-
directory_name=self.directory_name,
|
152
|
-
file_name=file,
|
153
|
-
)
|
167
|
+
azure_fileshare_hook.get_file_to_stream(stream=temp_file)
|
154
168
|
temp_file.flush()
|
155
169
|
|
156
170
|
# There will always be a '/' before file because it is
|
@@ -21,7 +21,6 @@ from __future__ import annotations
|
|
21
21
|
from typing import TYPE_CHECKING, Any, Sequence
|
22
22
|
|
23
23
|
from google.api_core.exceptions import Conflict
|
24
|
-
from google.api_core.retry import Retry
|
25
24
|
from google.cloud.bigquery import DEFAULT_RETRY, UnknownJob
|
26
25
|
|
27
26
|
from airflow import AirflowException
|
@@ -32,6 +31,8 @@ from airflow.providers.google.cloud.links.bigquery import BigQueryTableLink
|
|
32
31
|
from airflow.providers.google.cloud.triggers.bigquery import BigQueryInsertJobTrigger
|
33
32
|
|
34
33
|
if TYPE_CHECKING:
|
34
|
+
from google.api_core.retry import Retry
|
35
|
+
|
35
36
|
from airflow.utils.context import Context
|
36
37
|
|
37
38
|
|
@@ -45,7 +45,8 @@ class BigQueryToMsSqlOperator(BigQueryToSqlBaseOperator):
|
|
45
45
|
:param mssql_conn_id: reference to a specific mssql hook
|
46
46
|
"""
|
47
47
|
|
48
|
-
template_fields: Sequence[str] =
|
48
|
+
template_fields: Sequence[str] = (
|
49
|
+
*BigQueryToSqlBaseOperator.template_fields,
|
49
50
|
"source_project_dataset_table",
|
50
51
|
)
|
51
52
|
operator_extra_links = (BigQueryTableLink(),)
|
@@ -40,10 +40,7 @@ class BigQueryToMySqlOperator(BigQueryToSqlBaseOperator):
|
|
40
40
|
:param mysql_conn_id: Reference to :ref:`mysql connection id <howto/connection:mysql>`.
|
41
41
|
"""
|
42
42
|
|
43
|
-
template_fields: Sequence[str] =
|
44
|
-
"dataset_id",
|
45
|
-
"table_id",
|
46
|
-
)
|
43
|
+
template_fields: Sequence[str] = (*BigQueryToSqlBaseOperator.template_fields, "dataset_id", "table_id")
|
47
44
|
|
48
45
|
def __init__(
|
49
46
|
self,
|
@@ -36,10 +36,7 @@ class BigQueryToPostgresOperator(BigQueryToSqlBaseOperator):
|
|
36
36
|
:param postgres_conn_id: Reference to :ref:`postgres connection id <howto/connection:postgres>`.
|
37
37
|
"""
|
38
38
|
|
39
|
-
template_fields: Sequence[str] =
|
40
|
-
"dataset_id",
|
41
|
-
"table_id",
|
42
|
-
)
|
39
|
+
template_fields: Sequence[str] = (*BigQueryToSqlBaseOperator.template_fields, "dataset_id", "table_id")
|
43
40
|
|
44
41
|
def __init__(
|
45
42
|
self,
|
@@ -22,11 +22,11 @@ import abc
|
|
22
22
|
from typing import TYPE_CHECKING, Sequence
|
23
23
|
|
24
24
|
from airflow.models import BaseOperator
|
25
|
-
from airflow.providers.common.sql.hooks.sql import DbApiHook
|
26
25
|
from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook
|
27
26
|
from airflow.providers.google.cloud.utils.bigquery_get_data import bigquery_get_data
|
28
27
|
|
29
28
|
if TYPE_CHECKING:
|
29
|
+
from airflow.providers.common.sql.hooks.sql import DbApiHook
|
30
30
|
from airflow.utils.context import Context
|
31
31
|
|
32
32
|
|
@@ -17,14 +17,16 @@
|
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
19
|
import json
|
20
|
-
from datetime import datetime
|
21
20
|
from tempfile import NamedTemporaryFile
|
22
|
-
from typing import Any, Sequence
|
21
|
+
from typing import TYPE_CHECKING, Any, Sequence
|
23
22
|
|
24
23
|
from airflow.models import BaseOperator
|
25
24
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
26
25
|
from airflow.providers.google.suite.hooks.calendar import GoogleCalendarHook
|
27
26
|
|
27
|
+
if TYPE_CHECKING:
|
28
|
+
from datetime import datetime
|
29
|
+
|
28
30
|
|
29
31
|
class GoogleCalendarToGCSOperator(BaseOperator):
|
30
32
|
"""
|
@@ -258,9 +258,7 @@ class CassandraToGCSOperator(BaseOperator):
|
|
258
258
|
|
259
259
|
def convert_value(self, value: Any | None) -> Any | None:
|
260
260
|
"""Convert value to BQ type."""
|
261
|
-
if not value:
|
262
|
-
return value
|
263
|
-
elif isinstance(value, (str, int, float, bool, dict)):
|
261
|
+
if not value or isinstance(value, (str, int, float, bool, dict)):
|
264
262
|
return value
|
265
263
|
elif isinstance(value, bytes):
|
266
264
|
return b64encode(value).decode("ascii")
|
@@ -23,14 +23,14 @@ import tempfile
|
|
23
23
|
from enum import Enum
|
24
24
|
from typing import TYPE_CHECKING, Any, Sequence
|
25
25
|
|
26
|
-
from facebook_business.adobjects.adsinsights import AdsInsights
|
27
|
-
|
28
26
|
from airflow.exceptions import AirflowException
|
29
27
|
from airflow.models import BaseOperator
|
30
28
|
from airflow.providers.facebook.ads.hooks.ads import FacebookAdsReportingHook
|
31
29
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
32
30
|
|
33
31
|
if TYPE_CHECKING:
|
32
|
+
from facebook_business.adobjects.adsinsights import AdsInsights
|
33
|
+
|
34
34
|
from airflow.utils.context import Context
|
35
35
|
|
36
36
|
|
@@ -22,7 +22,6 @@ import json
|
|
22
22
|
from typing import TYPE_CHECKING, Any, Sequence
|
23
23
|
|
24
24
|
from google.api_core.exceptions import BadRequest, Conflict
|
25
|
-
from google.api_core.retry import Retry
|
26
25
|
from google.cloud.bigquery import (
|
27
26
|
DEFAULT_RETRY,
|
28
27
|
CopyJob,
|
@@ -44,6 +43,8 @@ from airflow.providers.google.cloud.links.bigquery import BigQueryTableLink
|
|
44
43
|
from airflow.providers.google.cloud.triggers.bigquery import BigQueryInsertJobTrigger
|
45
44
|
|
46
45
|
if TYPE_CHECKING:
|
46
|
+
from google.api_core.retry import Retry
|
47
|
+
|
47
48
|
from airflow.utils.context import Context
|
48
49
|
|
49
50
|
ALLOWED_FORMATS = [
|
@@ -17,14 +17,15 @@
|
|
17
17
|
# under the License.
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
|
-
from typing import Any
|
21
|
-
|
22
|
-
from prestodb.client import PrestoResult
|
23
|
-
from prestodb.dbapi import Cursor as PrestoCursor
|
20
|
+
from typing import TYPE_CHECKING, Any
|
24
21
|
|
25
22
|
from airflow.providers.google.cloud.transfers.sql_to_gcs import BaseSQLToGCSOperator
|
26
23
|
from airflow.providers.presto.hooks.presto import PrestoHook
|
27
24
|
|
25
|
+
if TYPE_CHECKING:
|
26
|
+
from prestodb.client import PrestoResult
|
27
|
+
from prestodb.dbapi import Cursor as PrestoCursor
|
28
|
+
|
28
29
|
|
29
30
|
class _PrestoToGCSPrestoCursorAdapter:
|
30
31
|
"""
|
@@ -235,7 +235,7 @@ class BaseSQLToGCSOperator(BaseOperator):
|
|
235
235
|
names in GCS, and values are file handles to local files that
|
236
236
|
contain the data for the GCS objects.
|
237
237
|
"""
|
238
|
-
org_schema =
|
238
|
+
org_schema = [schema_tuple[0] for schema_tuple in cursor.description]
|
239
239
|
schema = [column for column in org_schema if column not in self.exclude_columns]
|
240
240
|
|
241
241
|
col_type_dict = self._get_col_type_dict()
|
@@ -17,14 +17,15 @@
|
|
17
17
|
# under the License.
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
|
-
from typing import Any
|
21
|
-
|
22
|
-
from trino.client import TrinoResult
|
23
|
-
from trino.dbapi import Cursor as TrinoCursor
|
20
|
+
from typing import TYPE_CHECKING, Any
|
24
21
|
|
25
22
|
from airflow.providers.google.cloud.transfers.sql_to_gcs import BaseSQLToGCSOperator
|
26
23
|
from airflow.providers.trino.hooks.trino import TrinoHook
|
27
24
|
|
25
|
+
if TYPE_CHECKING:
|
26
|
+
from trino.client import TrinoResult
|
27
|
+
from trino.dbapi import Cursor as TrinoCursor
|
28
|
+
|
28
29
|
|
29
30
|
class _TrinoToGCSTrinoCursorAdapter:
|
30
31
|
"""
|