apache-airflow-providers-google 10.26.0rc1__py3-none-any.whl → 11.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/__init__.py +1 -1
- airflow/providers/google/ads/hooks/ads.py +2 -1
- airflow/providers/google/ads/operators/ads.py +2 -1
- airflow/providers/google/ads/transfers/ads_to_gcs.py +2 -1
- airflow/providers/google/assets/gcs.py +17 -1
- airflow/providers/google/cloud/hooks/automl.py +3 -6
- airflow/providers/google/cloud/hooks/bigquery.py +41 -1486
- airflow/providers/google/cloud/hooks/bigquery_dts.py +4 -11
- airflow/providers/google/cloud/hooks/bigtable.py +3 -6
- airflow/providers/google/cloud/hooks/cloud_batch.py +6 -3
- airflow/providers/google/cloud/hooks/cloud_build.py +3 -15
- airflow/providers/google/cloud/hooks/cloud_composer.py +2 -17
- airflow/providers/google/cloud/hooks/cloud_memorystore.py +5 -6
- airflow/providers/google/cloud/hooks/cloud_run.py +10 -5
- airflow/providers/google/cloud/hooks/cloud_sql.py +5 -7
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +3 -7
- airflow/providers/google/cloud/hooks/compute.py +3 -6
- airflow/providers/google/cloud/hooks/compute_ssh.py +0 -5
- airflow/providers/google/cloud/hooks/datacatalog.py +3 -6
- airflow/providers/google/cloud/hooks/dataflow.py +3 -14
- airflow/providers/google/cloud/hooks/dataform.py +2 -9
- airflow/providers/google/cloud/hooks/datafusion.py +4 -15
- airflow/providers/google/cloud/hooks/dataplex.py +4 -7
- airflow/providers/google/cloud/hooks/dataprep.py +2 -2
- airflow/providers/google/cloud/hooks/dataproc.py +77 -22
- airflow/providers/google/cloud/hooks/dataproc_metastore.py +2 -9
- airflow/providers/google/cloud/hooks/datastore.py +3 -6
- airflow/providers/google/cloud/hooks/dlp.py +3 -6
- airflow/providers/google/cloud/hooks/functions.py +2 -6
- airflow/providers/google/cloud/hooks/gcs.py +2 -18
- airflow/providers/google/cloud/hooks/gdm.py +1 -17
- airflow/providers/google/cloud/hooks/kms.py +3 -6
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +7 -97
- airflow/providers/google/cloud/hooks/life_sciences.py +2 -6
- airflow/providers/google/cloud/hooks/looker.py +2 -1
- airflow/providers/google/cloud/hooks/mlengine.py +0 -8
- airflow/providers/google/cloud/hooks/natural_language.py +3 -6
- airflow/providers/google/cloud/hooks/os_login.py +3 -6
- airflow/providers/google/cloud/hooks/pubsub.py +3 -6
- airflow/providers/google/cloud/hooks/secret_manager.py +3 -73
- airflow/providers/google/cloud/hooks/spanner.py +3 -6
- airflow/providers/google/cloud/hooks/speech_to_text.py +3 -6
- airflow/providers/google/cloud/hooks/stackdriver.py +3 -6
- airflow/providers/google/cloud/hooks/tasks.py +3 -6
- airflow/providers/google/cloud/hooks/text_to_speech.py +3 -6
- airflow/providers/google/cloud/hooks/translate.py +455 -9
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +3 -6
- airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +3 -6
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +3 -6
- airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +2 -9
- airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +2 -9
- airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +1 -14
- airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +3 -6
- airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +2 -9
- airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +3 -1
- airflow/providers/google/cloud/hooks/vertex_ai/prediction_service.py +2 -1
- airflow/providers/google/cloud/hooks/video_intelligence.py +3 -6
- airflow/providers/google/cloud/hooks/vision.py +3 -6
- airflow/providers/google/cloud/hooks/workflows.py +2 -9
- airflow/providers/google/cloud/links/dataproc.py +0 -1
- airflow/providers/google/cloud/links/translate.py +91 -0
- airflow/providers/google/cloud/log/gcs_task_handler.py +2 -1
- airflow/providers/google/cloud/log/stackdriver_task_handler.py +11 -3
- airflow/providers/google/cloud/openlineage/utils.py +54 -21
- airflow/providers/google/cloud/operators/automl.py +5 -4
- airflow/providers/google/cloud/operators/bigquery.py +2 -341
- airflow/providers/google/cloud/operators/bigquery_dts.py +2 -1
- airflow/providers/google/cloud/operators/bigtable.py +2 -1
- airflow/providers/google/cloud/operators/cloud_batch.py +2 -1
- airflow/providers/google/cloud/operators/cloud_build.py +2 -1
- airflow/providers/google/cloud/operators/cloud_composer.py +2 -1
- airflow/providers/google/cloud/operators/cloud_memorystore.py +2 -1
- airflow/providers/google/cloud/operators/cloud_run.py +2 -1
- airflow/providers/google/cloud/operators/cloud_sql.py +2 -1
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +2 -1
- airflow/providers/google/cloud/operators/compute.py +2 -1
- airflow/providers/google/cloud/operators/datacatalog.py +2 -1
- airflow/providers/google/cloud/operators/dataflow.py +2 -517
- airflow/providers/google/cloud/operators/dataform.py +2 -1
- airflow/providers/google/cloud/operators/datafusion.py +2 -1
- airflow/providers/google/cloud/operators/dataplex.py +37 -31
- airflow/providers/google/cloud/operators/dataprep.py +2 -1
- airflow/providers/google/cloud/operators/dataproc.py +3 -633
- airflow/providers/google/cloud/operators/dataproc_metastore.py +2 -1
- airflow/providers/google/cloud/operators/datastore.py +2 -1
- airflow/providers/google/cloud/operators/dlp.py +2 -1
- airflow/providers/google/cloud/operators/functions.py +2 -1
- airflow/providers/google/cloud/operators/gcs.py +5 -4
- airflow/providers/google/cloud/operators/kubernetes_engine.py +2 -11
- airflow/providers/google/cloud/operators/life_sciences.py +2 -1
- airflow/providers/google/cloud/operators/mlengine.py +2 -1
- airflow/providers/google/cloud/operators/natural_language.py +3 -2
- airflow/providers/google/cloud/operators/pubsub.py +2 -1
- airflow/providers/google/cloud/operators/spanner.py +2 -1
- airflow/providers/google/cloud/operators/speech_to_text.py +2 -1
- airflow/providers/google/cloud/operators/stackdriver.py +2 -1
- airflow/providers/google/cloud/operators/tasks.py +3 -2
- airflow/providers/google/cloud/operators/text_to_speech.py +2 -1
- airflow/providers/google/cloud/operators/translate.py +622 -32
- airflow/providers/google/cloud/operators/translate_speech.py +2 -1
- airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +2 -93
- airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +3 -13
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +2 -17
- airflow/providers/google/cloud/operators/vertex_ai/dataset.py +2 -1
- airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +2 -1
- airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +2 -1
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +3 -13
- airflow/providers/google/cloud/operators/vertex_ai/model_service.py +2 -1
- airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +2 -1
- airflow/providers/google/cloud/operators/video_intelligence.py +2 -1
- airflow/providers/google/cloud/operators/vision.py +3 -2
- airflow/providers/google/cloud/operators/workflows.py +3 -2
- airflow/providers/google/cloud/secrets/secret_manager.py +2 -19
- airflow/providers/google/cloud/sensors/bigquery.py +2 -81
- airflow/providers/google/cloud/sensors/bigquery_dts.py +2 -1
- airflow/providers/google/cloud/sensors/bigtable.py +2 -1
- airflow/providers/google/cloud/sensors/cloud_composer.py +8 -94
- airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +2 -1
- airflow/providers/google/cloud/sensors/dataflow.py +2 -1
- airflow/providers/google/cloud/sensors/dataform.py +2 -1
- airflow/providers/google/cloud/sensors/datafusion.py +2 -1
- airflow/providers/google/cloud/sensors/dataplex.py +2 -1
- airflow/providers/google/cloud/sensors/dataprep.py +2 -1
- airflow/providers/google/cloud/sensors/dataproc.py +2 -1
- airflow/providers/google/cloud/sensors/dataproc_metastore.py +2 -1
- airflow/providers/google/cloud/sensors/gcs.py +4 -36
- airflow/providers/google/cloud/sensors/pubsub.py +2 -1
- airflow/providers/google/cloud/sensors/tasks.py +2 -1
- airflow/providers/google/cloud/sensors/workflows.py +2 -1
- airflow/providers/google/cloud/transfers/adls_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +75 -18
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +9 -7
- airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +2 -1
- airflow/providers/google/cloud/transfers/bigquery_to_mysql.py +1 -1
- airflow/providers/google/cloud/transfers/bigquery_to_sql.py +2 -1
- airflow/providers/google/cloud/transfers/calendar_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +13 -9
- airflow/providers/google/cloud/transfers/gcs_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/gcs_to_local.py +2 -1
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +2 -1
- airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/gdrive_to_local.py +2 -1
- airflow/providers/google/cloud/transfers/local_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/s3_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/salesforce_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/sheets_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +2 -1
- airflow/providers/google/cloud/triggers/bigquery.py +2 -1
- airflow/providers/google/cloud/triggers/bigquery_dts.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_batch.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_build.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_composer.py +3 -2
- airflow/providers/google/cloud/triggers/cloud_run.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_sql.py +1 -1
- airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +2 -1
- airflow/providers/google/cloud/triggers/dataflow.py +2 -1
- airflow/providers/google/cloud/triggers/datafusion.py +2 -1
- airflow/providers/google/cloud/triggers/dataplex.py +1 -1
- airflow/providers/google/cloud/triggers/dataproc.py +2 -1
- airflow/providers/google/cloud/triggers/gcs.py +3 -2
- airflow/providers/google/cloud/triggers/kubernetes_engine.py +2 -1
- airflow/providers/google/cloud/triggers/mlengine.py +2 -1
- airflow/providers/google/cloud/triggers/pubsub.py +2 -1
- airflow/providers/google/cloud/triggers/vertex_ai.py +2 -1
- airflow/providers/google/cloud/utils/credentials_provider.py +1 -1
- airflow/providers/google/cloud/utils/dataform.py +1 -1
- airflow/providers/google/cloud/utils/field_validator.py +2 -1
- airflow/providers/google/cloud/utils/mlengine_operator_utils.py +2 -1
- airflow/providers/google/common/hooks/base_google.py +4 -11
- airflow/providers/google/common/hooks/discovery_api.py +1 -6
- airflow/providers/google/firebase/hooks/firestore.py +1 -1
- airflow/providers/google/firebase/operators/firestore.py +2 -1
- airflow/providers/google/get_provider_info.py +7 -22
- airflow/providers/google/marketing_platform/hooks/analytics_admin.py +2 -1
- airflow/providers/google/marketing_platform/hooks/campaign_manager.py +2 -3
- airflow/providers/google/marketing_platform/hooks/display_video.py +4 -3
- airflow/providers/google/marketing_platform/hooks/search_ads.py +6 -6
- airflow/providers/google/marketing_platform/operators/analytics_admin.py +2 -1
- airflow/providers/google/marketing_platform/operators/campaign_manager.py +2 -42
- airflow/providers/google/marketing_platform/operators/display_video.py +2 -47
- airflow/providers/google/marketing_platform/operators/search_ads.py +2 -1
- airflow/providers/google/marketing_platform/sensors/campaign_manager.py +2 -7
- airflow/providers/google/marketing_platform/sensors/display_video.py +2 -13
- airflow/providers/google/suite/hooks/calendar.py +2 -8
- airflow/providers/google/suite/hooks/drive.py +2 -6
- airflow/providers/google/suite/hooks/sheets.py +2 -7
- airflow/providers/google/suite/operators/sheets.py +2 -7
- airflow/providers/google/suite/sensors/drive.py +2 -7
- airflow/providers/google/suite/transfers/gcs_to_gdrive.py +2 -7
- airflow/providers/google/suite/transfers/gcs_to_sheets.py +2 -7
- airflow/providers/google/suite/transfers/local_to_drive.py +2 -7
- airflow/providers/google/suite/transfers/sql_to_sheets.py +2 -7
- {apache_airflow_providers_google-10.26.0rc1.dist-info → apache_airflow_providers_google-11.0.0.dist-info}/METADATA +17 -17
- apache_airflow_providers_google-11.0.0.dist-info/RECORD +315 -0
- airflow/providers/google/marketing_platform/hooks/analytics.py +0 -211
- airflow/providers/google/marketing_platform/operators/analytics.py +0 -551
- apache_airflow_providers_google-10.26.0rc1.dist-info/RECORD +0 -317
- {apache_airflow_providers_google-10.26.0rc1.dist-info → apache_airflow_providers_google-11.0.0.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_google-10.26.0rc1.dist-info → apache_airflow_providers_google-11.0.0.dist-info}/entry_points.txt +0 -0
@@ -20,7 +20,8 @@
|
|
20
20
|
from __future__ import annotations
|
21
21
|
|
22
22
|
import warnings
|
23
|
-
from
|
23
|
+
from collections.abc import Sequence
|
24
|
+
from typing import TYPE_CHECKING
|
24
25
|
|
25
26
|
from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
|
26
27
|
from airflow.models import BaseOperator
|
@@ -16,7 +16,8 @@
|
|
16
16
|
# under the License.
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
|
-
from
|
19
|
+
from collections.abc import Sequence
|
20
|
+
from typing import TYPE_CHECKING
|
20
21
|
|
21
22
|
from airflow.exceptions import AirflowException
|
22
23
|
from airflow.models import BaseOperator
|
@@ -20,8 +20,9 @@
|
|
20
20
|
from __future__ import annotations
|
21
21
|
|
22
22
|
import os
|
23
|
+
from collections.abc import Sequence
|
23
24
|
from tempfile import NamedTemporaryFile
|
24
|
-
from typing import TYPE_CHECKING
|
25
|
+
from typing import TYPE_CHECKING
|
25
26
|
|
26
27
|
from airflow.exceptions import AirflowException
|
27
28
|
from airflow.models import BaseOperator
|
@@ -16,7 +16,8 @@
|
|
16
16
|
# under the License.
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
|
-
from
|
19
|
+
from collections.abc import Sequence
|
20
|
+
from typing import TYPE_CHECKING
|
20
21
|
|
21
22
|
from airflow.models import BaseOperator
|
22
23
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
@@ -16,7 +16,8 @@
|
|
16
16
|
# under the License.
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
|
-
from
|
19
|
+
from collections.abc import Sequence
|
20
|
+
from typing import TYPE_CHECKING
|
20
21
|
|
21
22
|
from airflow.models import BaseOperator
|
22
23
|
from airflow.providers.google.suite.hooks.drive import GoogleDriveHook
|
@@ -20,8 +20,9 @@
|
|
20
20
|
from __future__ import annotations
|
21
21
|
|
22
22
|
import os
|
23
|
+
from collections.abc import Sequence
|
23
24
|
from glob import glob
|
24
|
-
from typing import TYPE_CHECKING
|
25
|
+
from typing import TYPE_CHECKING
|
25
26
|
|
26
27
|
from airflow.models import BaseOperator
|
27
28
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
@@ -21,7 +21,7 @@ from __future__ import annotations
|
|
21
21
|
|
22
22
|
import datetime
|
23
23
|
import decimal
|
24
|
-
from
|
24
|
+
from collections.abc import Sequence
|
25
25
|
|
26
26
|
from airflow.providers.google.cloud.transfers.sql_to_gcs import BaseSQLToGCSOperator
|
27
27
|
from airflow.providers.microsoft.mssql.hooks.mssql import MsSqlHook
|
@@ -17,9 +17,10 @@
|
|
17
17
|
# under the License.
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
|
+
from collections.abc import Sequence
|
20
21
|
from datetime import datetime, timezone
|
21
22
|
from tempfile import NamedTemporaryFile
|
22
|
-
from typing import TYPE_CHECKING, Any
|
23
|
+
from typing import TYPE_CHECKING, Any
|
23
24
|
|
24
25
|
from airflow.configuration import conf
|
25
26
|
from airflow.exceptions import AirflowException
|
@@ -18,7 +18,8 @@ from __future__ import annotations
|
|
18
18
|
|
19
19
|
import os
|
20
20
|
import tempfile
|
21
|
-
from
|
21
|
+
from collections.abc import Sequence
|
22
|
+
from typing import TYPE_CHECKING
|
22
23
|
|
23
24
|
from airflow.models import BaseOperator
|
24
25
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
@@ -20,8 +20,9 @@
|
|
20
20
|
from __future__ import annotations
|
21
21
|
|
22
22
|
import os
|
23
|
+
from collections.abc import Sequence
|
23
24
|
from tempfile import NamedTemporaryFile
|
24
|
-
from typing import TYPE_CHECKING
|
25
|
+
from typing import TYPE_CHECKING
|
25
26
|
|
26
27
|
from airflow.exceptions import AirflowException
|
27
28
|
from airflow.models import BaseOperator
|
@@ -17,8 +17,9 @@
|
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
19
|
import csv
|
20
|
+
from collections.abc import Sequence
|
20
21
|
from tempfile import NamedTemporaryFile
|
21
|
-
from typing import TYPE_CHECKING, Any
|
22
|
+
from typing import TYPE_CHECKING, Any
|
22
23
|
|
23
24
|
from airflow.models import BaseOperator
|
24
25
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
@@ -23,8 +23,9 @@ import abc
|
|
23
23
|
import csv
|
24
24
|
import json
|
25
25
|
import os
|
26
|
+
from collections.abc import Sequence
|
26
27
|
from tempfile import NamedTemporaryFile
|
27
|
-
from typing import TYPE_CHECKING, Any
|
28
|
+
from typing import TYPE_CHECKING, Any
|
28
29
|
|
29
30
|
import pyarrow as pa
|
30
31
|
import pyarrow.parquet as pq
|
@@ -17,7 +17,8 @@
|
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
19
|
import asyncio
|
20
|
-
from
|
20
|
+
from collections.abc import AsyncIterator, Sequence
|
21
|
+
from typing import TYPE_CHECKING, Any, SupportsAbs
|
21
22
|
|
22
23
|
from aiohttp import ClientSession
|
23
24
|
from aiohttp.client_exceptions import ClientResponseError
|
@@ -18,7 +18,8 @@
|
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
20
|
import asyncio
|
21
|
-
from
|
21
|
+
from collections.abc import AsyncIterator, Sequence
|
22
|
+
from typing import Any
|
22
23
|
|
23
24
|
from google.cloud.bigquery_datatransfer_v1 import TransferRun, TransferState
|
24
25
|
|
@@ -17,7 +17,8 @@
|
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
19
|
import asyncio
|
20
|
-
from
|
20
|
+
from collections.abc import AsyncIterator, Sequence
|
21
|
+
from typing import Any
|
21
22
|
|
22
23
|
from google.cloud.devtools.cloudbuild_v1.types import Build
|
23
24
|
|
@@ -20,8 +20,9 @@ from __future__ import annotations
|
|
20
20
|
|
21
21
|
import asyncio
|
22
22
|
import json
|
23
|
+
from collections.abc import Sequence
|
23
24
|
from datetime import datetime
|
24
|
-
from typing import Any
|
25
|
+
from typing import Any
|
25
26
|
|
26
27
|
from dateutil import parser
|
27
28
|
from google.cloud.orchestration.airflow.service_v1.types import ExecuteAirflowCommandResponse
|
@@ -231,7 +232,7 @@ class CloudComposerDAGRunTrigger(BaseTrigger):
|
|
231
232
|
for dag_run in dag_runs:
|
232
233
|
if (
|
233
234
|
start_date.timestamp()
|
234
|
-
< parser.parse(dag_run["
|
235
|
+
< parser.parse(dag_run["logical_date"]).timestamp()
|
235
236
|
< end_date.timestamp()
|
236
237
|
) and dag_run["state"] not in self.allowed_states:
|
237
238
|
return False
|
@@ -17,8 +17,9 @@
|
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
19
|
import asyncio
|
20
|
+
from collections.abc import AsyncIterator, Sequence
|
20
21
|
from enum import Enum
|
21
|
-
from typing import TYPE_CHECKING, Any
|
22
|
+
from typing import TYPE_CHECKING, Any
|
22
23
|
|
23
24
|
from airflow.exceptions import AirflowException
|
24
25
|
from airflow.providers.google.cloud.hooks.cloud_run import CloudRunAsyncHook
|
@@ -20,7 +20,7 @@
|
|
20
20
|
from __future__ import annotations
|
21
21
|
|
22
22
|
import asyncio
|
23
|
-
from
|
23
|
+
from collections.abc import Sequence
|
24
24
|
|
25
25
|
from airflow.providers.google.cloud.hooks.cloud_sql import CloudSQLAsyncHook, CloudSqlOperationStatus
|
26
26
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
@@ -18,7 +18,8 @@
|
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
20
|
import asyncio
|
21
|
-
from
|
21
|
+
from collections.abc import AsyncIterator, Iterable
|
22
|
+
from typing import Any
|
22
23
|
|
23
24
|
from google.api_core.exceptions import GoogleAPIError
|
24
25
|
from google.cloud.storage_transfer_v1.types import TransferOperation
|
@@ -18,8 +18,9 @@
|
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
20
|
import asyncio
|
21
|
+
from collections.abc import Sequence
|
21
22
|
from functools import cached_property
|
22
|
-
from typing import TYPE_CHECKING, Any
|
23
|
+
from typing import TYPE_CHECKING, Any
|
23
24
|
|
24
25
|
from google.cloud.dataflow_v1beta3 import JobState
|
25
26
|
from google.cloud.dataflow_v1beta3.types import (
|
@@ -17,7 +17,8 @@
|
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
19
|
import asyncio
|
20
|
-
from
|
20
|
+
from collections.abc import AsyncIterator, Sequence
|
21
|
+
from typing import Any
|
21
22
|
|
22
23
|
from airflow.providers.google.cloud.hooks.datafusion import DataFusionAsyncHook
|
23
24
|
from airflow.providers.google.cloud.utils.datafusion import DataFusionPipelineType
|
@@ -22,7 +22,8 @@ from __future__ import annotations
|
|
22
22
|
import asyncio
|
23
23
|
import re
|
24
24
|
import time
|
25
|
-
from
|
25
|
+
from collections.abc import AsyncIterator, Sequence
|
26
|
+
from typing import TYPE_CHECKING, Any
|
26
27
|
|
27
28
|
from google.api_core.exceptions import NotFound
|
28
29
|
from google.cloud.dataproc_v1 import Batch, Cluster, ClusterStatus, JobStatus
|
@@ -19,8 +19,9 @@ from __future__ import annotations
|
|
19
19
|
|
20
20
|
import asyncio
|
21
21
|
import os
|
22
|
+
from collections.abc import AsyncIterator
|
22
23
|
from datetime import datetime
|
23
|
-
from typing import Any
|
24
|
+
from typing import Any
|
24
25
|
|
25
26
|
from aiohttp import ClientSession
|
26
27
|
|
@@ -122,7 +123,7 @@ class GCSCheckBlobUpdateTimeTrigger(BaseTrigger):
|
|
122
123
|
:param target_date: context datetime to compare with blob object updated time
|
123
124
|
:param poke_interval: polling period in seconds to check for file/folder
|
124
125
|
:param google_cloud_conn_id: reference to the Google Connection
|
125
|
-
:param hook_params: dict object that has
|
126
|
+
:param hook_params: dict object that has impersonation_chain
|
126
127
|
"""
|
127
128
|
|
128
129
|
def __init__(
|
@@ -19,8 +19,9 @@ from __future__ import annotations
|
|
19
19
|
|
20
20
|
import asyncio
|
21
21
|
import warnings
|
22
|
+
from collections.abc import AsyncIterator, Sequence
|
22
23
|
from functools import cached_property
|
23
|
-
from typing import TYPE_CHECKING, Any
|
24
|
+
from typing import TYPE_CHECKING, Any
|
24
25
|
|
25
26
|
from google.cloud.container_v1.types import Operation
|
26
27
|
from packaging.version import parse as parse_version
|
@@ -17,7 +17,8 @@
|
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
19
|
import asyncio
|
20
|
-
from
|
20
|
+
from collections.abc import AsyncIterator, Sequence
|
21
|
+
from typing import Any
|
21
22
|
|
22
23
|
from airflow.providers.google.cloud.hooks.mlengine import MLEngineAsyncHook
|
23
24
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
@@ -19,8 +19,9 @@
|
|
19
19
|
from __future__ import annotations
|
20
20
|
|
21
21
|
import asyncio
|
22
|
+
from collections.abc import AsyncIterator, Sequence
|
22
23
|
from functools import cached_property
|
23
|
-
from typing import Any
|
24
|
+
from typing import Any
|
24
25
|
|
25
26
|
from google.cloud.pubsub_v1.types import ReceivedMessage
|
26
27
|
|
@@ -16,8 +16,9 @@
|
|
16
16
|
# under the License.
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
|
+
from collections.abc import AsyncIterator, Sequence
|
19
20
|
from functools import cached_property
|
20
|
-
from typing import TYPE_CHECKING, Any
|
21
|
+
from typing import TYPE_CHECKING, Any
|
21
22
|
|
22
23
|
from google.cloud.aiplatform_v1 import (
|
23
24
|
BatchPredictionJob,
|
@@ -23,8 +23,8 @@ import json
|
|
23
23
|
import logging
|
24
24
|
import os
|
25
25
|
import tempfile
|
26
|
+
from collections.abc import Collection, Generator, Sequence
|
26
27
|
from contextlib import ExitStack, contextmanager
|
27
|
-
from typing import Collection, Generator, Sequence
|
28
28
|
from urllib.parse import urlencode
|
29
29
|
|
30
30
|
import google.auth
|
@@ -18,8 +18,8 @@
|
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
20
|
import json
|
21
|
+
from collections.abc import Mapping
|
21
22
|
from enum import Enum
|
22
|
-
from typing import Mapping
|
23
23
|
|
24
24
|
from airflow.providers.google.cloud.operators.dataform import (
|
25
25
|
DataformInstallNpmPackagesOperator,
|
@@ -134,7 +134,8 @@ Here are the guidelines that you should follow to make validation forward-compat
|
|
134
134
|
from __future__ import annotations
|
135
135
|
|
136
136
|
import re
|
137
|
-
from
|
137
|
+
from collections.abc import Sequence
|
138
|
+
from typing import Callable
|
138
139
|
|
139
140
|
from airflow.exceptions import AirflowException
|
140
141
|
from airflow.utils.log.logging_mixin import LoggingMixin
|
@@ -22,7 +22,8 @@ import base64
|
|
22
22
|
import json
|
23
23
|
import os
|
24
24
|
import re
|
25
|
-
from
|
25
|
+
from collections.abc import Iterable
|
26
|
+
from typing import TYPE_CHECKING, Callable, TypeVar
|
26
27
|
from urllib.parse import urlsplit
|
27
28
|
|
28
29
|
import dill
|
@@ -26,9 +26,10 @@ import json
|
|
26
26
|
import logging
|
27
27
|
import os
|
28
28
|
import tempfile
|
29
|
+
from collections.abc import Generator, Sequence
|
29
30
|
from contextlib import ExitStack, contextmanager
|
30
31
|
from subprocess import check_output
|
31
|
-
from typing import TYPE_CHECKING, Any, Callable,
|
32
|
+
from typing import TYPE_CHECKING, Any, Callable, TypeVar, cast
|
32
33
|
|
33
34
|
import google.auth
|
34
35
|
import google.oauth2.service_account
|
@@ -197,12 +198,6 @@ class GoogleBaseHook(BaseHook):
|
|
197
198
|
JSON data provided in the UI: Specify 'Keyfile JSON'.
|
198
199
|
|
199
200
|
:param gcp_conn_id: The connection ID to use when fetching connection info.
|
200
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
201
|
-
if any. For this to work, the service account making the request must have
|
202
|
-
domain-wide delegation enabled. The usage of this parameter should be limited only to Google Workspace
|
203
|
-
(gsuite) and marketing platform operators and hooks. It is deprecated for usage by Google Cloud
|
204
|
-
and Firebase operators and hooks, as well as transfer operators in other providers that involve
|
205
|
-
Google cloud.
|
206
201
|
:param impersonation_chain: Optional service account to impersonate using short-term
|
207
202
|
credentials, or chained list of accounts required to get the access_token
|
208
203
|
of the last account in the list, which will be impersonated in the request.
|
@@ -279,12 +274,11 @@ class GoogleBaseHook(BaseHook):
|
|
279
274
|
def __init__(
|
280
275
|
self,
|
281
276
|
gcp_conn_id: str = "google_cloud_default",
|
282
|
-
delegate_to: str | None = None,
|
283
277
|
impersonation_chain: str | Sequence[str] | None = None,
|
278
|
+
**kwargs,
|
284
279
|
) -> None:
|
285
|
-
super().__init__()
|
280
|
+
super().__init__(**kwargs)
|
286
281
|
self.gcp_conn_id = gcp_conn_id
|
287
|
-
self.delegate_to = delegate_to
|
288
282
|
self.impersonation_chain = impersonation_chain
|
289
283
|
self.extras: dict = self.get_connection(self.gcp_conn_id).extra_dejson
|
290
284
|
self._cached_credentials: Credentials | None = None
|
@@ -339,7 +333,6 @@ class GoogleBaseHook(BaseHook):
|
|
339
333
|
key_secret_name=key_secret_name,
|
340
334
|
key_secret_project_id=key_secret_project_id,
|
341
335
|
scopes=self.scopes,
|
342
|
-
delegate_to=self.delegate_to,
|
343
336
|
target_principal=target_principal,
|
344
337
|
delegates=delegates,
|
345
338
|
is_anonymous=is_anonymous,
|
@@ -19,7 +19,7 @@
|
|
19
19
|
|
20
20
|
from __future__ import annotations
|
21
21
|
|
22
|
-
from
|
22
|
+
from collections.abc import Sequence
|
23
23
|
|
24
24
|
from googleapiclient.discovery import Resource, build
|
25
25
|
|
@@ -34,9 +34,6 @@ class GoogleDiscoveryApiHook(GoogleBaseHook):
|
|
34
34
|
for example 'youtube'.
|
35
35
|
:param api_version: The version of the api that will be requested for example 'v3'.
|
36
36
|
:param gcp_conn_id: The connection ID to use when fetching connection info.
|
37
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
38
|
-
if any. For this to work, the service account making the request must have
|
39
|
-
domain-wide delegation enabled.
|
40
37
|
:param impersonation_chain: Optional service account to impersonate using short-term
|
41
38
|
credentials, or chained list of accounts required to get the access_token
|
42
39
|
of the last account in the list, which will be impersonated in the request.
|
@@ -54,12 +51,10 @@ class GoogleDiscoveryApiHook(GoogleBaseHook):
|
|
54
51
|
api_service_name: str,
|
55
52
|
api_version: str,
|
56
53
|
gcp_conn_id: str = "google_cloud_default",
|
57
|
-
delegate_to: str | None = None,
|
58
54
|
impersonation_chain: str | Sequence[str] | None = None,
|
59
55
|
) -> None:
|
60
56
|
super().__init__(
|
61
57
|
gcp_conn_id=gcp_conn_id,
|
62
|
-
delegate_to=delegate_to,
|
63
58
|
impersonation_chain=impersonation_chain,
|
64
59
|
)
|
65
60
|
self.api_service_name = api_service_name
|
@@ -16,7 +16,8 @@
|
|
16
16
|
# under the License.
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
|
-
from
|
19
|
+
from collections.abc import Sequence
|
20
|
+
from typing import TYPE_CHECKING
|
20
21
|
|
21
22
|
from airflow.exceptions import AirflowException
|
22
23
|
from airflow.models import BaseOperator
|
@@ -28,8 +28,9 @@ def get_provider_info():
|
|
28
28
|
"name": "Google",
|
29
29
|
"description": "Google services including:\n\n - `Google Ads <https://ads.google.com/>`__\n - `Google Cloud (GCP) <https://cloud.google.com/>`__\n - `Google Firebase <https://firebase.google.com/>`__\n - `Google LevelDB <https://github.com/google/leveldb/>`__\n - `Google Marketing Platform <https://marketingplatform.google.com/>`__\n - `Google Workspace <https://workspace.google.com/>`__ (formerly Google Suite)\n",
|
30
30
|
"state": "ready",
|
31
|
-
"source-date-epoch":
|
31
|
+
"source-date-epoch": 1732429547,
|
32
32
|
"versions": [
|
33
|
+
"11.0.0",
|
33
34
|
"10.26.0",
|
34
35
|
"10.25.0",
|
35
36
|
"10.24.0",
|
@@ -119,7 +120,7 @@ def get_provider_info():
|
|
119
120
|
"google-cloud-build>=3.22.0",
|
120
121
|
"google-cloud-compute>=1.10.0",
|
121
122
|
"google-cloud-container>=2.17.4",
|
122
|
-
"google-cloud-datacatalog>=3.
|
123
|
+
"google-cloud-datacatalog>=3.23.0",
|
123
124
|
"google-cloud-dataflow-client>=0.8.6",
|
124
125
|
"google-cloud-dataform>=0.5.0",
|
125
126
|
"google-cloud-dataplex>=1.10.0",
|
@@ -142,7 +143,7 @@ def get_provider_info():
|
|
142
143
|
"google-cloud-storage-transfer>=1.4.1",
|
143
144
|
"google-cloud-tasks>=2.13.0",
|
144
145
|
"google-cloud-texttospeech>=2.14.1",
|
145
|
-
"google-cloud-translate>=3.
|
146
|
+
"google-cloud-translate>=3.16.0",
|
146
147
|
"google-cloud-videointelligence>=2.11.0",
|
147
148
|
"google-cloud-vision>=3.4.0",
|
148
149
|
"google-cloud-workflows>=1.10.0",
|
@@ -182,15 +183,6 @@ def get_provider_info():
|
|
182
183
|
],
|
183
184
|
"tags": ["gmp"],
|
184
185
|
},
|
185
|
-
{
|
186
|
-
"integration-name": "Google Analytics360",
|
187
|
-
"external-doc-url": "https://analytics.google.com/",
|
188
|
-
"logo": "/integration-logos/gcp/Google-Analytics.png",
|
189
|
-
"how-to-guide": [
|
190
|
-
"/docs/apache-airflow-providers-google/operators/marketing_platform/analytics.rst"
|
191
|
-
],
|
192
|
-
"tags": ["gmp"],
|
193
|
-
},
|
194
186
|
{
|
195
187
|
"integration-name": "Google Ads",
|
196
188
|
"external-doc-url": "https://ads.google.com/",
|
@@ -791,10 +783,6 @@ def get_provider_info():
|
|
791
783
|
"integration-name": "Google Analytics (GA4)",
|
792
784
|
"python-modules": ["airflow.providers.google.marketing_platform.operators.analytics_admin"],
|
793
785
|
},
|
794
|
-
{
|
795
|
-
"integration-name": "Google Analytics360",
|
796
|
-
"python-modules": ["airflow.providers.google.marketing_platform.operators.analytics"],
|
797
|
-
},
|
798
786
|
{
|
799
787
|
"integration-name": "Google Campaign Manager",
|
800
788
|
"python-modules": ["airflow.providers.google.marketing_platform.operators.campaign_manager"],
|
@@ -1130,10 +1118,6 @@ def get_provider_info():
|
|
1130
1118
|
"integration-name": "Google Analytics (GA4)",
|
1131
1119
|
"python-modules": ["airflow.providers.google.marketing_platform.hooks.analytics_admin"],
|
1132
1120
|
},
|
1133
|
-
{
|
1134
|
-
"integration-name": "Google Analytics360",
|
1135
|
-
"python-modules": ["airflow.providers.google.marketing_platform.hooks.analytics"],
|
1136
|
-
},
|
1137
1121
|
{
|
1138
1122
|
"integration-name": "Google Campaign Manager",
|
1139
1123
|
"python-modules": ["airflow.providers.google.marketing_platform.hooks.campaign_manager"],
|
@@ -1498,8 +1482,6 @@ def get_provider_info():
|
|
1498
1482
|
},
|
1499
1483
|
],
|
1500
1484
|
"extra-links": [
|
1501
|
-
"airflow.providers.google.cloud.operators.bigquery.BigQueryConsoleLink",
|
1502
|
-
"airflow.providers.google.cloud.operators.bigquery.BigQueryConsoleIndexableLink",
|
1503
1485
|
"airflow.providers.google.cloud.links.dataform.DataformRepositoryLink",
|
1504
1486
|
"airflow.providers.google.cloud.links.dataform.DataformWorkspaceLink",
|
1505
1487
|
"airflow.providers.google.cloud.links.dataform.DataformWorkflowInvocationLink",
|
@@ -1613,6 +1595,9 @@ def get_provider_info():
|
|
1613
1595
|
"airflow.providers.google.cloud.links.translate.TranslationLegacyModelLink",
|
1614
1596
|
"airflow.providers.google.cloud.links.translate.TranslationLegacyModelTrainLink",
|
1615
1597
|
"airflow.providers.google.cloud.links.translate.TranslationLegacyModelPredictLink",
|
1598
|
+
"airflow.providers.google.cloud.links.translate.TranslateTextBatchLink",
|
1599
|
+
"airflow.providers.google.cloud.links.translate.TranslationNativeDatasetLink",
|
1600
|
+
"airflow.providers.google.cloud.links.translate.TranslationDatasetsListLink",
|
1616
1601
|
],
|
1617
1602
|
"secrets-backends": [
|
1618
1603
|
"airflow.providers.google.cloud.secrets.secret_manager.CloudSecretManagerBackend"
|
@@ -27,7 +27,8 @@ Hooks for Google Analytics (GA4) Admin service.
|
|
27
27
|
|
28
28
|
from __future__ import annotations
|
29
29
|
|
30
|
-
from
|
30
|
+
from collections.abc import Sequence
|
31
|
+
from typing import TYPE_CHECKING
|
31
32
|
|
32
33
|
from google.analytics.admin_v1beta import (
|
33
34
|
AnalyticsAdminServiceClient,
|
@@ -19,7 +19,8 @@
|
|
19
19
|
|
20
20
|
from __future__ import annotations
|
21
21
|
|
22
|
-
from
|
22
|
+
from collections.abc import Sequence
|
23
|
+
from typing import TYPE_CHECKING, Any
|
23
24
|
|
24
25
|
from googleapiclient.discovery import Resource, build
|
25
26
|
|
@@ -39,12 +40,10 @@ class GoogleCampaignManagerHook(GoogleBaseHook):
|
|
39
40
|
self,
|
40
41
|
api_version: str = "v4",
|
41
42
|
gcp_conn_id: str = "google_cloud_default",
|
42
|
-
delegate_to: str | None = None,
|
43
43
|
impersonation_chain: str | Sequence[str] | None = None,
|
44
44
|
) -> None:
|
45
45
|
super().__init__(
|
46
46
|
gcp_conn_id=gcp_conn_id,
|
47
|
-
delegate_to=delegate_to,
|
48
47
|
impersonation_chain=impersonation_chain,
|
49
48
|
)
|
50
49
|
self.api_version = api_version
|
@@ -19,7 +19,8 @@
|
|
19
19
|
|
20
20
|
from __future__ import annotations
|
21
21
|
|
22
|
-
from
|
22
|
+
from collections.abc import Sequence
|
23
|
+
from typing import Any
|
23
24
|
|
24
25
|
from googleapiclient.discovery import Resource, build
|
25
26
|
|
@@ -35,13 +36,13 @@ class GoogleDisplayVideo360Hook(GoogleBaseHook):
|
|
35
36
|
self,
|
36
37
|
api_version: str = "v2",
|
37
38
|
gcp_conn_id: str = "google_cloud_default",
|
38
|
-
delegate_to: str | None = None,
|
39
39
|
impersonation_chain: str | Sequence[str] | None = None,
|
40
|
+
**kwargs,
|
40
41
|
) -> None:
|
41
42
|
super().__init__(
|
42
43
|
gcp_conn_id=gcp_conn_id,
|
43
|
-
delegate_to=delegate_to,
|
44
44
|
impersonation_chain=impersonation_chain,
|
45
|
+
**kwargs,
|
45
46
|
)
|
46
47
|
self.api_version = api_version
|
47
48
|
|