apache-airflow-providers-google 16.0.0rc1__py3-none-any.whl → 16.1.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/__init__.py +1 -1
- airflow/providers/google/ads/hooks/ads.py +9 -5
- airflow/providers/google/ads/operators/ads.py +1 -1
- airflow/providers/google/ads/transfers/ads_to_gcs.py +1 -1
- airflow/providers/google/cloud/hooks/bigquery.py +2 -3
- airflow/providers/google/cloud/hooks/cloud_sql.py +8 -4
- airflow/providers/google/cloud/hooks/datacatalog.py +9 -1
- airflow/providers/google/cloud/hooks/dataflow.py +2 -2
- airflow/providers/google/cloud/hooks/dataplex.py +1 -1
- airflow/providers/google/cloud/hooks/dataprep.py +4 -1
- airflow/providers/google/cloud/hooks/gcs.py +2 -2
- airflow/providers/google/cloud/hooks/looker.py +5 -1
- airflow/providers/google/cloud/hooks/mlengine.py +2 -1
- airflow/providers/google/cloud/hooks/secret_manager.py +102 -10
- airflow/providers/google/cloud/hooks/spanner.py +2 -2
- airflow/providers/google/cloud/hooks/translate.py +1 -1
- airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +307 -7
- airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +43 -14
- airflow/providers/google/cloud/hooks/vertex_ai/ray.py +11 -2
- airflow/providers/google/cloud/hooks/vision.py +2 -2
- airflow/providers/google/cloud/links/alloy_db.py +0 -46
- airflow/providers/google/cloud/links/base.py +75 -11
- airflow/providers/google/cloud/links/bigquery.py +0 -47
- airflow/providers/google/cloud/links/bigquery_dts.py +0 -20
- airflow/providers/google/cloud/links/bigtable.py +0 -48
- airflow/providers/google/cloud/links/cloud_build.py +0 -73
- airflow/providers/google/cloud/links/cloud_functions.py +0 -33
- airflow/providers/google/cloud/links/cloud_memorystore.py +0 -58
- airflow/providers/google/cloud/links/cloud_run.py +1 -33
- airflow/providers/google/cloud/links/cloud_sql.py +0 -33
- airflow/providers/google/cloud/links/cloud_storage_transfer.py +16 -43
- airflow/providers/google/cloud/links/cloud_tasks.py +6 -25
- airflow/providers/google/cloud/links/compute.py +0 -58
- airflow/providers/google/cloud/links/data_loss_prevention.py +0 -169
- airflow/providers/google/cloud/links/datacatalog.py +23 -54
- airflow/providers/google/cloud/links/dataflow.py +0 -34
- airflow/providers/google/cloud/links/dataform.py +0 -64
- airflow/providers/google/cloud/links/datafusion.py +1 -96
- airflow/providers/google/cloud/links/dataplex.py +0 -154
- airflow/providers/google/cloud/links/dataprep.py +0 -24
- airflow/providers/google/cloud/links/dataproc.py +14 -90
- airflow/providers/google/cloud/links/datastore.py +0 -31
- airflow/providers/google/cloud/links/kubernetes_engine.py +5 -59
- airflow/providers/google/cloud/links/life_sciences.py +0 -19
- airflow/providers/google/cloud/links/managed_kafka.py +0 -70
- airflow/providers/google/cloud/links/mlengine.py +0 -70
- airflow/providers/google/cloud/links/pubsub.py +0 -32
- airflow/providers/google/cloud/links/spanner.py +0 -33
- airflow/providers/google/cloud/links/stackdriver.py +0 -30
- airflow/providers/google/cloud/links/translate.py +16 -186
- airflow/providers/google/cloud/links/vertex_ai.py +8 -224
- airflow/providers/google/cloud/links/workflows.py +0 -52
- airflow/providers/google/cloud/operators/alloy_db.py +69 -54
- airflow/providers/google/cloud/operators/automl.py +16 -14
- airflow/providers/google/cloud/operators/bigquery.py +0 -15
- airflow/providers/google/cloud/operators/bigquery_dts.py +2 -4
- airflow/providers/google/cloud/operators/bigtable.py +35 -6
- airflow/providers/google/cloud/operators/cloud_base.py +21 -1
- airflow/providers/google/cloud/operators/cloud_build.py +74 -31
- airflow/providers/google/cloud/operators/cloud_composer.py +34 -35
- airflow/providers/google/cloud/operators/cloud_memorystore.py +68 -42
- airflow/providers/google/cloud/operators/cloud_run.py +0 -1
- airflow/providers/google/cloud/operators/cloud_sql.py +11 -15
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +0 -2
- airflow/providers/google/cloud/operators/compute.py +7 -39
- airflow/providers/google/cloud/operators/datacatalog.py +156 -20
- airflow/providers/google/cloud/operators/dataflow.py +37 -14
- airflow/providers/google/cloud/operators/dataform.py +14 -4
- airflow/providers/google/cloud/operators/datafusion.py +4 -12
- airflow/providers/google/cloud/operators/dataplex.py +180 -96
- airflow/providers/google/cloud/operators/dataprep.py +0 -4
- airflow/providers/google/cloud/operators/dataproc.py +10 -16
- airflow/providers/google/cloud/operators/dataproc_metastore.py +95 -87
- airflow/providers/google/cloud/operators/datastore.py +21 -5
- airflow/providers/google/cloud/operators/dlp.py +3 -26
- airflow/providers/google/cloud/operators/functions.py +15 -6
- airflow/providers/google/cloud/operators/gcs.py +0 -7
- airflow/providers/google/cloud/operators/kubernetes_engine.py +50 -7
- airflow/providers/google/cloud/operators/life_sciences.py +0 -1
- airflow/providers/google/cloud/operators/managed_kafka.py +106 -51
- airflow/providers/google/cloud/operators/mlengine.py +0 -1
- airflow/providers/google/cloud/operators/pubsub.py +2 -4
- airflow/providers/google/cloud/operators/spanner.py +0 -4
- airflow/providers/google/cloud/operators/speech_to_text.py +0 -1
- airflow/providers/google/cloud/operators/stackdriver.py +0 -8
- airflow/providers/google/cloud/operators/tasks.py +0 -11
- airflow/providers/google/cloud/operators/text_to_speech.py +0 -1
- airflow/providers/google/cloud/operators/translate.py +37 -13
- airflow/providers/google/cloud/operators/translate_speech.py +0 -1
- airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +31 -18
- airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +28 -8
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +38 -25
- airflow/providers/google/cloud/operators/vertex_ai/dataset.py +69 -7
- airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +42 -8
- airflow/providers/google/cloud/operators/vertex_ai/feature_store.py +531 -0
- airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +93 -25
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +10 -8
- airflow/providers/google/cloud/operators/vertex_ai/model_service.py +56 -10
- airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +25 -6
- airflow/providers/google/cloud/operators/vertex_ai/ray.py +9 -6
- airflow/providers/google/cloud/operators/workflows.py +1 -9
- airflow/providers/google/cloud/sensors/bigquery.py +1 -1
- airflow/providers/google/cloud/sensors/bigquery_dts.py +6 -1
- airflow/providers/google/cloud/sensors/bigtable.py +15 -3
- airflow/providers/google/cloud/sensors/cloud_composer.py +6 -1
- airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +6 -1
- airflow/providers/google/cloud/sensors/dataflow.py +3 -3
- airflow/providers/google/cloud/sensors/dataform.py +6 -1
- airflow/providers/google/cloud/sensors/datafusion.py +6 -1
- airflow/providers/google/cloud/sensors/dataplex.py +6 -1
- airflow/providers/google/cloud/sensors/dataprep.py +6 -1
- airflow/providers/google/cloud/sensors/dataproc.py +6 -1
- airflow/providers/google/cloud/sensors/dataproc_metastore.py +6 -1
- airflow/providers/google/cloud/sensors/gcs.py +9 -3
- airflow/providers/google/cloud/sensors/looker.py +6 -1
- airflow/providers/google/cloud/sensors/pubsub.py +8 -3
- airflow/providers/google/cloud/sensors/tasks.py +6 -1
- airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py +6 -1
- airflow/providers/google/cloud/sensors/workflows.py +6 -1
- airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +1 -2
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +1 -2
- airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +0 -1
- airflow/providers/google/cloud/transfers/bigquery_to_sql.py +1 -1
- airflow/providers/google/cloud/transfers/calendar_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +1 -2
- airflow/providers/google/cloud/transfers/gcs_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/gcs_to_local.py +1 -1
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +1 -1
- airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +5 -1
- airflow/providers/google/cloud/transfers/gdrive_to_local.py +1 -1
- airflow/providers/google/cloud/transfers/http_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/local_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/salesforce_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/sheets_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +1 -1
- airflow/providers/google/cloud/utils/field_validator.py +1 -2
- airflow/providers/google/common/auth_backend/google_openid.py +2 -1
- airflow/providers/google/common/deprecated.py +2 -1
- airflow/providers/google/common/hooks/base_google.py +7 -3
- airflow/providers/google/common/links/storage.py +0 -22
- airflow/providers/google/firebase/operators/firestore.py +1 -1
- airflow/providers/google/get_provider_info.py +0 -11
- airflow/providers/google/leveldb/hooks/leveldb.py +5 -1
- airflow/providers/google/leveldb/operators/leveldb.py +1 -1
- airflow/providers/google/marketing_platform/links/analytics_admin.py +3 -6
- airflow/providers/google/marketing_platform/operators/analytics_admin.py +0 -1
- airflow/providers/google/marketing_platform/operators/campaign_manager.py +4 -4
- airflow/providers/google/marketing_platform/operators/display_video.py +6 -6
- airflow/providers/google/marketing_platform/operators/search_ads.py +1 -1
- airflow/providers/google/marketing_platform/sensors/campaign_manager.py +6 -1
- airflow/providers/google/marketing_platform/sensors/display_video.py +6 -1
- airflow/providers/google/suite/operators/sheets.py +3 -3
- airflow/providers/google/suite/sensors/drive.py +6 -1
- airflow/providers/google/suite/transfers/gcs_to_gdrive.py +1 -1
- airflow/providers/google/suite/transfers/gcs_to_sheets.py +1 -1
- airflow/providers/google/suite/transfers/local_to_drive.py +1 -1
- airflow/providers/google/version_compat.py +28 -0
- {apache_airflow_providers_google-16.0.0rc1.dist-info → apache_airflow_providers_google-16.1.0rc1.dist-info}/METADATA +19 -20
- {apache_airflow_providers_google-16.0.0rc1.dist-info → apache_airflow_providers_google-16.1.0rc1.dist-info}/RECORD +166 -166
- {apache_airflow_providers_google-16.0.0rc1.dist-info → apache_airflow_providers_google-16.1.0rc1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_google-16.0.0rc1.dist-info → apache_airflow_providers_google-16.1.0rc1.dist-info}/entry_points.txt +0 -0
@@ -33,7 +33,12 @@ from airflow.exceptions import AirflowException
|
|
33
33
|
from airflow.providers.google.cloud.hooks.cloud_composer import CloudComposerHook
|
34
34
|
from airflow.providers.google.cloud.triggers.cloud_composer import CloudComposerDAGRunTrigger
|
35
35
|
from airflow.providers.google.common.consts import GOOGLE_DEFAULT_DEFERRABLE_METHOD_NAME
|
36
|
-
from airflow.
|
36
|
+
from airflow.providers.google.version_compat import AIRFLOW_V_3_0_PLUS
|
37
|
+
|
38
|
+
if AIRFLOW_V_3_0_PLUS:
|
39
|
+
from airflow.sdk import BaseSensorOperator
|
40
|
+
else:
|
41
|
+
from airflow.sensors.base import BaseSensorOperator # type: ignore[no-redef]
|
37
42
|
from airflow.utils.state import TaskInstanceState
|
38
43
|
|
39
44
|
if TYPE_CHECKING:
|
@@ -35,7 +35,12 @@ from airflow.providers.google.cloud.triggers.cloud_storage_transfer_service impo
|
|
35
35
|
CloudStorageTransferServiceCheckJobStatusTrigger,
|
36
36
|
)
|
37
37
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
38
|
-
from airflow.
|
38
|
+
from airflow.providers.google.version_compat import AIRFLOW_V_3_0_PLUS
|
39
|
+
|
40
|
+
if AIRFLOW_V_3_0_PLUS:
|
41
|
+
from airflow.sdk import BaseSensorOperator
|
42
|
+
else:
|
43
|
+
from airflow.sensors.base import BaseSensorOperator # type: ignore[no-redef]
|
39
44
|
|
40
45
|
if TYPE_CHECKING:
|
41
46
|
from airflow.utils.context import Context
|
@@ -19,9 +19,9 @@
|
|
19
19
|
|
20
20
|
from __future__ import annotations
|
21
21
|
|
22
|
-
from collections.abc import Sequence
|
22
|
+
from collections.abc import Callable, Sequence
|
23
23
|
from functools import cached_property
|
24
|
-
from typing import TYPE_CHECKING, Any
|
24
|
+
from typing import TYPE_CHECKING, Any
|
25
25
|
|
26
26
|
from airflow.configuration import conf
|
27
27
|
from airflow.exceptions import AirflowException
|
@@ -37,7 +37,7 @@ from airflow.providers.google.cloud.triggers.dataflow import (
|
|
37
37
|
DataflowJobStatusTrigger,
|
38
38
|
)
|
39
39
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
40
|
-
from airflow.
|
40
|
+
from airflow.providers.google.version_compat import BaseSensorOperator
|
41
41
|
|
42
42
|
if TYPE_CHECKING:
|
43
43
|
from airflow.utils.context import Context
|
@@ -24,7 +24,12 @@ from typing import TYPE_CHECKING
|
|
24
24
|
|
25
25
|
from airflow.exceptions import AirflowException
|
26
26
|
from airflow.providers.google.cloud.hooks.dataform import DataformHook
|
27
|
-
from airflow.
|
27
|
+
from airflow.providers.google.version_compat import AIRFLOW_V_3_0_PLUS
|
28
|
+
|
29
|
+
if AIRFLOW_V_3_0_PLUS:
|
30
|
+
from airflow.sdk import BaseSensorOperator
|
31
|
+
else:
|
32
|
+
from airflow.sensors.base import BaseSensorOperator # type: ignore[no-redef]
|
28
33
|
|
29
34
|
if TYPE_CHECKING:
|
30
35
|
from airflow.utils.context import Context
|
@@ -25,7 +25,12 @@ from typing import TYPE_CHECKING
|
|
25
25
|
from airflow.exceptions import AirflowException, AirflowNotFoundException
|
26
26
|
from airflow.providers.google.cloud.hooks.datafusion import DataFusionHook
|
27
27
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
28
|
-
from airflow.
|
28
|
+
from airflow.providers.google.version_compat import AIRFLOW_V_3_0_PLUS
|
29
|
+
|
30
|
+
if AIRFLOW_V_3_0_PLUS:
|
31
|
+
from airflow.sdk import BaseSensorOperator
|
32
|
+
else:
|
33
|
+
from airflow.sensors.base import BaseSensorOperator # type: ignore[no-redef]
|
29
34
|
|
30
35
|
if TYPE_CHECKING:
|
31
36
|
from airflow.utils.context import Context
|
@@ -37,7 +37,12 @@ from airflow.providers.google.cloud.hooks.dataplex import (
|
|
37
37
|
AirflowDataQualityScanResultTimeoutException,
|
38
38
|
DataplexHook,
|
39
39
|
)
|
40
|
-
from airflow.
|
40
|
+
from airflow.providers.google.version_compat import AIRFLOW_V_3_0_PLUS
|
41
|
+
|
42
|
+
if AIRFLOW_V_3_0_PLUS:
|
43
|
+
from airflow.sdk import BaseSensorOperator
|
44
|
+
else:
|
45
|
+
from airflow.sensors.base import BaseSensorOperator # type: ignore[no-redef]
|
41
46
|
|
42
47
|
|
43
48
|
class TaskState:
|
@@ -23,7 +23,12 @@ from collections.abc import Sequence
|
|
23
23
|
from typing import TYPE_CHECKING
|
24
24
|
|
25
25
|
from airflow.providers.google.cloud.hooks.dataprep import GoogleDataprepHook, JobGroupStatuses
|
26
|
-
from airflow.
|
26
|
+
from airflow.providers.google.version_compat import AIRFLOW_V_3_0_PLUS
|
27
|
+
|
28
|
+
if AIRFLOW_V_3_0_PLUS:
|
29
|
+
from airflow.sdk import BaseSensorOperator
|
30
|
+
else:
|
31
|
+
from airflow.sensors.base import BaseSensorOperator # type: ignore[no-redef]
|
27
32
|
|
28
33
|
if TYPE_CHECKING:
|
29
34
|
from airflow.utils.context import Context
|
@@ -29,7 +29,12 @@ from google.cloud.dataproc_v1.types import Batch, JobStatus
|
|
29
29
|
from airflow.exceptions import AirflowException
|
30
30
|
from airflow.providers.google.cloud.hooks.dataproc import DataprocHook
|
31
31
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
32
|
-
from airflow.
|
32
|
+
from airflow.providers.google.version_compat import AIRFLOW_V_3_0_PLUS
|
33
|
+
|
34
|
+
if AIRFLOW_V_3_0_PLUS:
|
35
|
+
from airflow.sdk import BaseSensorOperator
|
36
|
+
else:
|
37
|
+
from airflow.sensors.base import BaseSensorOperator # type: ignore[no-redef]
|
33
38
|
|
34
39
|
if TYPE_CHECKING:
|
35
40
|
from airflow.utils.context import Context
|
@@ -23,7 +23,12 @@ from typing import TYPE_CHECKING
|
|
23
23
|
from airflow.exceptions import AirflowException
|
24
24
|
from airflow.providers.google.cloud.hooks.dataproc_metastore import DataprocMetastoreHook
|
25
25
|
from airflow.providers.google.cloud.hooks.gcs import parse_json_from_gcs
|
26
|
-
from airflow.
|
26
|
+
from airflow.providers.google.version_compat import AIRFLOW_V_3_0_PLUS
|
27
|
+
|
28
|
+
if AIRFLOW_V_3_0_PLUS:
|
29
|
+
from airflow.sdk import BaseSensorOperator
|
30
|
+
else:
|
31
|
+
from airflow.sensors.base import BaseSensorOperator # type: ignore[no-redef]
|
27
32
|
|
28
33
|
if TYPE_CHECKING:
|
29
34
|
from google.api_core.operation import Operation
|
@@ -21,9 +21,9 @@ from __future__ import annotations
|
|
21
21
|
|
22
22
|
import os
|
23
23
|
import textwrap
|
24
|
-
from collections.abc import Sequence
|
24
|
+
from collections.abc import Callable, Sequence
|
25
25
|
from datetime import datetime, timedelta
|
26
|
-
from typing import TYPE_CHECKING, Any
|
26
|
+
from typing import TYPE_CHECKING, Any
|
27
27
|
|
28
28
|
from google.cloud.storage.retry import DEFAULT_RETRY
|
29
29
|
|
@@ -36,7 +36,13 @@ from airflow.providers.google.cloud.triggers.gcs import (
|
|
36
36
|
GCSPrefixBlobTrigger,
|
37
37
|
GCSUploadSessionTrigger,
|
38
38
|
)
|
39
|
-
from airflow.
|
39
|
+
from airflow.providers.google.version_compat import AIRFLOW_V_3_0_PLUS
|
40
|
+
|
41
|
+
if AIRFLOW_V_3_0_PLUS:
|
42
|
+
from airflow.sdk import BaseSensorOperator
|
43
|
+
from airflow.sdk.bases.sensor import poke_mode_only
|
44
|
+
else:
|
45
|
+
from airflow.sensors.base import BaseSensorOperator, poke_mode_only # type: ignore[no-redef]
|
40
46
|
|
41
47
|
if TYPE_CHECKING:
|
42
48
|
from google.api_core.retry import Retry
|
@@ -23,7 +23,12 @@ from typing import TYPE_CHECKING
|
|
23
23
|
|
24
24
|
from airflow.exceptions import AirflowException
|
25
25
|
from airflow.providers.google.cloud.hooks.looker import JobStatus, LookerHook
|
26
|
-
from airflow.
|
26
|
+
from airflow.providers.google.version_compat import AIRFLOW_V_3_0_PLUS
|
27
|
+
|
28
|
+
if AIRFLOW_V_3_0_PLUS:
|
29
|
+
from airflow.sdk import BaseSensorOperator
|
30
|
+
else:
|
31
|
+
from airflow.sensors.base import BaseSensorOperator # type: ignore[no-redef]
|
27
32
|
|
28
33
|
if TYPE_CHECKING:
|
29
34
|
from airflow.utils.context import Context
|
@@ -19,9 +19,9 @@
|
|
19
19
|
|
20
20
|
from __future__ import annotations
|
21
21
|
|
22
|
-
from collections.abc import Sequence
|
22
|
+
from collections.abc import Callable, Sequence
|
23
23
|
from datetime import timedelta
|
24
|
-
from typing import TYPE_CHECKING, Any
|
24
|
+
from typing import TYPE_CHECKING, Any
|
25
25
|
|
26
26
|
from google.cloud import pubsub_v1
|
27
27
|
from google.cloud.pubsub_v1.types import ReceivedMessage
|
@@ -30,7 +30,12 @@ from airflow.configuration import conf
|
|
30
30
|
from airflow.exceptions import AirflowException
|
31
31
|
from airflow.providers.google.cloud.hooks.pubsub import PubSubHook
|
32
32
|
from airflow.providers.google.cloud.triggers.pubsub import PubsubPullTrigger
|
33
|
-
from airflow.
|
33
|
+
from airflow.providers.google.version_compat import AIRFLOW_V_3_0_PLUS
|
34
|
+
|
35
|
+
if AIRFLOW_V_3_0_PLUS:
|
36
|
+
from airflow.sdk import BaseSensorOperator
|
37
|
+
else:
|
38
|
+
from airflow.sensors.base import BaseSensorOperator # type: ignore[no-redef]
|
34
39
|
|
35
40
|
if TYPE_CHECKING:
|
36
41
|
from airflow.utils.context import Context
|
@@ -24,7 +24,12 @@ from typing import TYPE_CHECKING
|
|
24
24
|
|
25
25
|
from airflow.providers.google.cloud.hooks.tasks import CloudTasksHook
|
26
26
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
27
|
-
from airflow.
|
27
|
+
from airflow.providers.google.version_compat import AIRFLOW_V_3_0_PLUS
|
28
|
+
|
29
|
+
if AIRFLOW_V_3_0_PLUS:
|
30
|
+
from airflow.sdk import BaseSensorOperator
|
31
|
+
else:
|
32
|
+
from airflow.sensors.base import BaseSensorOperator # type: ignore[no-redef]
|
28
33
|
|
29
34
|
if TYPE_CHECKING:
|
30
35
|
from airflow.utils.context import Context
|
@@ -25,7 +25,12 @@ from typing import TYPE_CHECKING
|
|
25
25
|
|
26
26
|
from airflow.exceptions import AirflowException
|
27
27
|
from airflow.providers.google.cloud.hooks.vertex_ai.feature_store import FeatureStoreHook
|
28
|
-
from airflow.
|
28
|
+
from airflow.providers.google.version_compat import AIRFLOW_V_3_0_PLUS
|
29
|
+
|
30
|
+
if AIRFLOW_V_3_0_PLUS:
|
31
|
+
from airflow.sdk import BaseSensorOperator
|
32
|
+
else:
|
33
|
+
from airflow.sensors.base import BaseSensorOperator # type: ignore[no-redef]
|
29
34
|
|
30
35
|
if TYPE_CHECKING:
|
31
36
|
from airflow.utils.context import Context
|
@@ -25,7 +25,12 @@ from google.cloud.workflows.executions_v1beta import Execution
|
|
25
25
|
from airflow.exceptions import AirflowException
|
26
26
|
from airflow.providers.google.cloud.hooks.workflows import WorkflowsHook
|
27
27
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
28
|
-
from airflow.
|
28
|
+
from airflow.providers.google.version_compat import AIRFLOW_V_3_0_PLUS
|
29
|
+
|
30
|
+
if AIRFLOW_V_3_0_PLUS:
|
31
|
+
from airflow.sdk import BaseSensorOperator
|
32
|
+
else:
|
33
|
+
from airflow.sensors.base import BaseSensorOperator # type: ignore[no-redef]
|
29
34
|
|
30
35
|
if TYPE_CHECKING:
|
31
36
|
from google.api_core.retry import Retry
|
@@ -21,8 +21,8 @@ import tempfile
|
|
21
21
|
from collections.abc import Sequence
|
22
22
|
from typing import TYPE_CHECKING
|
23
23
|
|
24
|
-
from airflow.models import BaseOperator
|
25
24
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
25
|
+
from airflow.providers.google.version_compat import BaseOperator
|
26
26
|
|
27
27
|
try:
|
28
28
|
from airflow.providers.microsoft.azure.hooks.wasb import WasbHook
|
@@ -23,8 +23,8 @@ from tempfile import NamedTemporaryFile
|
|
23
23
|
from typing import TYPE_CHECKING
|
24
24
|
|
25
25
|
from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
|
26
|
-
from airflow.models import BaseOperator
|
27
26
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook, _parse_gcs_url, gcs_object_is_directory
|
27
|
+
from airflow.providers.google.version_compat import BaseOperator
|
28
28
|
|
29
29
|
try:
|
30
30
|
from airflow.providers.microsoft.azure.hooks.fileshare import AzureFileShareHook
|
@@ -22,10 +22,10 @@ from __future__ import annotations
|
|
22
22
|
from collections.abc import Sequence
|
23
23
|
from typing import TYPE_CHECKING
|
24
24
|
|
25
|
-
from airflow.models import BaseOperator
|
26
25
|
from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook
|
27
26
|
from airflow.providers.google.cloud.links.bigquery import BigQueryTableLink
|
28
27
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
28
|
+
from airflow.providers.google.version_compat import BaseOperator
|
29
29
|
|
30
30
|
if TYPE_CHECKING:
|
31
31
|
from airflow.utils.context import Context
|
@@ -183,7 +183,6 @@ class BigQueryToBigQueryOperator(BaseOperator):
|
|
183
183
|
dest_table_info = self._job_conf["configuration"]["copy"]["destinationTable"]
|
184
184
|
BigQueryTableLink.persist(
|
185
185
|
context=context,
|
186
|
-
task_instance=self,
|
187
186
|
dataset_id=dest_table_info["datasetId"],
|
188
187
|
project_id=dest_table_info["projectId"],
|
189
188
|
table_id=dest_table_info["tableId"],
|
@@ -27,11 +27,11 @@ from google.cloud.bigquery import DEFAULT_RETRY, UnknownJob
|
|
27
27
|
|
28
28
|
from airflow.configuration import conf
|
29
29
|
from airflow.exceptions import AirflowException
|
30
|
-
from airflow.models import BaseOperator
|
31
30
|
from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook, BigQueryJob
|
32
31
|
from airflow.providers.google.cloud.links.bigquery import BigQueryTableLink
|
33
32
|
from airflow.providers.google.cloud.triggers.bigquery import BigQueryInsertJobTrigger
|
34
33
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
34
|
+
from airflow.providers.google.version_compat import BaseOperator
|
35
35
|
from airflow.utils.helpers import merge_dicts
|
36
36
|
|
37
37
|
if TYPE_CHECKING:
|
@@ -254,7 +254,6 @@ class BigQueryToGCSOperator(BaseOperator):
|
|
254
254
|
dataset_id, project_id, table_id = conf["datasetId"], conf["projectId"], conf["tableId"]
|
255
255
|
BigQueryTableLink.persist(
|
256
256
|
context=context,
|
257
|
-
task_instance=self,
|
258
257
|
dataset_id=dataset_id,
|
259
258
|
project_id=project_id,
|
260
259
|
table_id=table_id,
|
@@ -101,7 +101,6 @@ class BigQueryToMsSqlOperator(BigQueryToSqlBaseOperator):
|
|
101
101
|
project_id, dataset_id, table_id = self.source_project_dataset_table.split(".")
|
102
102
|
BigQueryTableLink.persist(
|
103
103
|
context=context,
|
104
|
-
task_instance=self,
|
105
104
|
dataset_id=dataset_id,
|
106
105
|
project_id=project_id,
|
107
106
|
table_id=table_id,
|
@@ -23,9 +23,9 @@ import abc
|
|
23
23
|
from collections.abc import Sequence
|
24
24
|
from typing import TYPE_CHECKING
|
25
25
|
|
26
|
-
from airflow.models import BaseOperator
|
27
26
|
from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook
|
28
27
|
from airflow.providers.google.cloud.utils.bigquery_get_data import bigquery_get_data
|
28
|
+
from airflow.providers.google.version_compat import BaseOperator
|
29
29
|
|
30
30
|
if TYPE_CHECKING:
|
31
31
|
from airflow.providers.common.sql.hooks.sql import DbApiHook
|
@@ -21,9 +21,9 @@ from collections.abc import Sequence
|
|
21
21
|
from tempfile import NamedTemporaryFile
|
22
22
|
from typing import TYPE_CHECKING, Any
|
23
23
|
|
24
|
-
from airflow.models import BaseOperator
|
25
24
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
26
25
|
from airflow.providers.google.suite.hooks.calendar import GoogleCalendarHook
|
26
|
+
from airflow.providers.google.version_compat import BaseOperator
|
27
27
|
|
28
28
|
if TYPE_CHECKING:
|
29
29
|
from datetime import datetime
|
@@ -31,9 +31,9 @@ from uuid import UUID
|
|
31
31
|
from cassandra.util import Date, OrderedMapSerializedKey, SortedSet, Time
|
32
32
|
|
33
33
|
from airflow.exceptions import AirflowException
|
34
|
-
from airflow.models import BaseOperator
|
35
34
|
from airflow.providers.apache.cassandra.hooks.cassandra import CassandraHook
|
36
35
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
36
|
+
from airflow.providers.google.version_compat import BaseOperator
|
37
37
|
|
38
38
|
if TYPE_CHECKING:
|
39
39
|
from airflow.utils.context import Context
|
@@ -26,9 +26,9 @@ from enum import Enum
|
|
26
26
|
from typing import TYPE_CHECKING, Any
|
27
27
|
|
28
28
|
from airflow.exceptions import AirflowException
|
29
|
-
from airflow.models import BaseOperator
|
30
29
|
from airflow.providers.facebook.ads.hooks.ads import FacebookAdsReportingHook
|
31
30
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
31
|
+
from airflow.providers.google.version_compat import BaseOperator
|
32
32
|
|
33
33
|
if TYPE_CHECKING:
|
34
34
|
from facebook_business.adobjects.adsinsights import AdsInsights
|
@@ -38,12 +38,12 @@ from google.cloud.bigquery.table import EncryptionConfiguration, Table, TableRef
|
|
38
38
|
|
39
39
|
from airflow.configuration import conf
|
40
40
|
from airflow.exceptions import AirflowException
|
41
|
-
from airflow.models import BaseOperator
|
42
41
|
from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook, BigQueryJob
|
43
42
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
44
43
|
from airflow.providers.google.cloud.links.bigquery import BigQueryTableLink
|
45
44
|
from airflow.providers.google.cloud.triggers.bigquery import BigQueryInsertJobTrigger
|
46
45
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
46
|
+
from airflow.providers.google.version_compat import BaseOperator
|
47
47
|
from airflow.utils.helpers import merge_dicts
|
48
48
|
|
49
49
|
if TYPE_CHECKING:
|
@@ -373,7 +373,6 @@ class GCSToBigQueryOperator(BaseOperator):
|
|
373
373
|
|
374
374
|
BigQueryTableLink.persist(
|
375
375
|
context=context,
|
376
|
-
task_instance=self,
|
377
376
|
dataset_id=table_obj_api_repr["tableReference"]["datasetId"],
|
378
377
|
project_id=table_obj_api_repr["tableReference"]["projectId"],
|
379
378
|
table_id=table_obj_api_repr["tableReference"]["tableId"],
|
@@ -24,8 +24,8 @@ from collections.abc import Sequence
|
|
24
24
|
from typing import TYPE_CHECKING
|
25
25
|
|
26
26
|
from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
|
27
|
-
from airflow.models import BaseOperator
|
28
27
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
28
|
+
from airflow.providers.google.version_compat import BaseOperator
|
29
29
|
|
30
30
|
WILDCARD = "*"
|
31
31
|
|
@@ -20,9 +20,9 @@ from collections.abc import Sequence
|
|
20
20
|
from typing import TYPE_CHECKING
|
21
21
|
|
22
22
|
from airflow.exceptions import AirflowException
|
23
|
-
from airflow.models import BaseOperator
|
24
23
|
from airflow.models.xcom import MAX_XCOM_SIZE
|
25
24
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
25
|
+
from airflow.providers.google.version_compat import BaseOperator
|
26
26
|
|
27
27
|
if TYPE_CHECKING:
|
28
28
|
from airflow.utils.context import Context
|
@@ -26,8 +26,8 @@ from tempfile import NamedTemporaryFile
|
|
26
26
|
from typing import TYPE_CHECKING
|
27
27
|
|
28
28
|
from airflow.exceptions import AirflowException
|
29
|
-
from airflow.models import BaseOperator
|
30
29
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
30
|
+
from airflow.providers.google.version_compat import BaseOperator
|
31
31
|
from airflow.providers.sftp.hooks.sftp import SFTPHook
|
32
32
|
|
33
33
|
WILDCARD = "*"
|
@@ -19,9 +19,9 @@ from __future__ import annotations
|
|
19
19
|
from collections.abc import Sequence
|
20
20
|
from typing import TYPE_CHECKING
|
21
21
|
|
22
|
-
from airflow.models import BaseOperator
|
23
22
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
24
23
|
from airflow.providers.google.suite.hooks.drive import GoogleDriveHook
|
24
|
+
from airflow.providers.google.version_compat import BaseOperator
|
25
25
|
|
26
26
|
if TYPE_CHECKING:
|
27
27
|
from airflow.utils.context import Context
|
@@ -99,3 +99,7 @@ class GoogleDriveToGCSOperator(BaseOperator):
|
|
99
99
|
bucket_name=self.bucket_name, object_name=self.object_name
|
100
100
|
) as file:
|
101
101
|
gdrive_hook.download_file(file_id=file_metadata["id"], file_handle=file)
|
102
|
+
|
103
|
+
def dry_run(self):
|
104
|
+
"""Perform a dry run of the operator."""
|
105
|
+
return None
|
@@ -19,8 +19,8 @@ from __future__ import annotations
|
|
19
19
|
from collections.abc import Sequence
|
20
20
|
from typing import TYPE_CHECKING
|
21
21
|
|
22
|
-
from airflow.models import BaseOperator
|
23
22
|
from airflow.providers.google.suite.hooks.drive import GoogleDriveHook
|
23
|
+
from airflow.providers.google.version_compat import BaseOperator
|
24
24
|
|
25
25
|
if TYPE_CHECKING:
|
26
26
|
from airflow.utils.context import Context
|
@@ -22,8 +22,8 @@ from __future__ import annotations
|
|
22
22
|
from functools import cached_property
|
23
23
|
from typing import TYPE_CHECKING, Any
|
24
24
|
|
25
|
-
from airflow.models import BaseOperator
|
26
25
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
26
|
+
from airflow.providers.google.version_compat import BaseOperator
|
27
27
|
from airflow.providers.http.hooks.http import HttpHook
|
28
28
|
|
29
29
|
if TYPE_CHECKING:
|
@@ -24,8 +24,8 @@ from collections.abc import Sequence
|
|
24
24
|
from glob import glob
|
25
25
|
from typing import TYPE_CHECKING
|
26
26
|
|
27
|
-
from airflow.models import BaseOperator
|
28
27
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
28
|
+
from airflow.providers.google.version_compat import BaseOperator
|
29
29
|
|
30
30
|
if TYPE_CHECKING:
|
31
31
|
from airflow.utils.context import Context
|
@@ -21,8 +21,8 @@ import tempfile
|
|
21
21
|
from collections.abc import Sequence
|
22
22
|
from typing import TYPE_CHECKING
|
23
23
|
|
24
|
-
from airflow.models import BaseOperator
|
25
24
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
25
|
+
from airflow.providers.google.version_compat import BaseOperator
|
26
26
|
from airflow.providers.salesforce.hooks.salesforce import SalesforceHook
|
27
27
|
|
28
28
|
if TYPE_CHECKING:
|
@@ -26,8 +26,8 @@ from tempfile import NamedTemporaryFile
|
|
26
26
|
from typing import TYPE_CHECKING
|
27
27
|
|
28
28
|
from airflow.exceptions import AirflowException
|
29
|
-
from airflow.models import BaseOperator
|
30
29
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
30
|
+
from airflow.providers.google.version_compat import BaseOperator
|
31
31
|
from airflow.providers.sftp.hooks.sftp import SFTPHook
|
32
32
|
|
33
33
|
if TYPE_CHECKING:
|
@@ -21,9 +21,9 @@ from collections.abc import Sequence
|
|
21
21
|
from tempfile import NamedTemporaryFile
|
22
22
|
from typing import TYPE_CHECKING, Any
|
23
23
|
|
24
|
-
from airflow.models import BaseOperator
|
25
24
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
26
25
|
from airflow.providers.google.suite.hooks.sheets import GSheetsHook
|
26
|
+
from airflow.providers.google.version_compat import BaseOperator
|
27
27
|
|
28
28
|
if TYPE_CHECKING:
|
29
29
|
from airflow.utils.context import Context
|
@@ -130,5 +130,5 @@ class GoogleSheetsToGCSOperator(BaseOperator):
|
|
130
130
|
gcs_path_to_file = self._upload_data(gcs_hook, sheet_hook, sheet_range, data)
|
131
131
|
destination_array.append(gcs_path_to_file)
|
132
132
|
|
133
|
-
|
133
|
+
context["ti"].xcom_push(key="destination_objects", value=destination_array)
|
134
134
|
return destination_array
|
@@ -30,8 +30,8 @@ from typing import TYPE_CHECKING, Any
|
|
30
30
|
import pyarrow as pa
|
31
31
|
import pyarrow.parquet as pq
|
32
32
|
|
33
|
-
from airflow.models import BaseOperator
|
34
33
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
34
|
+
from airflow.providers.google.version_compat import BaseOperator
|
35
35
|
|
36
36
|
if TYPE_CHECKING:
|
37
37
|
from airflow.providers.common.compat.openlineage.facet import OutputDataset
|
@@ -134,8 +134,7 @@ Here are the guidelines that you should follow to make validation forward-compat
|
|
134
134
|
from __future__ import annotations
|
135
135
|
|
136
136
|
import re
|
137
|
-
from collections.abc import Sequence
|
138
|
-
from typing import Callable
|
137
|
+
from collections.abc import Callable, Sequence
|
139
138
|
|
140
139
|
from airflow.exceptions import AirflowException
|
141
140
|
from airflow.utils.log.logging_mixin import LoggingMixin
|
@@ -20,8 +20,9 @@
|
|
20
20
|
from __future__ import annotations
|
21
21
|
|
22
22
|
import logging
|
23
|
+
from collections.abc import Callable
|
23
24
|
from functools import wraps
|
24
|
-
from typing import
|
25
|
+
from typing import TypeVar, cast
|
25
26
|
|
26
27
|
import google
|
27
28
|
import google.auth.transport.requests
|
@@ -18,8 +18,9 @@ from __future__ import annotations
|
|
18
18
|
|
19
19
|
import inspect
|
20
20
|
import re
|
21
|
+
from collections.abc import Callable
|
21
22
|
from datetime import date, datetime
|
22
|
-
from typing import Any
|
23
|
+
from typing import Any
|
23
24
|
|
24
25
|
from deprecated import deprecated as standard_deprecated
|
25
26
|
from deprecated.classic import ClassicAdapter
|
@@ -26,10 +26,10 @@ import json
|
|
26
26
|
import logging
|
27
27
|
import os
|
28
28
|
import tempfile
|
29
|
-
from collections.abc import Generator, Sequence
|
29
|
+
from collections.abc import Callable, Generator, Sequence
|
30
30
|
from contextlib import ExitStack, contextmanager
|
31
31
|
from subprocess import check_output
|
32
|
-
from typing import TYPE_CHECKING, Any,
|
32
|
+
from typing import TYPE_CHECKING, Any, TypeVar, cast
|
33
33
|
|
34
34
|
import google.auth
|
35
35
|
import google.oauth2.service_account
|
@@ -50,12 +50,16 @@ from requests import Session
|
|
50
50
|
|
51
51
|
from airflow import version
|
52
52
|
from airflow.exceptions import AirflowException
|
53
|
-
from airflow.hooks.base import BaseHook
|
54
53
|
from airflow.providers.google.cloud.utils.credentials_provider import (
|
55
54
|
_get_scopes,
|
56
55
|
_get_target_principal_and_delegates,
|
57
56
|
get_credentials_and_project_id,
|
58
57
|
)
|
58
|
+
|
59
|
+
try:
|
60
|
+
from airflow.sdk import BaseHook
|
61
|
+
except ImportError:
|
62
|
+
from airflow.hooks.base import BaseHook # type: ignore[attr-defined,no-redef]
|
59
63
|
from airflow.utils.process_utils import patch_environ
|
60
64
|
|
61
65
|
if TYPE_CHECKING:
|