apache-airflow-providers-google 15.1.0rc1__py3-none-any.whl → 19.1.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/3rd-party-licenses/NOTICE +2 -12
- airflow/providers/google/__init__.py +3 -3
- airflow/providers/google/ads/hooks/ads.py +39 -5
- airflow/providers/google/ads/operators/ads.py +2 -2
- airflow/providers/google/ads/transfers/ads_to_gcs.py +2 -2
- airflow/providers/google/assets/gcs.py +1 -11
- airflow/providers/google/cloud/bundles/__init__.py +16 -0
- airflow/providers/google/cloud/bundles/gcs.py +161 -0
- airflow/providers/google/cloud/hooks/bigquery.py +166 -281
- airflow/providers/google/cloud/hooks/cloud_composer.py +287 -14
- airflow/providers/google/cloud/hooks/cloud_logging.py +109 -0
- airflow/providers/google/cloud/hooks/cloud_run.py +17 -9
- airflow/providers/google/cloud/hooks/cloud_sql.py +101 -22
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +27 -6
- airflow/providers/google/cloud/hooks/compute_ssh.py +5 -1
- airflow/providers/google/cloud/hooks/datacatalog.py +9 -1
- airflow/providers/google/cloud/hooks/dataflow.py +71 -94
- airflow/providers/google/cloud/hooks/datafusion.py +1 -1
- airflow/providers/google/cloud/hooks/dataplex.py +1 -1
- airflow/providers/google/cloud/hooks/dataprep.py +1 -1
- airflow/providers/google/cloud/hooks/dataproc.py +72 -71
- airflow/providers/google/cloud/hooks/gcs.py +111 -14
- airflow/providers/google/cloud/hooks/gen_ai.py +196 -0
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +2 -2
- airflow/providers/google/cloud/hooks/looker.py +6 -1
- airflow/providers/google/cloud/hooks/mlengine.py +3 -2
- airflow/providers/google/cloud/hooks/secret_manager.py +102 -10
- airflow/providers/google/cloud/hooks/spanner.py +73 -8
- airflow/providers/google/cloud/hooks/stackdriver.py +10 -8
- airflow/providers/google/cloud/hooks/translate.py +1 -1
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +0 -209
- airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +2 -2
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +27 -1
- airflow/providers/google/cloud/hooks/vertex_ai/experiment_service.py +202 -0
- airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +307 -7
- airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +79 -75
- airflow/providers/google/cloud/hooks/vertex_ai/ray.py +223 -0
- airflow/providers/google/cloud/hooks/vision.py +2 -2
- airflow/providers/google/cloud/hooks/workflows.py +1 -1
- airflow/providers/google/cloud/links/alloy_db.py +0 -46
- airflow/providers/google/cloud/links/base.py +77 -13
- airflow/providers/google/cloud/links/bigquery.py +0 -47
- airflow/providers/google/cloud/links/bigquery_dts.py +0 -20
- airflow/providers/google/cloud/links/bigtable.py +0 -48
- airflow/providers/google/cloud/links/cloud_build.py +0 -73
- airflow/providers/google/cloud/links/cloud_functions.py +0 -33
- airflow/providers/google/cloud/links/cloud_memorystore.py +0 -58
- airflow/providers/google/cloud/links/{life_sciences.py → cloud_run.py} +5 -27
- airflow/providers/google/cloud/links/cloud_sql.py +0 -33
- airflow/providers/google/cloud/links/cloud_storage_transfer.py +17 -44
- airflow/providers/google/cloud/links/cloud_tasks.py +7 -26
- airflow/providers/google/cloud/links/compute.py +0 -58
- airflow/providers/google/cloud/links/data_loss_prevention.py +0 -169
- airflow/providers/google/cloud/links/datacatalog.py +23 -54
- airflow/providers/google/cloud/links/dataflow.py +0 -34
- airflow/providers/google/cloud/links/dataform.py +0 -64
- airflow/providers/google/cloud/links/datafusion.py +1 -96
- airflow/providers/google/cloud/links/dataplex.py +0 -154
- airflow/providers/google/cloud/links/dataprep.py +0 -24
- airflow/providers/google/cloud/links/dataproc.py +11 -95
- airflow/providers/google/cloud/links/datastore.py +0 -31
- airflow/providers/google/cloud/links/kubernetes_engine.py +9 -60
- airflow/providers/google/cloud/links/managed_kafka.py +0 -70
- airflow/providers/google/cloud/links/mlengine.py +0 -70
- airflow/providers/google/cloud/links/pubsub.py +0 -32
- airflow/providers/google/cloud/links/spanner.py +0 -33
- airflow/providers/google/cloud/links/stackdriver.py +0 -30
- airflow/providers/google/cloud/links/translate.py +17 -187
- airflow/providers/google/cloud/links/vertex_ai.py +28 -195
- airflow/providers/google/cloud/links/workflows.py +0 -52
- airflow/providers/google/cloud/log/gcs_task_handler.py +17 -9
- airflow/providers/google/cloud/log/stackdriver_task_handler.py +9 -6
- airflow/providers/google/cloud/openlineage/CloudStorageTransferJobFacet.json +68 -0
- airflow/providers/google/cloud/openlineage/CloudStorageTransferRunFacet.json +60 -0
- airflow/providers/google/cloud/openlineage/DataFusionRunFacet.json +32 -0
- airflow/providers/google/cloud/openlineage/facets.py +102 -1
- airflow/providers/google/cloud/openlineage/mixins.py +10 -8
- airflow/providers/google/cloud/openlineage/utils.py +15 -1
- airflow/providers/google/cloud/operators/alloy_db.py +70 -55
- airflow/providers/google/cloud/operators/bigquery.py +73 -636
- airflow/providers/google/cloud/operators/bigquery_dts.py +3 -5
- airflow/providers/google/cloud/operators/bigtable.py +36 -7
- airflow/providers/google/cloud/operators/cloud_base.py +21 -1
- airflow/providers/google/cloud/operators/cloud_batch.py +2 -2
- airflow/providers/google/cloud/operators/cloud_build.py +75 -32
- airflow/providers/google/cloud/operators/cloud_composer.py +128 -40
- airflow/providers/google/cloud/operators/cloud_logging_sink.py +341 -0
- airflow/providers/google/cloud/operators/cloud_memorystore.py +69 -43
- airflow/providers/google/cloud/operators/cloud_run.py +23 -5
- airflow/providers/google/cloud/operators/cloud_sql.py +8 -16
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +92 -11
- airflow/providers/google/cloud/operators/compute.py +8 -40
- airflow/providers/google/cloud/operators/datacatalog.py +157 -21
- airflow/providers/google/cloud/operators/dataflow.py +38 -15
- airflow/providers/google/cloud/operators/dataform.py +15 -5
- airflow/providers/google/cloud/operators/datafusion.py +41 -20
- airflow/providers/google/cloud/operators/dataplex.py +193 -109
- airflow/providers/google/cloud/operators/dataprep.py +1 -5
- airflow/providers/google/cloud/operators/dataproc.py +78 -35
- airflow/providers/google/cloud/operators/dataproc_metastore.py +96 -88
- airflow/providers/google/cloud/operators/datastore.py +22 -6
- airflow/providers/google/cloud/operators/dlp.py +6 -29
- airflow/providers/google/cloud/operators/functions.py +16 -7
- airflow/providers/google/cloud/operators/gcs.py +10 -8
- airflow/providers/google/cloud/operators/gen_ai.py +389 -0
- airflow/providers/google/cloud/operators/kubernetes_engine.py +60 -99
- airflow/providers/google/cloud/operators/looker.py +1 -1
- airflow/providers/google/cloud/operators/managed_kafka.py +107 -52
- airflow/providers/google/cloud/operators/natural_language.py +1 -1
- airflow/providers/google/cloud/operators/pubsub.py +60 -14
- airflow/providers/google/cloud/operators/spanner.py +25 -12
- airflow/providers/google/cloud/operators/speech_to_text.py +1 -2
- airflow/providers/google/cloud/operators/stackdriver.py +1 -9
- airflow/providers/google/cloud/operators/tasks.py +1 -12
- airflow/providers/google/cloud/operators/text_to_speech.py +1 -2
- airflow/providers/google/cloud/operators/translate.py +40 -16
- airflow/providers/google/cloud/operators/translate_speech.py +1 -2
- airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +39 -19
- airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +29 -9
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +54 -26
- airflow/providers/google/cloud/operators/vertex_ai/dataset.py +70 -8
- airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +43 -9
- airflow/providers/google/cloud/operators/vertex_ai/experiment_service.py +435 -0
- airflow/providers/google/cloud/operators/vertex_ai/feature_store.py +532 -1
- airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +135 -116
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +11 -9
- airflow/providers/google/cloud/operators/vertex_ai/model_service.py +57 -11
- airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +30 -7
- airflow/providers/google/cloud/operators/vertex_ai/ray.py +393 -0
- airflow/providers/google/cloud/operators/video_intelligence.py +1 -1
- airflow/providers/google/cloud/operators/vision.py +2 -2
- airflow/providers/google/cloud/operators/workflows.py +18 -15
- airflow/providers/google/cloud/sensors/bigquery.py +2 -2
- airflow/providers/google/cloud/sensors/bigquery_dts.py +2 -2
- airflow/providers/google/cloud/sensors/bigtable.py +11 -4
- airflow/providers/google/cloud/sensors/cloud_composer.py +533 -29
- airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +2 -2
- airflow/providers/google/cloud/sensors/dataflow.py +26 -9
- airflow/providers/google/cloud/sensors/dataform.py +2 -2
- airflow/providers/google/cloud/sensors/datafusion.py +4 -4
- airflow/providers/google/cloud/sensors/dataplex.py +2 -2
- airflow/providers/google/cloud/sensors/dataprep.py +2 -2
- airflow/providers/google/cloud/sensors/dataproc.py +2 -2
- airflow/providers/google/cloud/sensors/dataproc_metastore.py +2 -2
- airflow/providers/google/cloud/sensors/gcs.py +4 -4
- airflow/providers/google/cloud/sensors/looker.py +2 -2
- airflow/providers/google/cloud/sensors/pubsub.py +4 -4
- airflow/providers/google/cloud/sensors/tasks.py +2 -2
- airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py +2 -2
- airflow/providers/google/cloud/sensors/workflows.py +2 -2
- airflow/providers/google/cloud/transfers/adls_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +11 -8
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +4 -4
- airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +7 -3
- airflow/providers/google/cloud/transfers/bigquery_to_mysql.py +12 -1
- airflow/providers/google/cloud/transfers/bigquery_to_postgres.py +24 -10
- airflow/providers/google/cloud/transfers/bigquery_to_sql.py +104 -5
- airflow/providers/google/cloud/transfers/calendar_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +20 -12
- airflow/providers/google/cloud/transfers/gcs_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/gcs_to_local.py +5 -3
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +10 -4
- airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +6 -2
- airflow/providers/google/cloud/transfers/gdrive_to_local.py +2 -2
- airflow/providers/google/cloud/transfers/http_to_gcs.py +193 -0
- airflow/providers/google/cloud/transfers/local_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/oracle_to_gcs.py +36 -11
- airflow/providers/google/cloud/transfers/postgres_to_gcs.py +42 -9
- airflow/providers/google/cloud/transfers/s3_to_gcs.py +12 -6
- airflow/providers/google/cloud/transfers/salesforce_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +13 -4
- airflow/providers/google/cloud/transfers/sheets_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +10 -10
- airflow/providers/google/cloud/triggers/bigquery.py +75 -34
- airflow/providers/google/cloud/triggers/cloud_build.py +1 -1
- airflow/providers/google/cloud/triggers/cloud_composer.py +302 -46
- airflow/providers/google/cloud/triggers/cloud_run.py +2 -2
- airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +91 -1
- airflow/providers/google/cloud/triggers/dataflow.py +122 -0
- airflow/providers/google/cloud/triggers/datafusion.py +1 -1
- airflow/providers/google/cloud/triggers/dataplex.py +14 -2
- airflow/providers/google/cloud/triggers/dataproc.py +122 -52
- airflow/providers/google/cloud/triggers/kubernetes_engine.py +45 -27
- airflow/providers/google/cloud/triggers/mlengine.py +1 -1
- airflow/providers/google/cloud/triggers/pubsub.py +15 -19
- airflow/providers/google/cloud/utils/bigquery_get_data.py +1 -1
- airflow/providers/google/cloud/utils/credentials_provider.py +1 -1
- airflow/providers/google/cloud/utils/field_validator.py +1 -2
- airflow/providers/google/common/auth_backend/google_openid.py +4 -4
- airflow/providers/google/common/deprecated.py +2 -1
- airflow/providers/google/common/hooks/base_google.py +27 -8
- airflow/providers/google/common/links/storage.py +0 -22
- airflow/providers/google/common/utils/get_secret.py +31 -0
- airflow/providers/google/common/utils/id_token_credentials.py +3 -4
- airflow/providers/google/firebase/operators/firestore.py +2 -2
- airflow/providers/google/get_provider_info.py +56 -52
- airflow/providers/google/go_module_utils.py +35 -3
- airflow/providers/google/leveldb/hooks/leveldb.py +26 -1
- airflow/providers/google/leveldb/operators/leveldb.py +2 -2
- airflow/providers/google/marketing_platform/hooks/display_video.py +3 -109
- airflow/providers/google/marketing_platform/links/analytics_admin.py +5 -14
- airflow/providers/google/marketing_platform/operators/analytics_admin.py +1 -2
- airflow/providers/google/marketing_platform/operators/campaign_manager.py +5 -5
- airflow/providers/google/marketing_platform/operators/display_video.py +28 -489
- airflow/providers/google/marketing_platform/operators/search_ads.py +2 -2
- airflow/providers/google/marketing_platform/sensors/campaign_manager.py +2 -2
- airflow/providers/google/marketing_platform/sensors/display_video.py +3 -63
- airflow/providers/google/suite/hooks/calendar.py +1 -1
- airflow/providers/google/suite/hooks/sheets.py +15 -1
- airflow/providers/google/suite/operators/sheets.py +8 -3
- airflow/providers/google/suite/sensors/drive.py +2 -2
- airflow/providers/google/suite/transfers/gcs_to_gdrive.py +2 -2
- airflow/providers/google/suite/transfers/gcs_to_sheets.py +1 -1
- airflow/providers/google/suite/transfers/local_to_drive.py +3 -3
- airflow/providers/google/suite/transfers/sql_to_sheets.py +5 -4
- airflow/providers/google/version_compat.py +15 -1
- {apache_airflow_providers_google-15.1.0rc1.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/METADATA +92 -48
- apache_airflow_providers_google-19.1.0rc1.dist-info/RECORD +331 -0
- apache_airflow_providers_google-19.1.0rc1.dist-info/licenses/NOTICE +5 -0
- airflow/providers/google/cloud/hooks/automl.py +0 -673
- airflow/providers/google/cloud/hooks/life_sciences.py +0 -159
- airflow/providers/google/cloud/links/automl.py +0 -193
- airflow/providers/google/cloud/operators/automl.py +0 -1362
- airflow/providers/google/cloud/operators/life_sciences.py +0 -119
- airflow/providers/google/cloud/operators/mlengine.py +0 -112
- apache_airflow_providers_google-15.1.0rc1.dist-info/RECORD +0 -321
- {apache_airflow_providers_google-15.1.0rc1.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_google-15.1.0rc1.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/entry_points.txt +0 -0
- {airflow/providers/google → apache_airflow_providers_google-19.1.0rc1.dist-info/licenses}/LICENSE +0 -0
|
@@ -29,7 +29,7 @@ from urllib.parse import urlencode
|
|
|
29
29
|
|
|
30
30
|
import google.auth
|
|
31
31
|
import google.oauth2.service_account
|
|
32
|
-
from google.auth import impersonated_credentials
|
|
32
|
+
from google.auth import impersonated_credentials
|
|
33
33
|
from google.auth.credentials import AnonymousCredentials, Credentials
|
|
34
34
|
from google.auth.environment_vars import CREDENTIALS, LEGACY_PROJECT, PROJECT
|
|
35
35
|
|
|
@@ -134,8 +134,7 @@ Here are the guidelines that you should follow to make validation forward-compat
|
|
|
134
134
|
from __future__ import annotations
|
|
135
135
|
|
|
136
136
|
import re
|
|
137
|
-
from collections.abc import Sequence
|
|
138
|
-
from typing import Callable
|
|
137
|
+
from collections.abc import Callable, Sequence
|
|
139
138
|
|
|
140
139
|
from airflow.exceptions import AirflowException
|
|
141
140
|
from airflow.utils.log.logging_mixin import LoggingMixin
|
|
@@ -20,15 +20,15 @@
|
|
|
20
20
|
from __future__ import annotations
|
|
21
21
|
|
|
22
22
|
import logging
|
|
23
|
+
from collections.abc import Callable
|
|
23
24
|
from functools import wraps
|
|
24
|
-
from typing import
|
|
25
|
+
from typing import TypeVar, cast
|
|
25
26
|
|
|
26
|
-
import google
|
|
27
27
|
import google.auth.transport.requests
|
|
28
28
|
import google.oauth2.id_token
|
|
29
29
|
|
|
30
30
|
try:
|
|
31
|
-
from flask import Response, current_app, request as flask_request
|
|
31
|
+
from flask import Response, current_app, request as flask_request
|
|
32
32
|
except ImportError:
|
|
33
33
|
raise ImportError(
|
|
34
34
|
"Google requires FAB provider to be installed in order to use this auth backend. "
|
|
@@ -121,7 +121,7 @@ def _lookup_user(user_email: str):
|
|
|
121
121
|
|
|
122
122
|
|
|
123
123
|
def _set_current_user(user):
|
|
124
|
-
current_app.appbuilder.sm.lm._update_request_context_with_user(user=user)
|
|
124
|
+
current_app.appbuilder.sm.lm._update_request_context_with_user(user=user)
|
|
125
125
|
|
|
126
126
|
|
|
127
127
|
T = TypeVar("T", bound=Callable)
|
|
@@ -18,8 +18,9 @@ from __future__ import annotations
|
|
|
18
18
|
|
|
19
19
|
import inspect
|
|
20
20
|
import re
|
|
21
|
+
from collections.abc import Callable
|
|
21
22
|
from datetime import date, datetime
|
|
22
|
-
from typing import Any
|
|
23
|
+
from typing import Any
|
|
23
24
|
|
|
24
25
|
from deprecated import deprecated as standard_deprecated
|
|
25
26
|
from deprecated.classic import ClassicAdapter
|
|
@@ -26,20 +26,19 @@ import json
|
|
|
26
26
|
import logging
|
|
27
27
|
import os
|
|
28
28
|
import tempfile
|
|
29
|
-
from collections.abc import Generator, Sequence
|
|
29
|
+
from collections.abc import Callable, Generator, Sequence
|
|
30
30
|
from contextlib import ExitStack, contextmanager
|
|
31
31
|
from subprocess import check_output
|
|
32
|
-
from typing import TYPE_CHECKING, Any,
|
|
32
|
+
from typing import TYPE_CHECKING, Any, TypeVar, cast
|
|
33
33
|
|
|
34
34
|
import google.auth
|
|
35
|
-
import google.oauth2.service_account
|
|
36
35
|
import google_auth_httplib2
|
|
37
36
|
import requests
|
|
38
37
|
import tenacity
|
|
39
38
|
from asgiref.sync import sync_to_async
|
|
40
39
|
from gcloud.aio.auth.token import Token, TokenResponse
|
|
41
40
|
from google.api_core.exceptions import Forbidden, ResourceExhausted, TooManyRequests
|
|
42
|
-
from google.auth import _cloud_sdk, compute_engine
|
|
41
|
+
from google.auth import _cloud_sdk, compute_engine
|
|
43
42
|
from google.auth.environment_vars import CLOUD_SDK_CONFIG_DIR, CREDENTIALS
|
|
44
43
|
from google.auth.exceptions import RefreshError
|
|
45
44
|
from google.auth.transport import _http_client
|
|
@@ -50,7 +49,7 @@ from requests import Session
|
|
|
50
49
|
|
|
51
50
|
from airflow import version
|
|
52
51
|
from airflow.exceptions import AirflowException
|
|
53
|
-
from airflow.
|
|
52
|
+
from airflow.providers.common.compat.sdk import BaseHook
|
|
54
53
|
from airflow.providers.google.cloud.utils.credentials_provider import (
|
|
55
54
|
_get_scopes,
|
|
56
55
|
_get_target_principal_and_delegates,
|
|
@@ -155,6 +154,9 @@ PROVIDE_PROJECT_ID: str = cast("str", None)
|
|
|
155
154
|
T = TypeVar("T", bound=Callable)
|
|
156
155
|
RT = TypeVar("RT")
|
|
157
156
|
|
|
157
|
+
# Sentinel value to distinguish "parameter not provided" from "parameter explicitly set to a value"
|
|
158
|
+
_UNSET = object()
|
|
159
|
+
|
|
158
160
|
|
|
159
161
|
def get_field(extras: dict, field_name: str) -> str | None:
|
|
160
162
|
"""Get field from extra, first checking short name, then for backcompat we check for prefixed name."""
|
|
@@ -164,9 +166,11 @@ def get_field(extras: dict, field_name: str) -> str | None:
|
|
|
164
166
|
"when using this method."
|
|
165
167
|
)
|
|
166
168
|
if field_name in extras:
|
|
167
|
-
|
|
169
|
+
value = extras[field_name]
|
|
170
|
+
return None if value == "" else value
|
|
168
171
|
prefixed_name = f"extra__google_cloud_platform__{field_name}"
|
|
169
|
-
|
|
172
|
+
value = extras.get(prefixed_name)
|
|
173
|
+
return None if value == "" else value
|
|
170
174
|
|
|
171
175
|
|
|
172
176
|
class GoogleBaseHook(BaseHook):
|
|
@@ -406,7 +410,22 @@ class GoogleBaseHook(BaseHook):
|
|
|
406
410
|
custom UI elements to the hook page, which allow admins to specify
|
|
407
411
|
service_account, key_path, etc. They get formatted as shown below.
|
|
408
412
|
"""
|
|
409
|
-
|
|
413
|
+
# New behavior: If default is _UNSET, parameter was not provided
|
|
414
|
+
# Check connection extras first, return None if not found (caller handles default)
|
|
415
|
+
if default is _UNSET:
|
|
416
|
+
if hasattr(self, "extras"):
|
|
417
|
+
value = get_field(self.extras, f)
|
|
418
|
+
if value is not None:
|
|
419
|
+
return value
|
|
420
|
+
return None
|
|
421
|
+
|
|
422
|
+
# Old behavior (for backward compatibility):
|
|
423
|
+
# Check connection extras, but properly handle False values
|
|
424
|
+
if hasattr(self, "extras"):
|
|
425
|
+
value = get_field(self.extras, f)
|
|
426
|
+
if value is not None:
|
|
427
|
+
return value
|
|
428
|
+
return default
|
|
410
429
|
|
|
411
430
|
@property
|
|
412
431
|
def project_id(self) -> str:
|
|
@@ -18,18 +18,12 @@
|
|
|
18
18
|
|
|
19
19
|
from __future__ import annotations
|
|
20
20
|
|
|
21
|
-
from typing import TYPE_CHECKING
|
|
22
|
-
|
|
23
21
|
from airflow.providers.google.cloud.links.base import BaseGoogleLink
|
|
24
22
|
|
|
25
23
|
BASE_LINK = "https://console.cloud.google.com"
|
|
26
24
|
GCS_STORAGE_LINK = BASE_LINK + "/storage/browser/{uri};tab=objects?project={project_id}"
|
|
27
25
|
GCS_FILE_DETAILS_LINK = BASE_LINK + "/storage/browser/_details/{uri};tab=live_object?project={project_id}"
|
|
28
26
|
|
|
29
|
-
if TYPE_CHECKING:
|
|
30
|
-
from airflow.models import BaseOperator
|
|
31
|
-
from airflow.utils.context import Context
|
|
32
|
-
|
|
33
27
|
|
|
34
28
|
class StorageLink(BaseGoogleLink):
|
|
35
29
|
"""Helper class for constructing GCS Storage link."""
|
|
@@ -38,14 +32,6 @@ class StorageLink(BaseGoogleLink):
|
|
|
38
32
|
key = "storage_conf"
|
|
39
33
|
format_str = GCS_STORAGE_LINK
|
|
40
34
|
|
|
41
|
-
@staticmethod
|
|
42
|
-
def persist(context: Context, task_instance, uri: str, project_id: str | None):
|
|
43
|
-
task_instance.xcom_push(
|
|
44
|
-
context=context,
|
|
45
|
-
key=StorageLink.key,
|
|
46
|
-
value={"uri": uri, "project_id": project_id},
|
|
47
|
-
)
|
|
48
|
-
|
|
49
35
|
|
|
50
36
|
class FileDetailsLink(BaseGoogleLink):
|
|
51
37
|
"""Helper class for constructing GCS file details link."""
|
|
@@ -53,11 +39,3 @@ class FileDetailsLink(BaseGoogleLink):
|
|
|
53
39
|
name = "GCS File Details"
|
|
54
40
|
key = "file_details"
|
|
55
41
|
format_str = GCS_FILE_DETAILS_LINK
|
|
56
|
-
|
|
57
|
-
@staticmethod
|
|
58
|
-
def persist(context: Context, task_instance: BaseOperator, uri: str, project_id: str | None):
|
|
59
|
-
task_instance.xcom_push(
|
|
60
|
-
context=context,
|
|
61
|
-
key=FileDetailsLink.key,
|
|
62
|
-
value={"uri": uri, "project_id": project_id},
|
|
63
|
-
)
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
|
3
|
+
# distributed with this work for additional information
|
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
|
6
|
+
# "License"); you may not use this file except in compliance
|
|
7
|
+
# with the License. You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
|
12
|
+
# software distributed under the License is distributed on an
|
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
+
# KIND, either express or implied. See the License for the
|
|
15
|
+
# specific language governing permissions and limitations
|
|
16
|
+
# under the License.
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from google.cloud.exceptions import NotFound
|
|
21
|
+
|
|
22
|
+
from airflow.providers.google.cloud.hooks.secret_manager import (
|
|
23
|
+
GoogleCloudSecretManagerHook,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def get_secret(secret_id: str) -> str:
|
|
28
|
+
hook = GoogleCloudSecretManagerHook()
|
|
29
|
+
if hook.secret_exists(secret_id=secret_id):
|
|
30
|
+
return hook.access_secret(secret_id=secret_id).payload.data.decode()
|
|
31
|
+
raise NotFound("The secret '%s' not found", secret_id)
|
|
@@ -35,12 +35,11 @@ import json
|
|
|
35
35
|
import os
|
|
36
36
|
from typing import TYPE_CHECKING
|
|
37
37
|
|
|
38
|
-
import google.auth.transport
|
|
39
38
|
from google.auth import credentials as google_auth_credentials, environment_vars, exceptions
|
|
40
|
-
from google.oauth2 import credentials as oauth2_credentials, service_account
|
|
39
|
+
from google.oauth2 import credentials as oauth2_credentials, service_account
|
|
41
40
|
|
|
42
41
|
if TYPE_CHECKING:
|
|
43
|
-
import google.
|
|
42
|
+
import google.auth.transport
|
|
44
43
|
|
|
45
44
|
# Valid types accepted for file-based credentials.
|
|
46
45
|
# They are taken from "google.auth._default" and since they are all "protected" and the imports might
|
|
@@ -147,7 +146,7 @@ def _get_gcloud_sdk_credentials(
|
|
|
147
146
|
target_audience: str | None,
|
|
148
147
|
) -> google_auth_credentials.Credentials | None:
|
|
149
148
|
"""Get the credentials and project ID from the Cloud SDK."""
|
|
150
|
-
from google.auth import _cloud_sdk
|
|
149
|
+
from google.auth import _cloud_sdk
|
|
151
150
|
|
|
152
151
|
# Check if application default credentials exist.
|
|
153
152
|
credentials_filename = _cloud_sdk.get_application_default_credentials_path()
|
|
@@ -20,12 +20,12 @@ from collections.abc import Sequence
|
|
|
20
20
|
from typing import TYPE_CHECKING
|
|
21
21
|
|
|
22
22
|
from airflow.exceptions import AirflowException
|
|
23
|
-
from airflow.models import BaseOperator
|
|
24
23
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
|
25
24
|
from airflow.providers.google.firebase.hooks.firestore import CloudFirestoreHook
|
|
25
|
+
from airflow.providers.google.version_compat import BaseOperator
|
|
26
26
|
|
|
27
27
|
if TYPE_CHECKING:
|
|
28
|
-
from airflow.
|
|
28
|
+
from airflow.providers.common.compat.sdk import Context
|
|
29
29
|
|
|
30
30
|
|
|
31
31
|
class CloudFirestoreExportDatabaseOperator(BaseOperator):
|
|
@@ -43,13 +43,6 @@ def get_provider_info():
|
|
|
43
43
|
"how-to-guide": ["/docs/apache-airflow-providers-google/operators/ads.rst"],
|
|
44
44
|
"tags": ["gmp"],
|
|
45
45
|
},
|
|
46
|
-
{
|
|
47
|
-
"integration-name": "Google AutoML",
|
|
48
|
-
"external-doc-url": "https://cloud.google.com/automl/",
|
|
49
|
-
"how-to-guide": ["/docs/apache-airflow-providers-google/operators/cloud/automl.rst"],
|
|
50
|
-
"logo": "/docs/integration-logos/Cloud-AutoML.png",
|
|
51
|
-
"tags": ["gcp"],
|
|
52
|
-
},
|
|
53
46
|
{
|
|
54
47
|
"integration-name": "Google BigQuery Data Transfer Service",
|
|
55
48
|
"external-doc-url": "https://cloud.google.com/bigquery/transfer/",
|
|
@@ -143,13 +136,6 @@ def get_provider_info():
|
|
|
143
136
|
"logo": "/docs/integration-logos/Key-Management-Service.png",
|
|
144
137
|
"tags": ["gcp"],
|
|
145
138
|
},
|
|
146
|
-
{
|
|
147
|
-
"integration-name": "Google Cloud Life Sciences",
|
|
148
|
-
"external-doc-url": "https://cloud.google.com/life-sciences/",
|
|
149
|
-
"how-to-guide": ["/docs/apache-airflow-providers-google/operators/cloud/life_sciences.rst"],
|
|
150
|
-
"logo": "/docs/integration-logos/Google-Cloud-Life-Sciences.png",
|
|
151
|
-
"tags": ["gcp"],
|
|
152
|
-
},
|
|
153
139
|
{
|
|
154
140
|
"integration-name": "Google Cloud Managed Service for Apache Kafka",
|
|
155
141
|
"external-doc-url": "https://cloud.google.com/managed-service-for-apache-kafka/docs/",
|
|
@@ -226,7 +212,10 @@ def get_provider_info():
|
|
|
226
212
|
{
|
|
227
213
|
"integration-name": "Google Cloud Tasks",
|
|
228
214
|
"external-doc-url": "https://cloud.google.com/tasks/",
|
|
229
|
-
"how-to-guide": [
|
|
215
|
+
"how-to-guide": [
|
|
216
|
+
"/docs/apache-airflow-providers-google/operators/cloud/tasks.rst",
|
|
217
|
+
"/docs/apache-airflow-providers-google/sensors/google-cloud-tasks.rst",
|
|
218
|
+
],
|
|
230
219
|
"logo": "/docs/integration-logos/Cloud-Tasks.png",
|
|
231
220
|
"tags": ["gcp"],
|
|
232
221
|
},
|
|
@@ -470,6 +459,21 @@ def get_provider_info():
|
|
|
470
459
|
"how-to-guide": ["/docs/apache-airflow-providers-google/operators/cloud/looker.rst"],
|
|
471
460
|
"tags": ["gcp"],
|
|
472
461
|
},
|
|
462
|
+
{
|
|
463
|
+
"integration-name": "Google Cloud Logging Sink",
|
|
464
|
+
"external-doc-url": "https://cloud.google.com/logging",
|
|
465
|
+
"logo": "/docs/integration-logos/Cloud-Logging-Sink.png",
|
|
466
|
+
"how-to-guide": [
|
|
467
|
+
"/docs/apache-airflow-providers-google/operators/cloud/cloud_logging_sink.rst"
|
|
468
|
+
],
|
|
469
|
+
"tags": ["gcp"],
|
|
470
|
+
},
|
|
471
|
+
{
|
|
472
|
+
"integration-name": "Google Cloud Generative AI",
|
|
473
|
+
"external-doc-url": "https://cloud.google.com/generative-ai-studio",
|
|
474
|
+
"how-to-guide": ["/docs/apache-airflow-providers-google/operators/cloud/gen_ai.rst"],
|
|
475
|
+
"tags": ["gcp"],
|
|
476
|
+
},
|
|
473
477
|
],
|
|
474
478
|
"operators": [
|
|
475
479
|
{
|
|
@@ -484,10 +488,6 @@ def get_provider_info():
|
|
|
484
488
|
"integration-name": "Google Cloud AlloyDB",
|
|
485
489
|
"python-modules": ["airflow.providers.google.cloud.operators.alloy_db"],
|
|
486
490
|
},
|
|
487
|
-
{
|
|
488
|
-
"integration-name": "Google AutoML",
|
|
489
|
-
"python-modules": ["airflow.providers.google.cloud.operators.automl"],
|
|
490
|
-
},
|
|
491
491
|
{
|
|
492
492
|
"integration-name": "Google BigQuery",
|
|
493
493
|
"python-modules": ["airflow.providers.google.cloud.operators.bigquery"],
|
|
@@ -528,10 +528,6 @@ def get_provider_info():
|
|
|
528
528
|
"integration-name": "Google Compute Engine",
|
|
529
529
|
"python-modules": ["airflow.providers.google.cloud.operators.compute"],
|
|
530
530
|
},
|
|
531
|
-
{
|
|
532
|
-
"integration-name": "Google Data Catalog",
|
|
533
|
-
"python-modules": ["airflow.providers.google.cloud.operators.datacatalog"],
|
|
534
|
-
},
|
|
535
531
|
{
|
|
536
532
|
"integration-name": "Google Dataflow",
|
|
537
533
|
"python-modules": ["airflow.providers.google.cloud.operators.dataflow"],
|
|
@@ -576,14 +572,6 @@ def get_provider_info():
|
|
|
576
572
|
"integration-name": "Google Kubernetes Engine",
|
|
577
573
|
"python-modules": ["airflow.providers.google.cloud.operators.kubernetes_engine"],
|
|
578
574
|
},
|
|
579
|
-
{
|
|
580
|
-
"integration-name": "Google Cloud Life Sciences",
|
|
581
|
-
"python-modules": ["airflow.providers.google.cloud.operators.life_sciences"],
|
|
582
|
-
},
|
|
583
|
-
{
|
|
584
|
-
"integration-name": "Google Machine Learning Engine",
|
|
585
|
-
"python-modules": ["airflow.providers.google.cloud.operators.mlengine"],
|
|
586
|
-
},
|
|
587
575
|
{
|
|
588
576
|
"integration-name": "Google Cloud Natural Language",
|
|
589
577
|
"python-modules": ["airflow.providers.google.cloud.operators.natural_language"],
|
|
@@ -670,11 +658,13 @@ def get_provider_info():
|
|
|
670
658
|
"airflow.providers.google.cloud.operators.vertex_ai.auto_ml",
|
|
671
659
|
"airflow.providers.google.cloud.operators.vertex_ai.batch_prediction_job",
|
|
672
660
|
"airflow.providers.google.cloud.operators.vertex_ai.endpoint_service",
|
|
661
|
+
"airflow.providers.google.cloud.operators.vertex_ai.experiment_service",
|
|
673
662
|
"airflow.providers.google.cloud.operators.vertex_ai.hyperparameter_tuning_job",
|
|
674
663
|
"airflow.providers.google.cloud.operators.vertex_ai.model_service",
|
|
675
664
|
"airflow.providers.google.cloud.operators.vertex_ai.pipeline_job",
|
|
676
665
|
"airflow.providers.google.cloud.operators.vertex_ai.generative_model",
|
|
677
666
|
"airflow.providers.google.cloud.operators.vertex_ai.feature_store",
|
|
667
|
+
"airflow.providers.google.cloud.operators.vertex_ai.ray",
|
|
678
668
|
],
|
|
679
669
|
},
|
|
680
670
|
{
|
|
@@ -693,6 +683,14 @@ def get_provider_info():
|
|
|
693
683
|
"integration-name": "Google Cloud Managed Service for Apache Kafka",
|
|
694
684
|
"python-modules": ["airflow.providers.google.cloud.operators.managed_kafka"],
|
|
695
685
|
},
|
|
686
|
+
{
|
|
687
|
+
"integration-name": "Google Cloud Logging Sink",
|
|
688
|
+
"python-modules": ["airflow.providers.google.cloud.operators.cloud_logging_sink"],
|
|
689
|
+
},
|
|
690
|
+
{
|
|
691
|
+
"integration-name": "Google Cloud Generative AI",
|
|
692
|
+
"python-modules": ["airflow.providers.google.cloud.operators.gen_ai"],
|
|
693
|
+
},
|
|
696
694
|
],
|
|
697
695
|
"sensors": [
|
|
698
696
|
{
|
|
@@ -813,10 +811,6 @@ def get_provider_info():
|
|
|
813
811
|
],
|
|
814
812
|
"hooks": [
|
|
815
813
|
{"integration-name": "Google Ads", "python-modules": ["airflow.providers.google.ads.hooks.ads"]},
|
|
816
|
-
{
|
|
817
|
-
"integration-name": "Google AutoML",
|
|
818
|
-
"python-modules": ["airflow.providers.google.cloud.hooks.automl"],
|
|
819
|
-
},
|
|
820
814
|
{
|
|
821
815
|
"integration-name": "Google BigQuery",
|
|
822
816
|
"python-modules": ["airflow.providers.google.cloud.hooks.bigquery"],
|
|
@@ -864,10 +858,6 @@ def get_provider_info():
|
|
|
864
858
|
"airflow.providers.google.cloud.hooks.compute_ssh",
|
|
865
859
|
],
|
|
866
860
|
},
|
|
867
|
-
{
|
|
868
|
-
"integration-name": "Google Data Catalog",
|
|
869
|
-
"python-modules": ["airflow.providers.google.cloud.hooks.datacatalog"],
|
|
870
|
-
},
|
|
871
861
|
{
|
|
872
862
|
"integration-name": "Google Dataflow",
|
|
873
863
|
"python-modules": ["airflow.providers.google.cloud.hooks.dataflow"],
|
|
@@ -920,10 +910,6 @@ def get_provider_info():
|
|
|
920
910
|
"integration-name": "Google Kubernetes Engine",
|
|
921
911
|
"python-modules": ["airflow.providers.google.cloud.hooks.kubernetes_engine"],
|
|
922
912
|
},
|
|
923
|
-
{
|
|
924
|
-
"integration-name": "Google Cloud Life Sciences",
|
|
925
|
-
"python-modules": ["airflow.providers.google.cloud.hooks.life_sciences"],
|
|
926
|
-
},
|
|
927
913
|
{
|
|
928
914
|
"integration-name": "Google Machine Learning Engine",
|
|
929
915
|
"python-modules": ["airflow.providers.google.cloud.hooks.mlengine"],
|
|
@@ -1035,12 +1021,14 @@ def get_provider_info():
|
|
|
1035
1021
|
"airflow.providers.google.cloud.hooks.vertex_ai.auto_ml",
|
|
1036
1022
|
"airflow.providers.google.cloud.hooks.vertex_ai.batch_prediction_job",
|
|
1037
1023
|
"airflow.providers.google.cloud.hooks.vertex_ai.endpoint_service",
|
|
1024
|
+
"airflow.providers.google.cloud.hooks.vertex_ai.experiment_service",
|
|
1038
1025
|
"airflow.providers.google.cloud.hooks.vertex_ai.hyperparameter_tuning_job",
|
|
1039
1026
|
"airflow.providers.google.cloud.hooks.vertex_ai.model_service",
|
|
1040
1027
|
"airflow.providers.google.cloud.hooks.vertex_ai.pipeline_job",
|
|
1041
1028
|
"airflow.providers.google.cloud.hooks.vertex_ai.generative_model",
|
|
1042
1029
|
"airflow.providers.google.cloud.hooks.vertex_ai.prediction_service",
|
|
1043
1030
|
"airflow.providers.google.cloud.hooks.vertex_ai.feature_store",
|
|
1031
|
+
"airflow.providers.google.cloud.hooks.vertex_ai.ray",
|
|
1044
1032
|
],
|
|
1045
1033
|
},
|
|
1046
1034
|
{
|
|
@@ -1059,6 +1047,14 @@ def get_provider_info():
|
|
|
1059
1047
|
"integration-name": "Google Cloud Managed Service for Apache Kafka",
|
|
1060
1048
|
"python-modules": ["airflow.providers.google.cloud.hooks.managed_kafka"],
|
|
1061
1049
|
},
|
|
1050
|
+
{
|
|
1051
|
+
"integration-name": "Google Cloud Logging Sink",
|
|
1052
|
+
"python-modules": ["airflow.providers.google.cloud.hooks.cloud_logging"],
|
|
1053
|
+
},
|
|
1054
|
+
{
|
|
1055
|
+
"integration-name": "Google Cloud Generative AI",
|
|
1056
|
+
"python-modules": ["airflow.providers.google.cloud.hooks.gen_ai"],
|
|
1057
|
+
},
|
|
1062
1058
|
],
|
|
1063
1059
|
"triggers": [
|
|
1064
1060
|
{
|
|
@@ -1336,6 +1332,12 @@ def get_provider_info():
|
|
|
1336
1332
|
"python-module": "airflow.providers.google.cloud.transfers.azure_blob_to_gcs",
|
|
1337
1333
|
"how-to-guide": "/docs/apache-airflow-providers-google/operators/transfer/azure_blob_to_gcs.rst",
|
|
1338
1334
|
},
|
|
1335
|
+
{
|
|
1336
|
+
"source-integration-name": "Hypertext Transfer Protocol (HTTP)",
|
|
1337
|
+
"target-integration-name": "Google Cloud Storage (GCS)",
|
|
1338
|
+
"python-module": "airflow.providers.google.cloud.transfers.http_to_gcs",
|
|
1339
|
+
"how-to-guide": "/docs/apache-airflow-providers-google/operators/transfer/http_to_gcs.rst",
|
|
1340
|
+
},
|
|
1339
1341
|
],
|
|
1340
1342
|
"connection-types": [
|
|
1341
1343
|
{
|
|
@@ -1366,6 +1368,14 @@ def get_provider_info():
|
|
|
1366
1368
|
"hook-class-name": "airflow.providers.google.leveldb.hooks.leveldb.LevelDBHook",
|
|
1367
1369
|
"connection-type": "leveldb",
|
|
1368
1370
|
},
|
|
1371
|
+
{
|
|
1372
|
+
"hook-class-name": "airflow.providers.google.ads.hooks.ads.GoogleAdsHook",
|
|
1373
|
+
"connection-type": "google_ads",
|
|
1374
|
+
},
|
|
1375
|
+
{
|
|
1376
|
+
"hook-class-name": "airflow.providers.google.cloud.hooks.looker.LookerHook",
|
|
1377
|
+
"connection-type": "gcp_looker",
|
|
1378
|
+
},
|
|
1369
1379
|
],
|
|
1370
1380
|
"extra-links": [
|
|
1371
1381
|
"airflow.providers.google.cloud.links.alloy_db.AlloyDBBackupsLink",
|
|
@@ -1396,11 +1406,9 @@ def get_provider_info():
|
|
|
1396
1406
|
"airflow.providers.google.cloud.links.compute.ComputeInstanceDetailsLink",
|
|
1397
1407
|
"airflow.providers.google.cloud.links.compute.ComputeInstanceTemplateDetailsLink",
|
|
1398
1408
|
"airflow.providers.google.cloud.links.compute.ComputeInstanceGroupManagerDetailsLink",
|
|
1409
|
+
"airflow.providers.google.cloud.links.cloud_run.CloudRunJobLoggingLink",
|
|
1399
1410
|
"airflow.providers.google.cloud.links.cloud_tasks.CloudTasksQueueLink",
|
|
1400
1411
|
"airflow.providers.google.cloud.links.cloud_tasks.CloudTasksLink",
|
|
1401
|
-
"airflow.providers.google.cloud.links.datacatalog.DataCatalogEntryGroupLink",
|
|
1402
|
-
"airflow.providers.google.cloud.links.datacatalog.DataCatalogEntryLink",
|
|
1403
|
-
"airflow.providers.google.cloud.links.datacatalog.DataCatalogTagTemplateLink",
|
|
1404
1412
|
"airflow.providers.google.cloud.links.dataproc.DataprocLink",
|
|
1405
1413
|
"airflow.providers.google.cloud.links.dataproc.DataprocListLink",
|
|
1406
1414
|
"airflow.providers.google.cloud.links.dataproc.DataprocClusterLink",
|
|
@@ -1427,6 +1435,8 @@ def get_provider_info():
|
|
|
1427
1435
|
"airflow.providers.google.cloud.links.vertex_ai.VertexAIEndpointListLink",
|
|
1428
1436
|
"airflow.providers.google.cloud.links.vertex_ai.VertexAIPipelineJobLink",
|
|
1429
1437
|
"airflow.providers.google.cloud.links.vertex_ai.VertexAIPipelineJobListLink",
|
|
1438
|
+
"airflow.providers.google.cloud.links.vertex_ai.VertexAIRayClusterLink",
|
|
1439
|
+
"airflow.providers.google.cloud.links.vertex_ai.VertexAIRayClusterListLink",
|
|
1430
1440
|
"airflow.providers.google.cloud.links.workflows.WorkflowsWorkflowDetailsLink",
|
|
1431
1441
|
"airflow.providers.google.cloud.links.workflows.WorkflowsListOfWorkflowsLink",
|
|
1432
1442
|
"airflow.providers.google.cloud.links.workflows.WorkflowsExecutionLink",
|
|
@@ -1457,12 +1467,6 @@ def get_provider_info():
|
|
|
1457
1467
|
"airflow.providers.google.cloud.links.cloud_build.CloudBuildListLink",
|
|
1458
1468
|
"airflow.providers.google.cloud.links.cloud_build.CloudBuildTriggersListLink",
|
|
1459
1469
|
"airflow.providers.google.cloud.links.cloud_build.CloudBuildTriggerDetailsLink",
|
|
1460
|
-
"airflow.providers.google.cloud.links.automl.AutoMLDatasetLink",
|
|
1461
|
-
"airflow.providers.google.cloud.links.automl.AutoMLDatasetListLink",
|
|
1462
|
-
"airflow.providers.google.cloud.links.automl.AutoMLModelLink",
|
|
1463
|
-
"airflow.providers.google.cloud.links.automl.AutoMLModelTrainLink",
|
|
1464
|
-
"airflow.providers.google.cloud.links.automl.AutoMLModelPredictLink",
|
|
1465
|
-
"airflow.providers.google.cloud.links.life_sciences.LifeSciencesLink",
|
|
1466
1470
|
"airflow.providers.google.cloud.links.cloud_functions.CloudFunctionsDetailsLink",
|
|
1467
1471
|
"airflow.providers.google.cloud.links.cloud_functions.CloudFunctionsListLink",
|
|
1468
1472
|
"airflow.providers.google.cloud.links.cloud_storage_transfer.CloudStorageTransferListLink",
|
|
@@ -19,9 +19,41 @@
|
|
|
19
19
|
|
|
20
20
|
from __future__ import annotations
|
|
21
21
|
|
|
22
|
+
import logging
|
|
22
23
|
import os
|
|
24
|
+
import shlex
|
|
25
|
+
import subprocess
|
|
23
26
|
|
|
24
|
-
|
|
27
|
+
|
|
28
|
+
def _execute_in_subprocess(cmd: list[str], cwd: str | None = None, env: dict[str, str] | None = None) -> None:
|
|
29
|
+
"""
|
|
30
|
+
Execute a process and stream output to logger.
|
|
31
|
+
|
|
32
|
+
:param cmd: command and arguments to run
|
|
33
|
+
:param cwd: Current working directory passed to the Popen constructor
|
|
34
|
+
:param env: Additional environment variables to set for the subprocess.
|
|
35
|
+
"""
|
|
36
|
+
log = logging.getLogger(__name__)
|
|
37
|
+
|
|
38
|
+
log.info("Executing cmd: %s", " ".join(shlex.quote(c) for c in cmd))
|
|
39
|
+
with subprocess.Popen(
|
|
40
|
+
cmd,
|
|
41
|
+
stdout=subprocess.PIPE,
|
|
42
|
+
stderr=subprocess.STDOUT,
|
|
43
|
+
bufsize=0,
|
|
44
|
+
close_fds=True,
|
|
45
|
+
cwd=cwd,
|
|
46
|
+
env=env,
|
|
47
|
+
) as proc:
|
|
48
|
+
log.info("Output:")
|
|
49
|
+
if proc.stdout:
|
|
50
|
+
with proc.stdout:
|
|
51
|
+
for line in iter(proc.stdout.readline, b""):
|
|
52
|
+
log.info("%s", line.decode().rstrip())
|
|
53
|
+
|
|
54
|
+
exit_code = proc.wait()
|
|
55
|
+
if exit_code != 0:
|
|
56
|
+
raise subprocess.CalledProcessError(exit_code, cmd)
|
|
25
57
|
|
|
26
58
|
|
|
27
59
|
def init_module(go_module_name: str, go_module_path: str) -> None:
|
|
@@ -36,7 +68,7 @@ def init_module(go_module_name: str, go_module_path: str) -> None:
|
|
|
36
68
|
if os.path.isfile(os.path.join(go_module_path, "go.mod")):
|
|
37
69
|
return
|
|
38
70
|
go_mod_init_cmd = ["go", "mod", "init", go_module_name]
|
|
39
|
-
|
|
71
|
+
_execute_in_subprocess(go_mod_init_cmd, cwd=go_module_path)
|
|
40
72
|
|
|
41
73
|
|
|
42
74
|
def install_dependencies(go_module_path: str) -> None:
|
|
@@ -46,4 +78,4 @@ def install_dependencies(go_module_path: str) -> None:
|
|
|
46
78
|
:param go_module_path: The path to the directory containing the Go module.
|
|
47
79
|
"""
|
|
48
80
|
go_mod_tidy = ["go", "mod", "tidy"]
|
|
49
|
-
|
|
81
|
+
_execute_in_subprocess(go_mod_tidy, cwd=go_module_path)
|
|
@@ -18,8 +18,10 @@
|
|
|
18
18
|
|
|
19
19
|
from __future__ import annotations
|
|
20
20
|
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
21
23
|
from airflow.exceptions import AirflowException, AirflowOptionalProviderFeatureException
|
|
22
|
-
from airflow.
|
|
24
|
+
from airflow.providers.common.compat.sdk import BaseHook
|
|
23
25
|
|
|
24
26
|
try:
|
|
25
27
|
import plyvel
|
|
@@ -46,6 +48,29 @@ class LevelDBHook(BaseHook):
|
|
|
46
48
|
conn_type = "leveldb"
|
|
47
49
|
hook_name = "LevelDB"
|
|
48
50
|
|
|
51
|
+
@classmethod
|
|
52
|
+
def get_connection_form_widgets(cls) -> dict[str, Any]:
|
|
53
|
+
"""Return connection widgets to add to LevelDB connection form."""
|
|
54
|
+
from flask_babel import lazy_gettext
|
|
55
|
+
from wtforms import BooleanField
|
|
56
|
+
|
|
57
|
+
return {
|
|
58
|
+
"create_if_missing": BooleanField(
|
|
59
|
+
lazy_gettext("Create a database if it does not exist"), default=False
|
|
60
|
+
),
|
|
61
|
+
"error_if_exists": BooleanField(
|
|
62
|
+
lazy_gettext("Raise an exception if the database already exists"), default=False
|
|
63
|
+
),
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
@classmethod
|
|
67
|
+
def get_ui_field_behaviour(cls) -> dict[str, Any]:
|
|
68
|
+
"""Return custom UI field behaviour for LevelDB connection."""
|
|
69
|
+
return {
|
|
70
|
+
"hidden_fields": ["login", "password", "schema", "port"],
|
|
71
|
+
"relabeling": {},
|
|
72
|
+
}
|
|
73
|
+
|
|
49
74
|
def __init__(self, leveldb_conn_id: str = default_conn_name):
|
|
50
75
|
super().__init__()
|
|
51
76
|
self.leveldb_conn_id = leveldb_conn_id
|
|
@@ -18,11 +18,11 @@ from __future__ import annotations
|
|
|
18
18
|
|
|
19
19
|
from typing import TYPE_CHECKING, Any
|
|
20
20
|
|
|
21
|
-
from airflow.models import BaseOperator
|
|
22
21
|
from airflow.providers.google.leveldb.hooks.leveldb import LevelDBHook
|
|
22
|
+
from airflow.providers.google.version_compat import BaseOperator
|
|
23
23
|
|
|
24
24
|
if TYPE_CHECKING:
|
|
25
|
-
from airflow.
|
|
25
|
+
from airflow.providers.common.compat.sdk import Context
|
|
26
26
|
|
|
27
27
|
|
|
28
28
|
class LevelDBOperator(BaseOperator):
|