apache-airflow-providers-google 10.14.0rc2__py3-none-any.whl → 10.15.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/__init__.py +1 -1
- airflow/providers/google/cloud/hooks/automl.py +13 -13
- airflow/providers/google/cloud/hooks/bigquery.py +193 -246
- airflow/providers/google/cloud/hooks/bigquery_dts.py +6 -6
- airflow/providers/google/cloud/hooks/bigtable.py +8 -8
- airflow/providers/google/cloud/hooks/cloud_batch.py +1 -1
- airflow/providers/google/cloud/hooks/cloud_build.py +19 -20
- airflow/providers/google/cloud/hooks/cloud_composer.py +4 -4
- airflow/providers/google/cloud/hooks/cloud_memorystore.py +10 -10
- airflow/providers/google/cloud/hooks/cloud_run.py +1 -1
- airflow/providers/google/cloud/hooks/cloud_sql.py +17 -17
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +3 -3
- airflow/providers/google/cloud/hooks/compute.py +16 -16
- airflow/providers/google/cloud/hooks/compute_ssh.py +1 -1
- airflow/providers/google/cloud/hooks/datacatalog.py +22 -22
- airflow/providers/google/cloud/hooks/dataflow.py +48 -49
- airflow/providers/google/cloud/hooks/dataform.py +16 -16
- airflow/providers/google/cloud/hooks/datafusion.py +15 -15
- airflow/providers/google/cloud/hooks/datapipeline.py +3 -3
- airflow/providers/google/cloud/hooks/dataplex.py +19 -19
- airflow/providers/google/cloud/hooks/dataprep.py +8 -8
- airflow/providers/google/cloud/hooks/dataproc.py +88 -0
- airflow/providers/google/cloud/hooks/dataproc_metastore.py +13 -13
- airflow/providers/google/cloud/hooks/datastore.py +3 -3
- airflow/providers/google/cloud/hooks/dlp.py +25 -25
- airflow/providers/google/cloud/hooks/gcs.py +25 -23
- airflow/providers/google/cloud/hooks/gdm.py +3 -3
- airflow/providers/google/cloud/hooks/kms.py +3 -3
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +63 -48
- airflow/providers/google/cloud/hooks/life_sciences.py +13 -12
- airflow/providers/google/cloud/hooks/looker.py +7 -7
- airflow/providers/google/cloud/hooks/mlengine.py +12 -12
- airflow/providers/google/cloud/hooks/natural_language.py +2 -2
- airflow/providers/google/cloud/hooks/os_login.py +1 -1
- airflow/providers/google/cloud/hooks/pubsub.py +9 -9
- airflow/providers/google/cloud/hooks/secret_manager.py +1 -1
- airflow/providers/google/cloud/hooks/spanner.py +11 -11
- airflow/providers/google/cloud/hooks/speech_to_text.py +1 -1
- airflow/providers/google/cloud/hooks/stackdriver.py +7 -7
- airflow/providers/google/cloud/hooks/tasks.py +11 -11
- airflow/providers/google/cloud/hooks/text_to_speech.py +1 -1
- airflow/providers/google/cloud/hooks/translate.py +1 -1
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +13 -13
- airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +6 -6
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +45 -50
- airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +13 -13
- airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +9 -9
- airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +128 -11
- airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +10 -10
- airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +8 -8
- airflow/providers/google/cloud/hooks/video_intelligence.py +2 -2
- airflow/providers/google/cloud/hooks/vision.py +1 -1
- airflow/providers/google/cloud/hooks/workflows.py +10 -10
- airflow/providers/google/cloud/links/datafusion.py +12 -5
- airflow/providers/google/cloud/operators/bigquery.py +9 -11
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +3 -1
- airflow/providers/google/cloud/operators/dataflow.py +16 -16
- airflow/providers/google/cloud/operators/datafusion.py +9 -1
- airflow/providers/google/cloud/operators/dataproc.py +298 -65
- airflow/providers/google/cloud/operators/kubernetes_engine.py +6 -6
- airflow/providers/google/cloud/operators/life_sciences.py +10 -9
- airflow/providers/google/cloud/operators/mlengine.py +96 -96
- airflow/providers/google/cloud/operators/pubsub.py +2 -0
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +33 -3
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +59 -2
- airflow/providers/google/cloud/secrets/secret_manager.py +8 -7
- airflow/providers/google/cloud/sensors/bigquery.py +20 -16
- airflow/providers/google/cloud/sensors/cloud_composer.py +11 -8
- airflow/providers/google/cloud/sensors/gcs.py +8 -7
- airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +4 -4
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +1 -1
- airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/mysql_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/oracle_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/postgres_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/presto_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/s3_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/trino_to_gcs.py +1 -1
- airflow/providers/google/cloud/triggers/bigquery.py +12 -12
- airflow/providers/google/cloud/triggers/bigquery_dts.py +1 -1
- airflow/providers/google/cloud/triggers/cloud_batch.py +3 -1
- airflow/providers/google/cloud/triggers/cloud_build.py +2 -2
- airflow/providers/google/cloud/triggers/cloud_run.py +1 -1
- airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +6 -6
- airflow/providers/google/cloud/triggers/dataflow.py +3 -1
- airflow/providers/google/cloud/triggers/datafusion.py +2 -2
- airflow/providers/google/cloud/triggers/dataplex.py +2 -2
- airflow/providers/google/cloud/triggers/dataproc.py +2 -2
- airflow/providers/google/cloud/triggers/gcs.py +12 -8
- airflow/providers/google/cloud/triggers/kubernetes_engine.py +2 -2
- airflow/providers/google/cloud/triggers/mlengine.py +2 -2
- airflow/providers/google/cloud/triggers/pubsub.py +1 -1
- airflow/providers/google/cloud/triggers/vertex_ai.py +99 -0
- airflow/providers/google/cloud/utils/bigquery.py +2 -2
- airflow/providers/google/cloud/utils/credentials_provider.py +2 -2
- airflow/providers/google/cloud/utils/dataform.py +1 -1
- airflow/providers/google/cloud/utils/field_validator.py +2 -2
- airflow/providers/google/cloud/utils/helpers.py +2 -2
- airflow/providers/google/cloud/utils/mlengine_operator_utils.py +1 -1
- airflow/providers/google/cloud/utils/mlengine_prediction_summary.py +1 -1
- airflow/providers/google/common/auth_backend/google_openid.py +2 -2
- airflow/providers/google/common/hooks/base_google.py +29 -22
- airflow/providers/google/common/hooks/discovery_api.py +2 -2
- airflow/providers/google/common/utils/id_token_credentials.py +5 -5
- airflow/providers/google/firebase/hooks/firestore.py +3 -3
- airflow/providers/google/get_provider_info.py +7 -2
- airflow/providers/google/leveldb/hooks/leveldb.py +2 -2
- airflow/providers/google/marketing_platform/hooks/analytics.py +11 -14
- airflow/providers/google/marketing_platform/hooks/campaign_manager.py +11 -11
- airflow/providers/google/marketing_platform/hooks/display_video.py +13 -13
- airflow/providers/google/marketing_platform/hooks/search_ads.py +4 -4
- airflow/providers/google/marketing_platform/operators/analytics.py +37 -32
- airflow/providers/google/suite/hooks/calendar.py +2 -2
- airflow/providers/google/suite/hooks/drive.py +7 -7
- airflow/providers/google/suite/hooks/sheets.py +8 -8
- {apache_airflow_providers_google-10.14.0rc2.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/METADATA +11 -11
- {apache_airflow_providers_google-10.14.0rc2.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/RECORD +121 -120
- {apache_airflow_providers_google-10.14.0rc2.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_google-10.14.0rc2.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/entry_points.txt +0 -0
@@ -28,10 +28,10 @@ from __future__ import annotations
|
|
28
28
|
import contextlib
|
29
29
|
import json
|
30
30
|
import time
|
31
|
-
import warnings
|
32
31
|
from functools import cached_property
|
33
32
|
from typing import TYPE_CHECKING, Sequence
|
34
33
|
|
34
|
+
from deprecated import deprecated
|
35
35
|
from gcloud.aio.auth import Token
|
36
36
|
from google.api_core.exceptions import NotFound
|
37
37
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
@@ -48,6 +48,7 @@ from urllib3.exceptions import HTTPError
|
|
48
48
|
|
49
49
|
from airflow import version
|
50
50
|
from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
|
51
|
+
from airflow.providers.cncf.kubernetes.kube_client import _enable_tcp_keepalive
|
51
52
|
from airflow.providers.cncf.kubernetes.utils.pod_manager import PodOperatorHookProtocol
|
52
53
|
from airflow.providers.google.common.consts import CLIENT_INFO
|
53
54
|
from airflow.providers.google.common.hooks.base_google import (
|
@@ -98,22 +99,23 @@ class GKEHook(GoogleBaseHook):
|
|
98
99
|
|
99
100
|
# To preserve backward compatibility
|
100
101
|
# TODO: remove one day
|
102
|
+
@deprecated(
|
103
|
+
reason=(
|
104
|
+
"The get_conn method has been deprecated. "
|
105
|
+
"You should use the get_cluster_manager_client method."
|
106
|
+
),
|
107
|
+
category=AirflowProviderDeprecationWarning,
|
108
|
+
)
|
101
109
|
def get_conn(self) -> container_v1.ClusterManagerClient:
|
102
|
-
warnings.warn(
|
103
|
-
"The get_conn method has been deprecated. You should use the get_cluster_manager_client method.",
|
104
|
-
AirflowProviderDeprecationWarning,
|
105
|
-
stacklevel=2,
|
106
|
-
)
|
107
110
|
return self.get_cluster_manager_client()
|
108
111
|
|
109
112
|
# To preserve backward compatibility
|
110
113
|
# TODO: remove one day
|
114
|
+
@deprecated(
|
115
|
+
reason="The get_client method has been deprecated. You should use the get_conn method.",
|
116
|
+
category=AirflowProviderDeprecationWarning,
|
117
|
+
)
|
111
118
|
def get_client(self) -> ClusterManagerClient:
|
112
|
-
warnings.warn(
|
113
|
-
"The get_client method has been deprecated. You should use the get_conn method.",
|
114
|
-
AirflowProviderDeprecationWarning,
|
115
|
-
stacklevel=2,
|
116
|
-
)
|
117
119
|
return self.get_conn()
|
118
120
|
|
119
121
|
def wait_for_operation(self, operation: Operation, project_id: str | None = None) -> Operation:
|
@@ -176,7 +178,7 @@ class GKEHook(GoogleBaseHook):
|
|
176
178
|
retry: Retry | _MethodDefault = DEFAULT,
|
177
179
|
timeout: float | None = None,
|
178
180
|
) -> Operation | None:
|
179
|
-
"""
|
181
|
+
"""Delete the cluster, the Kubernetes endpoint, and all worker nodes.
|
180
182
|
|
181
183
|
Firewalls and routes that were configured during cluster creation are
|
182
184
|
also deleted. Other Google Compute Engine resources that might be in use
|
@@ -352,6 +354,7 @@ class GKEPodHook(GoogleBaseHook, PodOperatorHookProtocol):
|
|
352
354
|
self,
|
353
355
|
cluster_url: str,
|
354
356
|
ssl_ca_cert: str,
|
357
|
+
disable_tcp_keepalive: bool | None = None,
|
355
358
|
gcp_conn_id: str = "google_cloud_default",
|
356
359
|
impersonation_chain: str | Sequence[str] | None = None,
|
357
360
|
**kwargs,
|
@@ -363,6 +366,7 @@ class GKEPodHook(GoogleBaseHook, PodOperatorHookProtocol):
|
|
363
366
|
)
|
364
367
|
self._cluster_url = cluster_url
|
365
368
|
self._ssl_ca_cert = ssl_ca_cert
|
369
|
+
self.disable_tcp_keepalive = disable_tcp_keepalive
|
366
370
|
|
367
371
|
@cached_property
|
368
372
|
def api_client(self) -> client.ApiClient:
|
@@ -397,6 +401,10 @@ class GKEPodHook(GoogleBaseHook, PodOperatorHookProtocol):
|
|
397
401
|
def get_conn(self) -> client.ApiClient:
|
398
402
|
configuration = self._get_config()
|
399
403
|
configuration.refresh_api_key_hook = self._refresh_api_key_hook
|
404
|
+
|
405
|
+
if self.disable_tcp_keepalive is not True:
|
406
|
+
_enable_tcp_keepalive()
|
407
|
+
|
400
408
|
return client.ApiClient(configuration)
|
401
409
|
|
402
410
|
def _refresh_api_key_hook(self, configuration: client.configuration.Configuration):
|
@@ -500,14 +508,15 @@ class GKEPodAsyncHook(GoogleBaseAsyncHook):
|
|
500
508
|
:param name: Name of the pod.
|
501
509
|
:param namespace: Name of the pod's namespace.
|
502
510
|
"""
|
503
|
-
|
504
|
-
async with self.
|
505
|
-
|
506
|
-
|
507
|
-
|
508
|
-
|
509
|
-
|
510
|
-
|
511
|
+
with await self.service_file_as_context() as service_file: # type: ignore[attr-defined]
|
512
|
+
async with Token(scopes=self.scopes, service_file=service_file) as token:
|
513
|
+
async with self.get_conn(token) as connection:
|
514
|
+
v1_api = async_client.CoreV1Api(connection)
|
515
|
+
pod: V1Pod = await v1_api.read_namespaced_pod(
|
516
|
+
name=name,
|
517
|
+
namespace=namespace,
|
518
|
+
)
|
519
|
+
return pod
|
511
520
|
|
512
521
|
async def delete_pod(self, name: str, namespace: str):
|
513
522
|
"""Delete a pod.
|
@@ -515,18 +524,21 @@ class GKEPodAsyncHook(GoogleBaseAsyncHook):
|
|
515
524
|
:param name: Name of the pod.
|
516
525
|
:param namespace: Name of the pod's namespace.
|
517
526
|
"""
|
518
|
-
|
519
|
-
|
520
|
-
|
521
|
-
|
522
|
-
|
523
|
-
|
524
|
-
|
525
|
-
|
526
|
-
|
527
|
-
|
528
|
-
|
529
|
-
|
527
|
+
with await self.service_file_as_context() as service_file: # type: ignore[attr-defined]
|
528
|
+
async with Token(scopes=self.scopes, service_file=service_file) as token, self.get_conn(
|
529
|
+
token
|
530
|
+
) as connection:
|
531
|
+
try:
|
532
|
+
v1_api = async_client.CoreV1Api(connection)
|
533
|
+
await v1_api.delete_namespaced_pod(
|
534
|
+
name=name,
|
535
|
+
namespace=namespace,
|
536
|
+
body=client.V1DeleteOptions(),
|
537
|
+
)
|
538
|
+
except async_client.ApiException as e:
|
539
|
+
# If the pod is already deleted
|
540
|
+
if e.status != 404:
|
541
|
+
raise
|
530
542
|
|
531
543
|
async def read_logs(self, name: str, namespace: str):
|
532
544
|
"""Read logs inside the pod while starting containers inside.
|
@@ -539,19 +551,22 @@ class GKEPodAsyncHook(GoogleBaseAsyncHook):
|
|
539
551
|
:param name: Name of the pod.
|
540
552
|
:param namespace: Name of the pod's namespace.
|
541
553
|
"""
|
542
|
-
|
543
|
-
|
544
|
-
|
545
|
-
|
546
|
-
|
547
|
-
|
548
|
-
|
549
|
-
|
550
|
-
|
551
|
-
|
552
|
-
|
553
|
-
|
554
|
-
|
555
|
-
|
556
|
-
|
557
|
-
|
554
|
+
with await self.service_file_as_context() as service_file: # type: ignore[attr-defined]
|
555
|
+
async with Token(scopes=self.scopes, service_file=service_file) as token, self.get_conn(
|
556
|
+
token
|
557
|
+
) as connection:
|
558
|
+
try:
|
559
|
+
v1_api = async_client.CoreV1Api(connection)
|
560
|
+
logs = await v1_api.read_namespaced_pod_log(
|
561
|
+
name=name,
|
562
|
+
namespace=namespace,
|
563
|
+
follow=False,
|
564
|
+
timestamps=True,
|
565
|
+
)
|
566
|
+
logs = logs.splitlines()
|
567
|
+
for line in logs:
|
568
|
+
self.log.info("Container logs from %s", line)
|
569
|
+
return logs
|
570
|
+
except HTTPError:
|
571
|
+
self.log.exception("There was an error reading the kubernetes API.")
|
572
|
+
raise
|
@@ -19,10 +19,10 @@
|
|
19
19
|
from __future__ import annotations
|
20
20
|
|
21
21
|
import time
|
22
|
-
import warnings
|
23
22
|
from typing import Sequence
|
24
23
|
|
25
24
|
import google.api_core.path_template
|
25
|
+
from deprecated import deprecated
|
26
26
|
from googleapiclient.discovery import build
|
27
27
|
|
28
28
|
from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
|
@@ -32,6 +32,15 @@ from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
|
|
32
32
|
TIME_TO_SLEEP_IN_SECONDS = 5
|
33
33
|
|
34
34
|
|
35
|
+
@deprecated(
|
36
|
+
reason=(
|
37
|
+
"This hook is deprecated. Consider using "
|
38
|
+
"Google Cloud Batch Operators' hook instead. "
|
39
|
+
"The Life Sciences API (beta) will be discontinued "
|
40
|
+
"on July 8, 2025 in favor of Google Cloud Batch."
|
41
|
+
),
|
42
|
+
category=AirflowProviderDeprecationWarning,
|
43
|
+
)
|
35
44
|
class LifeSciencesHook(GoogleBaseHook):
|
36
45
|
"""
|
37
46
|
Hook for the Google Cloud Life Sciences APIs.
|
@@ -76,17 +85,9 @@ class LifeSciencesHook(GoogleBaseHook):
|
|
76
85
|
)
|
77
86
|
self.api_version = api_version
|
78
87
|
|
79
|
-
warnings.warn(
|
80
|
-
"""This hook is deprecated. Consider using Google Cloud Batch Operators' hook instead.
|
81
|
-
The Life Sciences API (beta) will be discontinued on July 8, 2025 in favor
|
82
|
-
of Google Cloud Batch.""",
|
83
|
-
AirflowProviderDeprecationWarning,
|
84
|
-
stacklevel=3,
|
85
|
-
)
|
86
|
-
|
87
88
|
def get_conn(self) -> build:
|
88
89
|
"""
|
89
|
-
|
90
|
+
Retrieve the connection to Cloud Life Sciences.
|
90
91
|
|
91
92
|
:return: Google Cloud Life Sciences service object.
|
92
93
|
"""
|
@@ -98,7 +99,7 @@ class LifeSciencesHook(GoogleBaseHook):
|
|
98
99
|
@GoogleBaseHook.fallback_to_default_project_id
|
99
100
|
def run_pipeline(self, body: dict, location: str, project_id: str) -> dict:
|
100
101
|
"""
|
101
|
-
|
102
|
+
Run a pipeline.
|
102
103
|
|
103
104
|
:param body: The request body.
|
104
105
|
:param location: The location of the project. For example: "us-east1".
|
@@ -136,7 +137,7 @@ class LifeSciencesHook(GoogleBaseHook):
|
|
136
137
|
|
137
138
|
def _wait_for_operation_to_complete(self, operation_name: str) -> None:
|
138
139
|
"""
|
139
|
-
|
140
|
+
Wait for the named operation to complete - checks status of the asynchronous call.
|
140
141
|
|
141
142
|
:param operation_name: The name of the operation.
|
142
143
|
:return: The response returned by the operation.
|
@@ -54,7 +54,7 @@ class LookerHook(BaseHook):
|
|
54
54
|
query_params: dict | None = None,
|
55
55
|
):
|
56
56
|
"""
|
57
|
-
|
57
|
+
Submit a PDT materialization job to Looker.
|
58
58
|
|
59
59
|
:param model: Required. The model of the PDT to start building.
|
60
60
|
:param view: Required. The view of the PDT to start building.
|
@@ -83,7 +83,7 @@ class LookerHook(BaseHook):
|
|
83
83
|
materialization_id: str,
|
84
84
|
):
|
85
85
|
"""
|
86
|
-
|
86
|
+
Get the PDT materialization job status from Looker.
|
87
87
|
|
88
88
|
:param materialization_id: Required. The materialization id to check status for.
|
89
89
|
"""
|
@@ -100,7 +100,7 @@ class LookerHook(BaseHook):
|
|
100
100
|
materialization_id: str,
|
101
101
|
) -> dict:
|
102
102
|
"""
|
103
|
-
|
103
|
+
Get the PDT materialization job status.
|
104
104
|
|
105
105
|
:param materialization_id: Required. The materialization id to check status for.
|
106
106
|
"""
|
@@ -120,7 +120,7 @@ class LookerHook(BaseHook):
|
|
120
120
|
materialization_id: str,
|
121
121
|
):
|
122
122
|
"""
|
123
|
-
|
123
|
+
Start a PDT materialization job cancellation request.
|
124
124
|
|
125
125
|
:param materialization_id: Required. The materialization id to stop.
|
126
126
|
"""
|
@@ -140,7 +140,7 @@ class LookerHook(BaseHook):
|
|
140
140
|
timeout: int | None = None,
|
141
141
|
) -> None:
|
142
142
|
"""
|
143
|
-
|
143
|
+
Poll a PDT materialization job to check if it finishes.
|
144
144
|
|
145
145
|
:param materialization_id: Required. The materialization id to wait for.
|
146
146
|
:param wait_time: Optional. Number of seconds between checks.
|
@@ -185,7 +185,7 @@ class LookerHook(BaseHook):
|
|
185
185
|
self.log.info("PDT materialization job completed successfully. Job id: %s.", materialization_id)
|
186
186
|
|
187
187
|
def get_looker_sdk(self):
|
188
|
-
"""
|
188
|
+
"""Return Looker SDK client for Looker API 4.0."""
|
189
189
|
conn = self.get_connection(self.looker_conn_id)
|
190
190
|
settings = LookerApiSettings(conn)
|
191
191
|
|
@@ -211,7 +211,7 @@ class LookerApiSettings(api_settings.ApiSettings):
|
|
211
211
|
|
212
212
|
def read_config(self):
|
213
213
|
"""
|
214
|
-
|
214
|
+
Fetch the connection settings from Airflow's connection object.
|
215
215
|
|
216
216
|
Overrides the default logic of getting connection settings.
|
217
217
|
"""
|
@@ -97,7 +97,7 @@ class MLEngineHook(GoogleBaseHook):
|
|
97
97
|
|
98
98
|
def get_conn(self) -> Resource:
|
99
99
|
"""
|
100
|
-
|
100
|
+
Retrieve the connection to MLEngine.
|
101
101
|
|
102
102
|
:return: Google MLEngine services object.
|
103
103
|
"""
|
@@ -107,7 +107,7 @@ class MLEngineHook(GoogleBaseHook):
|
|
107
107
|
@GoogleBaseHook.fallback_to_default_project_id
|
108
108
|
def create_job(self, job: dict, project_id: str, use_existing_job_fn: Callable | None = None) -> dict:
|
109
109
|
"""
|
110
|
-
|
110
|
+
Launch a MLEngine job and wait for it to reach a terminal state.
|
111
111
|
|
112
112
|
:param project_id: The Google Cloud project id within which MLEngine
|
113
113
|
job will be launched. If set to None or missing, the default project_id from the Google Cloud
|
@@ -169,7 +169,7 @@ class MLEngineHook(GoogleBaseHook):
|
|
169
169
|
project_id: str,
|
170
170
|
):
|
171
171
|
"""
|
172
|
-
|
172
|
+
Launch a MLEngine job and wait for it to reach a terminal state.
|
173
173
|
|
174
174
|
:param project_id: The Google Cloud project id within which MLEngine
|
175
175
|
job will be launched. If set to None or missing, the default project_id from the Google Cloud
|
@@ -202,7 +202,7 @@ class MLEngineHook(GoogleBaseHook):
|
|
202
202
|
project_id: str,
|
203
203
|
) -> dict:
|
204
204
|
"""
|
205
|
-
|
205
|
+
Cancel a MLEngine job.
|
206
206
|
|
207
207
|
:param project_id: The Google Cloud project id within which MLEngine
|
208
208
|
job will be cancelled. If set to None or missing, the default project_id from the Google Cloud
|
@@ -231,7 +231,7 @@ class MLEngineHook(GoogleBaseHook):
|
|
231
231
|
|
232
232
|
def get_job(self, project_id: str, job_id: str) -> dict:
|
233
233
|
"""
|
234
|
-
|
234
|
+
Get a MLEngine job based on the job id.
|
235
235
|
|
236
236
|
:param project_id: The project in which the Job is located. If set to None or missing, the default
|
237
237
|
project_id from the Google Cloud connection is used. (templated)
|
@@ -255,7 +255,7 @@ class MLEngineHook(GoogleBaseHook):
|
|
255
255
|
|
256
256
|
def _wait_for_job_done(self, project_id: str, job_id: str, interval: int = 30):
|
257
257
|
"""
|
258
|
-
|
258
|
+
Wait for the Job to reach a terminal state.
|
259
259
|
|
260
260
|
This method will periodically check the job state until the job reach
|
261
261
|
a terminal state.
|
@@ -284,7 +284,7 @@ class MLEngineHook(GoogleBaseHook):
|
|
284
284
|
project_id: str,
|
285
285
|
) -> dict:
|
286
286
|
"""
|
287
|
-
|
287
|
+
Create the Version on Google Cloud ML Engine.
|
288
288
|
|
289
289
|
:param version_spec: A dictionary containing the information about the version. (templated)
|
290
290
|
:param model_name: The name of the Google Cloud ML Engine model that the version belongs to.
|
@@ -320,7 +320,7 @@ class MLEngineHook(GoogleBaseHook):
|
|
320
320
|
project_id: str,
|
321
321
|
) -> dict:
|
322
322
|
"""
|
323
|
-
|
323
|
+
Set a version to be the default. Blocks until finished.
|
324
324
|
|
325
325
|
:param model_name: The name of the Google Cloud ML Engine model that the version belongs to.
|
326
326
|
(templated)
|
@@ -351,7 +351,7 @@ class MLEngineHook(GoogleBaseHook):
|
|
351
351
|
project_id: str,
|
352
352
|
) -> list[dict]:
|
353
353
|
"""
|
354
|
-
|
354
|
+
List all available versions of a model. Blocks until finished.
|
355
355
|
|
356
356
|
:param model_name: The name of the Google Cloud ML Engine model that the version
|
357
357
|
belongs to. (templated)
|
@@ -387,7 +387,7 @@ class MLEngineHook(GoogleBaseHook):
|
|
387
387
|
project_id: str,
|
388
388
|
) -> dict:
|
389
389
|
"""
|
390
|
-
|
390
|
+
Delete the given version of a model. Blocks until finished.
|
391
391
|
|
392
392
|
:param model_name: The name of the Google Cloud ML Engine model that the version
|
393
393
|
belongs to. (templated)
|
@@ -467,7 +467,7 @@ class MLEngineHook(GoogleBaseHook):
|
|
467
467
|
project_id: str,
|
468
468
|
) -> dict | None:
|
469
469
|
"""
|
470
|
-
|
470
|
+
Get a Model. Blocks until finished.
|
471
471
|
|
472
472
|
:param model_name: The name of the model.
|
473
473
|
:param project_id: The Google Cloud project name to which MLEngine model belongs. If set to None
|
@@ -581,7 +581,7 @@ class MLEngineAsyncHook(GoogleBaseAsyncHook):
|
|
581
581
|
project_id: str | None = None,
|
582
582
|
) -> str | None:
|
583
583
|
"""
|
584
|
-
|
584
|
+
Poll for job status asynchronously using gcloud-aio.
|
585
585
|
|
586
586
|
Note that an OSError is raised when Job results are still pending.
|
587
587
|
Exception means that Job finished with errors
|
@@ -74,7 +74,7 @@ class CloudNaturalLanguageHook(GoogleBaseHook):
|
|
74
74
|
|
75
75
|
def get_conn(self) -> LanguageServiceClient:
|
76
76
|
"""
|
77
|
-
|
77
|
+
Retrieve connection to Cloud Natural Language service.
|
78
78
|
|
79
79
|
:return: Cloud Natural Language service object
|
80
80
|
"""
|
@@ -92,7 +92,7 @@ class CloudNaturalLanguageHook(GoogleBaseHook):
|
|
92
92
|
metadata: Sequence[tuple[str, str]] = (),
|
93
93
|
) -> AnalyzeEntitiesResponse:
|
94
94
|
"""
|
95
|
-
|
95
|
+
Find named entities in the text along with various properties.
|
96
96
|
|
97
97
|
Examples properties: entity types, salience, mentions for each entity, and others.
|
98
98
|
|
@@ -78,7 +78,7 @@ class OSLoginHook(GoogleBaseHook):
|
|
78
78
|
metadata: Sequence[tuple[str, str]] = (),
|
79
79
|
) -> ImportSshPublicKeyResponse:
|
80
80
|
"""
|
81
|
-
|
81
|
+
Add an SSH public key and returns the profile information.
|
82
82
|
|
83
83
|
Default POSIX account information is set when no username and UID exist as part of the login profile.
|
84
84
|
|
@@ -93,7 +93,7 @@ class PubSubHook(GoogleBaseHook):
|
|
93
93
|
|
94
94
|
def get_conn(self) -> PublisherClient:
|
95
95
|
"""
|
96
|
-
|
96
|
+
Retrieve connection to Google Cloud Pub/Sub.
|
97
97
|
|
98
98
|
:return: Google Cloud Pub/Sub client object.
|
99
99
|
"""
|
@@ -118,7 +118,7 @@ class PubSubHook(GoogleBaseHook):
|
|
118
118
|
project_id: str = PROVIDE_PROJECT_ID,
|
119
119
|
) -> None:
|
120
120
|
"""
|
121
|
-
|
121
|
+
Publish messages to a Pub/Sub topic.
|
122
122
|
|
123
123
|
:param topic: the Pub/Sub topic to which to publish; do not
|
124
124
|
include the ``projects/{project}/topics/`` prefix.
|
@@ -191,7 +191,7 @@ class PubSubHook(GoogleBaseHook):
|
|
191
191
|
metadata: Sequence[tuple[str, str]] = (),
|
192
192
|
) -> None:
|
193
193
|
"""
|
194
|
-
|
194
|
+
Create a Pub/Sub topic, if it does not already exist.
|
195
195
|
|
196
196
|
:param topic: the Pub/Sub topic name to create; do not
|
197
197
|
include the ``projects/{project}/topics/`` prefix.
|
@@ -264,7 +264,7 @@ class PubSubHook(GoogleBaseHook):
|
|
264
264
|
metadata: Sequence[tuple[str, str]] = (),
|
265
265
|
) -> None:
|
266
266
|
"""
|
267
|
-
|
267
|
+
Delete a Pub/Sub topic if it exists.
|
268
268
|
|
269
269
|
:param topic: the Pub/Sub topic name to delete; do not
|
270
270
|
include the ``projects/{project}/topics/`` prefix.
|
@@ -318,7 +318,7 @@ class PubSubHook(GoogleBaseHook):
|
|
318
318
|
metadata: Sequence[tuple[str, str]] = (),
|
319
319
|
) -> str:
|
320
320
|
"""
|
321
|
-
|
321
|
+
Create a Pub/Sub subscription, if it does not already exist.
|
322
322
|
|
323
323
|
:param topic: the Pub/Sub topic name that the subscription will be bound
|
324
324
|
to create; do not include the ``projects/{project}/subscriptions/`` prefix.
|
@@ -435,7 +435,7 @@ class PubSubHook(GoogleBaseHook):
|
|
435
435
|
metadata: Sequence[tuple[str, str]] = (),
|
436
436
|
) -> None:
|
437
437
|
"""
|
438
|
-
|
438
|
+
Delete a Pub/Sub subscription, if it exists.
|
439
439
|
|
440
440
|
:param subscription: the Pub/Sub subscription name to delete; do not
|
441
441
|
include the ``projects/{project}/subscriptions/`` prefix.
|
@@ -483,7 +483,7 @@ class PubSubHook(GoogleBaseHook):
|
|
483
483
|
metadata: Sequence[tuple[str, str]] = (),
|
484
484
|
) -> list[ReceivedMessage]:
|
485
485
|
"""
|
486
|
-
|
486
|
+
Pull up to ``max_messages`` messages from Pub/Sub subscription.
|
487
487
|
|
488
488
|
:param subscription: the Pub/Sub subscription name to pull from; do not
|
489
489
|
include the 'projects/{project}/topics/' prefix.
|
@@ -596,7 +596,7 @@ class PubSubAsyncHook(GoogleBaseAsyncHook):
|
|
596
596
|
|
597
597
|
async def _get_subscriber_client(self) -> SubscriberAsyncClient:
|
598
598
|
"""
|
599
|
-
|
599
|
+
Return async connection to the Google PubSub.
|
600
600
|
|
601
601
|
:return: Google Pub/Sub asynchronous client.
|
602
602
|
"""
|
@@ -671,7 +671,7 @@ class PubSubAsyncHook(GoogleBaseAsyncHook):
|
|
671
671
|
metadata: Sequence[tuple[str, str]] = (),
|
672
672
|
) -> list[ReceivedMessage]:
|
673
673
|
"""
|
674
|
-
|
674
|
+
Pull up to ``max_messages`` messages from Pub/Sub subscription.
|
675
675
|
|
676
676
|
:param subscription: the Pub/Sub subscription name to pull from; do not
|
677
677
|
include the 'projects/{project}/topics/' prefix.
|
@@ -76,7 +76,7 @@ class SpannerHook(GoogleBaseHook, DbApiHook):
|
|
76
76
|
|
77
77
|
def _get_client(self, project_id: str) -> Client:
|
78
78
|
"""
|
79
|
-
|
79
|
+
Provide a client for interacting with the Cloud Spanner API.
|
80
80
|
|
81
81
|
:param project_id: The ID of the Google Cloud project.
|
82
82
|
:return: Client
|
@@ -122,7 +122,7 @@ class SpannerHook(GoogleBaseHook, DbApiHook):
|
|
122
122
|
project_id: str,
|
123
123
|
) -> Instance | None:
|
124
124
|
"""
|
125
|
-
|
125
|
+
Get information about a particular instance.
|
126
126
|
|
127
127
|
:param project_id: Optional, The ID of the Google Cloud project that owns the Cloud Spanner
|
128
128
|
database. If set to None or missing, the default project_id from the Google Cloud connection
|
@@ -145,7 +145,7 @@ class SpannerHook(GoogleBaseHook, DbApiHook):
|
|
145
145
|
func: Callable[[Instance], Operation],
|
146
146
|
) -> None:
|
147
147
|
"""
|
148
|
-
|
148
|
+
Invoke a method on a given instance by applying a specified Callable.
|
149
149
|
|
150
150
|
:param project_id: The ID of the Google Cloud project that owns the Cloud Spanner database.
|
151
151
|
:param instance_id: The ID of the instance.
|
@@ -183,7 +183,7 @@ class SpannerHook(GoogleBaseHook, DbApiHook):
|
|
183
183
|
project_id: str,
|
184
184
|
) -> None:
|
185
185
|
"""
|
186
|
-
|
186
|
+
Create a new Cloud Spanner instance.
|
187
187
|
|
188
188
|
:param instance_id: The ID of the Cloud Spanner instance.
|
189
189
|
:param configuration_name: The name of the instance configuration defining how the
|
@@ -213,7 +213,7 @@ class SpannerHook(GoogleBaseHook, DbApiHook):
|
|
213
213
|
project_id: str,
|
214
214
|
) -> None:
|
215
215
|
"""
|
216
|
-
|
216
|
+
Update an existing Cloud Spanner instance.
|
217
217
|
|
218
218
|
:param instance_id: The ID of the Cloud Spanner instance.
|
219
219
|
:param configuration_name: The name of the instance configuration defining how the
|
@@ -236,7 +236,7 @@ class SpannerHook(GoogleBaseHook, DbApiHook):
|
|
236
236
|
@GoogleBaseHook.fallback_to_default_project_id
|
237
237
|
def delete_instance(self, instance_id: str, project_id: str) -> None:
|
238
238
|
"""
|
239
|
-
|
239
|
+
Delete an existing Cloud Spanner instance.
|
240
240
|
|
241
241
|
:param instance_id: The ID of the Cloud Spanner instance.
|
242
242
|
:param project_id: Optional, the ID of the Google Cloud project that owns the Cloud Spanner
|
@@ -260,7 +260,7 @@ class SpannerHook(GoogleBaseHook, DbApiHook):
|
|
260
260
|
project_id: str,
|
261
261
|
) -> Database | None:
|
262
262
|
"""
|
263
|
-
|
263
|
+
Retrieve a database in Cloud Spanner; return None if the database does not exist in the instance.
|
264
264
|
|
265
265
|
:param instance_id: The ID of the Cloud Spanner instance.
|
266
266
|
:param database_id: The ID of the database in Cloud Spanner.
|
@@ -287,7 +287,7 @@ class SpannerHook(GoogleBaseHook, DbApiHook):
|
|
287
287
|
project_id: str,
|
288
288
|
) -> None:
|
289
289
|
"""
|
290
|
-
|
290
|
+
Create a new database in Cloud Spanner.
|
291
291
|
|
292
292
|
:param instance_id: The ID of the Cloud Spanner instance.
|
293
293
|
:param database_id: The ID of the database to create in Cloud Spanner.
|
@@ -321,7 +321,7 @@ class SpannerHook(GoogleBaseHook, DbApiHook):
|
|
321
321
|
operation_id: str | None = None,
|
322
322
|
) -> None:
|
323
323
|
"""
|
324
|
-
|
324
|
+
Update DDL of a database in Cloud Spanner.
|
325
325
|
|
326
326
|
:param instance_id: The ID of the Cloud Spanner instance.
|
327
327
|
:param database_id: The ID of the database in Cloud Spanner.
|
@@ -357,7 +357,7 @@ class SpannerHook(GoogleBaseHook, DbApiHook):
|
|
357
357
|
@GoogleBaseHook.fallback_to_default_project_id
|
358
358
|
def delete_database(self, instance_id: str, database_id, project_id: str) -> bool:
|
359
359
|
"""
|
360
|
-
|
360
|
+
Drop a database in Cloud Spanner.
|
361
361
|
|
362
362
|
:param instance_id: The ID of the Cloud Spanner instance.
|
363
363
|
:param database_id: The ID of the database in Cloud Spanner.
|
@@ -392,7 +392,7 @@ class SpannerHook(GoogleBaseHook, DbApiHook):
|
|
392
392
|
project_id: str,
|
393
393
|
) -> None:
|
394
394
|
"""
|
395
|
-
|
395
|
+
Execute an arbitrary DML query (INSERT, UPDATE, DELETE).
|
396
396
|
|
397
397
|
:param instance_id: The ID of the Cloud Spanner instance.
|
398
398
|
:param database_id: The ID of the database in Cloud Spanner.
|