apache-airflow-providers-google 10.14.0rc2__py3-none-any.whl → 10.15.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/__init__.py +1 -1
- airflow/providers/google/cloud/hooks/automl.py +13 -13
- airflow/providers/google/cloud/hooks/bigquery.py +193 -246
- airflow/providers/google/cloud/hooks/bigquery_dts.py +6 -6
- airflow/providers/google/cloud/hooks/bigtable.py +8 -8
- airflow/providers/google/cloud/hooks/cloud_batch.py +1 -1
- airflow/providers/google/cloud/hooks/cloud_build.py +19 -20
- airflow/providers/google/cloud/hooks/cloud_composer.py +4 -4
- airflow/providers/google/cloud/hooks/cloud_memorystore.py +10 -10
- airflow/providers/google/cloud/hooks/cloud_run.py +1 -1
- airflow/providers/google/cloud/hooks/cloud_sql.py +17 -17
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +3 -3
- airflow/providers/google/cloud/hooks/compute.py +16 -16
- airflow/providers/google/cloud/hooks/compute_ssh.py +1 -1
- airflow/providers/google/cloud/hooks/datacatalog.py +22 -22
- airflow/providers/google/cloud/hooks/dataflow.py +48 -49
- airflow/providers/google/cloud/hooks/dataform.py +16 -16
- airflow/providers/google/cloud/hooks/datafusion.py +15 -15
- airflow/providers/google/cloud/hooks/datapipeline.py +3 -3
- airflow/providers/google/cloud/hooks/dataplex.py +19 -19
- airflow/providers/google/cloud/hooks/dataprep.py +8 -8
- airflow/providers/google/cloud/hooks/dataproc.py +88 -0
- airflow/providers/google/cloud/hooks/dataproc_metastore.py +13 -13
- airflow/providers/google/cloud/hooks/datastore.py +3 -3
- airflow/providers/google/cloud/hooks/dlp.py +25 -25
- airflow/providers/google/cloud/hooks/gcs.py +25 -23
- airflow/providers/google/cloud/hooks/gdm.py +3 -3
- airflow/providers/google/cloud/hooks/kms.py +3 -3
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +63 -48
- airflow/providers/google/cloud/hooks/life_sciences.py +13 -12
- airflow/providers/google/cloud/hooks/looker.py +7 -7
- airflow/providers/google/cloud/hooks/mlengine.py +12 -12
- airflow/providers/google/cloud/hooks/natural_language.py +2 -2
- airflow/providers/google/cloud/hooks/os_login.py +1 -1
- airflow/providers/google/cloud/hooks/pubsub.py +9 -9
- airflow/providers/google/cloud/hooks/secret_manager.py +1 -1
- airflow/providers/google/cloud/hooks/spanner.py +11 -11
- airflow/providers/google/cloud/hooks/speech_to_text.py +1 -1
- airflow/providers/google/cloud/hooks/stackdriver.py +7 -7
- airflow/providers/google/cloud/hooks/tasks.py +11 -11
- airflow/providers/google/cloud/hooks/text_to_speech.py +1 -1
- airflow/providers/google/cloud/hooks/translate.py +1 -1
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +13 -13
- airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +6 -6
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +45 -50
- airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +13 -13
- airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +9 -9
- airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +128 -11
- airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +10 -10
- airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +8 -8
- airflow/providers/google/cloud/hooks/video_intelligence.py +2 -2
- airflow/providers/google/cloud/hooks/vision.py +1 -1
- airflow/providers/google/cloud/hooks/workflows.py +10 -10
- airflow/providers/google/cloud/links/datafusion.py +12 -5
- airflow/providers/google/cloud/operators/bigquery.py +9 -11
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +3 -1
- airflow/providers/google/cloud/operators/dataflow.py +16 -16
- airflow/providers/google/cloud/operators/datafusion.py +9 -1
- airflow/providers/google/cloud/operators/dataproc.py +298 -65
- airflow/providers/google/cloud/operators/kubernetes_engine.py +6 -6
- airflow/providers/google/cloud/operators/life_sciences.py +10 -9
- airflow/providers/google/cloud/operators/mlengine.py +96 -96
- airflow/providers/google/cloud/operators/pubsub.py +2 -0
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +33 -3
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +59 -2
- airflow/providers/google/cloud/secrets/secret_manager.py +8 -7
- airflow/providers/google/cloud/sensors/bigquery.py +20 -16
- airflow/providers/google/cloud/sensors/cloud_composer.py +11 -8
- airflow/providers/google/cloud/sensors/gcs.py +8 -7
- airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +4 -4
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +1 -1
- airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/mysql_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/oracle_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/postgres_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/presto_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/s3_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/trino_to_gcs.py +1 -1
- airflow/providers/google/cloud/triggers/bigquery.py +12 -12
- airflow/providers/google/cloud/triggers/bigquery_dts.py +1 -1
- airflow/providers/google/cloud/triggers/cloud_batch.py +3 -1
- airflow/providers/google/cloud/triggers/cloud_build.py +2 -2
- airflow/providers/google/cloud/triggers/cloud_run.py +1 -1
- airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +6 -6
- airflow/providers/google/cloud/triggers/dataflow.py +3 -1
- airflow/providers/google/cloud/triggers/datafusion.py +2 -2
- airflow/providers/google/cloud/triggers/dataplex.py +2 -2
- airflow/providers/google/cloud/triggers/dataproc.py +2 -2
- airflow/providers/google/cloud/triggers/gcs.py +12 -8
- airflow/providers/google/cloud/triggers/kubernetes_engine.py +2 -2
- airflow/providers/google/cloud/triggers/mlengine.py +2 -2
- airflow/providers/google/cloud/triggers/pubsub.py +1 -1
- airflow/providers/google/cloud/triggers/vertex_ai.py +99 -0
- airflow/providers/google/cloud/utils/bigquery.py +2 -2
- airflow/providers/google/cloud/utils/credentials_provider.py +2 -2
- airflow/providers/google/cloud/utils/dataform.py +1 -1
- airflow/providers/google/cloud/utils/field_validator.py +2 -2
- airflow/providers/google/cloud/utils/helpers.py +2 -2
- airflow/providers/google/cloud/utils/mlengine_operator_utils.py +1 -1
- airflow/providers/google/cloud/utils/mlengine_prediction_summary.py +1 -1
- airflow/providers/google/common/auth_backend/google_openid.py +2 -2
- airflow/providers/google/common/hooks/base_google.py +29 -22
- airflow/providers/google/common/hooks/discovery_api.py +2 -2
- airflow/providers/google/common/utils/id_token_credentials.py +5 -5
- airflow/providers/google/firebase/hooks/firestore.py +3 -3
- airflow/providers/google/get_provider_info.py +7 -2
- airflow/providers/google/leveldb/hooks/leveldb.py +2 -2
- airflow/providers/google/marketing_platform/hooks/analytics.py +11 -14
- airflow/providers/google/marketing_platform/hooks/campaign_manager.py +11 -11
- airflow/providers/google/marketing_platform/hooks/display_video.py +13 -13
- airflow/providers/google/marketing_platform/hooks/search_ads.py +4 -4
- airflow/providers/google/marketing_platform/operators/analytics.py +37 -32
- airflow/providers/google/suite/hooks/calendar.py +2 -2
- airflow/providers/google/suite/hooks/drive.py +7 -7
- airflow/providers/google/suite/hooks/sheets.py +8 -8
- {apache_airflow_providers_google-10.14.0rc2.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/METADATA +11 -11
- {apache_airflow_providers_google-10.14.0rc2.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/RECORD +121 -120
- {apache_airflow_providers_google-10.14.0rc2.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_google-10.14.0rc2.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/entry_points.txt +0 -0
@@ -48,7 +48,7 @@ class DataprocMetastoreHook(GoogleBaseHook):
|
|
48
48
|
super().__init__(**kwargs)
|
49
49
|
|
50
50
|
def get_dataproc_metastore_client(self) -> DataprocMetastoreClient:
|
51
|
-
"""
|
51
|
+
"""Return DataprocMetastoreClient."""
|
52
52
|
client_options = ClientOptions(api_endpoint="metastore.googleapis.com:443")
|
53
53
|
|
54
54
|
return DataprocMetastoreClient(
|
@@ -56,7 +56,7 @@ class DataprocMetastoreHook(GoogleBaseHook):
|
|
56
56
|
)
|
57
57
|
|
58
58
|
def get_dataproc_metastore_client_v1beta(self):
|
59
|
-
"""
|
59
|
+
"""Return DataprocMetastoreClient (from v1 beta)."""
|
60
60
|
from google.cloud.metastore_v1beta import DataprocMetastoreClient
|
61
61
|
|
62
62
|
client_options = ClientOptions(api_endpoint="metastore.googleapis.com:443")
|
@@ -65,7 +65,7 @@ class DataprocMetastoreHook(GoogleBaseHook):
|
|
65
65
|
)
|
66
66
|
|
67
67
|
def wait_for_operation(self, timeout: float | None, operation: Operation):
|
68
|
-
"""
|
68
|
+
"""Wait for long-lasting operation to complete."""
|
69
69
|
try:
|
70
70
|
return operation.result(timeout=timeout)
|
71
71
|
except Exception:
|
@@ -86,7 +86,7 @@ class DataprocMetastoreHook(GoogleBaseHook):
|
|
86
86
|
metadata: Sequence[tuple[str, str]] = (),
|
87
87
|
):
|
88
88
|
"""
|
89
|
-
|
89
|
+
Create a new backup in a given project and location.
|
90
90
|
|
91
91
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
92
92
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -143,7 +143,7 @@ class DataprocMetastoreHook(GoogleBaseHook):
|
|
143
143
|
metadata: Sequence[tuple[str, str]] = (),
|
144
144
|
):
|
145
145
|
"""
|
146
|
-
|
146
|
+
Create a new MetadataImport in a given project and location.
|
147
147
|
|
148
148
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
149
149
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -200,7 +200,7 @@ class DataprocMetastoreHook(GoogleBaseHook):
|
|
200
200
|
metadata: Sequence[tuple[str, str]] = (),
|
201
201
|
):
|
202
202
|
"""
|
203
|
-
|
203
|
+
Create a metastore service in a project and location.
|
204
204
|
|
205
205
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
206
206
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
@@ -250,7 +250,7 @@ class DataprocMetastoreHook(GoogleBaseHook):
|
|
250
250
|
metadata: Sequence[tuple[str, str]] = (),
|
251
251
|
):
|
252
252
|
"""
|
253
|
-
|
253
|
+
Delete a single backup.
|
254
254
|
|
255
255
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
256
256
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -298,7 +298,7 @@ class DataprocMetastoreHook(GoogleBaseHook):
|
|
298
298
|
metadata: Sequence[tuple[str, str]] = (),
|
299
299
|
):
|
300
300
|
"""
|
301
|
-
|
301
|
+
Delete a single service.
|
302
302
|
|
303
303
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
304
304
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -342,7 +342,7 @@ class DataprocMetastoreHook(GoogleBaseHook):
|
|
342
342
|
metadata: Sequence[tuple[str, str]] = (),
|
343
343
|
):
|
344
344
|
"""
|
345
|
-
|
345
|
+
Export metadata from a service.
|
346
346
|
|
347
347
|
:param destination_gcs_folder: A Cloud Storage URI of a folder, in the format
|
348
348
|
``gs://<bucket_name>/<path_inside_bucket>``. A sub-folder
|
@@ -391,7 +391,7 @@ class DataprocMetastoreHook(GoogleBaseHook):
|
|
391
391
|
metadata: Sequence[tuple[str, str]] = (),
|
392
392
|
):
|
393
393
|
"""
|
394
|
-
|
394
|
+
Get the details of a single service.
|
395
395
|
|
396
396
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
397
397
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -474,7 +474,7 @@ class DataprocMetastoreHook(GoogleBaseHook):
|
|
474
474
|
metadata: Sequence[tuple[str, str]] = (),
|
475
475
|
):
|
476
476
|
"""
|
477
|
-
|
477
|
+
List backups in a service.
|
478
478
|
|
479
479
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
480
480
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -603,7 +603,7 @@ class DataprocMetastoreHook(GoogleBaseHook):
|
|
603
603
|
metadata: Sequence[tuple[str, str]] = (),
|
604
604
|
):
|
605
605
|
"""
|
606
|
-
|
606
|
+
Update the parameters of a single service.
|
607
607
|
|
608
608
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
609
609
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -660,7 +660,7 @@ class DataprocMetastoreHook(GoogleBaseHook):
|
|
660
660
|
partition_names: list[str] | None = None,
|
661
661
|
) -> Operation:
|
662
662
|
"""
|
663
|
-
|
663
|
+
List Hive partitions.
|
664
664
|
|
665
665
|
:param project_id: Optional. The ID of a dbt Cloud project.
|
666
666
|
:param service_id: Required. Dataproc Metastore service id.
|
@@ -57,7 +57,7 @@ class DatastoreHook(GoogleBaseHook):
|
|
57
57
|
|
58
58
|
def get_conn(self) -> Resource:
|
59
59
|
"""
|
60
|
-
|
60
|
+
Establish a connection to the Google API.
|
61
61
|
|
62
62
|
:return: a Google Cloud Datastore service object.
|
63
63
|
"""
|
@@ -200,7 +200,7 @@ class DatastoreHook(GoogleBaseHook):
|
|
200
200
|
|
201
201
|
def get_operation(self, name: str) -> dict:
|
202
202
|
"""
|
203
|
-
|
203
|
+
Get the latest state of a long-running operation.
|
204
204
|
|
205
205
|
.. seealso::
|
206
206
|
https://cloud.google.com/datastore/docs/reference/data/rest/v1/projects.operations/get
|
@@ -216,7 +216,7 @@ class DatastoreHook(GoogleBaseHook):
|
|
216
216
|
|
217
217
|
def delete_operation(self, name: str) -> dict:
|
218
218
|
"""
|
219
|
-
|
219
|
+
Delete the long-running operation.
|
220
220
|
|
221
221
|
.. seealso::
|
222
222
|
https://cloud.google.com/datastore/docs/reference/data/rest/v1/projects.operations/delete
|
@@ -103,7 +103,7 @@ class CloudDLPHook(GoogleBaseHook):
|
|
103
103
|
|
104
104
|
def get_conn(self) -> DlpServiceClient:
|
105
105
|
"""
|
106
|
-
|
106
|
+
Provide a client for interacting with the Cloud DLP API.
|
107
107
|
|
108
108
|
:return: Google Cloud DLP API Client
|
109
109
|
"""
|
@@ -130,7 +130,7 @@ class CloudDLPHook(GoogleBaseHook):
|
|
130
130
|
metadata: Sequence[tuple[str, str]] = (),
|
131
131
|
) -> None:
|
132
132
|
"""
|
133
|
-
|
133
|
+
Start asynchronous cancellation on a long-running DLP job.
|
134
134
|
|
135
135
|
:param dlp_job_id: ID of the DLP job resource to be cancelled.
|
136
136
|
:param project_id: (Optional) Google Cloud project ID where the
|
@@ -221,7 +221,7 @@ class CloudDLPHook(GoogleBaseHook):
|
|
221
221
|
time_to_sleep_in_seconds: int = 60,
|
222
222
|
) -> DlpJob:
|
223
223
|
"""
|
224
|
-
|
224
|
+
Create a new job to inspect storage or calculate risk metrics.
|
225
225
|
|
226
226
|
:param project_id: (Optional) Google Cloud project ID where the
|
227
227
|
DLP Instance exists. If set to None or missing, the default
|
@@ -383,7 +383,7 @@ class CloudDLPHook(GoogleBaseHook):
|
|
383
383
|
metadata: Sequence[tuple[str, str]] = (),
|
384
384
|
) -> StoredInfoType:
|
385
385
|
"""
|
386
|
-
|
386
|
+
Create a pre-built stored info type to be used for inspection.
|
387
387
|
|
388
388
|
:param organization_id: (Optional) The organization ID. Required to set this
|
389
389
|
field if parent resource is an organization.
|
@@ -480,7 +480,7 @@ class CloudDLPHook(GoogleBaseHook):
|
|
480
480
|
self, template_id, organization_id=None, project_id=None, retry=DEFAULT, timeout=None, metadata=()
|
481
481
|
) -> None:
|
482
482
|
"""
|
483
|
-
|
483
|
+
Delete a deidentify template.
|
484
484
|
|
485
485
|
:param template_id: The ID of deidentify template to be deleted.
|
486
486
|
:param organization_id: (Optional) The organization ID. Required to set this
|
@@ -529,7 +529,7 @@ class CloudDLPHook(GoogleBaseHook):
|
|
529
529
|
metadata: Sequence[tuple[str, str]] = (),
|
530
530
|
) -> None:
|
531
531
|
"""
|
532
|
-
|
532
|
+
Delete a long-running DLP job.
|
533
533
|
|
534
534
|
This method indicates that the client is no longer interested in the DLP job result.
|
535
535
|
The job will be cancelled if possible.
|
@@ -570,7 +570,7 @@ class CloudDLPHook(GoogleBaseHook):
|
|
570
570
|
metadata: Sequence[tuple[str, str]] = (),
|
571
571
|
) -> None:
|
572
572
|
"""
|
573
|
-
|
573
|
+
Delete an inspect template.
|
574
574
|
|
575
575
|
:param template_id: The ID of the inspect template to be deleted.
|
576
576
|
:param organization_id: (Optional) The organization ID. Required to set this
|
@@ -619,7 +619,7 @@ class CloudDLPHook(GoogleBaseHook):
|
|
619
619
|
metadata: Sequence[tuple[str, str]] = (),
|
620
620
|
) -> None:
|
621
621
|
"""
|
622
|
-
|
622
|
+
Delete a job trigger.
|
623
623
|
|
624
624
|
:param job_trigger_id: The ID of the DLP job trigger to be deleted.
|
625
625
|
:param project_id: (Optional) Google Cloud project ID where the
|
@@ -657,7 +657,7 @@ class CloudDLPHook(GoogleBaseHook):
|
|
657
657
|
metadata: Sequence[tuple[str, str]] = (),
|
658
658
|
) -> None:
|
659
659
|
"""
|
660
|
-
|
660
|
+
Delete a stored info type.
|
661
661
|
|
662
662
|
:param stored_info_type_id: The ID of the stored info type to be deleted.
|
663
663
|
:param organization_id: (Optional) The organization ID. Required to set this
|
@@ -706,7 +706,7 @@ class CloudDLPHook(GoogleBaseHook):
|
|
706
706
|
metadata: Sequence[tuple[str, str]] = (),
|
707
707
|
) -> DeidentifyTemplate:
|
708
708
|
"""
|
709
|
-
|
709
|
+
Get a deidentify template.
|
710
710
|
|
711
711
|
:param template_id: The ID of deidentify template to be read.
|
712
712
|
:param organization_id: (Optional) The organization ID. Required to set this
|
@@ -755,7 +755,7 @@ class CloudDLPHook(GoogleBaseHook):
|
|
755
755
|
metadata: Sequence[tuple[str, str]] = (),
|
756
756
|
) -> DlpJob:
|
757
757
|
"""
|
758
|
-
|
758
|
+
Get the latest state of a long-running Dlp Job.
|
759
759
|
|
760
760
|
:param dlp_job_id: The ID of the DLP job resource to be read.
|
761
761
|
:param project_id: (Optional) Google Cloud project ID where the
|
@@ -793,7 +793,7 @@ class CloudDLPHook(GoogleBaseHook):
|
|
793
793
|
metadata: Sequence[tuple[str, str]] = (),
|
794
794
|
) -> InspectTemplate:
|
795
795
|
"""
|
796
|
-
|
796
|
+
Get an inspect template.
|
797
797
|
|
798
798
|
:param template_id: The ID of inspect template to be read.
|
799
799
|
:param organization_id: (Optional) The organization ID. Required to set this
|
@@ -842,7 +842,7 @@ class CloudDLPHook(GoogleBaseHook):
|
|
842
842
|
metadata: Sequence[tuple[str, str]] = (),
|
843
843
|
) -> JobTrigger:
|
844
844
|
"""
|
845
|
-
|
845
|
+
Get a DLP job trigger.
|
846
846
|
|
847
847
|
:param job_trigger_id: The ID of the DLP job trigger to be read.
|
848
848
|
:param project_id: (Optional) Google Cloud project ID where the
|
@@ -880,7 +880,7 @@ class CloudDLPHook(GoogleBaseHook):
|
|
880
880
|
metadata: Sequence[tuple[str, str]] = (),
|
881
881
|
) -> StoredInfoType:
|
882
882
|
"""
|
883
|
-
|
883
|
+
Get a stored info type.
|
884
884
|
|
885
885
|
:param stored_info_type_id: The ID of the stored info type to be read.
|
886
886
|
:param organization_id: (Optional) The organization ID. Required to set this
|
@@ -931,7 +931,7 @@ class CloudDLPHook(GoogleBaseHook):
|
|
931
931
|
metadata: Sequence[tuple[str, str]] = (),
|
932
932
|
) -> InspectContentResponse:
|
933
933
|
"""
|
934
|
-
|
934
|
+
Find potentially sensitive info in content; limits input size, processing time, and output size.
|
935
935
|
|
936
936
|
:param project_id: (Optional) Google Cloud project ID where the
|
937
937
|
DLP Instance exists. If set to None or missing, the default
|
@@ -974,7 +974,7 @@ class CloudDLPHook(GoogleBaseHook):
|
|
974
974
|
metadata: Sequence[tuple[str, str]] = (),
|
975
975
|
) -> list[DeidentifyTemplate]:
|
976
976
|
"""
|
977
|
-
|
977
|
+
List deidentify templates.
|
978
978
|
|
979
979
|
:param organization_id: (Optional) The organization ID. Required to set this
|
980
980
|
field if parent resource is an organization.
|
@@ -1030,7 +1030,7 @@ class CloudDLPHook(GoogleBaseHook):
|
|
1030
1030
|
metadata: Sequence[tuple[str, str]] = (),
|
1031
1031
|
) -> list[DlpJob]:
|
1032
1032
|
"""
|
1033
|
-
|
1033
|
+
List DLP jobs that match the specified filter in the request.
|
1034
1034
|
|
1035
1035
|
:param project_id: (Optional) Google Cloud project ID where the
|
1036
1036
|
DLP Instance exists. If set to None or missing, the default
|
@@ -1074,7 +1074,7 @@ class CloudDLPHook(GoogleBaseHook):
|
|
1074
1074
|
metadata: Sequence[tuple[str, str]] = (),
|
1075
1075
|
) -> ListInfoTypesResponse:
|
1076
1076
|
"""
|
1077
|
-
|
1077
|
+
Return a list of the sensitive information types that the DLP API supports.
|
1078
1078
|
|
1079
1079
|
:param language_code: (Optional) Optional BCP-47 language code for localized info
|
1080
1080
|
type friendly names. If omitted, or if localized strings are not available,
|
@@ -1110,7 +1110,7 @@ class CloudDLPHook(GoogleBaseHook):
|
|
1110
1110
|
metadata: Sequence[tuple[str, str]] = (),
|
1111
1111
|
) -> list[InspectTemplate]:
|
1112
1112
|
"""
|
1113
|
-
|
1113
|
+
List inspect templates.
|
1114
1114
|
|
1115
1115
|
:param organization_id: (Optional) The organization ID. Required to set this
|
1116
1116
|
field if parent resource is an organization.
|
@@ -1164,7 +1164,7 @@ class CloudDLPHook(GoogleBaseHook):
|
|
1164
1164
|
metadata: Sequence[tuple[str, str]] = (),
|
1165
1165
|
) -> list[JobTrigger]:
|
1166
1166
|
"""
|
1167
|
-
|
1167
|
+
List job triggers.
|
1168
1168
|
|
1169
1169
|
:param project_id: (Optional) Google Cloud project ID where the
|
1170
1170
|
DLP Instance exists. If set to None or missing, the default
|
@@ -1208,7 +1208,7 @@ class CloudDLPHook(GoogleBaseHook):
|
|
1208
1208
|
metadata: Sequence[tuple[str, str]] = (),
|
1209
1209
|
) -> list[StoredInfoType]:
|
1210
1210
|
"""
|
1211
|
-
|
1211
|
+
List stored info types.
|
1212
1212
|
|
1213
1213
|
:param organization_id: (Optional) The organization ID. Required to set this
|
1214
1214
|
field if parent resource is an organization.
|
@@ -1363,7 +1363,7 @@ class CloudDLPHook(GoogleBaseHook):
|
|
1363
1363
|
metadata: Sequence[tuple[str, str]] = (),
|
1364
1364
|
) -> DeidentifyTemplate:
|
1365
1365
|
"""
|
1366
|
-
|
1366
|
+
Update the deidentify template.
|
1367
1367
|
|
1368
1368
|
:param template_id: The ID of deidentify template to be updated.
|
1369
1369
|
:param organization_id: (Optional) The organization ID. Required to set this
|
@@ -1418,7 +1418,7 @@ class CloudDLPHook(GoogleBaseHook):
|
|
1418
1418
|
metadata: Sequence[tuple[str, str]] = (),
|
1419
1419
|
) -> InspectTemplate:
|
1420
1420
|
"""
|
1421
|
-
|
1421
|
+
Update the inspect template.
|
1422
1422
|
|
1423
1423
|
:param template_id: The ID of the inspect template to be updated.
|
1424
1424
|
:param organization_id: (Optional) The organization ID. Required to set this
|
@@ -1472,7 +1472,7 @@ class CloudDLPHook(GoogleBaseHook):
|
|
1472
1472
|
metadata: Sequence[tuple[str, str]] = (),
|
1473
1473
|
) -> JobTrigger:
|
1474
1474
|
"""
|
1475
|
-
|
1475
|
+
Update a job trigger.
|
1476
1476
|
|
1477
1477
|
:param job_trigger_id: The ID of the DLP job trigger to be updated.
|
1478
1478
|
:param project_id: (Optional) Google Cloud project ID where the
|
@@ -1520,7 +1520,7 @@ class CloudDLPHook(GoogleBaseHook):
|
|
1520
1520
|
metadata: Sequence[tuple[str, str]] = (),
|
1521
1521
|
) -> StoredInfoType:
|
1522
1522
|
"""
|
1523
|
-
|
1523
|
+
Update the stored info type by creating a new version.
|
1524
1524
|
|
1525
1525
|
:param stored_info_type_id: The ID of the stored info type to be updated.
|
1526
1526
|
:param organization_id: (Optional) The organization ID. Required to set this
|
@@ -74,7 +74,9 @@ def _fallback_object_url_to_object_name_and_bucket_name(
|
|
74
74
|
object_name_keyword_arg_name="object_name",
|
75
75
|
) -> Callable[[T], T]:
|
76
76
|
"""
|
77
|
-
|
77
|
+
Convert object URL parameter to object name and bucket name parameter.
|
78
|
+
|
79
|
+
This method is a Decorator factory.
|
78
80
|
|
79
81
|
:param object_url_keyword_arg_name: Name of the object URL parameter
|
80
82
|
:param bucket_name_keyword_arg_name: Name of the bucket name parameter
|
@@ -160,7 +162,7 @@ class GCSHook(GoogleBaseHook):
|
|
160
162
|
)
|
161
163
|
|
162
164
|
def get_conn(self) -> storage.Client:
|
163
|
-
"""
|
165
|
+
"""Return a Google Cloud Storage service object."""
|
164
166
|
if not self._conn:
|
165
167
|
self._conn = storage.Client(
|
166
168
|
credentials=self.get_credentials(), client_info=CLIENT_INFO, project=self.project_id
|
@@ -176,7 +178,7 @@ class GCSHook(GoogleBaseHook):
|
|
176
178
|
destination_object: str | None = None,
|
177
179
|
) -> None:
|
178
180
|
"""
|
179
|
-
|
181
|
+
Copy an object from a bucket to another, with renaming if requested.
|
180
182
|
|
181
183
|
destination_bucket or destination_object can be omitted, in which case
|
182
184
|
source bucket/object is used, but not both.
|
@@ -304,7 +306,7 @@ class GCSHook(GoogleBaseHook):
|
|
304
306
|
user_project: str | None = None,
|
305
307
|
) -> str | bytes:
|
306
308
|
"""
|
307
|
-
|
309
|
+
Download a file from Google Cloud Storage.
|
308
310
|
|
309
311
|
When no filename is supplied, the operator loads the file into memory and returns its
|
310
312
|
content. When a filename is supplied, it writes the file to the specified location and
|
@@ -366,7 +368,7 @@ class GCSHook(GoogleBaseHook):
|
|
366
368
|
num_max_attempts: int | None = 1,
|
367
369
|
) -> bytes:
|
368
370
|
"""
|
369
|
-
|
371
|
+
Download a file from Google Cloud Storage.
|
370
372
|
|
371
373
|
When no filename is supplied, the operator loads the file into memory and returns its
|
372
374
|
content. When a filename is supplied, it writes the file to the specified location and
|
@@ -399,7 +401,7 @@ class GCSHook(GoogleBaseHook):
|
|
399
401
|
user_project: str | None = None,
|
400
402
|
) -> Generator[IO[bytes], None, None]:
|
401
403
|
"""
|
402
|
-
|
404
|
+
Download the file to a temporary directory and returns a file handle.
|
403
405
|
|
404
406
|
You can use this method by passing the bucket_name and object_name parameters
|
405
407
|
or just object_url parameter.
|
@@ -435,7 +437,7 @@ class GCSHook(GoogleBaseHook):
|
|
435
437
|
user_project: str | None = None,
|
436
438
|
) -> Generator[IO[bytes], None, None]:
|
437
439
|
"""
|
438
|
-
|
440
|
+
Create temporary file, returns a file handle and uploads the files content on close.
|
439
441
|
|
440
442
|
You can use this method by passing the bucket_name and object_name parameters
|
441
443
|
or just object_url parameter.
|
@@ -478,7 +480,7 @@ class GCSHook(GoogleBaseHook):
|
|
478
480
|
user_project: str | None = None,
|
479
481
|
) -> None:
|
480
482
|
"""
|
481
|
-
|
483
|
+
Upload a local file or file data as string or bytes to Google Cloud Storage.
|
482
484
|
|
483
485
|
:param bucket_name: The bucket to upload to.
|
484
486
|
:param object_name: The object name to set when uploading the file.
|
@@ -498,7 +500,7 @@ class GCSHook(GoogleBaseHook):
|
|
498
500
|
|
499
501
|
def _call_with_retry(f: Callable[[], None]) -> None:
|
500
502
|
"""
|
501
|
-
|
503
|
+
Upload a file or a string with a retry mechanism and exponential back-off.
|
502
504
|
|
503
505
|
:param f: Callable that should be retried.
|
504
506
|
"""
|
@@ -572,7 +574,7 @@ class GCSHook(GoogleBaseHook):
|
|
572
574
|
|
573
575
|
def exists(self, bucket_name: str, object_name: str, retry: Retry = DEFAULT_RETRY) -> bool:
|
574
576
|
"""
|
575
|
-
|
577
|
+
Check for the existence of a file in Google Cloud Storage.
|
576
578
|
|
577
579
|
:param bucket_name: The Google Cloud Storage bucket where the object is.
|
578
580
|
:param object_name: The name of the blob_name to check in the Google cloud
|
@@ -601,7 +603,7 @@ class GCSHook(GoogleBaseHook):
|
|
601
603
|
|
602
604
|
def is_updated_after(self, bucket_name: str, object_name: str, ts: datetime) -> bool:
|
603
605
|
"""
|
604
|
-
|
606
|
+
Check if an blob_name is updated in Google Cloud Storage.
|
605
607
|
|
606
608
|
:param bucket_name: The Google Cloud Storage bucket where the object is.
|
607
609
|
:param object_name: The name of the object to check in the Google cloud
|
@@ -621,7 +623,7 @@ class GCSHook(GoogleBaseHook):
|
|
621
623
|
self, bucket_name: str, object_name: str, min_ts: datetime, max_ts: datetime
|
622
624
|
) -> bool:
|
623
625
|
"""
|
624
|
-
|
626
|
+
Check if an blob_name is updated in Google Cloud Storage.
|
625
627
|
|
626
628
|
:param bucket_name: The Google Cloud Storage bucket where the object is.
|
627
629
|
:param object_name: The name of the object to check in the Google cloud
|
@@ -642,7 +644,7 @@ class GCSHook(GoogleBaseHook):
|
|
642
644
|
|
643
645
|
def is_updated_before(self, bucket_name: str, object_name: str, ts: datetime) -> bool:
|
644
646
|
"""
|
645
|
-
|
647
|
+
Check if an blob_name is updated before given time in Google Cloud Storage.
|
646
648
|
|
647
649
|
:param bucket_name: The Google Cloud Storage bucket where the object is.
|
648
650
|
:param object_name: The name of the object to check in the Google cloud
|
@@ -680,7 +682,7 @@ class GCSHook(GoogleBaseHook):
|
|
680
682
|
|
681
683
|
def delete(self, bucket_name: str, object_name: str) -> None:
|
682
684
|
"""
|
683
|
-
|
685
|
+
Delete an object from the bucket.
|
684
686
|
|
685
687
|
:param bucket_name: name of the bucket, where the object resides
|
686
688
|
:param object_name: name of the object to delete
|
@@ -954,7 +956,7 @@ class GCSHook(GoogleBaseHook):
|
|
954
956
|
|
955
957
|
def get_size(self, bucket_name: str, object_name: str) -> int:
|
956
958
|
"""
|
957
|
-
|
959
|
+
Get the size of a file in Google Cloud Storage.
|
958
960
|
|
959
961
|
:param bucket_name: The Google Cloud Storage bucket where the blob_name is.
|
960
962
|
:param object_name: The name of the object to check in the Google
|
@@ -971,7 +973,7 @@ class GCSHook(GoogleBaseHook):
|
|
971
973
|
|
972
974
|
def get_crc32c(self, bucket_name: str, object_name: str):
|
973
975
|
"""
|
974
|
-
|
976
|
+
Get the CRC32c checksum of an object in Google Cloud Storage.
|
975
977
|
|
976
978
|
:param bucket_name: The Google Cloud Storage bucket where the blob_name is.
|
977
979
|
:param object_name: The name of the object to check in the Google cloud
|
@@ -991,7 +993,7 @@ class GCSHook(GoogleBaseHook):
|
|
991
993
|
|
992
994
|
def get_md5hash(self, bucket_name: str, object_name: str) -> str:
|
993
995
|
"""
|
994
|
-
|
996
|
+
Get the MD5 hash of an object in Google Cloud Storage.
|
995
997
|
|
996
998
|
:param bucket_name: The Google Cloud Storage bucket where the blob_name is.
|
997
999
|
:param object_name: The name of the object to check in the Google cloud
|
@@ -1016,7 +1018,7 @@ class GCSHook(GoogleBaseHook):
|
|
1016
1018
|
labels: dict | None = None,
|
1017
1019
|
) -> str:
|
1018
1020
|
"""
|
1019
|
-
|
1021
|
+
Create a new bucket.
|
1020
1022
|
|
1021
1023
|
Google Cloud Storage uses a flat namespace, so you can't
|
1022
1024
|
create a bucket with a name that is already in use.
|
@@ -1076,7 +1078,7 @@ class GCSHook(GoogleBaseHook):
|
|
1076
1078
|
self, bucket_name: str, entity: str, role: str, user_project: str | None = None
|
1077
1079
|
) -> None:
|
1078
1080
|
"""
|
1079
|
-
|
1081
|
+
Create a new ACL entry on the specified bucket_name.
|
1080
1082
|
|
1081
1083
|
See: https://cloud.google.com/storage/docs/json_api/v1/bucketAccessControls/insert
|
1082
1084
|
|
@@ -1111,7 +1113,7 @@ class GCSHook(GoogleBaseHook):
|
|
1111
1113
|
user_project: str | None = None,
|
1112
1114
|
) -> None:
|
1113
1115
|
"""
|
1114
|
-
|
1116
|
+
Create a new ACL entry on the specified object.
|
1115
1117
|
|
1116
1118
|
See: https://cloud.google.com/storage/docs/json_api/v1/objectAccessControls/insert
|
1117
1119
|
|
@@ -1184,7 +1186,7 @@ class GCSHook(GoogleBaseHook):
|
|
1184
1186
|
delete_extra_files: bool = False,
|
1185
1187
|
) -> None:
|
1186
1188
|
"""
|
1187
|
-
|
1189
|
+
Synchronize the contents of the buckets.
|
1188
1190
|
|
1189
1191
|
Parameters ``source_object`` and ``destination_object`` describe the root sync directories. If they
|
1190
1192
|
are not passed, the entire bucket will be synchronized. If they are passed, they should point
|
@@ -1342,7 +1344,7 @@ def parse_json_from_gcs(
|
|
1342
1344
|
impersonation_chain: str | Sequence[str] | None = None,
|
1343
1345
|
) -> Any:
|
1344
1346
|
"""
|
1345
|
-
|
1347
|
+
Download and parses json file from Google cloud Storage.
|
1346
1348
|
|
1347
1349
|
:param gcp_conn_id: Airflow Google Cloud connection ID.
|
1348
1350
|
:param file_uri: full path to json file
|
@@ -1397,7 +1399,7 @@ class GCSAsyncHook(GoogleBaseAsyncHook):
|
|
1397
1399
|
sync_hook_class = GCSHook
|
1398
1400
|
|
1399
1401
|
async def get_storage_client(self, session: ClientSession) -> Storage:
|
1400
|
-
"""
|
1402
|
+
"""Return a Google Cloud Storage service object."""
|
1401
1403
|
token = await self.get_token(session=session)
|
1402
1404
|
return Storage(
|
1403
1405
|
token=token,
|
@@ -49,7 +49,7 @@ class GoogleDeploymentManagerHook(GoogleBaseHook):
|
|
49
49
|
)
|
50
50
|
|
51
51
|
def get_conn(self) -> Resource:
|
52
|
-
"""
|
52
|
+
"""Return a Google Deployment Manager service object."""
|
53
53
|
http_authorized = self._authorize()
|
54
54
|
return build("deploymentmanager", "v2", http=http_authorized, cache_discovery=False)
|
55
55
|
|
@@ -61,7 +61,7 @@ class GoogleDeploymentManagerHook(GoogleBaseHook):
|
|
61
61
|
order_by: str | None = None,
|
62
62
|
) -> list[dict[str, Any]]:
|
63
63
|
"""
|
64
|
-
|
64
|
+
List deployments in a google cloud project.
|
65
65
|
|
66
66
|
:param project_id: The project ID for this request.
|
67
67
|
:param deployment_filter: A filter expression which limits resources returned in the response.
|
@@ -84,7 +84,7 @@ class GoogleDeploymentManagerHook(GoogleBaseHook):
|
|
84
84
|
self, project_id: str | None, deployment: str | None = None, delete_policy: str | None = None
|
85
85
|
) -> None:
|
86
86
|
"""
|
87
|
-
|
87
|
+
Delete a deployment and all associated resources in a google cloud project.
|
88
88
|
|
89
89
|
:param project_id: The project ID for this request.
|
90
90
|
:param deployment: The name of the deployment for this request.
|
@@ -32,12 +32,12 @@ if TYPE_CHECKING:
|
|
32
32
|
|
33
33
|
|
34
34
|
def _b64encode(s: bytes) -> str:
|
35
|
-
"""
|
35
|
+
"""Encode a Base64 bytes object to a string."""
|
36
36
|
return base64.b64encode(s).decode("ascii")
|
37
37
|
|
38
38
|
|
39
39
|
def _b64decode(s: str) -> bytes:
|
40
|
-
"""
|
40
|
+
"""Decode a Base64 string to bytes."""
|
41
41
|
return base64.b64decode(s.encode("utf-8"))
|
42
42
|
|
43
43
|
|
@@ -75,7 +75,7 @@ class CloudKMSHook(GoogleBaseHook):
|
|
75
75
|
|
76
76
|
def get_conn(self) -> KeyManagementServiceClient:
|
77
77
|
"""
|
78
|
-
|
78
|
+
Retrieve connection to Cloud Key Management service.
|
79
79
|
|
80
80
|
:return: Cloud Key Management service object
|
81
81
|
"""
|