apache-airflow-providers-google 10.17.0rc1__py3-none-any.whl → 10.18.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/__init__.py +3 -3
- airflow/providers/google/cloud/hooks/automl.py +1 -1
- airflow/providers/google/cloud/hooks/bigquery.py +64 -33
- airflow/providers/google/cloud/hooks/cloud_composer.py +250 -2
- airflow/providers/google/cloud/hooks/cloud_sql.py +154 -7
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +7 -2
- airflow/providers/google/cloud/hooks/compute_ssh.py +2 -1
- airflow/providers/google/cloud/hooks/dataflow.py +246 -32
- airflow/providers/google/cloud/hooks/dataplex.py +6 -2
- airflow/providers/google/cloud/hooks/dlp.py +14 -14
- airflow/providers/google/cloud/hooks/gcs.py +6 -2
- airflow/providers/google/cloud/hooks/gdm.py +2 -2
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +2 -2
- airflow/providers/google/cloud/hooks/mlengine.py +8 -4
- airflow/providers/google/cloud/hooks/pubsub.py +1 -1
- airflow/providers/google/cloud/hooks/secret_manager.py +252 -4
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +1431 -74
- airflow/providers/google/cloud/links/vertex_ai.py +2 -1
- airflow/providers/google/cloud/log/gcs_task_handler.py +2 -1
- airflow/providers/google/cloud/operators/automl.py +13 -12
- airflow/providers/google/cloud/operators/bigquery.py +36 -22
- airflow/providers/google/cloud/operators/bigquery_dts.py +4 -3
- airflow/providers/google/cloud/operators/bigtable.py +7 -6
- airflow/providers/google/cloud/operators/cloud_build.py +12 -11
- airflow/providers/google/cloud/operators/cloud_composer.py +147 -2
- airflow/providers/google/cloud/operators/cloud_memorystore.py +17 -16
- airflow/providers/google/cloud/operators/cloud_sql.py +60 -17
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +35 -16
- airflow/providers/google/cloud/operators/compute.py +12 -11
- airflow/providers/google/cloud/operators/datacatalog.py +21 -20
- airflow/providers/google/cloud/operators/dataflow.py +59 -42
- airflow/providers/google/cloud/operators/datafusion.py +11 -10
- airflow/providers/google/cloud/operators/datapipeline.py +3 -2
- airflow/providers/google/cloud/operators/dataprep.py +5 -4
- airflow/providers/google/cloud/operators/dataproc.py +19 -16
- airflow/providers/google/cloud/operators/datastore.py +8 -7
- airflow/providers/google/cloud/operators/dlp.py +31 -30
- airflow/providers/google/cloud/operators/functions.py +4 -3
- airflow/providers/google/cloud/operators/gcs.py +66 -41
- airflow/providers/google/cloud/operators/kubernetes_engine.py +232 -12
- airflow/providers/google/cloud/operators/life_sciences.py +2 -1
- airflow/providers/google/cloud/operators/mlengine.py +11 -10
- airflow/providers/google/cloud/operators/pubsub.py +6 -5
- airflow/providers/google/cloud/operators/spanner.py +7 -6
- airflow/providers/google/cloud/operators/speech_to_text.py +2 -1
- airflow/providers/google/cloud/operators/stackdriver.py +11 -10
- airflow/providers/google/cloud/operators/tasks.py +14 -13
- airflow/providers/google/cloud/operators/text_to_speech.py +2 -1
- airflow/providers/google/cloud/operators/translate_speech.py +2 -1
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +333 -26
- airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +20 -12
- airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +0 -1
- airflow/providers/google/cloud/operators/vision.py +13 -12
- airflow/providers/google/cloud/operators/workflows.py +10 -9
- airflow/providers/google/cloud/secrets/secret_manager.py +2 -1
- airflow/providers/google/cloud/sensors/bigquery_dts.py +2 -1
- airflow/providers/google/cloud/sensors/bigtable.py +2 -1
- airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +2 -1
- airflow/providers/google/cloud/sensors/dataflow.py +239 -52
- airflow/providers/google/cloud/sensors/datafusion.py +2 -1
- airflow/providers/google/cloud/sensors/dataproc.py +3 -2
- airflow/providers/google/cloud/sensors/gcs.py +14 -12
- airflow/providers/google/cloud/sensors/tasks.py +2 -1
- airflow/providers/google/cloud/sensors/workflows.py +2 -1
- airflow/providers/google/cloud/transfers/adls_to_gcs.py +8 -2
- airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +7 -1
- airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +7 -1
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +1 -1
- airflow/providers/google/cloud/transfers/bigquery_to_sql.py +1 -0
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +5 -6
- airflow/providers/google/cloud/transfers/gcs_to_gcs.py +22 -12
- airflow/providers/google/cloud/triggers/bigquery.py +14 -3
- airflow/providers/google/cloud/triggers/cloud_composer.py +68 -0
- airflow/providers/google/cloud/triggers/cloud_sql.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +2 -1
- airflow/providers/google/cloud/triggers/dataflow.py +504 -4
- airflow/providers/google/cloud/triggers/dataproc.py +110 -26
- airflow/providers/google/cloud/triggers/mlengine.py +2 -1
- airflow/providers/google/cloud/triggers/vertex_ai.py +94 -0
- airflow/providers/google/common/hooks/base_google.py +45 -7
- airflow/providers/google/firebase/hooks/firestore.py +2 -2
- airflow/providers/google/firebase/operators/firestore.py +2 -1
- airflow/providers/google/get_provider_info.py +3 -2
- {apache_airflow_providers_google-10.17.0rc1.dist-info → apache_airflow_providers_google-10.18.0rc1.dist-info}/METADATA +8 -8
- {apache_airflow_providers_google-10.17.0rc1.dist-info → apache_airflow_providers_google-10.18.0rc1.dist-info}/RECORD +88 -89
- airflow/providers/google/cloud/example_dags/example_cloud_sql_query.py +0 -289
- {apache_airflow_providers_google-10.17.0rc1.dist-info → apache_airflow_providers_google-10.18.0rc1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_google-10.17.0rc1.dist-info → apache_airflow_providers_google-10.18.0rc1.dist-info}/entry_points.txt +0 -0
@@ -24,6 +24,7 @@ from typing import TYPE_CHECKING, Sequence
|
|
24
24
|
from airflow.providers.google.cloud.hooks.dataprep import GoogleDataprepHook
|
25
25
|
from airflow.providers.google.cloud.links.dataprep import DataprepFlowLink, DataprepJobGroupLink
|
26
26
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
27
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
27
28
|
|
28
29
|
if TYPE_CHECKING:
|
29
30
|
from airflow.utils.context import Context
|
@@ -92,7 +93,7 @@ class DataprepGetJobGroupOperator(GoogleCloudBaseOperator):
|
|
92
93
|
self,
|
93
94
|
*,
|
94
95
|
dataprep_conn_id: str = "dataprep_default",
|
95
|
-
project_id: str
|
96
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
96
97
|
job_group_id: int | str,
|
97
98
|
embed: str,
|
98
99
|
include_deleted: bool,
|
@@ -149,7 +150,7 @@ class DataprepRunJobGroupOperator(GoogleCloudBaseOperator):
|
|
149
150
|
def __init__(
|
150
151
|
self,
|
151
152
|
*,
|
152
|
-
project_id: str
|
153
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
153
154
|
dataprep_conn_id: str = "dataprep_default",
|
154
155
|
body_request: dict,
|
155
156
|
**kwargs,
|
@@ -198,7 +199,7 @@ class DataprepCopyFlowOperator(GoogleCloudBaseOperator):
|
|
198
199
|
def __init__(
|
199
200
|
self,
|
200
201
|
*,
|
201
|
-
project_id: str
|
202
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
202
203
|
dataprep_conn_id: str = "dataprep_default",
|
203
204
|
flow_id: int | str,
|
204
205
|
name: str = "",
|
@@ -280,7 +281,7 @@ class DataprepRunFlowOperator(GoogleCloudBaseOperator):
|
|
280
281
|
def __init__(
|
281
282
|
self,
|
282
283
|
*,
|
283
|
-
project_id: str
|
284
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
284
285
|
flow_id: int | str,
|
285
286
|
body_request: dict,
|
286
287
|
dataprep_conn_id: str = "dataprep_default",
|
@@ -63,6 +63,7 @@ from airflow.providers.google.cloud.triggers.dataproc import (
|
|
63
63
|
DataprocSubmitTrigger,
|
64
64
|
)
|
65
65
|
from airflow.providers.google.cloud.utils.dataproc import DataprocOperationType
|
66
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
66
67
|
from airflow.utils import timezone
|
67
68
|
|
68
69
|
if TYPE_CHECKING:
|
@@ -627,7 +628,7 @@ class DataprocCreateClusterOperator(GoogleCloudBaseOperator):
|
|
627
628
|
*,
|
628
629
|
cluster_name: str,
|
629
630
|
region: str,
|
630
|
-
project_id: str
|
631
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
631
632
|
cluster_config: dict | Cluster | None = None,
|
632
633
|
virtual_cluster_config: dict | None = None,
|
633
634
|
labels: dict | None = None,
|
@@ -815,6 +816,7 @@ class DataprocCreateClusterOperator(GoogleCloudBaseOperator):
|
|
815
816
|
gcp_conn_id=self.gcp_conn_id,
|
816
817
|
impersonation_chain=self.impersonation_chain,
|
817
818
|
polling_interval_seconds=self.polling_interval_seconds,
|
819
|
+
delete_on_error=self.delete_on_error,
|
818
820
|
),
|
819
821
|
method_name="execute_complete",
|
820
822
|
)
|
@@ -928,7 +930,7 @@ class DataprocScaleClusterOperator(GoogleCloudBaseOperator):
|
|
928
930
|
self,
|
929
931
|
*,
|
930
932
|
cluster_name: str,
|
931
|
-
project_id: str
|
933
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
932
934
|
region: str = "global",
|
933
935
|
num_workers: int = 2,
|
934
936
|
num_preemptible_workers: int = 0,
|
@@ -1047,7 +1049,7 @@ class DataprocDeleteClusterOperator(GoogleCloudBaseOperator):
|
|
1047
1049
|
*,
|
1048
1050
|
region: str,
|
1049
1051
|
cluster_name: str,
|
1050
|
-
project_id: str
|
1052
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
1051
1053
|
cluster_uuid: str | None = None,
|
1052
1054
|
request_id: str | None = None,
|
1053
1055
|
retry: AsyncRetry | _MethodDefault = DEFAULT,
|
@@ -1173,7 +1175,7 @@ class _DataprocStartStopClusterBaseOperator(GoogleCloudBaseOperator):
|
|
1173
1175
|
*,
|
1174
1176
|
cluster_name: str,
|
1175
1177
|
region: str,
|
1176
|
-
project_id: str
|
1178
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
1177
1179
|
cluster_uuid: str | None = None,
|
1178
1180
|
request_id: str | None = None,
|
1179
1181
|
retry: AsyncRetry | _MethodDefault = DEFAULT,
|
@@ -1372,7 +1374,7 @@ class DataprocJobBaseOperator(GoogleCloudBaseOperator):
|
|
1372
1374
|
region: str,
|
1373
1375
|
job_name: str = "{{task.task_id}}_{{ds_nodash}}",
|
1374
1376
|
cluster_name: str = "cluster-1",
|
1375
|
-
project_id: str
|
1377
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
1376
1378
|
dataproc_properties: dict | None = None,
|
1377
1379
|
dataproc_jars: list[str] | None = None,
|
1378
1380
|
gcp_conn_id: str = "google_cloud_default",
|
@@ -2135,7 +2137,7 @@ class DataprocCreateWorkflowTemplateOperator(GoogleCloudBaseOperator):
|
|
2135
2137
|
*,
|
2136
2138
|
template: dict,
|
2137
2139
|
region: str,
|
2138
|
-
project_id: str
|
2140
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
2139
2141
|
retry: Retry | _MethodDefault = DEFAULT,
|
2140
2142
|
timeout: float | None = None,
|
2141
2143
|
metadata: Sequence[tuple[str, str]] = (),
|
@@ -2229,7 +2231,7 @@ class DataprocInstantiateWorkflowTemplateOperator(GoogleCloudBaseOperator):
|
|
2229
2231
|
*,
|
2230
2232
|
template_id: str,
|
2231
2233
|
region: str,
|
2232
|
-
project_id: str
|
2234
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
2233
2235
|
version: int | None = None,
|
2234
2236
|
request_id: str | None = None,
|
2235
2237
|
parameters: dict[str, str] | None = None,
|
@@ -2376,7 +2378,7 @@ class DataprocInstantiateInlineWorkflowTemplateOperator(GoogleCloudBaseOperator)
|
|
2376
2378
|
*,
|
2377
2379
|
template: dict,
|
2378
2380
|
region: str,
|
2379
|
-
project_id: str
|
2381
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
2380
2382
|
request_id: str | None = None,
|
2381
2383
|
retry: AsyncRetry | _MethodDefault = DEFAULT,
|
2382
2384
|
timeout: float | None = None,
|
@@ -2513,7 +2515,7 @@ class DataprocSubmitJobOperator(GoogleCloudBaseOperator):
|
|
2513
2515
|
*,
|
2514
2516
|
job: dict,
|
2515
2517
|
region: str,
|
2516
|
-
project_id: str
|
2518
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
2517
2519
|
request_id: str | None = None,
|
2518
2520
|
retry: Retry | _MethodDefault = DEFAULT,
|
2519
2521
|
timeout: float | None = None,
|
@@ -2590,6 +2592,7 @@ class DataprocSubmitJobOperator(GoogleCloudBaseOperator):
|
|
2590
2592
|
gcp_conn_id=self.gcp_conn_id,
|
2591
2593
|
impersonation_chain=self.impersonation_chain,
|
2592
2594
|
polling_interval_seconds=self.polling_interval_seconds,
|
2595
|
+
cancel_on_kill=self.cancel_on_kill,
|
2593
2596
|
),
|
2594
2597
|
method_name="execute_complete",
|
2595
2598
|
)
|
@@ -2682,7 +2685,7 @@ class DataprocUpdateClusterOperator(GoogleCloudBaseOperator):
|
|
2682
2685
|
graceful_decommission_timeout: dict | Duration,
|
2683
2686
|
region: str,
|
2684
2687
|
request_id: str | None = None,
|
2685
|
-
project_id: str
|
2688
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
2686
2689
|
retry: AsyncRetry | _MethodDefault = DEFAULT,
|
2687
2690
|
timeout: float | None = None,
|
2688
2691
|
metadata: Sequence[tuple[str, str]] = (),
|
@@ -2816,7 +2819,7 @@ class DataprocDiagnoseClusterOperator(GoogleCloudBaseOperator):
|
|
2816
2819
|
*,
|
2817
2820
|
region: str,
|
2818
2821
|
cluster_name: str,
|
2819
|
-
project_id: str
|
2822
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
2820
2823
|
tarball_gcs_dir: str | None = None,
|
2821
2824
|
diagnosis_interval: dict | Interval | None = None,
|
2822
2825
|
jobs: MutableSequence[str] | None = None,
|
@@ -2954,7 +2957,7 @@ class DataprocCreateBatchOperator(GoogleCloudBaseOperator):
|
|
2954
2957
|
self,
|
2955
2958
|
*,
|
2956
2959
|
region: str | None = None,
|
2957
|
-
project_id: str
|
2960
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
2958
2961
|
batch: dict | Batch,
|
2959
2962
|
batch_id: str,
|
2960
2963
|
request_id: str | None = None,
|
@@ -3146,7 +3149,7 @@ class DataprocDeleteBatchOperator(GoogleCloudBaseOperator):
|
|
3146
3149
|
*,
|
3147
3150
|
batch_id: str,
|
3148
3151
|
region: str,
|
3149
|
-
project_id: str
|
3152
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
3150
3153
|
retry: Retry | _MethodDefault = DEFAULT,
|
3151
3154
|
timeout: float | None = None,
|
3152
3155
|
metadata: Sequence[tuple[str, str]] = (),
|
@@ -3210,7 +3213,7 @@ class DataprocGetBatchOperator(GoogleCloudBaseOperator):
|
|
3210
3213
|
*,
|
3211
3214
|
batch_id: str,
|
3212
3215
|
region: str,
|
3213
|
-
project_id: str
|
3216
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
3214
3217
|
retry: Retry | _MethodDefault = DEFAULT,
|
3215
3218
|
timeout: float | None = None,
|
3216
3219
|
metadata: Sequence[tuple[str, str]] = (),
|
@@ -3285,7 +3288,7 @@ class DataprocListBatchesOperator(GoogleCloudBaseOperator):
|
|
3285
3288
|
self,
|
3286
3289
|
*,
|
3287
3290
|
region: str,
|
3288
|
-
project_id: str
|
3291
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
3289
3292
|
page_size: int | None = None,
|
3290
3293
|
page_token: str | None = None,
|
3291
3294
|
retry: Retry | _MethodDefault = DEFAULT,
|
@@ -3358,7 +3361,7 @@ class DataprocCancelOperationOperator(GoogleCloudBaseOperator):
|
|
3358
3361
|
*,
|
3359
3362
|
operation_name: str,
|
3360
3363
|
region: str,
|
3361
|
-
project_id: str
|
3364
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
3362
3365
|
retry: Retry | _MethodDefault = DEFAULT,
|
3363
3366
|
timeout: float | None = None,
|
3364
3367
|
metadata: Sequence[tuple[str, str]] = (),
|
@@ -29,6 +29,7 @@ from airflow.providers.google.cloud.links.datastore import (
|
|
29
29
|
CloudDatastoreImportExportLink,
|
30
30
|
)
|
31
31
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
32
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
32
33
|
from airflow.providers.google.common.links.storage import StorageLink
|
33
34
|
|
34
35
|
if TYPE_CHECKING:
|
@@ -90,7 +91,7 @@ class CloudDatastoreExportEntitiesOperator(GoogleCloudBaseOperator):
|
|
90
91
|
labels: dict | None = None,
|
91
92
|
polling_interval_in_seconds: int = 10,
|
92
93
|
overwrite_existing: bool = False,
|
93
|
-
project_id: str
|
94
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
94
95
|
impersonation_chain: str | Sequence[str] | None = None,
|
95
96
|
**kwargs,
|
96
97
|
) -> None:
|
@@ -194,7 +195,7 @@ class CloudDatastoreImportEntitiesOperator(GoogleCloudBaseOperator):
|
|
194
195
|
labels: dict | None = None,
|
195
196
|
datastore_conn_id: str = "google_cloud_default",
|
196
197
|
polling_interval_in_seconds: float = 10,
|
197
|
-
project_id: str
|
198
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
198
199
|
impersonation_chain: str | Sequence[str] | None = None,
|
199
200
|
**kwargs,
|
200
201
|
) -> None:
|
@@ -268,7 +269,7 @@ class CloudDatastoreAllocateIdsOperator(GoogleCloudBaseOperator):
|
|
268
269
|
self,
|
269
270
|
*,
|
270
271
|
partial_keys: list,
|
271
|
-
project_id: str
|
272
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
272
273
|
gcp_conn_id: str = "google_cloud_default",
|
273
274
|
impersonation_chain: str | Sequence[str] | None = None,
|
274
275
|
**kwargs,
|
@@ -326,7 +327,7 @@ class CloudDatastoreBeginTransactionOperator(GoogleCloudBaseOperator):
|
|
326
327
|
self,
|
327
328
|
*,
|
328
329
|
transaction_options: dict[str, Any],
|
329
|
-
project_id: str
|
330
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
330
331
|
gcp_conn_id: str = "google_cloud_default",
|
331
332
|
impersonation_chain: str | Sequence[str] | None = None,
|
332
333
|
**kwargs,
|
@@ -384,7 +385,7 @@ class CloudDatastoreCommitOperator(GoogleCloudBaseOperator):
|
|
384
385
|
self,
|
385
386
|
*,
|
386
387
|
body: dict[str, Any],
|
387
|
-
project_id: str
|
388
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
388
389
|
gcp_conn_id: str = "google_cloud_default",
|
389
390
|
impersonation_chain: str | Sequence[str] | None = None,
|
390
391
|
**kwargs,
|
@@ -442,7 +443,7 @@ class CloudDatastoreRollbackOperator(GoogleCloudBaseOperator):
|
|
442
443
|
self,
|
443
444
|
*,
|
444
445
|
transaction: str,
|
445
|
-
project_id: str
|
446
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
446
447
|
gcp_conn_id: str = "google_cloud_default",
|
447
448
|
impersonation_chain: str | Sequence[str] | None = None,
|
448
449
|
**kwargs,
|
@@ -498,7 +499,7 @@ class CloudDatastoreRunQueryOperator(GoogleCloudBaseOperator):
|
|
498
499
|
self,
|
499
500
|
*,
|
500
501
|
body: dict[str, Any],
|
501
|
-
project_id: str
|
502
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
502
503
|
gcp_conn_id: str = "google_cloud_default",
|
503
504
|
impersonation_chain: str | Sequence[str] | None = None,
|
504
505
|
**kwargs,
|
@@ -59,6 +59,7 @@ from airflow.providers.google.cloud.links.data_loss_prevention import (
|
|
59
59
|
CloudDLPPossibleInfoTypesListLink,
|
60
60
|
)
|
61
61
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
62
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
62
63
|
|
63
64
|
if TYPE_CHECKING:
|
64
65
|
from google.api_core.retry import Retry
|
@@ -108,7 +109,7 @@ class CloudDLPCancelDLPJobOperator(GoogleCloudBaseOperator):
|
|
108
109
|
self,
|
109
110
|
*,
|
110
111
|
dlp_job_id: str,
|
111
|
-
project_id: str
|
112
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
112
113
|
retry: Retry | _MethodDefault = DEFAULT,
|
113
114
|
timeout: float | None = None,
|
114
115
|
metadata: Sequence[tuple[str, str]] = (),
|
@@ -195,7 +196,7 @@ class CloudDLPCreateDeidentifyTemplateOperator(GoogleCloudBaseOperator):
|
|
195
196
|
self,
|
196
197
|
*,
|
197
198
|
organization_id: str | None = None,
|
198
|
-
project_id: str
|
199
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
199
200
|
deidentify_template: dict | DeidentifyTemplate | None = None,
|
200
201
|
template_id: str | None = None,
|
201
202
|
retry: Retry | _MethodDefault = DEFAULT,
|
@@ -304,7 +305,7 @@ class CloudDLPCreateDLPJobOperator(GoogleCloudBaseOperator):
|
|
304
305
|
def __init__(
|
305
306
|
self,
|
306
307
|
*,
|
307
|
-
project_id: str
|
308
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
308
309
|
inspect_job: dict | InspectJobConfig | None = None,
|
309
310
|
risk_job: dict | RiskAnalysisJobConfig | None = None,
|
310
311
|
job_id: str | None = None,
|
@@ -416,7 +417,7 @@ class CloudDLPCreateInspectTemplateOperator(GoogleCloudBaseOperator):
|
|
416
417
|
self,
|
417
418
|
*,
|
418
419
|
organization_id: str | None = None,
|
419
|
-
project_id: str
|
420
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
420
421
|
inspect_template: InspectTemplate | None = None,
|
421
422
|
template_id: str | None = None,
|
422
423
|
retry: Retry | _MethodDefault = DEFAULT,
|
@@ -522,7 +523,7 @@ class CloudDLPCreateJobTriggerOperator(GoogleCloudBaseOperator):
|
|
522
523
|
def __init__(
|
523
524
|
self,
|
524
525
|
*,
|
525
|
-
project_id: str
|
526
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
526
527
|
job_trigger: dict | JobTrigger | None = None,
|
527
528
|
trigger_id: str | None = None,
|
528
529
|
retry: Retry | _MethodDefault = DEFAULT,
|
@@ -631,7 +632,7 @@ class CloudDLPCreateStoredInfoTypeOperator(GoogleCloudBaseOperator):
|
|
631
632
|
self,
|
632
633
|
*,
|
633
634
|
organization_id: str | None = None,
|
634
|
-
project_id: str
|
635
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
635
636
|
config: StoredInfoTypeConfig | None = None,
|
636
637
|
stored_info_type_id: str | None = None,
|
637
638
|
retry: Retry | _MethodDefault = DEFAULT,
|
@@ -752,7 +753,7 @@ class CloudDLPDeidentifyContentOperator(GoogleCloudBaseOperator):
|
|
752
753
|
def __init__(
|
753
754
|
self,
|
754
755
|
*,
|
755
|
-
project_id: str
|
756
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
756
757
|
deidentify_config: dict | DeidentifyConfig | None = None,
|
757
758
|
inspect_config: dict | InspectConfig | None = None,
|
758
759
|
item: dict | ContentItem | None = None,
|
@@ -842,7 +843,7 @@ class CloudDLPDeleteDeidentifyTemplateOperator(GoogleCloudBaseOperator):
|
|
842
843
|
*,
|
843
844
|
template_id: str,
|
844
845
|
organization_id: str | None = None,
|
845
|
-
project_id: str
|
846
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
846
847
|
retry: Retry | _MethodDefault = DEFAULT,
|
847
848
|
timeout: float | None = None,
|
848
849
|
metadata: Sequence[tuple[str, str]] = (),
|
@@ -929,7 +930,7 @@ class CloudDLPDeleteDLPJobOperator(GoogleCloudBaseOperator):
|
|
929
930
|
self,
|
930
931
|
*,
|
931
932
|
dlp_job_id: str,
|
932
|
-
project_id: str
|
933
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
933
934
|
retry: Retry | _MethodDefault = DEFAULT,
|
934
935
|
timeout: float | None = None,
|
935
936
|
metadata: Sequence[tuple[str, str]] = (),
|
@@ -1017,7 +1018,7 @@ class CloudDLPDeleteInspectTemplateOperator(GoogleCloudBaseOperator):
|
|
1017
1018
|
*,
|
1018
1019
|
template_id: str,
|
1019
1020
|
organization_id: str | None = None,
|
1020
|
-
project_id: str
|
1021
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
1021
1022
|
retry: Retry | _MethodDefault = DEFAULT,
|
1022
1023
|
timeout: float | None = None,
|
1023
1024
|
metadata: Sequence[tuple[str, str]] = (),
|
@@ -1103,7 +1104,7 @@ class CloudDLPDeleteJobTriggerOperator(GoogleCloudBaseOperator):
|
|
1103
1104
|
self,
|
1104
1105
|
*,
|
1105
1106
|
job_trigger_id: str,
|
1106
|
-
project_id: str
|
1107
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
1107
1108
|
retry: Retry | _MethodDefault = DEFAULT,
|
1108
1109
|
timeout: float | None = None,
|
1109
1110
|
metadata: Sequence[tuple[str, str]] = (),
|
@@ -1191,7 +1192,7 @@ class CloudDLPDeleteStoredInfoTypeOperator(GoogleCloudBaseOperator):
|
|
1191
1192
|
*,
|
1192
1193
|
stored_info_type_id: str,
|
1193
1194
|
organization_id: str | None = None,
|
1194
|
-
project_id: str
|
1195
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
1195
1196
|
retry: Retry | _MethodDefault = DEFAULT,
|
1196
1197
|
timeout: float | None = None,
|
1197
1198
|
metadata: Sequence[tuple[str, str]] = (),
|
@@ -1281,7 +1282,7 @@ class CloudDLPGetDeidentifyTemplateOperator(GoogleCloudBaseOperator):
|
|
1281
1282
|
*,
|
1282
1283
|
template_id: str,
|
1283
1284
|
organization_id: str | None = None,
|
1284
|
-
project_id: str
|
1285
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
1285
1286
|
retry: Retry | _MethodDefault = DEFAULT,
|
1286
1287
|
timeout: float | None = None,
|
1287
1288
|
metadata: Sequence[tuple[str, str]] = (),
|
@@ -1364,7 +1365,7 @@ class CloudDLPGetDLPJobOperator(GoogleCloudBaseOperator):
|
|
1364
1365
|
self,
|
1365
1366
|
*,
|
1366
1367
|
dlp_job_id: str,
|
1367
|
-
project_id: str
|
1368
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
1368
1369
|
retry: Retry | _MethodDefault = DEFAULT,
|
1369
1370
|
timeout: float | None = None,
|
1370
1371
|
metadata: Sequence[tuple[str, str]] = (),
|
@@ -1452,7 +1453,7 @@ class CloudDLPGetInspectTemplateOperator(GoogleCloudBaseOperator):
|
|
1452
1453
|
*,
|
1453
1454
|
template_id: str,
|
1454
1455
|
organization_id: str | None = None,
|
1455
|
-
project_id: str
|
1456
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
1456
1457
|
retry: Retry | _MethodDefault = DEFAULT,
|
1457
1458
|
timeout: float | None = None,
|
1458
1459
|
metadata: Sequence[tuple[str, str]] = (),
|
@@ -1538,7 +1539,7 @@ class CloudDLPGetDLPJobTriggerOperator(GoogleCloudBaseOperator):
|
|
1538
1539
|
self,
|
1539
1540
|
*,
|
1540
1541
|
job_trigger_id: str,
|
1541
|
-
project_id: str
|
1542
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
1542
1543
|
retry: Retry | _MethodDefault = DEFAULT,
|
1543
1544
|
timeout: float | None = None,
|
1544
1545
|
metadata: Sequence[tuple[str, str]] = (),
|
@@ -1626,7 +1627,7 @@ class CloudDLPGetStoredInfoTypeOperator(GoogleCloudBaseOperator):
|
|
1626
1627
|
*,
|
1627
1628
|
stored_info_type_id: str,
|
1628
1629
|
organization_id: str | None = None,
|
1629
|
-
project_id: str
|
1630
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
1630
1631
|
retry: Retry | _MethodDefault = DEFAULT,
|
1631
1632
|
timeout: float | None = None,
|
1632
1633
|
metadata: Sequence[tuple[str, str]] = (),
|
@@ -1716,7 +1717,7 @@ class CloudDLPInspectContentOperator(GoogleCloudBaseOperator):
|
|
1716
1717
|
def __init__(
|
1717
1718
|
self,
|
1718
1719
|
*,
|
1719
|
-
project_id: str
|
1720
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
1720
1721
|
inspect_config: dict | InspectConfig | None = None,
|
1721
1722
|
item: dict | ContentItem | None = None,
|
1722
1723
|
inspect_template_name: str | None = None,
|
@@ -1802,7 +1803,7 @@ class CloudDLPListDeidentifyTemplatesOperator(GoogleCloudBaseOperator):
|
|
1802
1803
|
self,
|
1803
1804
|
*,
|
1804
1805
|
organization_id: str | None = None,
|
1805
|
-
project_id: str
|
1806
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
1806
1807
|
page_size: int | None = None,
|
1807
1808
|
order_by: str | None = None,
|
1808
1809
|
retry: Retry | _MethodDefault = DEFAULT,
|
@@ -1894,7 +1895,7 @@ class CloudDLPListDLPJobsOperator(GoogleCloudBaseOperator):
|
|
1894
1895
|
def __init__(
|
1895
1896
|
self,
|
1896
1897
|
*,
|
1897
|
-
project_id: str
|
1898
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
1898
1899
|
results_filter: str | None = None,
|
1899
1900
|
page_size: int | None = None,
|
1900
1901
|
job_type: str | None = None,
|
@@ -1986,7 +1987,7 @@ class CloudDLPListInfoTypesOperator(GoogleCloudBaseOperator):
|
|
1986
1987
|
def __init__(
|
1987
1988
|
self,
|
1988
1989
|
*,
|
1989
|
-
project_id: str
|
1990
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
1990
1991
|
language_code: str | None = None,
|
1991
1992
|
results_filter: str | None = None,
|
1992
1993
|
retry: Retry | _MethodDefault = DEFAULT,
|
@@ -2077,7 +2078,7 @@ class CloudDLPListInspectTemplatesOperator(GoogleCloudBaseOperator):
|
|
2077
2078
|
self,
|
2078
2079
|
*,
|
2079
2080
|
organization_id: str | None = None,
|
2080
|
-
project_id: str
|
2081
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
2081
2082
|
page_size: int | None = None,
|
2082
2083
|
order_by: str | None = None,
|
2083
2084
|
retry: Retry | _MethodDefault = DEFAULT,
|
@@ -2168,7 +2169,7 @@ class CloudDLPListJobTriggersOperator(GoogleCloudBaseOperator):
|
|
2168
2169
|
def __init__(
|
2169
2170
|
self,
|
2170
2171
|
*,
|
2171
|
-
project_id: str
|
2172
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
2172
2173
|
page_size: int | None = None,
|
2173
2174
|
order_by: str | None = None,
|
2174
2175
|
results_filter: str | None = None,
|
@@ -2263,7 +2264,7 @@ class CloudDLPListStoredInfoTypesOperator(GoogleCloudBaseOperator):
|
|
2263
2264
|
self,
|
2264
2265
|
*,
|
2265
2266
|
organization_id: str | None = None,
|
2266
|
-
project_id: str
|
2267
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
2267
2268
|
page_size: int | None = None,
|
2268
2269
|
order_by: str | None = None,
|
2269
2270
|
retry: Retry | _MethodDefault = DEFAULT,
|
@@ -2359,7 +2360,7 @@ class CloudDLPRedactImageOperator(GoogleCloudBaseOperator):
|
|
2359
2360
|
def __init__(
|
2360
2361
|
self,
|
2361
2362
|
*,
|
2362
|
-
project_id: str
|
2363
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
2363
2364
|
inspect_config: dict | InspectConfig | None = None,
|
2364
2365
|
image_redaction_configs: None | (list[dict] | list[RedactImageRequest.ImageRedactionConfig]) = None,
|
2365
2366
|
include_findings: bool | None = None,
|
@@ -2453,7 +2454,7 @@ class CloudDLPReidentifyContentOperator(GoogleCloudBaseOperator):
|
|
2453
2454
|
def __init__(
|
2454
2455
|
self,
|
2455
2456
|
*,
|
2456
|
-
project_id: str
|
2457
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
2457
2458
|
reidentify_config: dict | DeidentifyConfig | None = None,
|
2458
2459
|
inspect_config: dict | InspectConfig | None = None,
|
2459
2460
|
item: dict | ContentItem | None = None,
|
@@ -2548,7 +2549,7 @@ class CloudDLPUpdateDeidentifyTemplateOperator(GoogleCloudBaseOperator):
|
|
2548
2549
|
*,
|
2549
2550
|
template_id: str,
|
2550
2551
|
organization_id: str | None = None,
|
2551
|
-
project_id: str
|
2552
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
2552
2553
|
deidentify_template: dict | DeidentifyTemplate | None = None,
|
2553
2554
|
update_mask: dict | FieldMask | None = None,
|
2554
2555
|
retry: Retry | _MethodDefault = DEFAULT,
|
@@ -2648,7 +2649,7 @@ class CloudDLPUpdateInspectTemplateOperator(GoogleCloudBaseOperator):
|
|
2648
2649
|
*,
|
2649
2650
|
template_id: str,
|
2650
2651
|
organization_id: str | None = None,
|
2651
|
-
project_id: str
|
2652
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
2652
2653
|
inspect_template: dict | InspectTemplate | None = None,
|
2653
2654
|
update_mask: dict | FieldMask | None = None,
|
2654
2655
|
retry: Retry | _MethodDefault = DEFAULT,
|
@@ -2744,7 +2745,7 @@ class CloudDLPUpdateJobTriggerOperator(GoogleCloudBaseOperator):
|
|
2744
2745
|
self,
|
2745
2746
|
*,
|
2746
2747
|
job_trigger_id,
|
2747
|
-
project_id: str
|
2748
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
2748
2749
|
job_trigger: dict | JobTrigger | None = None,
|
2749
2750
|
update_mask: dict | FieldMask | None = None,
|
2750
2751
|
retry: Retry | _MethodDefault = DEFAULT,
|
@@ -2843,7 +2844,7 @@ class CloudDLPUpdateStoredInfoTypeOperator(GoogleCloudBaseOperator):
|
|
2843
2844
|
*,
|
2844
2845
|
stored_info_type_id,
|
2845
2846
|
organization_id: str | None = None,
|
2846
|
-
project_id: str
|
2847
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
2847
2848
|
config: dict | StoredInfoTypeConfig | None = None,
|
2848
2849
|
update_mask: dict | FieldMask | None = None,
|
2849
2850
|
retry: Retry | _MethodDefault = DEFAULT,
|
@@ -35,6 +35,7 @@ from airflow.providers.google.cloud.utils.field_validator import (
|
|
35
35
|
GcpBodyFieldValidator,
|
36
36
|
GcpFieldValidationException,
|
37
37
|
)
|
38
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
38
39
|
from airflow.version import version
|
39
40
|
|
40
41
|
if TYPE_CHECKING:
|
@@ -155,7 +156,7 @@ class CloudFunctionDeployFunctionOperator(GoogleCloudBaseOperator):
|
|
155
156
|
*,
|
156
157
|
location: str,
|
157
158
|
body: dict,
|
158
|
-
project_id: str
|
159
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
159
160
|
gcp_conn_id: str = "google_cloud_default",
|
160
161
|
api_version: str = "v1",
|
161
162
|
zip_path: str | None = None,
|
@@ -363,7 +364,7 @@ class CloudFunctionDeleteFunctionOperator(GoogleCloudBaseOperator):
|
|
363
364
|
gcp_conn_id: str = "google_cloud_default",
|
364
365
|
api_version: str = "v1",
|
365
366
|
impersonation_chain: str | Sequence[str] | None = None,
|
366
|
-
project_id: str
|
367
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
367
368
|
**kwargs,
|
368
369
|
) -> None:
|
369
370
|
self.name = name
|
@@ -447,7 +448,7 @@ class CloudFunctionInvokeFunctionOperator(GoogleCloudBaseOperator):
|
|
447
448
|
function_id: str,
|
448
449
|
input_data: dict,
|
449
450
|
location: str,
|
450
|
-
project_id: str
|
451
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
451
452
|
gcp_conn_id: str = "google_cloud_default",
|
452
453
|
api_version: str = "v1",
|
453
454
|
impersonation_chain: str | Sequence[str] | None = None,
|