apache-airflow-providers-google 10.17.0rc1__py3-none-any.whl → 10.18.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/__init__.py +3 -3
- airflow/providers/google/cloud/hooks/automl.py +1 -1
- airflow/providers/google/cloud/hooks/bigquery.py +64 -33
- airflow/providers/google/cloud/hooks/cloud_composer.py +250 -2
- airflow/providers/google/cloud/hooks/cloud_sql.py +154 -7
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +7 -2
- airflow/providers/google/cloud/hooks/compute_ssh.py +2 -1
- airflow/providers/google/cloud/hooks/dataflow.py +246 -32
- airflow/providers/google/cloud/hooks/dataplex.py +6 -2
- airflow/providers/google/cloud/hooks/dlp.py +14 -14
- airflow/providers/google/cloud/hooks/gcs.py +6 -2
- airflow/providers/google/cloud/hooks/gdm.py +2 -2
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +2 -2
- airflow/providers/google/cloud/hooks/mlengine.py +8 -4
- airflow/providers/google/cloud/hooks/pubsub.py +1 -1
- airflow/providers/google/cloud/hooks/secret_manager.py +252 -4
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +1431 -74
- airflow/providers/google/cloud/links/vertex_ai.py +2 -1
- airflow/providers/google/cloud/log/gcs_task_handler.py +2 -1
- airflow/providers/google/cloud/operators/automl.py +13 -12
- airflow/providers/google/cloud/operators/bigquery.py +36 -22
- airflow/providers/google/cloud/operators/bigquery_dts.py +4 -3
- airflow/providers/google/cloud/operators/bigtable.py +7 -6
- airflow/providers/google/cloud/operators/cloud_build.py +12 -11
- airflow/providers/google/cloud/operators/cloud_composer.py +147 -2
- airflow/providers/google/cloud/operators/cloud_memorystore.py +17 -16
- airflow/providers/google/cloud/operators/cloud_sql.py +60 -17
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +35 -16
- airflow/providers/google/cloud/operators/compute.py +12 -11
- airflow/providers/google/cloud/operators/datacatalog.py +21 -20
- airflow/providers/google/cloud/operators/dataflow.py +59 -42
- airflow/providers/google/cloud/operators/datafusion.py +11 -10
- airflow/providers/google/cloud/operators/datapipeline.py +3 -2
- airflow/providers/google/cloud/operators/dataprep.py +5 -4
- airflow/providers/google/cloud/operators/dataproc.py +19 -16
- airflow/providers/google/cloud/operators/datastore.py +8 -7
- airflow/providers/google/cloud/operators/dlp.py +31 -30
- airflow/providers/google/cloud/operators/functions.py +4 -3
- airflow/providers/google/cloud/operators/gcs.py +66 -41
- airflow/providers/google/cloud/operators/kubernetes_engine.py +232 -12
- airflow/providers/google/cloud/operators/life_sciences.py +2 -1
- airflow/providers/google/cloud/operators/mlengine.py +11 -10
- airflow/providers/google/cloud/operators/pubsub.py +6 -5
- airflow/providers/google/cloud/operators/spanner.py +7 -6
- airflow/providers/google/cloud/operators/speech_to_text.py +2 -1
- airflow/providers/google/cloud/operators/stackdriver.py +11 -10
- airflow/providers/google/cloud/operators/tasks.py +14 -13
- airflow/providers/google/cloud/operators/text_to_speech.py +2 -1
- airflow/providers/google/cloud/operators/translate_speech.py +2 -1
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +333 -26
- airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +20 -12
- airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +0 -1
- airflow/providers/google/cloud/operators/vision.py +13 -12
- airflow/providers/google/cloud/operators/workflows.py +10 -9
- airflow/providers/google/cloud/secrets/secret_manager.py +2 -1
- airflow/providers/google/cloud/sensors/bigquery_dts.py +2 -1
- airflow/providers/google/cloud/sensors/bigtable.py +2 -1
- airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +2 -1
- airflow/providers/google/cloud/sensors/dataflow.py +239 -52
- airflow/providers/google/cloud/sensors/datafusion.py +2 -1
- airflow/providers/google/cloud/sensors/dataproc.py +3 -2
- airflow/providers/google/cloud/sensors/gcs.py +14 -12
- airflow/providers/google/cloud/sensors/tasks.py +2 -1
- airflow/providers/google/cloud/sensors/workflows.py +2 -1
- airflow/providers/google/cloud/transfers/adls_to_gcs.py +8 -2
- airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +7 -1
- airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +7 -1
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +1 -1
- airflow/providers/google/cloud/transfers/bigquery_to_sql.py +1 -0
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +5 -6
- airflow/providers/google/cloud/transfers/gcs_to_gcs.py +22 -12
- airflow/providers/google/cloud/triggers/bigquery.py +14 -3
- airflow/providers/google/cloud/triggers/cloud_composer.py +68 -0
- airflow/providers/google/cloud/triggers/cloud_sql.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +2 -1
- airflow/providers/google/cloud/triggers/dataflow.py +504 -4
- airflow/providers/google/cloud/triggers/dataproc.py +110 -26
- airflow/providers/google/cloud/triggers/mlengine.py +2 -1
- airflow/providers/google/cloud/triggers/vertex_ai.py +94 -0
- airflow/providers/google/common/hooks/base_google.py +45 -7
- airflow/providers/google/firebase/hooks/firestore.py +2 -2
- airflow/providers/google/firebase/operators/firestore.py +2 -1
- airflow/providers/google/get_provider_info.py +3 -2
- {apache_airflow_providers_google-10.17.0rc1.dist-info → apache_airflow_providers_google-10.18.0rc1.dist-info}/METADATA +8 -8
- {apache_airflow_providers_google-10.17.0rc1.dist-info → apache_airflow_providers_google-10.18.0rc1.dist-info}/RECORD +88 -89
- airflow/providers/google/cloud/example_dags/example_cloud_sql_query.py +0 -289
- {apache_airflow_providers_google-10.17.0rc1.dist-info → apache_airflow_providers_google-10.18.0rc1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_google-10.17.0rc1.dist-info → apache_airflow_providers_google-10.18.0rc1.dist-info}/entry_points.txt +0 -0
@@ -34,6 +34,7 @@ from google.cloud.vision_v1 import (
|
|
34
34
|
|
35
35
|
from airflow.providers.google.cloud.hooks.vision import CloudVisionHook
|
36
36
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
37
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
37
38
|
|
38
39
|
if TYPE_CHECKING:
|
39
40
|
from google.api_core.retry import Retry
|
@@ -94,7 +95,7 @@ class CloudVisionCreateProductSetOperator(GoogleCloudBaseOperator):
|
|
94
95
|
*,
|
95
96
|
product_set: dict | ProductSet,
|
96
97
|
location: str,
|
97
|
-
project_id: str
|
98
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
98
99
|
product_set_id: str | None = None,
|
99
100
|
retry: Retry | _MethodDefault = DEFAULT,
|
100
101
|
timeout: float | None = None,
|
@@ -181,7 +182,7 @@ class CloudVisionGetProductSetOperator(GoogleCloudBaseOperator):
|
|
181
182
|
*,
|
182
183
|
location: str,
|
183
184
|
product_set_id: str,
|
184
|
-
project_id: str
|
185
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
185
186
|
retry: Retry | _MethodDefault = DEFAULT,
|
186
187
|
timeout: float | None = None,
|
187
188
|
metadata: MetaData = (),
|
@@ -279,7 +280,7 @@ class CloudVisionUpdateProductSetOperator(GoogleCloudBaseOperator):
|
|
279
280
|
product_set: dict | ProductSet,
|
280
281
|
location: str | None = None,
|
281
282
|
product_set_id: str | None = None,
|
282
|
-
project_id: str
|
283
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
283
284
|
update_mask: dict | FieldMask | None = None,
|
284
285
|
retry: Retry | _MethodDefault = DEFAULT,
|
285
286
|
timeout: float | None = None,
|
@@ -367,7 +368,7 @@ class CloudVisionDeleteProductSetOperator(GoogleCloudBaseOperator):
|
|
367
368
|
*,
|
368
369
|
location: str,
|
369
370
|
product_set_id: str,
|
370
|
-
project_id: str
|
371
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
371
372
|
retry: Retry | _MethodDefault = DEFAULT,
|
372
373
|
timeout: float | None = None,
|
373
374
|
metadata: MetaData = (),
|
@@ -455,7 +456,7 @@ class CloudVisionCreateProductOperator(GoogleCloudBaseOperator):
|
|
455
456
|
*,
|
456
457
|
location: str,
|
457
458
|
product: str,
|
458
|
-
project_id: str
|
459
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
459
460
|
product_id: str | None = None,
|
460
461
|
retry: Retry | _MethodDefault = DEFAULT,
|
461
462
|
timeout: float | None = None,
|
@@ -545,7 +546,7 @@ class CloudVisionGetProductOperator(GoogleCloudBaseOperator):
|
|
545
546
|
*,
|
546
547
|
location: str,
|
547
548
|
product_id: str,
|
548
|
-
project_id: str
|
549
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
549
550
|
retry: Retry | _MethodDefault = DEFAULT,
|
550
551
|
timeout: float | None = None,
|
551
552
|
metadata: MetaData = (),
|
@@ -652,7 +653,7 @@ class CloudVisionUpdateProductOperator(GoogleCloudBaseOperator):
|
|
652
653
|
product: dict | Product,
|
653
654
|
location: str | None = None,
|
654
655
|
product_id: str | None = None,
|
655
|
-
project_id: str
|
656
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
656
657
|
update_mask: dict | FieldMask | None = None,
|
657
658
|
retry: Retry | _MethodDefault = DEFAULT,
|
658
659
|
timeout: float | None = None,
|
@@ -743,7 +744,7 @@ class CloudVisionDeleteProductOperator(GoogleCloudBaseOperator):
|
|
743
744
|
*,
|
744
745
|
location: str,
|
745
746
|
product_id: str,
|
746
|
-
project_id: str
|
747
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
747
748
|
retry: Retry | _MethodDefault = DEFAULT,
|
748
749
|
timeout: float | None = None,
|
749
750
|
metadata: MetaData = (),
|
@@ -899,7 +900,7 @@ class CloudVisionCreateReferenceImageOperator(GoogleCloudBaseOperator):
|
|
899
900
|
reference_image: dict | ReferenceImage,
|
900
901
|
product_id: str,
|
901
902
|
reference_image_id: str | None = None,
|
902
|
-
project_id: str
|
903
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
903
904
|
retry: Retry | _MethodDefault = DEFAULT,
|
904
905
|
timeout: float | None = None,
|
905
906
|
metadata: MetaData = (),
|
@@ -996,7 +997,7 @@ class CloudVisionDeleteReferenceImageOperator(GoogleCloudBaseOperator):
|
|
996
997
|
location: str,
|
997
998
|
product_id: str,
|
998
999
|
reference_image_id: str,
|
999
|
-
project_id: str
|
1000
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
1000
1001
|
retry: Retry | _MethodDefault = DEFAULT,
|
1001
1002
|
timeout: float | None = None,
|
1002
1003
|
metadata: MetaData = (),
|
@@ -1085,7 +1086,7 @@ class CloudVisionAddProductToProductSetOperator(GoogleCloudBaseOperator):
|
|
1085
1086
|
product_set_id: str,
|
1086
1087
|
product_id: str,
|
1087
1088
|
location: str,
|
1088
|
-
project_id: str
|
1089
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
1089
1090
|
retry: Retry | _MethodDefault = DEFAULT,
|
1090
1091
|
timeout: float | None = None,
|
1091
1092
|
metadata: MetaData = (),
|
@@ -1167,7 +1168,7 @@ class CloudVisionRemoveProductFromProductSetOperator(GoogleCloudBaseOperator):
|
|
1167
1168
|
product_set_id: str,
|
1168
1169
|
product_id: str,
|
1169
1170
|
location: str,
|
1170
|
-
project_id: str
|
1171
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
1171
1172
|
retry: Retry | _MethodDefault = DEFAULT,
|
1172
1173
|
timeout: float | None = None,
|
1173
1174
|
metadata: MetaData = (),
|
@@ -34,6 +34,7 @@ from airflow.providers.google.cloud.links.workflows import (
|
|
34
34
|
WorkflowsWorkflowDetailsLink,
|
35
35
|
)
|
36
36
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
37
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
37
38
|
|
38
39
|
if TYPE_CHECKING:
|
39
40
|
from google.api_core.retry import Retry
|
@@ -80,7 +81,7 @@ class WorkflowsCreateWorkflowOperator(GoogleCloudBaseOperator):
|
|
80
81
|
workflow: dict,
|
81
82
|
workflow_id: str,
|
82
83
|
location: str,
|
83
|
-
project_id: str
|
84
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
84
85
|
retry: Retry | _MethodDefault = DEFAULT,
|
85
86
|
timeout: float | None = None,
|
86
87
|
metadata: Sequence[tuple[str, str]] = (),
|
@@ -192,7 +193,7 @@ class WorkflowsUpdateWorkflowOperator(GoogleCloudBaseOperator):
|
|
192
193
|
*,
|
193
194
|
workflow_id: str,
|
194
195
|
location: str,
|
195
|
-
project_id: str
|
196
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
196
197
|
update_mask: FieldMask | None = None,
|
197
198
|
retry: Retry | _MethodDefault = DEFAULT,
|
198
199
|
timeout: float | None = None,
|
@@ -270,7 +271,7 @@ class WorkflowsDeleteWorkflowOperator(GoogleCloudBaseOperator):
|
|
270
271
|
*,
|
271
272
|
workflow_id: str,
|
272
273
|
location: str,
|
273
|
-
project_id: str
|
274
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
274
275
|
retry: Retry | _MethodDefault = DEFAULT,
|
275
276
|
timeout: float | None = None,
|
276
277
|
metadata: Sequence[tuple[str, str]] = (),
|
@@ -332,7 +333,7 @@ class WorkflowsListWorkflowsOperator(GoogleCloudBaseOperator):
|
|
332
333
|
self,
|
333
334
|
*,
|
334
335
|
location: str,
|
335
|
-
project_id: str
|
336
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
336
337
|
filter_: str | None = None,
|
337
338
|
order_by: str | None = None,
|
338
339
|
retry: Retry | _MethodDefault = DEFAULT,
|
@@ -402,7 +403,7 @@ class WorkflowsGetWorkflowOperator(GoogleCloudBaseOperator):
|
|
402
403
|
*,
|
403
404
|
workflow_id: str,
|
404
405
|
location: str,
|
405
|
-
project_id: str
|
406
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
406
407
|
retry: Retry | _MethodDefault = DEFAULT,
|
407
408
|
timeout: float | None = None,
|
408
409
|
metadata: Sequence[tuple[str, str]] = (),
|
@@ -473,7 +474,7 @@ class WorkflowsCreateExecutionOperator(GoogleCloudBaseOperator):
|
|
473
474
|
workflow_id: str,
|
474
475
|
execution: dict,
|
475
476
|
location: str,
|
476
|
-
project_id: str
|
477
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
477
478
|
retry: Retry | _MethodDefault = DEFAULT,
|
478
479
|
timeout: float | None = None,
|
479
480
|
metadata: Sequence[tuple[str, str]] = (),
|
@@ -548,7 +549,7 @@ class WorkflowsCancelExecutionOperator(GoogleCloudBaseOperator):
|
|
548
549
|
workflow_id: str,
|
549
550
|
execution_id: str,
|
550
551
|
location: str,
|
551
|
-
project_id: str
|
552
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
552
553
|
retry: Retry | _MethodDefault = DEFAULT,
|
553
554
|
timeout: float | None = None,
|
554
555
|
metadata: Sequence[tuple[str, str]] = (),
|
@@ -626,7 +627,7 @@ class WorkflowsListExecutionsOperator(GoogleCloudBaseOperator):
|
|
626
627
|
workflow_id: str,
|
627
628
|
location: str,
|
628
629
|
start_date_filter: datetime.datetime | None = None,
|
629
|
-
project_id: str
|
630
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
630
631
|
retry: Retry | _MethodDefault = DEFAULT,
|
631
632
|
timeout: float | None = None,
|
632
633
|
metadata: Sequence[tuple[str, str]] = (),
|
@@ -703,7 +704,7 @@ class WorkflowsGetExecutionOperator(GoogleCloudBaseOperator):
|
|
703
704
|
workflow_id: str,
|
704
705
|
execution_id: str,
|
705
706
|
location: str,
|
706
|
-
project_id: str
|
707
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
707
708
|
retry: Retry | _MethodDefault = DEFAULT,
|
708
709
|
timeout: float | None = None,
|
709
710
|
metadata: Sequence[tuple[str, str]] = (),
|
@@ -30,6 +30,7 @@ from airflow.providers.google.cloud.utils.credentials_provider import (
|
|
30
30
|
_get_target_principal_and_delegates,
|
31
31
|
get_credentials_and_project_id,
|
32
32
|
)
|
33
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
33
34
|
from airflow.secrets import BaseSecretsBackend
|
34
35
|
from airflow.utils.log.logging_mixin import LoggingMixin
|
35
36
|
|
@@ -94,7 +95,7 @@ class CloudSecretManagerBackend(BaseSecretsBackend, LoggingMixin):
|
|
94
95
|
gcp_key_path: str | None = None,
|
95
96
|
gcp_credential_config_file: dict[str, str] | str | None = None,
|
96
97
|
gcp_scopes: str | None = None,
|
97
|
-
project_id: str
|
98
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
98
99
|
sep: str = "-",
|
99
100
|
impersonation_chain: str | Sequence[str] | None = None,
|
100
101
|
**kwargs,
|
@@ -26,6 +26,7 @@ from google.cloud.bigquery_datatransfer_v1 import TransferState
|
|
26
26
|
|
27
27
|
from airflow.exceptions import AirflowException, AirflowSkipException
|
28
28
|
from airflow.providers.google.cloud.hooks.bigquery_dts import BiqQueryDataTransferServiceHook
|
29
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
29
30
|
from airflow.sensors.base import BaseSensorOperator
|
30
31
|
|
31
32
|
if TYPE_CHECKING:
|
@@ -83,7 +84,7 @@ class BigQueryDataTransferServiceTransferRunSensor(BaseSensorOperator):
|
|
83
84
|
expected_statuses: (
|
84
85
|
set[str | TransferState | int] | str | TransferState | int
|
85
86
|
) = TransferState.SUCCEEDED,
|
86
|
-
project_id: str
|
87
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
87
88
|
gcp_conn_id: str = "google_cloud_default",
|
88
89
|
retry: Retry | _MethodDefault = DEFAULT,
|
89
90
|
request_timeout: float | None = None,
|
@@ -28,6 +28,7 @@ from google.cloud.bigtable.table import ClusterState
|
|
28
28
|
from airflow.providers.google.cloud.hooks.bigtable import BigtableHook
|
29
29
|
from airflow.providers.google.cloud.links.bigtable import BigtableTablesLink
|
30
30
|
from airflow.providers.google.cloud.operators.bigtable import BigtableValidationMixin
|
31
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
31
32
|
from airflow.sensors.base import BaseSensorOperator
|
32
33
|
|
33
34
|
if TYPE_CHECKING:
|
@@ -74,7 +75,7 @@ class BigtableTableReplicationCompletedSensor(BaseSensorOperator, BigtableValida
|
|
74
75
|
*,
|
75
76
|
instance_id: str,
|
76
77
|
table_id: str,
|
77
|
-
project_id: str
|
78
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
78
79
|
gcp_conn_id: str = "google_cloud_default",
|
79
80
|
impersonation_chain: str | Sequence[str] | None = None,
|
80
81
|
**kwargs,
|
@@ -28,6 +28,7 @@ from airflow.providers.google.cloud.hooks.cloud_storage_transfer_service import
|
|
28
28
|
CloudDataTransferServiceHook,
|
29
29
|
)
|
30
30
|
from airflow.providers.google.cloud.links.cloud_storage_transfer import CloudStorageTransferJobLink
|
31
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
31
32
|
from airflow.sensors.base import BaseSensorOperator
|
32
33
|
|
33
34
|
if TYPE_CHECKING:
|
@@ -73,7 +74,7 @@ class CloudDataTransferServiceJobStatusSensor(BaseSensorOperator):
|
|
73
74
|
*,
|
74
75
|
job_name: str,
|
75
76
|
expected_statuses: set[str] | str,
|
76
|
-
project_id: str
|
77
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
77
78
|
gcp_conn_id: str = "google_cloud_default",
|
78
79
|
impersonation_chain: str | Sequence[str] | None = None,
|
79
80
|
**kwargs,
|