apache-airflow-providers-google 10.17.0rc1__py3-none-any.whl → 10.18.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. airflow/providers/google/__init__.py +5 -8
  2. airflow/providers/google/cloud/hooks/automl.py +35 -1
  3. airflow/providers/google/cloud/hooks/bigquery.py +126 -41
  4. airflow/providers/google/cloud/hooks/cloud_composer.py +250 -2
  5. airflow/providers/google/cloud/hooks/cloud_sql.py +154 -7
  6. airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +7 -2
  7. airflow/providers/google/cloud/hooks/compute_ssh.py +2 -1
  8. airflow/providers/google/cloud/hooks/dataflow.py +246 -32
  9. airflow/providers/google/cloud/hooks/dataplex.py +6 -2
  10. airflow/providers/google/cloud/hooks/dlp.py +14 -14
  11. airflow/providers/google/cloud/hooks/gcs.py +6 -2
  12. airflow/providers/google/cloud/hooks/gdm.py +2 -2
  13. airflow/providers/google/cloud/hooks/kubernetes_engine.py +2 -2
  14. airflow/providers/google/cloud/hooks/mlengine.py +8 -4
  15. airflow/providers/google/cloud/hooks/pubsub.py +1 -1
  16. airflow/providers/google/cloud/hooks/secret_manager.py +252 -4
  17. airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +1431 -74
  18. airflow/providers/google/cloud/hooks/vertex_ai/prediction_service.py +91 -0
  19. airflow/providers/google/cloud/links/vertex_ai.py +2 -1
  20. airflow/providers/google/cloud/log/gcs_task_handler.py +2 -1
  21. airflow/providers/google/cloud/operators/automl.py +243 -37
  22. airflow/providers/google/cloud/operators/bigquery.py +164 -62
  23. airflow/providers/google/cloud/operators/bigquery_dts.py +4 -3
  24. airflow/providers/google/cloud/operators/bigtable.py +7 -6
  25. airflow/providers/google/cloud/operators/cloud_build.py +12 -11
  26. airflow/providers/google/cloud/operators/cloud_composer.py +147 -2
  27. airflow/providers/google/cloud/operators/cloud_memorystore.py +17 -16
  28. airflow/providers/google/cloud/operators/cloud_sql.py +60 -17
  29. airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +35 -16
  30. airflow/providers/google/cloud/operators/compute.py +12 -11
  31. airflow/providers/google/cloud/operators/datacatalog.py +21 -20
  32. airflow/providers/google/cloud/operators/dataflow.py +59 -42
  33. airflow/providers/google/cloud/operators/datafusion.py +11 -10
  34. airflow/providers/google/cloud/operators/datapipeline.py +3 -2
  35. airflow/providers/google/cloud/operators/dataprep.py +5 -4
  36. airflow/providers/google/cloud/operators/dataproc.py +20 -17
  37. airflow/providers/google/cloud/operators/datastore.py +8 -7
  38. airflow/providers/google/cloud/operators/dlp.py +31 -30
  39. airflow/providers/google/cloud/operators/functions.py +4 -3
  40. airflow/providers/google/cloud/operators/gcs.py +66 -41
  41. airflow/providers/google/cloud/operators/kubernetes_engine.py +256 -49
  42. airflow/providers/google/cloud/operators/life_sciences.py +2 -1
  43. airflow/providers/google/cloud/operators/mlengine.py +11 -10
  44. airflow/providers/google/cloud/operators/pubsub.py +6 -5
  45. airflow/providers/google/cloud/operators/spanner.py +7 -6
  46. airflow/providers/google/cloud/operators/speech_to_text.py +2 -1
  47. airflow/providers/google/cloud/operators/stackdriver.py +11 -10
  48. airflow/providers/google/cloud/operators/tasks.py +14 -13
  49. airflow/providers/google/cloud/operators/text_to_speech.py +2 -1
  50. airflow/providers/google/cloud/operators/translate_speech.py +2 -1
  51. airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +333 -26
  52. airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +20 -12
  53. airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +0 -1
  54. airflow/providers/google/cloud/operators/vision.py +13 -12
  55. airflow/providers/google/cloud/operators/workflows.py +12 -14
  56. airflow/providers/google/cloud/secrets/secret_manager.py +2 -1
  57. airflow/providers/google/cloud/sensors/bigquery_dts.py +2 -1
  58. airflow/providers/google/cloud/sensors/bigtable.py +2 -1
  59. airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +2 -1
  60. airflow/providers/google/cloud/sensors/dataflow.py +239 -52
  61. airflow/providers/google/cloud/sensors/datafusion.py +2 -1
  62. airflow/providers/google/cloud/sensors/dataproc.py +3 -2
  63. airflow/providers/google/cloud/sensors/gcs.py +14 -12
  64. airflow/providers/google/cloud/sensors/tasks.py +2 -1
  65. airflow/providers/google/cloud/sensors/workflows.py +2 -1
  66. airflow/providers/google/cloud/transfers/adls_to_gcs.py +8 -2
  67. airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +7 -1
  68. airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +7 -1
  69. airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +2 -1
  70. airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +1 -1
  71. airflow/providers/google/cloud/transfers/bigquery_to_sql.py +1 -0
  72. airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +5 -6
  73. airflow/providers/google/cloud/transfers/gcs_to_gcs.py +22 -12
  74. airflow/providers/google/cloud/triggers/bigquery.py +75 -6
  75. airflow/providers/google/cloud/triggers/cloud_composer.py +68 -0
  76. airflow/providers/google/cloud/triggers/cloud_sql.py +2 -1
  77. airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +2 -1
  78. airflow/providers/google/cloud/triggers/dataflow.py +504 -4
  79. airflow/providers/google/cloud/triggers/dataproc.py +190 -27
  80. airflow/providers/google/cloud/triggers/kubernetes_engine.py +2 -3
  81. airflow/providers/google/cloud/triggers/mlengine.py +2 -1
  82. airflow/providers/google/cloud/triggers/vertex_ai.py +94 -0
  83. airflow/providers/google/common/hooks/base_google.py +45 -7
  84. airflow/providers/google/firebase/hooks/firestore.py +2 -2
  85. airflow/providers/google/firebase/operators/firestore.py +2 -1
  86. airflow/providers/google/get_provider_info.py +5 -3
  87. {apache_airflow_providers_google-10.17.0rc1.dist-info → apache_airflow_providers_google-10.18.0.dist-info}/METADATA +18 -18
  88. {apache_airflow_providers_google-10.17.0rc1.dist-info → apache_airflow_providers_google-10.18.0.dist-info}/RECORD +90 -90
  89. airflow/providers/google/cloud/example_dags/example_cloud_sql_query.py +0 -289
  90. {apache_airflow_providers_google-10.17.0rc1.dist-info → apache_airflow_providers_google-10.18.0.dist-info}/WHEEL +0 -0
  91. {apache_airflow_providers_google-10.17.0rc1.dist-info → apache_airflow_providers_google-10.18.0.dist-info}/entry_points.txt +0 -0
@@ -34,6 +34,7 @@ from google.cloud.vision_v1 import (
34
34
 
35
35
  from airflow.providers.google.cloud.hooks.vision import CloudVisionHook
36
36
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
37
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
37
38
 
38
39
  if TYPE_CHECKING:
39
40
  from google.api_core.retry import Retry
@@ -94,7 +95,7 @@ class CloudVisionCreateProductSetOperator(GoogleCloudBaseOperator):
94
95
  *,
95
96
  product_set: dict | ProductSet,
96
97
  location: str,
97
- project_id: str | None = None,
98
+ project_id: str = PROVIDE_PROJECT_ID,
98
99
  product_set_id: str | None = None,
99
100
  retry: Retry | _MethodDefault = DEFAULT,
100
101
  timeout: float | None = None,
@@ -181,7 +182,7 @@ class CloudVisionGetProductSetOperator(GoogleCloudBaseOperator):
181
182
  *,
182
183
  location: str,
183
184
  product_set_id: str,
184
- project_id: str | None = None,
185
+ project_id: str = PROVIDE_PROJECT_ID,
185
186
  retry: Retry | _MethodDefault = DEFAULT,
186
187
  timeout: float | None = None,
187
188
  metadata: MetaData = (),
@@ -279,7 +280,7 @@ class CloudVisionUpdateProductSetOperator(GoogleCloudBaseOperator):
279
280
  product_set: dict | ProductSet,
280
281
  location: str | None = None,
281
282
  product_set_id: str | None = None,
282
- project_id: str | None = None,
283
+ project_id: str = PROVIDE_PROJECT_ID,
283
284
  update_mask: dict | FieldMask | None = None,
284
285
  retry: Retry | _MethodDefault = DEFAULT,
285
286
  timeout: float | None = None,
@@ -367,7 +368,7 @@ class CloudVisionDeleteProductSetOperator(GoogleCloudBaseOperator):
367
368
  *,
368
369
  location: str,
369
370
  product_set_id: str,
370
- project_id: str | None = None,
371
+ project_id: str = PROVIDE_PROJECT_ID,
371
372
  retry: Retry | _MethodDefault = DEFAULT,
372
373
  timeout: float | None = None,
373
374
  metadata: MetaData = (),
@@ -455,7 +456,7 @@ class CloudVisionCreateProductOperator(GoogleCloudBaseOperator):
455
456
  *,
456
457
  location: str,
457
458
  product: str,
458
- project_id: str | None = None,
459
+ project_id: str = PROVIDE_PROJECT_ID,
459
460
  product_id: str | None = None,
460
461
  retry: Retry | _MethodDefault = DEFAULT,
461
462
  timeout: float | None = None,
@@ -545,7 +546,7 @@ class CloudVisionGetProductOperator(GoogleCloudBaseOperator):
545
546
  *,
546
547
  location: str,
547
548
  product_id: str,
548
- project_id: str | None = None,
549
+ project_id: str = PROVIDE_PROJECT_ID,
549
550
  retry: Retry | _MethodDefault = DEFAULT,
550
551
  timeout: float | None = None,
551
552
  metadata: MetaData = (),
@@ -652,7 +653,7 @@ class CloudVisionUpdateProductOperator(GoogleCloudBaseOperator):
652
653
  product: dict | Product,
653
654
  location: str | None = None,
654
655
  product_id: str | None = None,
655
- project_id: str | None = None,
656
+ project_id: str = PROVIDE_PROJECT_ID,
656
657
  update_mask: dict | FieldMask | None = None,
657
658
  retry: Retry | _MethodDefault = DEFAULT,
658
659
  timeout: float | None = None,
@@ -743,7 +744,7 @@ class CloudVisionDeleteProductOperator(GoogleCloudBaseOperator):
743
744
  *,
744
745
  location: str,
745
746
  product_id: str,
746
- project_id: str | None = None,
747
+ project_id: str = PROVIDE_PROJECT_ID,
747
748
  retry: Retry | _MethodDefault = DEFAULT,
748
749
  timeout: float | None = None,
749
750
  metadata: MetaData = (),
@@ -899,7 +900,7 @@ class CloudVisionCreateReferenceImageOperator(GoogleCloudBaseOperator):
899
900
  reference_image: dict | ReferenceImage,
900
901
  product_id: str,
901
902
  reference_image_id: str | None = None,
902
- project_id: str | None = None,
903
+ project_id: str = PROVIDE_PROJECT_ID,
903
904
  retry: Retry | _MethodDefault = DEFAULT,
904
905
  timeout: float | None = None,
905
906
  metadata: MetaData = (),
@@ -996,7 +997,7 @@ class CloudVisionDeleteReferenceImageOperator(GoogleCloudBaseOperator):
996
997
  location: str,
997
998
  product_id: str,
998
999
  reference_image_id: str,
999
- project_id: str | None = None,
1000
+ project_id: str = PROVIDE_PROJECT_ID,
1000
1001
  retry: Retry | _MethodDefault = DEFAULT,
1001
1002
  timeout: float | None = None,
1002
1003
  metadata: MetaData = (),
@@ -1085,7 +1086,7 @@ class CloudVisionAddProductToProductSetOperator(GoogleCloudBaseOperator):
1085
1086
  product_set_id: str,
1086
1087
  product_id: str,
1087
1088
  location: str,
1088
- project_id: str | None = None,
1089
+ project_id: str = PROVIDE_PROJECT_ID,
1089
1090
  retry: Retry | _MethodDefault = DEFAULT,
1090
1091
  timeout: float | None = None,
1091
1092
  metadata: MetaData = (),
@@ -1167,7 +1168,7 @@ class CloudVisionRemoveProductFromProductSetOperator(GoogleCloudBaseOperator):
1167
1168
  product_set_id: str,
1168
1169
  product_id: str,
1169
1170
  location: str,
1170
- project_id: str | None = None,
1171
+ project_id: str = PROVIDE_PROJECT_ID,
1171
1172
  retry: Retry | _MethodDefault = DEFAULT,
1172
1173
  timeout: float | None = None,
1173
1174
  metadata: MetaData = (),
@@ -34,17 +34,15 @@ from airflow.providers.google.cloud.links.workflows import (
34
34
  WorkflowsWorkflowDetailsLink,
35
35
  )
36
36
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
37
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
37
38
 
38
39
  if TYPE_CHECKING:
39
40
  from google.api_core.retry import Retry
40
41
  from google.protobuf.field_mask_pb2 import FieldMask
41
42
 
42
43
  from airflow.utils.context import Context
43
- try:
44
- from airflow.utils.hashlib_wrapper import md5
45
- except ModuleNotFoundError:
46
- # Remove when Airflow providers min Airflow version is "2.7.0"
47
- from hashlib import md5
44
+
45
+ from airflow.utils.hashlib_wrapper import md5
48
46
 
49
47
 
50
48
  class WorkflowsCreateWorkflowOperator(GoogleCloudBaseOperator):
@@ -80,7 +78,7 @@ class WorkflowsCreateWorkflowOperator(GoogleCloudBaseOperator):
80
78
  workflow: dict,
81
79
  workflow_id: str,
82
80
  location: str,
83
- project_id: str | None = None,
81
+ project_id: str = PROVIDE_PROJECT_ID,
84
82
  retry: Retry | _MethodDefault = DEFAULT,
85
83
  timeout: float | None = None,
86
84
  metadata: Sequence[tuple[str, str]] = (),
@@ -192,7 +190,7 @@ class WorkflowsUpdateWorkflowOperator(GoogleCloudBaseOperator):
192
190
  *,
193
191
  workflow_id: str,
194
192
  location: str,
195
- project_id: str | None = None,
193
+ project_id: str = PROVIDE_PROJECT_ID,
196
194
  update_mask: FieldMask | None = None,
197
195
  retry: Retry | _MethodDefault = DEFAULT,
198
196
  timeout: float | None = None,
@@ -270,7 +268,7 @@ class WorkflowsDeleteWorkflowOperator(GoogleCloudBaseOperator):
270
268
  *,
271
269
  workflow_id: str,
272
270
  location: str,
273
- project_id: str | None = None,
271
+ project_id: str = PROVIDE_PROJECT_ID,
274
272
  retry: Retry | _MethodDefault = DEFAULT,
275
273
  timeout: float | None = None,
276
274
  metadata: Sequence[tuple[str, str]] = (),
@@ -332,7 +330,7 @@ class WorkflowsListWorkflowsOperator(GoogleCloudBaseOperator):
332
330
  self,
333
331
  *,
334
332
  location: str,
335
- project_id: str | None = None,
333
+ project_id: str = PROVIDE_PROJECT_ID,
336
334
  filter_: str | None = None,
337
335
  order_by: str | None = None,
338
336
  retry: Retry | _MethodDefault = DEFAULT,
@@ -402,7 +400,7 @@ class WorkflowsGetWorkflowOperator(GoogleCloudBaseOperator):
402
400
  *,
403
401
  workflow_id: str,
404
402
  location: str,
405
- project_id: str | None = None,
403
+ project_id: str = PROVIDE_PROJECT_ID,
406
404
  retry: Retry | _MethodDefault = DEFAULT,
407
405
  timeout: float | None = None,
408
406
  metadata: Sequence[tuple[str, str]] = (),
@@ -473,7 +471,7 @@ class WorkflowsCreateExecutionOperator(GoogleCloudBaseOperator):
473
471
  workflow_id: str,
474
472
  execution: dict,
475
473
  location: str,
476
- project_id: str | None = None,
474
+ project_id: str = PROVIDE_PROJECT_ID,
477
475
  retry: Retry | _MethodDefault = DEFAULT,
478
476
  timeout: float | None = None,
479
477
  metadata: Sequence[tuple[str, str]] = (),
@@ -548,7 +546,7 @@ class WorkflowsCancelExecutionOperator(GoogleCloudBaseOperator):
548
546
  workflow_id: str,
549
547
  execution_id: str,
550
548
  location: str,
551
- project_id: str | None = None,
549
+ project_id: str = PROVIDE_PROJECT_ID,
552
550
  retry: Retry | _MethodDefault = DEFAULT,
553
551
  timeout: float | None = None,
554
552
  metadata: Sequence[tuple[str, str]] = (),
@@ -626,7 +624,7 @@ class WorkflowsListExecutionsOperator(GoogleCloudBaseOperator):
626
624
  workflow_id: str,
627
625
  location: str,
628
626
  start_date_filter: datetime.datetime | None = None,
629
- project_id: str | None = None,
627
+ project_id: str = PROVIDE_PROJECT_ID,
630
628
  retry: Retry | _MethodDefault = DEFAULT,
631
629
  timeout: float | None = None,
632
630
  metadata: Sequence[tuple[str, str]] = (),
@@ -703,7 +701,7 @@ class WorkflowsGetExecutionOperator(GoogleCloudBaseOperator):
703
701
  workflow_id: str,
704
702
  execution_id: str,
705
703
  location: str,
706
- project_id: str | None = None,
704
+ project_id: str = PROVIDE_PROJECT_ID,
707
705
  retry: Retry | _MethodDefault = DEFAULT,
708
706
  timeout: float | None = None,
709
707
  metadata: Sequence[tuple[str, str]] = (),
@@ -30,6 +30,7 @@ from airflow.providers.google.cloud.utils.credentials_provider import (
30
30
  _get_target_principal_and_delegates,
31
31
  get_credentials_and_project_id,
32
32
  )
33
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
33
34
  from airflow.secrets import BaseSecretsBackend
34
35
  from airflow.utils.log.logging_mixin import LoggingMixin
35
36
 
@@ -94,7 +95,7 @@ class CloudSecretManagerBackend(BaseSecretsBackend, LoggingMixin):
94
95
  gcp_key_path: str | None = None,
95
96
  gcp_credential_config_file: dict[str, str] | str | None = None,
96
97
  gcp_scopes: str | None = None,
97
- project_id: str | None = None,
98
+ project_id: str = PROVIDE_PROJECT_ID,
98
99
  sep: str = "-",
99
100
  impersonation_chain: str | Sequence[str] | None = None,
100
101
  **kwargs,
@@ -26,6 +26,7 @@ from google.cloud.bigquery_datatransfer_v1 import TransferState
26
26
 
27
27
  from airflow.exceptions import AirflowException, AirflowSkipException
28
28
  from airflow.providers.google.cloud.hooks.bigquery_dts import BiqQueryDataTransferServiceHook
29
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
29
30
  from airflow.sensors.base import BaseSensorOperator
30
31
 
31
32
  if TYPE_CHECKING:
@@ -83,7 +84,7 @@ class BigQueryDataTransferServiceTransferRunSensor(BaseSensorOperator):
83
84
  expected_statuses: (
84
85
  set[str | TransferState | int] | str | TransferState | int
85
86
  ) = TransferState.SUCCEEDED,
86
- project_id: str | None = None,
87
+ project_id: str = PROVIDE_PROJECT_ID,
87
88
  gcp_conn_id: str = "google_cloud_default",
88
89
  retry: Retry | _MethodDefault = DEFAULT,
89
90
  request_timeout: float | None = None,
@@ -28,6 +28,7 @@ from google.cloud.bigtable.table import ClusterState
28
28
  from airflow.providers.google.cloud.hooks.bigtable import BigtableHook
29
29
  from airflow.providers.google.cloud.links.bigtable import BigtableTablesLink
30
30
  from airflow.providers.google.cloud.operators.bigtable import BigtableValidationMixin
31
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
31
32
  from airflow.sensors.base import BaseSensorOperator
32
33
 
33
34
  if TYPE_CHECKING:
@@ -74,7 +75,7 @@ class BigtableTableReplicationCompletedSensor(BaseSensorOperator, BigtableValida
74
75
  *,
75
76
  instance_id: str,
76
77
  table_id: str,
77
- project_id: str | None = None,
78
+ project_id: str = PROVIDE_PROJECT_ID,
78
79
  gcp_conn_id: str = "google_cloud_default",
79
80
  impersonation_chain: str | Sequence[str] | None = None,
80
81
  **kwargs,
@@ -28,6 +28,7 @@ from airflow.providers.google.cloud.hooks.cloud_storage_transfer_service import
28
28
  CloudDataTransferServiceHook,
29
29
  )
30
30
  from airflow.providers.google.cloud.links.cloud_storage_transfer import CloudStorageTransferJobLink
31
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
31
32
  from airflow.sensors.base import BaseSensorOperator
32
33
 
33
34
  if TYPE_CHECKING:
@@ -73,7 +74,7 @@ class CloudDataTransferServiceJobStatusSensor(BaseSensorOperator):
73
74
  *,
74
75
  job_name: str,
75
76
  expected_statuses: set[str] | str,
76
- project_id: str | None = None,
77
+ project_id: str = PROVIDE_PROJECT_ID,
77
78
  gcp_conn_id: str = "google_cloud_default",
78
79
  impersonation_chain: str | Sequence[str] | None = None,
79
80
  **kwargs,