apache-airflow-providers-google 10.17.0rc1__py3-none-any.whl → 10.18.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. airflow/providers/google/__init__.py +3 -3
  2. airflow/providers/google/cloud/hooks/automl.py +1 -1
  3. airflow/providers/google/cloud/hooks/bigquery.py +64 -33
  4. airflow/providers/google/cloud/hooks/cloud_composer.py +250 -2
  5. airflow/providers/google/cloud/hooks/cloud_sql.py +154 -7
  6. airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +7 -2
  7. airflow/providers/google/cloud/hooks/compute_ssh.py +2 -1
  8. airflow/providers/google/cloud/hooks/dataflow.py +246 -32
  9. airflow/providers/google/cloud/hooks/dataplex.py +6 -2
  10. airflow/providers/google/cloud/hooks/dlp.py +14 -14
  11. airflow/providers/google/cloud/hooks/gcs.py +6 -2
  12. airflow/providers/google/cloud/hooks/gdm.py +2 -2
  13. airflow/providers/google/cloud/hooks/kubernetes_engine.py +2 -2
  14. airflow/providers/google/cloud/hooks/mlengine.py +8 -4
  15. airflow/providers/google/cloud/hooks/pubsub.py +1 -1
  16. airflow/providers/google/cloud/hooks/secret_manager.py +252 -4
  17. airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +1431 -74
  18. airflow/providers/google/cloud/links/vertex_ai.py +2 -1
  19. airflow/providers/google/cloud/log/gcs_task_handler.py +2 -1
  20. airflow/providers/google/cloud/operators/automl.py +13 -12
  21. airflow/providers/google/cloud/operators/bigquery.py +36 -22
  22. airflow/providers/google/cloud/operators/bigquery_dts.py +4 -3
  23. airflow/providers/google/cloud/operators/bigtable.py +7 -6
  24. airflow/providers/google/cloud/operators/cloud_build.py +12 -11
  25. airflow/providers/google/cloud/operators/cloud_composer.py +147 -2
  26. airflow/providers/google/cloud/operators/cloud_memorystore.py +17 -16
  27. airflow/providers/google/cloud/operators/cloud_sql.py +60 -17
  28. airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +35 -16
  29. airflow/providers/google/cloud/operators/compute.py +12 -11
  30. airflow/providers/google/cloud/operators/datacatalog.py +21 -20
  31. airflow/providers/google/cloud/operators/dataflow.py +59 -42
  32. airflow/providers/google/cloud/operators/datafusion.py +11 -10
  33. airflow/providers/google/cloud/operators/datapipeline.py +3 -2
  34. airflow/providers/google/cloud/operators/dataprep.py +5 -4
  35. airflow/providers/google/cloud/operators/dataproc.py +19 -16
  36. airflow/providers/google/cloud/operators/datastore.py +8 -7
  37. airflow/providers/google/cloud/operators/dlp.py +31 -30
  38. airflow/providers/google/cloud/operators/functions.py +4 -3
  39. airflow/providers/google/cloud/operators/gcs.py +66 -41
  40. airflow/providers/google/cloud/operators/kubernetes_engine.py +232 -12
  41. airflow/providers/google/cloud/operators/life_sciences.py +2 -1
  42. airflow/providers/google/cloud/operators/mlengine.py +11 -10
  43. airflow/providers/google/cloud/operators/pubsub.py +6 -5
  44. airflow/providers/google/cloud/operators/spanner.py +7 -6
  45. airflow/providers/google/cloud/operators/speech_to_text.py +2 -1
  46. airflow/providers/google/cloud/operators/stackdriver.py +11 -10
  47. airflow/providers/google/cloud/operators/tasks.py +14 -13
  48. airflow/providers/google/cloud/operators/text_to_speech.py +2 -1
  49. airflow/providers/google/cloud/operators/translate_speech.py +2 -1
  50. airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +333 -26
  51. airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +20 -12
  52. airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +0 -1
  53. airflow/providers/google/cloud/operators/vision.py +13 -12
  54. airflow/providers/google/cloud/operators/workflows.py +10 -9
  55. airflow/providers/google/cloud/secrets/secret_manager.py +2 -1
  56. airflow/providers/google/cloud/sensors/bigquery_dts.py +2 -1
  57. airflow/providers/google/cloud/sensors/bigtable.py +2 -1
  58. airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +2 -1
  59. airflow/providers/google/cloud/sensors/dataflow.py +239 -52
  60. airflow/providers/google/cloud/sensors/datafusion.py +2 -1
  61. airflow/providers/google/cloud/sensors/dataproc.py +3 -2
  62. airflow/providers/google/cloud/sensors/gcs.py +14 -12
  63. airflow/providers/google/cloud/sensors/tasks.py +2 -1
  64. airflow/providers/google/cloud/sensors/workflows.py +2 -1
  65. airflow/providers/google/cloud/transfers/adls_to_gcs.py +8 -2
  66. airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +7 -1
  67. airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +7 -1
  68. airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +2 -1
  69. airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +1 -1
  70. airflow/providers/google/cloud/transfers/bigquery_to_sql.py +1 -0
  71. airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +5 -6
  72. airflow/providers/google/cloud/transfers/gcs_to_gcs.py +22 -12
  73. airflow/providers/google/cloud/triggers/bigquery.py +14 -3
  74. airflow/providers/google/cloud/triggers/cloud_composer.py +68 -0
  75. airflow/providers/google/cloud/triggers/cloud_sql.py +2 -1
  76. airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +2 -1
  77. airflow/providers/google/cloud/triggers/dataflow.py +504 -4
  78. airflow/providers/google/cloud/triggers/dataproc.py +110 -26
  79. airflow/providers/google/cloud/triggers/mlengine.py +2 -1
  80. airflow/providers/google/cloud/triggers/vertex_ai.py +94 -0
  81. airflow/providers/google/common/hooks/base_google.py +45 -7
  82. airflow/providers/google/firebase/hooks/firestore.py +2 -2
  83. airflow/providers/google/firebase/operators/firestore.py +2 -1
  84. airflow/providers/google/get_provider_info.py +3 -2
  85. {apache_airflow_providers_google-10.17.0rc1.dist-info → apache_airflow_providers_google-10.18.0rc1.dist-info}/METADATA +8 -8
  86. {apache_airflow_providers_google-10.17.0rc1.dist-info → apache_airflow_providers_google-10.18.0rc1.dist-info}/RECORD +88 -89
  87. airflow/providers/google/cloud/example_dags/example_cloud_sql_query.py +0 -289
  88. {apache_airflow_providers_google-10.17.0rc1.dist-info → apache_airflow_providers_google-10.18.0rc1.dist-info}/WHEEL +0 -0
  89. {apache_airflow_providers_google-10.17.0rc1.dist-info → apache_airflow_providers_google-10.18.0rc1.dist-info}/entry_points.txt +0 -0
@@ -34,6 +34,7 @@ from google.cloud.vision_v1 import (
34
34
 
35
35
  from airflow.providers.google.cloud.hooks.vision import CloudVisionHook
36
36
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
37
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
37
38
 
38
39
  if TYPE_CHECKING:
39
40
  from google.api_core.retry import Retry
@@ -94,7 +95,7 @@ class CloudVisionCreateProductSetOperator(GoogleCloudBaseOperator):
94
95
  *,
95
96
  product_set: dict | ProductSet,
96
97
  location: str,
97
- project_id: str | None = None,
98
+ project_id: str = PROVIDE_PROJECT_ID,
98
99
  product_set_id: str | None = None,
99
100
  retry: Retry | _MethodDefault = DEFAULT,
100
101
  timeout: float | None = None,
@@ -181,7 +182,7 @@ class CloudVisionGetProductSetOperator(GoogleCloudBaseOperator):
181
182
  *,
182
183
  location: str,
183
184
  product_set_id: str,
184
- project_id: str | None = None,
185
+ project_id: str = PROVIDE_PROJECT_ID,
185
186
  retry: Retry | _MethodDefault = DEFAULT,
186
187
  timeout: float | None = None,
187
188
  metadata: MetaData = (),
@@ -279,7 +280,7 @@ class CloudVisionUpdateProductSetOperator(GoogleCloudBaseOperator):
279
280
  product_set: dict | ProductSet,
280
281
  location: str | None = None,
281
282
  product_set_id: str | None = None,
282
- project_id: str | None = None,
283
+ project_id: str = PROVIDE_PROJECT_ID,
283
284
  update_mask: dict | FieldMask | None = None,
284
285
  retry: Retry | _MethodDefault = DEFAULT,
285
286
  timeout: float | None = None,
@@ -367,7 +368,7 @@ class CloudVisionDeleteProductSetOperator(GoogleCloudBaseOperator):
367
368
  *,
368
369
  location: str,
369
370
  product_set_id: str,
370
- project_id: str | None = None,
371
+ project_id: str = PROVIDE_PROJECT_ID,
371
372
  retry: Retry | _MethodDefault = DEFAULT,
372
373
  timeout: float | None = None,
373
374
  metadata: MetaData = (),
@@ -455,7 +456,7 @@ class CloudVisionCreateProductOperator(GoogleCloudBaseOperator):
455
456
  *,
456
457
  location: str,
457
458
  product: str,
458
- project_id: str | None = None,
459
+ project_id: str = PROVIDE_PROJECT_ID,
459
460
  product_id: str | None = None,
460
461
  retry: Retry | _MethodDefault = DEFAULT,
461
462
  timeout: float | None = None,
@@ -545,7 +546,7 @@ class CloudVisionGetProductOperator(GoogleCloudBaseOperator):
545
546
  *,
546
547
  location: str,
547
548
  product_id: str,
548
- project_id: str | None = None,
549
+ project_id: str = PROVIDE_PROJECT_ID,
549
550
  retry: Retry | _MethodDefault = DEFAULT,
550
551
  timeout: float | None = None,
551
552
  metadata: MetaData = (),
@@ -652,7 +653,7 @@ class CloudVisionUpdateProductOperator(GoogleCloudBaseOperator):
652
653
  product: dict | Product,
653
654
  location: str | None = None,
654
655
  product_id: str | None = None,
655
- project_id: str | None = None,
656
+ project_id: str = PROVIDE_PROJECT_ID,
656
657
  update_mask: dict | FieldMask | None = None,
657
658
  retry: Retry | _MethodDefault = DEFAULT,
658
659
  timeout: float | None = None,
@@ -743,7 +744,7 @@ class CloudVisionDeleteProductOperator(GoogleCloudBaseOperator):
743
744
  *,
744
745
  location: str,
745
746
  product_id: str,
746
- project_id: str | None = None,
747
+ project_id: str = PROVIDE_PROJECT_ID,
747
748
  retry: Retry | _MethodDefault = DEFAULT,
748
749
  timeout: float | None = None,
749
750
  metadata: MetaData = (),
@@ -899,7 +900,7 @@ class CloudVisionCreateReferenceImageOperator(GoogleCloudBaseOperator):
899
900
  reference_image: dict | ReferenceImage,
900
901
  product_id: str,
901
902
  reference_image_id: str | None = None,
902
- project_id: str | None = None,
903
+ project_id: str = PROVIDE_PROJECT_ID,
903
904
  retry: Retry | _MethodDefault = DEFAULT,
904
905
  timeout: float | None = None,
905
906
  metadata: MetaData = (),
@@ -996,7 +997,7 @@ class CloudVisionDeleteReferenceImageOperator(GoogleCloudBaseOperator):
996
997
  location: str,
997
998
  product_id: str,
998
999
  reference_image_id: str,
999
- project_id: str | None = None,
1000
+ project_id: str = PROVIDE_PROJECT_ID,
1000
1001
  retry: Retry | _MethodDefault = DEFAULT,
1001
1002
  timeout: float | None = None,
1002
1003
  metadata: MetaData = (),
@@ -1085,7 +1086,7 @@ class CloudVisionAddProductToProductSetOperator(GoogleCloudBaseOperator):
1085
1086
  product_set_id: str,
1086
1087
  product_id: str,
1087
1088
  location: str,
1088
- project_id: str | None = None,
1089
+ project_id: str = PROVIDE_PROJECT_ID,
1089
1090
  retry: Retry | _MethodDefault = DEFAULT,
1090
1091
  timeout: float | None = None,
1091
1092
  metadata: MetaData = (),
@@ -1167,7 +1168,7 @@ class CloudVisionRemoveProductFromProductSetOperator(GoogleCloudBaseOperator):
1167
1168
  product_set_id: str,
1168
1169
  product_id: str,
1169
1170
  location: str,
1170
- project_id: str | None = None,
1171
+ project_id: str = PROVIDE_PROJECT_ID,
1171
1172
  retry: Retry | _MethodDefault = DEFAULT,
1172
1173
  timeout: float | None = None,
1173
1174
  metadata: MetaData = (),
@@ -34,6 +34,7 @@ from airflow.providers.google.cloud.links.workflows import (
34
34
  WorkflowsWorkflowDetailsLink,
35
35
  )
36
36
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
37
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
37
38
 
38
39
  if TYPE_CHECKING:
39
40
  from google.api_core.retry import Retry
@@ -80,7 +81,7 @@ class WorkflowsCreateWorkflowOperator(GoogleCloudBaseOperator):
80
81
  workflow: dict,
81
82
  workflow_id: str,
82
83
  location: str,
83
- project_id: str | None = None,
84
+ project_id: str = PROVIDE_PROJECT_ID,
84
85
  retry: Retry | _MethodDefault = DEFAULT,
85
86
  timeout: float | None = None,
86
87
  metadata: Sequence[tuple[str, str]] = (),
@@ -192,7 +193,7 @@ class WorkflowsUpdateWorkflowOperator(GoogleCloudBaseOperator):
192
193
  *,
193
194
  workflow_id: str,
194
195
  location: str,
195
- project_id: str | None = None,
196
+ project_id: str = PROVIDE_PROJECT_ID,
196
197
  update_mask: FieldMask | None = None,
197
198
  retry: Retry | _MethodDefault = DEFAULT,
198
199
  timeout: float | None = None,
@@ -270,7 +271,7 @@ class WorkflowsDeleteWorkflowOperator(GoogleCloudBaseOperator):
270
271
  *,
271
272
  workflow_id: str,
272
273
  location: str,
273
- project_id: str | None = None,
274
+ project_id: str = PROVIDE_PROJECT_ID,
274
275
  retry: Retry | _MethodDefault = DEFAULT,
275
276
  timeout: float | None = None,
276
277
  metadata: Sequence[tuple[str, str]] = (),
@@ -332,7 +333,7 @@ class WorkflowsListWorkflowsOperator(GoogleCloudBaseOperator):
332
333
  self,
333
334
  *,
334
335
  location: str,
335
- project_id: str | None = None,
336
+ project_id: str = PROVIDE_PROJECT_ID,
336
337
  filter_: str | None = None,
337
338
  order_by: str | None = None,
338
339
  retry: Retry | _MethodDefault = DEFAULT,
@@ -402,7 +403,7 @@ class WorkflowsGetWorkflowOperator(GoogleCloudBaseOperator):
402
403
  *,
403
404
  workflow_id: str,
404
405
  location: str,
405
- project_id: str | None = None,
406
+ project_id: str = PROVIDE_PROJECT_ID,
406
407
  retry: Retry | _MethodDefault = DEFAULT,
407
408
  timeout: float | None = None,
408
409
  metadata: Sequence[tuple[str, str]] = (),
@@ -473,7 +474,7 @@ class WorkflowsCreateExecutionOperator(GoogleCloudBaseOperator):
473
474
  workflow_id: str,
474
475
  execution: dict,
475
476
  location: str,
476
- project_id: str | None = None,
477
+ project_id: str = PROVIDE_PROJECT_ID,
477
478
  retry: Retry | _MethodDefault = DEFAULT,
478
479
  timeout: float | None = None,
479
480
  metadata: Sequence[tuple[str, str]] = (),
@@ -548,7 +549,7 @@ class WorkflowsCancelExecutionOperator(GoogleCloudBaseOperator):
548
549
  workflow_id: str,
549
550
  execution_id: str,
550
551
  location: str,
551
- project_id: str | None = None,
552
+ project_id: str = PROVIDE_PROJECT_ID,
552
553
  retry: Retry | _MethodDefault = DEFAULT,
553
554
  timeout: float | None = None,
554
555
  metadata: Sequence[tuple[str, str]] = (),
@@ -626,7 +627,7 @@ class WorkflowsListExecutionsOperator(GoogleCloudBaseOperator):
626
627
  workflow_id: str,
627
628
  location: str,
628
629
  start_date_filter: datetime.datetime | None = None,
629
- project_id: str | None = None,
630
+ project_id: str = PROVIDE_PROJECT_ID,
630
631
  retry: Retry | _MethodDefault = DEFAULT,
631
632
  timeout: float | None = None,
632
633
  metadata: Sequence[tuple[str, str]] = (),
@@ -703,7 +704,7 @@ class WorkflowsGetExecutionOperator(GoogleCloudBaseOperator):
703
704
  workflow_id: str,
704
705
  execution_id: str,
705
706
  location: str,
706
- project_id: str | None = None,
707
+ project_id: str = PROVIDE_PROJECT_ID,
707
708
  retry: Retry | _MethodDefault = DEFAULT,
708
709
  timeout: float | None = None,
709
710
  metadata: Sequence[tuple[str, str]] = (),
@@ -30,6 +30,7 @@ from airflow.providers.google.cloud.utils.credentials_provider import (
30
30
  _get_target_principal_and_delegates,
31
31
  get_credentials_and_project_id,
32
32
  )
33
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
33
34
  from airflow.secrets import BaseSecretsBackend
34
35
  from airflow.utils.log.logging_mixin import LoggingMixin
35
36
 
@@ -94,7 +95,7 @@ class CloudSecretManagerBackend(BaseSecretsBackend, LoggingMixin):
94
95
  gcp_key_path: str | None = None,
95
96
  gcp_credential_config_file: dict[str, str] | str | None = None,
96
97
  gcp_scopes: str | None = None,
97
- project_id: str | None = None,
98
+ project_id: str = PROVIDE_PROJECT_ID,
98
99
  sep: str = "-",
99
100
  impersonation_chain: str | Sequence[str] | None = None,
100
101
  **kwargs,
@@ -26,6 +26,7 @@ from google.cloud.bigquery_datatransfer_v1 import TransferState
26
26
 
27
27
  from airflow.exceptions import AirflowException, AirflowSkipException
28
28
  from airflow.providers.google.cloud.hooks.bigquery_dts import BiqQueryDataTransferServiceHook
29
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
29
30
  from airflow.sensors.base import BaseSensorOperator
30
31
 
31
32
  if TYPE_CHECKING:
@@ -83,7 +84,7 @@ class BigQueryDataTransferServiceTransferRunSensor(BaseSensorOperator):
83
84
  expected_statuses: (
84
85
  set[str | TransferState | int] | str | TransferState | int
85
86
  ) = TransferState.SUCCEEDED,
86
- project_id: str | None = None,
87
+ project_id: str = PROVIDE_PROJECT_ID,
87
88
  gcp_conn_id: str = "google_cloud_default",
88
89
  retry: Retry | _MethodDefault = DEFAULT,
89
90
  request_timeout: float | None = None,
@@ -28,6 +28,7 @@ from google.cloud.bigtable.table import ClusterState
28
28
  from airflow.providers.google.cloud.hooks.bigtable import BigtableHook
29
29
  from airflow.providers.google.cloud.links.bigtable import BigtableTablesLink
30
30
  from airflow.providers.google.cloud.operators.bigtable import BigtableValidationMixin
31
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
31
32
  from airflow.sensors.base import BaseSensorOperator
32
33
 
33
34
  if TYPE_CHECKING:
@@ -74,7 +75,7 @@ class BigtableTableReplicationCompletedSensor(BaseSensorOperator, BigtableValida
74
75
  *,
75
76
  instance_id: str,
76
77
  table_id: str,
77
- project_id: str | None = None,
78
+ project_id: str = PROVIDE_PROJECT_ID,
78
79
  gcp_conn_id: str = "google_cloud_default",
79
80
  impersonation_chain: str | Sequence[str] | None = None,
80
81
  **kwargs,
@@ -28,6 +28,7 @@ from airflow.providers.google.cloud.hooks.cloud_storage_transfer_service import
28
28
  CloudDataTransferServiceHook,
29
29
  )
30
30
  from airflow.providers.google.cloud.links.cloud_storage_transfer import CloudStorageTransferJobLink
31
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
31
32
  from airflow.sensors.base import BaseSensorOperator
32
33
 
33
34
  if TYPE_CHECKING:
@@ -73,7 +74,7 @@ class CloudDataTransferServiceJobStatusSensor(BaseSensorOperator):
73
74
  *,
74
75
  job_name: str,
75
76
  expected_statuses: set[str] | str,
76
- project_id: str | None = None,
77
+ project_id: str = PROVIDE_PROJECT_ID,
77
78
  gcp_conn_id: str = "google_cloud_default",
78
79
  impersonation_chain: str | Sequence[str] | None = None,
79
80
  **kwargs,