apache-airflow-providers-google 10.22.0rc1__py3-none-any.whl → 10.23.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. airflow/providers/google/__init__.py +1 -1
  2. airflow/providers/google/cloud/hooks/bigquery.py +91 -54
  3. airflow/providers/google/cloud/hooks/cloud_build.py +3 -2
  4. airflow/providers/google/cloud/hooks/dataflow.py +112 -47
  5. airflow/providers/google/cloud/hooks/datapipeline.py +3 -3
  6. airflow/providers/google/cloud/hooks/kubernetes_engine.py +15 -26
  7. airflow/providers/google/cloud/hooks/life_sciences.py +5 -7
  8. airflow/providers/google/cloud/hooks/secret_manager.py +3 -3
  9. airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +28 -8
  10. airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +11 -6
  11. airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +214 -34
  12. airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +11 -4
  13. airflow/providers/google/cloud/links/automl.py +13 -22
  14. airflow/providers/google/cloud/log/gcs_task_handler.py +1 -2
  15. airflow/providers/google/cloud/operators/bigquery.py +6 -4
  16. airflow/providers/google/cloud/operators/dataflow.py +186 -4
  17. airflow/providers/google/cloud/operators/datafusion.py +3 -2
  18. airflow/providers/google/cloud/operators/datapipeline.py +5 -6
  19. airflow/providers/google/cloud/operators/dataproc.py +30 -33
  20. airflow/providers/google/cloud/operators/gcs.py +4 -4
  21. airflow/providers/google/cloud/operators/kubernetes_engine.py +16 -2
  22. airflow/providers/google/cloud/operators/life_sciences.py +5 -7
  23. airflow/providers/google/cloud/operators/mlengine.py +42 -65
  24. airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +18 -4
  25. airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +5 -5
  26. airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +280 -9
  27. airflow/providers/google/cloud/operators/vertex_ai/model_service.py +4 -0
  28. airflow/providers/google/cloud/secrets/secret_manager.py +3 -5
  29. airflow/providers/google/cloud/sensors/bigquery.py +8 -27
  30. airflow/providers/google/cloud/sensors/bigquery_dts.py +1 -4
  31. airflow/providers/google/cloud/sensors/cloud_composer.py +9 -14
  32. airflow/providers/google/cloud/sensors/dataflow.py +1 -25
  33. airflow/providers/google/cloud/sensors/dataform.py +1 -4
  34. airflow/providers/google/cloud/sensors/datafusion.py +1 -7
  35. airflow/providers/google/cloud/sensors/dataplex.py +1 -31
  36. airflow/providers/google/cloud/sensors/dataproc.py +1 -16
  37. airflow/providers/google/cloud/sensors/dataproc_metastore.py +1 -7
  38. airflow/providers/google/cloud/sensors/gcs.py +5 -27
  39. airflow/providers/google/cloud/sensors/looker.py +1 -13
  40. airflow/providers/google/cloud/sensors/pubsub.py +11 -5
  41. airflow/providers/google/cloud/sensors/workflows.py +1 -4
  42. airflow/providers/google/cloud/transfers/sftp_to_gcs.py +6 -0
  43. airflow/providers/google/cloud/triggers/dataflow.py +145 -1
  44. airflow/providers/google/cloud/triggers/kubernetes_engine.py +66 -3
  45. airflow/providers/google/common/deprecated.py +176 -0
  46. airflow/providers/google/common/hooks/base_google.py +3 -2
  47. airflow/providers/google/get_provider_info.py +8 -10
  48. airflow/providers/google/marketing_platform/hooks/analytics.py +4 -2
  49. airflow/providers/google/marketing_platform/hooks/search_ads.py +169 -30
  50. airflow/providers/google/marketing_platform/operators/analytics.py +16 -33
  51. airflow/providers/google/marketing_platform/operators/search_ads.py +217 -156
  52. airflow/providers/google/marketing_platform/sensors/display_video.py +1 -4
  53. {apache_airflow_providers_google-10.22.0rc1.dist-info → apache_airflow_providers_google-10.23.0.dist-info}/METADATA +25 -23
  54. {apache_airflow_providers_google-10.22.0rc1.dist-info → apache_airflow_providers_google-10.23.0.dist-info}/RECORD +56 -56
  55. airflow/providers/google/marketing_platform/sensors/search_ads.py +0 -92
  56. {apache_airflow_providers_google-10.22.0rc1.dist-info → apache_airflow_providers_google-10.23.0.dist-info}/WHEEL +0 -0
  57. {apache_airflow_providers_google-10.22.0rc1.dist-info → apache_airflow_providers_google-10.23.0.dist-info}/entry_points.txt +0 -0
@@ -24,7 +24,6 @@ import re
24
24
  import time
25
25
  from typing import TYPE_CHECKING, Any, Sequence
26
26
 
27
- from deprecated import deprecated
28
27
  from googleapiclient.errors import HttpError
29
28
 
30
29
  from airflow.configuration import conf
@@ -39,6 +38,7 @@ from airflow.providers.google.cloud.links.mlengine import (
39
38
  )
40
39
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
41
40
  from airflow.providers.google.cloud.triggers.mlengine import MLEngineStartTrainingJobTrigger
41
+ from airflow.providers.google.common.deprecated import deprecated
42
42
  from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
43
43
 
44
44
  if TYPE_CHECKING:
@@ -80,11 +80,9 @@ def _normalize_mlengine_job_id(job_id: str) -> str:
80
80
 
81
81
 
82
82
  @deprecated(
83
- reason=(
84
- "This operator is deprecated. All the functionality of legacy "
85
- "MLEngine and new features are available on the Vertex AI platform. "
86
- "Please use `CreateBatchPredictionJobOperator`"
87
- ),
83
+ planned_removal_date="March 01, 2025",
84
+ use_instead="CreateBatchPredictionJobOperator",
85
+ reason="All the functionality of legacy MLEngine and new features are available on the Vertex AI.",
88
86
  category=AirflowProviderDeprecationWarning,
89
87
  )
90
88
  class MLEngineStartBatchPredictionJobOperator(GoogleCloudBaseOperator):
@@ -297,10 +295,8 @@ class MLEngineStartBatchPredictionJobOperator(GoogleCloudBaseOperator):
297
295
 
298
296
 
299
297
  @deprecated(
300
- reason=(
301
- "This operator is deprecated. Consider using operators for specific operations: "
302
- "MLEngineCreateModelOperator, MLEngineGetModelOperator."
303
- ),
298
+ planned_removal_date="March 01, 2025",
299
+ use_instead="MLEngineCreateModelOperator, MLEngineGetModelOperator",
304
300
  category=AirflowProviderDeprecationWarning,
305
301
  )
306
302
  class MLEngineManageModelOperator(GoogleCloudBaseOperator):
@@ -372,11 +368,9 @@ class MLEngineManageModelOperator(GoogleCloudBaseOperator):
372
368
 
373
369
 
374
370
  @deprecated(
375
- reason=(
376
- "This operator is deprecated. All the functionality of legacy "
377
- "MLEngine and new features are available on the Vertex AI platform. "
378
- "Please use appropriate VertexAI operator."
379
- ),
371
+ planned_removal_date="November 01, 2025",
372
+ use_instead="appropriate VertexAI operator",
373
+ reason="All the functionality of legacy MLEngine and new features are available on the Vertex AI.",
380
374
  category=AirflowProviderDeprecationWarning,
381
375
  )
382
376
  class MLEngineCreateModelOperator(GoogleCloudBaseOperator):
@@ -448,11 +442,9 @@ class MLEngineCreateModelOperator(GoogleCloudBaseOperator):
448
442
 
449
443
 
450
444
  @deprecated(
451
- reason=(
452
- "This operator is deprecated. All the functionality of legacy "
453
- "MLEngine and new features are available on the Vertex AI platform. "
454
- "Please use `GetModelOperator`"
455
- ),
445
+ planned_removal_date="March 01, 2025",
446
+ use_instead="GetModelOperator",
447
+ reason="All the functionality of legacy MLEngine and new features are available on the Vertex AI.",
456
448
  category=AirflowProviderDeprecationWarning,
457
449
  )
458
450
  class MLEngineGetModelOperator(GoogleCloudBaseOperator):
@@ -523,11 +515,9 @@ class MLEngineGetModelOperator(GoogleCloudBaseOperator):
523
515
 
524
516
 
525
517
  @deprecated(
526
- reason=(
527
- "This operator is deprecated. All the functionality of legacy "
528
- "MLEngine and new features are available on the Vertex AI platform. "
529
- "Please use `DeleteModelOperator`"
530
- ),
518
+ planned_removal_date="March 01, 2025",
519
+ use_instead="DeleteModelOperator",
520
+ reason="All the functionality of legacy MLEngine and new features are available on the Vertex AI.",
531
521
  category=AirflowProviderDeprecationWarning,
532
522
  )
533
523
  class MLEngineDeleteModelOperator(GoogleCloudBaseOperator):
@@ -606,11 +596,9 @@ class MLEngineDeleteModelOperator(GoogleCloudBaseOperator):
606
596
 
607
597
 
608
598
  @deprecated(
609
- reason=(
610
- "This operator is deprecated. Consider using operators for specific operations: "
611
- "MLEngineCreateVersion, MLEngineSetDefaultVersion, "
612
- "MLEngineListVersions, MLEngineDeleteVersion."
613
- ),
599
+ planned_removal_date="March 01, 2025",
600
+ use_instead="MLEngineCreateVersion, MLEngineSetDefaultVersion, MLEngineListVersions, "
601
+ "MLEngineDeleteVersion",
614
602
  category=AirflowProviderDeprecationWarning,
615
603
  )
616
604
  class MLEngineManageVersionOperator(GoogleCloudBaseOperator):
@@ -725,11 +713,9 @@ class MLEngineManageVersionOperator(GoogleCloudBaseOperator):
725
713
 
726
714
 
727
715
  @deprecated(
728
- reason=(
729
- "This operator is deprecated. All the functionality of legacy "
730
- "MLEngine and new features are available on the Vertex AI platform. "
731
- "Please use parent_model parameter for VertexAI operators instead."
732
- ),
716
+ planned_removal_date="March 01, 2025",
717
+ use_instead="parent_model parameter for VertexAI operators",
718
+ reason="All the functionality of legacy MLEngine and new features are available on the Vertex AI.",
733
719
  category=AirflowProviderDeprecationWarning,
734
720
  )
735
721
  class MLEngineCreateVersionOperator(GoogleCloudBaseOperator):
@@ -816,11 +802,9 @@ class MLEngineCreateVersionOperator(GoogleCloudBaseOperator):
816
802
 
817
803
 
818
804
  @deprecated(
819
- reason=(
820
- "This operator is deprecated. All the functionality of legacy "
821
- "MLEngine and new features are available on the Vertex AI platform. "
822
- "Please use `SetDefaultVersionOnModelOperator`"
823
- ),
805
+ planned_removal_date="March 01, 2025",
806
+ use_instead="SetDefaultVersionOnModelOperator",
807
+ reason="All the functionality of legacy MLEngine and new features are available on the Vertex AI.",
824
808
  category=AirflowProviderDeprecationWarning,
825
809
  )
826
810
  class MLEngineSetDefaultVersionOperator(GoogleCloudBaseOperator):
@@ -909,11 +893,9 @@ class MLEngineSetDefaultVersionOperator(GoogleCloudBaseOperator):
909
893
 
910
894
 
911
895
  @deprecated(
912
- reason=(
913
- "This operator is deprecated. All the functionality of legacy "
914
- "MLEngine and new features are available on the Vertex AI platform. "
915
- "Please use `ListModelVersionsOperator`"
916
- ),
896
+ planned_removal_date="March 01, 2025",
897
+ use_instead="istModelVersionsOperator",
898
+ reason="All the functionality of legacy MLEngine and new features are available on the Vertex AI.",
917
899
  category=AirflowProviderDeprecationWarning,
918
900
  )
919
901
  class MLEngineListVersionsOperator(GoogleCloudBaseOperator):
@@ -995,11 +977,9 @@ class MLEngineListVersionsOperator(GoogleCloudBaseOperator):
995
977
 
996
978
 
997
979
  @deprecated(
998
- reason=(
999
- "This operator is deprecated. All the functionality of legacy "
1000
- "MLEngine and new features are available on the Vertex AI platform. "
1001
- "Please use `DeleteModelVersionOperator`"
1002
- ),
980
+ planned_removal_date="March 01, 2025",
981
+ use_instead="DeleteModelVersionOperator",
982
+ reason="All the functionality of legacy MLEngine and new features are available on the Vertex AI.",
1003
983
  category=AirflowProviderDeprecationWarning,
1004
984
  )
1005
985
  class MLEngineDeleteVersionOperator(GoogleCloudBaseOperator):
@@ -1087,11 +1067,9 @@ class MLEngineDeleteVersionOperator(GoogleCloudBaseOperator):
1087
1067
 
1088
1068
 
1089
1069
  @deprecated(
1090
- reason=(
1091
- "This operator is deprecated. All the functionality of legacy "
1092
- "MLEngine and new features are available on the Vertex AI platform. "
1093
- "Please use `CreateCustomPythonPackageTrainingJobOperator`"
1094
- ),
1070
+ planned_removal_date="March 01, 2025",
1071
+ use_instead="CreateCustomPythonPackageTrainingJobOperator",
1072
+ reason="All the functionality of legacy MLEngine and new features are available on the Vertex AI.",
1095
1073
  category=AirflowProviderDeprecationWarning,
1096
1074
  )
1097
1075
  class MLEngineStartTrainingJobOperator(GoogleCloudBaseOperator):
@@ -1426,11 +1404,9 @@ class MLEngineStartTrainingJobOperator(GoogleCloudBaseOperator):
1426
1404
 
1427
1405
 
1428
1406
  @deprecated(
1429
- reason=(
1430
- "This operator is deprecated. All the functionality of legacy "
1431
- "MLEngine and new features are available on the Vertex AI platform. "
1432
- "Please use `CancelCustomTrainingJobOperator`"
1433
- ),
1407
+ planned_removal_date="March 01, 2025",
1408
+ use_instead="CancelCustomTrainingJobOperator",
1409
+ reason="All the functionality of legacy MLEngine and new features are available on the Vertex AI.",
1434
1410
  category=AirflowProviderDeprecationWarning,
1435
1411
  )
1436
1412
  class MLEngineTrainingCancelJobOperator(GoogleCloudBaseOperator):
@@ -1481,8 +1457,8 @@ class MLEngineTrainingCancelJobOperator(GoogleCloudBaseOperator):
1481
1457
 
1482
1458
  @property
1483
1459
  @deprecated(
1484
- reason="`_project_id` is deprecated and will be removed in the future. Please use `project_id`"
1485
- " instead.",
1460
+ planned_removal_date="March 01, 2025",
1461
+ use_instead="project_id",
1486
1462
  category=AirflowProviderDeprecationWarning,
1487
1463
  )
1488
1464
  def _project_id(self):
@@ -1491,7 +1467,8 @@ class MLEngineTrainingCancelJobOperator(GoogleCloudBaseOperator):
1491
1467
 
1492
1468
  @property
1493
1469
  @deprecated(
1494
- reason="`_job_id` is deprecated and will be removed in the future. Please use `job_id` instead.",
1470
+ planned_removal_date="March 01, 2025",
1471
+ use_instead="job_id",
1495
1472
  category=AirflowProviderDeprecationWarning,
1496
1473
  )
1497
1474
  def _job_id(self):
@@ -1500,8 +1477,8 @@ class MLEngineTrainingCancelJobOperator(GoogleCloudBaseOperator):
1500
1477
 
1501
1478
  @property
1502
1479
  @deprecated(
1503
- reason="`_impersonation_chain` is deprecated and will be removed in the future."
1504
- " Please use `impersonation_chain` instead.",
1480
+ planned_removal_date="March 01, 2025",
1481
+ use_instead="impersonation_chain",
1505
1482
  category=AirflowProviderDeprecationWarning,
1506
1483
  )
1507
1484
  def _impersonation_chain(self):
@@ -22,7 +22,6 @@ from __future__ import annotations
22
22
 
23
23
  from typing import TYPE_CHECKING, Sequence
24
24
 
25
- from deprecated import deprecated
26
25
  from google.api_core.exceptions import NotFound
27
26
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
28
27
  from google.cloud.aiplatform import datasets
@@ -37,6 +36,7 @@ from airflow.providers.google.cloud.links.vertex_ai import (
37
36
  VertexAITrainingPipelinesLink,
38
37
  )
39
38
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
39
+ from airflow.providers.google.common.deprecated import deprecated
40
40
 
41
41
  if TYPE_CHECKING:
42
42
  from google.api_core.retry import Retry
@@ -455,8 +455,22 @@ class CreateAutoMLTabularTrainingJobOperator(AutoMLTrainingJobBaseOperator):
455
455
  return result
456
456
 
457
457
 
458
+ @deprecated(
459
+ planned_removal_date="September 15, 2024",
460
+ use_instead="SupervisedFineTuningTrainOperator",
461
+ instructions=(
462
+ "Please consider using Fine Tuning over the Gemini model. "
463
+ "More info: https://cloud.google.com/vertex-ai/docs/start/automl-gemini-comparison"
464
+ ),
465
+ category=AirflowProviderDeprecationWarning,
466
+ )
458
467
  class CreateAutoMLTextTrainingJobOperator(AutoMLTrainingJobBaseOperator):
459
- """Create Auto ML Text Training job."""
468
+ """
469
+ Create Auto ML Text Training job.
470
+
471
+ WARNING: Text creation API is deprecated since September 15, 2024
472
+ (https://cloud.google.com/vertex-ai/docs/tutorials/text-classification-automl/overview).
473
+ """
460
474
 
461
475
  template_fields = [
462
476
  "parent_model",
@@ -642,8 +656,8 @@ class DeleteAutoMLTrainingJobOperator(GoogleCloudBaseOperator):
642
656
 
643
657
  @property
644
658
  @deprecated(
645
- reason="`training_pipeline` is deprecated and will be removed in the future. "
646
- "Please use `training_pipeline_id` instead.",
659
+ planned_removal_date="March 01, 2025",
660
+ use_instead="training_pipeline_id",
647
661
  category=AirflowProviderDeprecationWarning,
648
662
  )
649
663
  def training_pipeline(self):
@@ -23,7 +23,6 @@ import warnings
23
23
  from functools import cached_property
24
24
  from typing import TYPE_CHECKING, Any, Sequence
25
25
 
26
- from deprecated import deprecated
27
26
  from google.api_core.exceptions import NotFound
28
27
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
29
28
  from google.cloud.aiplatform.models import Model
@@ -44,6 +43,7 @@ from airflow.providers.google.cloud.triggers.vertex_ai import (
44
43
  CustomPythonPackageTrainingJobTrigger,
45
44
  CustomTrainingJobTrigger,
46
45
  )
46
+ from airflow.providers.google.common.deprecated import deprecated
47
47
 
48
48
  if TYPE_CHECKING:
49
49
  from google.api_core.retry import Retry
@@ -1633,8 +1633,8 @@ class DeleteCustomTrainingJobOperator(GoogleCloudBaseOperator):
1633
1633
 
1634
1634
  @property
1635
1635
  @deprecated(
1636
- reason="`training_pipeline` is deprecated and will be removed in the future. "
1637
- "Please use `training_pipeline_id` instead.",
1636
+ planned_removal_date="March 01, 2025",
1637
+ use_instead="training_pipeline_id",
1638
1638
  category=AirflowProviderDeprecationWarning,
1639
1639
  )
1640
1640
  def training_pipeline(self):
@@ -1643,8 +1643,8 @@ class DeleteCustomTrainingJobOperator(GoogleCloudBaseOperator):
1643
1643
 
1644
1644
  @property
1645
1645
  @deprecated(
1646
- reason="`custom_job` is deprecated and will be removed in the future. "
1647
- "Please use `custom_job_id` instead.",
1646
+ planned_removal_date="March 01, 2025",
1647
+ use_instead="custom_job_id",
1648
1648
  category=AirflowProviderDeprecationWarning,
1649
1649
  )
1650
1650
  def custom_job(self):