apache-airflow-providers-google 10.22.0__py3-none-any.whl → 10.23.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/__init__.py +1 -1
- airflow/providers/google/cloud/hooks/bigquery.py +91 -54
- airflow/providers/google/cloud/hooks/cloud_build.py +3 -2
- airflow/providers/google/cloud/hooks/dataflow.py +112 -47
- airflow/providers/google/cloud/hooks/datapipeline.py +3 -3
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +15 -26
- airflow/providers/google/cloud/hooks/life_sciences.py +5 -7
- airflow/providers/google/cloud/hooks/secret_manager.py +3 -3
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +28 -8
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +11 -6
- airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +214 -34
- airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +11 -4
- airflow/providers/google/cloud/links/automl.py +13 -22
- airflow/providers/google/cloud/log/gcs_task_handler.py +1 -2
- airflow/providers/google/cloud/operators/bigquery.py +6 -4
- airflow/providers/google/cloud/operators/dataflow.py +186 -4
- airflow/providers/google/cloud/operators/datafusion.py +3 -2
- airflow/providers/google/cloud/operators/datapipeline.py +5 -6
- airflow/providers/google/cloud/operators/dataproc.py +30 -33
- airflow/providers/google/cloud/operators/gcs.py +4 -4
- airflow/providers/google/cloud/operators/kubernetes_engine.py +16 -2
- airflow/providers/google/cloud/operators/life_sciences.py +5 -7
- airflow/providers/google/cloud/operators/mlengine.py +42 -65
- airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +18 -4
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +5 -5
- airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +280 -9
- airflow/providers/google/cloud/operators/vertex_ai/model_service.py +4 -0
- airflow/providers/google/cloud/secrets/secret_manager.py +3 -5
- airflow/providers/google/cloud/sensors/bigquery.py +8 -27
- airflow/providers/google/cloud/sensors/bigquery_dts.py +1 -4
- airflow/providers/google/cloud/sensors/cloud_composer.py +9 -14
- airflow/providers/google/cloud/sensors/dataflow.py +1 -25
- airflow/providers/google/cloud/sensors/dataform.py +1 -4
- airflow/providers/google/cloud/sensors/datafusion.py +1 -7
- airflow/providers/google/cloud/sensors/dataplex.py +1 -31
- airflow/providers/google/cloud/sensors/dataproc.py +1 -16
- airflow/providers/google/cloud/sensors/dataproc_metastore.py +1 -7
- airflow/providers/google/cloud/sensors/gcs.py +5 -27
- airflow/providers/google/cloud/sensors/looker.py +1 -13
- airflow/providers/google/cloud/sensors/pubsub.py +11 -5
- airflow/providers/google/cloud/sensors/workflows.py +1 -4
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +6 -0
- airflow/providers/google/cloud/triggers/dataflow.py +145 -1
- airflow/providers/google/cloud/triggers/kubernetes_engine.py +66 -3
- airflow/providers/google/common/deprecated.py +176 -0
- airflow/providers/google/common/hooks/base_google.py +3 -2
- airflow/providers/google/get_provider_info.py +8 -10
- airflow/providers/google/marketing_platform/hooks/analytics.py +4 -2
- airflow/providers/google/marketing_platform/hooks/search_ads.py +169 -30
- airflow/providers/google/marketing_platform/operators/analytics.py +16 -33
- airflow/providers/google/marketing_platform/operators/search_ads.py +217 -156
- airflow/providers/google/marketing_platform/sensors/display_video.py +1 -4
- {apache_airflow_providers_google-10.22.0.dist-info → apache_airflow_providers_google-10.23.0.dist-info}/METADATA +18 -16
- {apache_airflow_providers_google-10.22.0.dist-info → apache_airflow_providers_google-10.23.0.dist-info}/RECORD +56 -56
- airflow/providers/google/marketing_platform/sensors/search_ads.py +0 -92
- {apache_airflow_providers_google-10.22.0.dist-info → apache_airflow_providers_google-10.23.0.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_google-10.22.0.dist-info → apache_airflow_providers_google-10.23.0.dist-info}/entry_points.txt +0 -0
@@ -24,7 +24,6 @@ import re
|
|
24
24
|
import time
|
25
25
|
from typing import TYPE_CHECKING, Any, Sequence
|
26
26
|
|
27
|
-
from deprecated import deprecated
|
28
27
|
from googleapiclient.errors import HttpError
|
29
28
|
|
30
29
|
from airflow.configuration import conf
|
@@ -39,6 +38,7 @@ from airflow.providers.google.cloud.links.mlengine import (
|
|
39
38
|
)
|
40
39
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
41
40
|
from airflow.providers.google.cloud.triggers.mlengine import MLEngineStartTrainingJobTrigger
|
41
|
+
from airflow.providers.google.common.deprecated import deprecated
|
42
42
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
43
43
|
|
44
44
|
if TYPE_CHECKING:
|
@@ -80,11 +80,9 @@ def _normalize_mlengine_job_id(job_id: str) -> str:
|
|
80
80
|
|
81
81
|
|
82
82
|
@deprecated(
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
"Please use `CreateBatchPredictionJobOperator`"
|
87
|
-
),
|
83
|
+
planned_removal_date="March 01, 2025",
|
84
|
+
use_instead="CreateBatchPredictionJobOperator",
|
85
|
+
reason="All the functionality of legacy MLEngine and new features are available on the Vertex AI.",
|
88
86
|
category=AirflowProviderDeprecationWarning,
|
89
87
|
)
|
90
88
|
class MLEngineStartBatchPredictionJobOperator(GoogleCloudBaseOperator):
|
@@ -297,10 +295,8 @@ class MLEngineStartBatchPredictionJobOperator(GoogleCloudBaseOperator):
|
|
297
295
|
|
298
296
|
|
299
297
|
@deprecated(
|
300
|
-
|
301
|
-
|
302
|
-
"MLEngineCreateModelOperator, MLEngineGetModelOperator."
|
303
|
-
),
|
298
|
+
planned_removal_date="March 01, 2025",
|
299
|
+
use_instead="MLEngineCreateModelOperator, MLEngineGetModelOperator",
|
304
300
|
category=AirflowProviderDeprecationWarning,
|
305
301
|
)
|
306
302
|
class MLEngineManageModelOperator(GoogleCloudBaseOperator):
|
@@ -372,11 +368,9 @@ class MLEngineManageModelOperator(GoogleCloudBaseOperator):
|
|
372
368
|
|
373
369
|
|
374
370
|
@deprecated(
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
"Please use appropriate VertexAI operator."
|
379
|
-
),
|
371
|
+
planned_removal_date="November 01, 2025",
|
372
|
+
use_instead="appropriate VertexAI operator",
|
373
|
+
reason="All the functionality of legacy MLEngine and new features are available on the Vertex AI.",
|
380
374
|
category=AirflowProviderDeprecationWarning,
|
381
375
|
)
|
382
376
|
class MLEngineCreateModelOperator(GoogleCloudBaseOperator):
|
@@ -448,11 +442,9 @@ class MLEngineCreateModelOperator(GoogleCloudBaseOperator):
|
|
448
442
|
|
449
443
|
|
450
444
|
@deprecated(
|
451
|
-
|
452
|
-
|
453
|
-
|
454
|
-
"Please use `GetModelOperator`"
|
455
|
-
),
|
445
|
+
planned_removal_date="March 01, 2025",
|
446
|
+
use_instead="GetModelOperator",
|
447
|
+
reason="All the functionality of legacy MLEngine and new features are available on the Vertex AI.",
|
456
448
|
category=AirflowProviderDeprecationWarning,
|
457
449
|
)
|
458
450
|
class MLEngineGetModelOperator(GoogleCloudBaseOperator):
|
@@ -523,11 +515,9 @@ class MLEngineGetModelOperator(GoogleCloudBaseOperator):
|
|
523
515
|
|
524
516
|
|
525
517
|
@deprecated(
|
526
|
-
|
527
|
-
|
528
|
-
|
529
|
-
"Please use `DeleteModelOperator`"
|
530
|
-
),
|
518
|
+
planned_removal_date="March 01, 2025",
|
519
|
+
use_instead="DeleteModelOperator",
|
520
|
+
reason="All the functionality of legacy MLEngine and new features are available on the Vertex AI.",
|
531
521
|
category=AirflowProviderDeprecationWarning,
|
532
522
|
)
|
533
523
|
class MLEngineDeleteModelOperator(GoogleCloudBaseOperator):
|
@@ -606,11 +596,9 @@ class MLEngineDeleteModelOperator(GoogleCloudBaseOperator):
|
|
606
596
|
|
607
597
|
|
608
598
|
@deprecated(
|
609
|
-
|
610
|
-
|
611
|
-
|
612
|
-
"MLEngineListVersions, MLEngineDeleteVersion."
|
613
|
-
),
|
599
|
+
planned_removal_date="March 01, 2025",
|
600
|
+
use_instead="MLEngineCreateVersion, MLEngineSetDefaultVersion, MLEngineListVersions, "
|
601
|
+
"MLEngineDeleteVersion",
|
614
602
|
category=AirflowProviderDeprecationWarning,
|
615
603
|
)
|
616
604
|
class MLEngineManageVersionOperator(GoogleCloudBaseOperator):
|
@@ -725,11 +713,9 @@ class MLEngineManageVersionOperator(GoogleCloudBaseOperator):
|
|
725
713
|
|
726
714
|
|
727
715
|
@deprecated(
|
728
|
-
|
729
|
-
|
730
|
-
|
731
|
-
"Please use parent_model parameter for VertexAI operators instead."
|
732
|
-
),
|
716
|
+
planned_removal_date="March 01, 2025",
|
717
|
+
use_instead="parent_model parameter for VertexAI operators",
|
718
|
+
reason="All the functionality of legacy MLEngine and new features are available on the Vertex AI.",
|
733
719
|
category=AirflowProviderDeprecationWarning,
|
734
720
|
)
|
735
721
|
class MLEngineCreateVersionOperator(GoogleCloudBaseOperator):
|
@@ -816,11 +802,9 @@ class MLEngineCreateVersionOperator(GoogleCloudBaseOperator):
|
|
816
802
|
|
817
803
|
|
818
804
|
@deprecated(
|
819
|
-
|
820
|
-
|
821
|
-
|
822
|
-
"Please use `SetDefaultVersionOnModelOperator`"
|
823
|
-
),
|
805
|
+
planned_removal_date="March 01, 2025",
|
806
|
+
use_instead="SetDefaultVersionOnModelOperator",
|
807
|
+
reason="All the functionality of legacy MLEngine and new features are available on the Vertex AI.",
|
824
808
|
category=AirflowProviderDeprecationWarning,
|
825
809
|
)
|
826
810
|
class MLEngineSetDefaultVersionOperator(GoogleCloudBaseOperator):
|
@@ -909,11 +893,9 @@ class MLEngineSetDefaultVersionOperator(GoogleCloudBaseOperator):
|
|
909
893
|
|
910
894
|
|
911
895
|
@deprecated(
|
912
|
-
|
913
|
-
|
914
|
-
|
915
|
-
"Please use `ListModelVersionsOperator`"
|
916
|
-
),
|
896
|
+
planned_removal_date="March 01, 2025",
|
897
|
+
use_instead="istModelVersionsOperator",
|
898
|
+
reason="All the functionality of legacy MLEngine and new features are available on the Vertex AI.",
|
917
899
|
category=AirflowProviderDeprecationWarning,
|
918
900
|
)
|
919
901
|
class MLEngineListVersionsOperator(GoogleCloudBaseOperator):
|
@@ -995,11 +977,9 @@ class MLEngineListVersionsOperator(GoogleCloudBaseOperator):
|
|
995
977
|
|
996
978
|
|
997
979
|
@deprecated(
|
998
|
-
|
999
|
-
|
1000
|
-
|
1001
|
-
"Please use `DeleteModelVersionOperator`"
|
1002
|
-
),
|
980
|
+
planned_removal_date="March 01, 2025",
|
981
|
+
use_instead="DeleteModelVersionOperator",
|
982
|
+
reason="All the functionality of legacy MLEngine and new features are available on the Vertex AI.",
|
1003
983
|
category=AirflowProviderDeprecationWarning,
|
1004
984
|
)
|
1005
985
|
class MLEngineDeleteVersionOperator(GoogleCloudBaseOperator):
|
@@ -1087,11 +1067,9 @@ class MLEngineDeleteVersionOperator(GoogleCloudBaseOperator):
|
|
1087
1067
|
|
1088
1068
|
|
1089
1069
|
@deprecated(
|
1090
|
-
|
1091
|
-
|
1092
|
-
|
1093
|
-
"Please use `CreateCustomPythonPackageTrainingJobOperator`"
|
1094
|
-
),
|
1070
|
+
planned_removal_date="March 01, 2025",
|
1071
|
+
use_instead="CreateCustomPythonPackageTrainingJobOperator",
|
1072
|
+
reason="All the functionality of legacy MLEngine and new features are available on the Vertex AI.",
|
1095
1073
|
category=AirflowProviderDeprecationWarning,
|
1096
1074
|
)
|
1097
1075
|
class MLEngineStartTrainingJobOperator(GoogleCloudBaseOperator):
|
@@ -1426,11 +1404,9 @@ class MLEngineStartTrainingJobOperator(GoogleCloudBaseOperator):
|
|
1426
1404
|
|
1427
1405
|
|
1428
1406
|
@deprecated(
|
1429
|
-
|
1430
|
-
|
1431
|
-
|
1432
|
-
"Please use `CancelCustomTrainingJobOperator`"
|
1433
|
-
),
|
1407
|
+
planned_removal_date="March 01, 2025",
|
1408
|
+
use_instead="CancelCustomTrainingJobOperator",
|
1409
|
+
reason="All the functionality of legacy MLEngine and new features are available on the Vertex AI.",
|
1434
1410
|
category=AirflowProviderDeprecationWarning,
|
1435
1411
|
)
|
1436
1412
|
class MLEngineTrainingCancelJobOperator(GoogleCloudBaseOperator):
|
@@ -1481,8 +1457,8 @@ class MLEngineTrainingCancelJobOperator(GoogleCloudBaseOperator):
|
|
1481
1457
|
|
1482
1458
|
@property
|
1483
1459
|
@deprecated(
|
1484
|
-
|
1485
|
-
"
|
1460
|
+
planned_removal_date="March 01, 2025",
|
1461
|
+
use_instead="project_id",
|
1486
1462
|
category=AirflowProviderDeprecationWarning,
|
1487
1463
|
)
|
1488
1464
|
def _project_id(self):
|
@@ -1491,7 +1467,8 @@ class MLEngineTrainingCancelJobOperator(GoogleCloudBaseOperator):
|
|
1491
1467
|
|
1492
1468
|
@property
|
1493
1469
|
@deprecated(
|
1494
|
-
|
1470
|
+
planned_removal_date="March 01, 2025",
|
1471
|
+
use_instead="job_id",
|
1495
1472
|
category=AirflowProviderDeprecationWarning,
|
1496
1473
|
)
|
1497
1474
|
def _job_id(self):
|
@@ -1500,8 +1477,8 @@ class MLEngineTrainingCancelJobOperator(GoogleCloudBaseOperator):
|
|
1500
1477
|
|
1501
1478
|
@property
|
1502
1479
|
@deprecated(
|
1503
|
-
|
1504
|
-
"
|
1480
|
+
planned_removal_date="March 01, 2025",
|
1481
|
+
use_instead="impersonation_chain",
|
1505
1482
|
category=AirflowProviderDeprecationWarning,
|
1506
1483
|
)
|
1507
1484
|
def _impersonation_chain(self):
|
@@ -22,7 +22,6 @@ from __future__ import annotations
|
|
22
22
|
|
23
23
|
from typing import TYPE_CHECKING, Sequence
|
24
24
|
|
25
|
-
from deprecated import deprecated
|
26
25
|
from google.api_core.exceptions import NotFound
|
27
26
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
28
27
|
from google.cloud.aiplatform import datasets
|
@@ -37,6 +36,7 @@ from airflow.providers.google.cloud.links.vertex_ai import (
|
|
37
36
|
VertexAITrainingPipelinesLink,
|
38
37
|
)
|
39
38
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
39
|
+
from airflow.providers.google.common.deprecated import deprecated
|
40
40
|
|
41
41
|
if TYPE_CHECKING:
|
42
42
|
from google.api_core.retry import Retry
|
@@ -455,8 +455,22 @@ class CreateAutoMLTabularTrainingJobOperator(AutoMLTrainingJobBaseOperator):
|
|
455
455
|
return result
|
456
456
|
|
457
457
|
|
458
|
+
@deprecated(
|
459
|
+
planned_removal_date="September 15, 2024",
|
460
|
+
use_instead="SupervisedFineTuningTrainOperator",
|
461
|
+
instructions=(
|
462
|
+
"Please consider using Fine Tuning over the Gemini model. "
|
463
|
+
"More info: https://cloud.google.com/vertex-ai/docs/start/automl-gemini-comparison"
|
464
|
+
),
|
465
|
+
category=AirflowProviderDeprecationWarning,
|
466
|
+
)
|
458
467
|
class CreateAutoMLTextTrainingJobOperator(AutoMLTrainingJobBaseOperator):
|
459
|
-
"""
|
468
|
+
"""
|
469
|
+
Create Auto ML Text Training job.
|
470
|
+
|
471
|
+
WARNING: Text creation API is deprecated since September 15, 2024
|
472
|
+
(https://cloud.google.com/vertex-ai/docs/tutorials/text-classification-automl/overview).
|
473
|
+
"""
|
460
474
|
|
461
475
|
template_fields = [
|
462
476
|
"parent_model",
|
@@ -642,8 +656,8 @@ class DeleteAutoMLTrainingJobOperator(GoogleCloudBaseOperator):
|
|
642
656
|
|
643
657
|
@property
|
644
658
|
@deprecated(
|
645
|
-
|
646
|
-
"
|
659
|
+
planned_removal_date="March 01, 2025",
|
660
|
+
use_instead="training_pipeline_id",
|
647
661
|
category=AirflowProviderDeprecationWarning,
|
648
662
|
)
|
649
663
|
def training_pipeline(self):
|
@@ -23,7 +23,6 @@ import warnings
|
|
23
23
|
from functools import cached_property
|
24
24
|
from typing import TYPE_CHECKING, Any, Sequence
|
25
25
|
|
26
|
-
from deprecated import deprecated
|
27
26
|
from google.api_core.exceptions import NotFound
|
28
27
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
29
28
|
from google.cloud.aiplatform.models import Model
|
@@ -44,6 +43,7 @@ from airflow.providers.google.cloud.triggers.vertex_ai import (
|
|
44
43
|
CustomPythonPackageTrainingJobTrigger,
|
45
44
|
CustomTrainingJobTrigger,
|
46
45
|
)
|
46
|
+
from airflow.providers.google.common.deprecated import deprecated
|
47
47
|
|
48
48
|
if TYPE_CHECKING:
|
49
49
|
from google.api_core.retry import Retry
|
@@ -1633,8 +1633,8 @@ class DeleteCustomTrainingJobOperator(GoogleCloudBaseOperator):
|
|
1633
1633
|
|
1634
1634
|
@property
|
1635
1635
|
@deprecated(
|
1636
|
-
|
1637
|
-
"
|
1636
|
+
planned_removal_date="March 01, 2025",
|
1637
|
+
use_instead="training_pipeline_id",
|
1638
1638
|
category=AirflowProviderDeprecationWarning,
|
1639
1639
|
)
|
1640
1640
|
def training_pipeline(self):
|
@@ -1643,8 +1643,8 @@ class DeleteCustomTrainingJobOperator(GoogleCloudBaseOperator):
|
|
1643
1643
|
|
1644
1644
|
@property
|
1645
1645
|
@deprecated(
|
1646
|
-
|
1647
|
-
"
|
1646
|
+
planned_removal_date="March 01, 2025",
|
1647
|
+
use_instead="custom_job_id",
|
1648
1648
|
category=AirflowProviderDeprecationWarning,
|
1649
1649
|
)
|
1650
1650
|
def custom_job(self):
|