apache-airflow-providers-google 14.0.0__py3-none-any.whl → 19.1.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/3rd-party-licenses/LICENSES.txt +14 -0
- airflow/providers/google/3rd-party-licenses/NOTICE +5 -0
- airflow/providers/google/__init__.py +3 -3
- airflow/providers/google/_vendor/__init__.py +0 -0
- airflow/providers/google/_vendor/json_merge_patch.py +91 -0
- airflow/providers/google/ads/hooks/ads.py +52 -43
- airflow/providers/google/ads/operators/ads.py +2 -2
- airflow/providers/google/ads/transfers/ads_to_gcs.py +3 -19
- airflow/providers/google/assets/gcs.py +1 -11
- airflow/providers/google/cloud/_internal_client/secret_manager_client.py +3 -2
- airflow/providers/google/cloud/bundles/gcs.py +161 -0
- airflow/providers/google/cloud/hooks/alloy_db.py +2 -3
- airflow/providers/google/cloud/hooks/bigquery.py +195 -318
- airflow/providers/google/cloud/hooks/bigquery_dts.py +8 -8
- airflow/providers/google/cloud/hooks/bigtable.py +3 -2
- airflow/providers/google/cloud/hooks/cloud_batch.py +8 -9
- airflow/providers/google/cloud/hooks/cloud_build.py +6 -65
- airflow/providers/google/cloud/hooks/cloud_composer.py +292 -24
- airflow/providers/google/cloud/hooks/cloud_logging.py +109 -0
- airflow/providers/google/cloud/hooks/cloud_memorystore.py +4 -3
- airflow/providers/google/cloud/hooks/cloud_run.py +20 -11
- airflow/providers/google/cloud/hooks/cloud_sql.py +136 -64
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +35 -15
- airflow/providers/google/cloud/hooks/compute.py +7 -6
- airflow/providers/google/cloud/hooks/compute_ssh.py +7 -4
- airflow/providers/google/cloud/hooks/datacatalog.py +12 -3
- airflow/providers/google/cloud/hooks/dataflow.py +87 -242
- airflow/providers/google/cloud/hooks/dataform.py +9 -14
- airflow/providers/google/cloud/hooks/datafusion.py +7 -9
- airflow/providers/google/cloud/hooks/dataplex.py +13 -12
- airflow/providers/google/cloud/hooks/dataprep.py +2 -2
- airflow/providers/google/cloud/hooks/dataproc.py +76 -74
- airflow/providers/google/cloud/hooks/dataproc_metastore.py +4 -3
- airflow/providers/google/cloud/hooks/dlp.py +5 -4
- airflow/providers/google/cloud/hooks/gcs.py +144 -33
- airflow/providers/google/cloud/hooks/gen_ai.py +196 -0
- airflow/providers/google/cloud/hooks/kms.py +3 -2
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +22 -17
- airflow/providers/google/cloud/hooks/looker.py +6 -1
- airflow/providers/google/cloud/hooks/managed_kafka.py +227 -3
- airflow/providers/google/cloud/hooks/mlengine.py +7 -8
- airflow/providers/google/cloud/hooks/natural_language.py +3 -2
- airflow/providers/google/cloud/hooks/os_login.py +3 -2
- airflow/providers/google/cloud/hooks/pubsub.py +6 -6
- airflow/providers/google/cloud/hooks/secret_manager.py +105 -12
- airflow/providers/google/cloud/hooks/spanner.py +75 -10
- airflow/providers/google/cloud/hooks/speech_to_text.py +3 -2
- airflow/providers/google/cloud/hooks/stackdriver.py +18 -18
- airflow/providers/google/cloud/hooks/tasks.py +4 -3
- airflow/providers/google/cloud/hooks/text_to_speech.py +3 -2
- airflow/providers/google/cloud/hooks/translate.py +8 -17
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +8 -222
- airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +9 -15
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +33 -283
- airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +5 -12
- airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +6 -12
- airflow/providers/google/cloud/hooks/vertex_ai/experiment_service.py +202 -0
- airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +311 -10
- airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +79 -75
- airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +7 -13
- airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +8 -12
- airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +6 -12
- airflow/providers/google/cloud/hooks/vertex_ai/prediction_service.py +3 -2
- airflow/providers/google/cloud/hooks/vertex_ai/ray.py +223 -0
- airflow/providers/google/cloud/hooks/video_intelligence.py +3 -2
- airflow/providers/google/cloud/hooks/vision.py +7 -7
- airflow/providers/google/cloud/hooks/workflows.py +4 -3
- airflow/providers/google/cloud/links/alloy_db.py +0 -46
- airflow/providers/google/cloud/links/base.py +77 -7
- airflow/providers/google/cloud/links/bigquery.py +0 -47
- airflow/providers/google/cloud/links/bigquery_dts.py +0 -20
- airflow/providers/google/cloud/links/bigtable.py +0 -48
- airflow/providers/google/cloud/links/cloud_build.py +0 -73
- airflow/providers/google/cloud/links/cloud_functions.py +0 -33
- airflow/providers/google/cloud/links/cloud_memorystore.py +0 -58
- airflow/providers/google/cloud/links/{life_sciences.py → cloud_run.py} +5 -27
- airflow/providers/google/cloud/links/cloud_sql.py +0 -33
- airflow/providers/google/cloud/links/cloud_storage_transfer.py +17 -46
- airflow/providers/google/cloud/links/cloud_tasks.py +7 -26
- airflow/providers/google/cloud/links/compute.py +0 -58
- airflow/providers/google/cloud/links/data_loss_prevention.py +0 -169
- airflow/providers/google/cloud/links/datacatalog.py +23 -54
- airflow/providers/google/cloud/links/dataflow.py +0 -34
- airflow/providers/google/cloud/links/dataform.py +0 -64
- airflow/providers/google/cloud/links/datafusion.py +1 -90
- airflow/providers/google/cloud/links/dataplex.py +0 -154
- airflow/providers/google/cloud/links/dataprep.py +0 -24
- airflow/providers/google/cloud/links/dataproc.py +11 -89
- airflow/providers/google/cloud/links/datastore.py +0 -31
- airflow/providers/google/cloud/links/kubernetes_engine.py +11 -61
- airflow/providers/google/cloud/links/managed_kafka.py +11 -51
- airflow/providers/google/cloud/links/mlengine.py +0 -70
- airflow/providers/google/cloud/links/pubsub.py +0 -32
- airflow/providers/google/cloud/links/spanner.py +0 -33
- airflow/providers/google/cloud/links/stackdriver.py +0 -30
- airflow/providers/google/cloud/links/translate.py +17 -187
- airflow/providers/google/cloud/links/vertex_ai.py +28 -195
- airflow/providers/google/cloud/links/workflows.py +0 -52
- airflow/providers/google/cloud/log/gcs_task_handler.py +166 -118
- airflow/providers/google/cloud/log/stackdriver_task_handler.py +14 -9
- airflow/providers/google/cloud/openlineage/CloudStorageTransferJobFacet.json +68 -0
- airflow/providers/google/cloud/openlineage/CloudStorageTransferRunFacet.json +60 -0
- airflow/providers/google/cloud/openlineage/DataFusionRunFacet.json +32 -0
- airflow/providers/google/cloud/openlineage/facets.py +141 -40
- airflow/providers/google/cloud/openlineage/mixins.py +14 -13
- airflow/providers/google/cloud/openlineage/utils.py +19 -3
- airflow/providers/google/cloud/operators/alloy_db.py +76 -61
- airflow/providers/google/cloud/operators/bigquery.py +104 -667
- airflow/providers/google/cloud/operators/bigquery_dts.py +12 -12
- airflow/providers/google/cloud/operators/bigtable.py +38 -7
- airflow/providers/google/cloud/operators/cloud_base.py +22 -1
- airflow/providers/google/cloud/operators/cloud_batch.py +18 -18
- airflow/providers/google/cloud/operators/cloud_build.py +80 -36
- airflow/providers/google/cloud/operators/cloud_composer.py +157 -71
- airflow/providers/google/cloud/operators/cloud_logging_sink.py +341 -0
- airflow/providers/google/cloud/operators/cloud_memorystore.py +74 -46
- airflow/providers/google/cloud/operators/cloud_run.py +39 -20
- airflow/providers/google/cloud/operators/cloud_sql.py +46 -61
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +92 -14
- airflow/providers/google/cloud/operators/compute.py +18 -50
- airflow/providers/google/cloud/operators/datacatalog.py +167 -29
- airflow/providers/google/cloud/operators/dataflow.py +38 -15
- airflow/providers/google/cloud/operators/dataform.py +19 -7
- airflow/providers/google/cloud/operators/datafusion.py +43 -43
- airflow/providers/google/cloud/operators/dataplex.py +212 -126
- airflow/providers/google/cloud/operators/dataprep.py +1 -5
- airflow/providers/google/cloud/operators/dataproc.py +134 -207
- airflow/providers/google/cloud/operators/dataproc_metastore.py +102 -84
- airflow/providers/google/cloud/operators/datastore.py +22 -6
- airflow/providers/google/cloud/operators/dlp.py +24 -45
- airflow/providers/google/cloud/operators/functions.py +21 -14
- airflow/providers/google/cloud/operators/gcs.py +15 -12
- airflow/providers/google/cloud/operators/gen_ai.py +389 -0
- airflow/providers/google/cloud/operators/kubernetes_engine.py +115 -106
- airflow/providers/google/cloud/operators/looker.py +1 -1
- airflow/providers/google/cloud/operators/managed_kafka.py +362 -40
- airflow/providers/google/cloud/operators/natural_language.py +5 -3
- airflow/providers/google/cloud/operators/pubsub.py +69 -21
- airflow/providers/google/cloud/operators/spanner.py +53 -45
- airflow/providers/google/cloud/operators/speech_to_text.py +5 -4
- airflow/providers/google/cloud/operators/stackdriver.py +5 -11
- airflow/providers/google/cloud/operators/tasks.py +6 -15
- airflow/providers/google/cloud/operators/text_to_speech.py +4 -3
- airflow/providers/google/cloud/operators/translate.py +46 -20
- airflow/providers/google/cloud/operators/translate_speech.py +4 -3
- airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +44 -34
- airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +34 -12
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +62 -53
- airflow/providers/google/cloud/operators/vertex_ai/dataset.py +75 -11
- airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +48 -12
- airflow/providers/google/cloud/operators/vertex_ai/experiment_service.py +435 -0
- airflow/providers/google/cloud/operators/vertex_ai/feature_store.py +532 -1
- airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +135 -116
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +16 -12
- airflow/providers/google/cloud/operators/vertex_ai/model_service.py +62 -14
- airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +35 -10
- airflow/providers/google/cloud/operators/vertex_ai/ray.py +393 -0
- airflow/providers/google/cloud/operators/video_intelligence.py +5 -3
- airflow/providers/google/cloud/operators/vision.py +7 -5
- airflow/providers/google/cloud/operators/workflows.py +24 -19
- airflow/providers/google/cloud/secrets/secret_manager.py +2 -1
- airflow/providers/google/cloud/sensors/bigquery.py +2 -2
- airflow/providers/google/cloud/sensors/bigquery_dts.py +6 -4
- airflow/providers/google/cloud/sensors/bigtable.py +14 -6
- airflow/providers/google/cloud/sensors/cloud_composer.py +535 -33
- airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +6 -5
- airflow/providers/google/cloud/sensors/dataflow.py +27 -10
- airflow/providers/google/cloud/sensors/dataform.py +2 -2
- airflow/providers/google/cloud/sensors/datafusion.py +4 -4
- airflow/providers/google/cloud/sensors/dataplex.py +7 -5
- airflow/providers/google/cloud/sensors/dataprep.py +2 -2
- airflow/providers/google/cloud/sensors/dataproc.py +10 -9
- airflow/providers/google/cloud/sensors/dataproc_metastore.py +4 -3
- airflow/providers/google/cloud/sensors/gcs.py +22 -21
- airflow/providers/google/cloud/sensors/looker.py +5 -5
- airflow/providers/google/cloud/sensors/pubsub.py +20 -20
- airflow/providers/google/cloud/sensors/tasks.py +2 -2
- airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py +2 -2
- airflow/providers/google/cloud/sensors/workflows.py +6 -4
- airflow/providers/google/cloud/transfers/adls_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +11 -8
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +14 -13
- airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +7 -3
- airflow/providers/google/cloud/transfers/bigquery_to_mysql.py +12 -1
- airflow/providers/google/cloud/transfers/bigquery_to_postgres.py +24 -10
- airflow/providers/google/cloud/transfers/bigquery_to_sql.py +104 -5
- airflow/providers/google/cloud/transfers/calendar_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +18 -22
- airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +4 -5
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +45 -38
- airflow/providers/google/cloud/transfers/gcs_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/gcs_to_local.py +5 -3
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +10 -4
- airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +6 -2
- airflow/providers/google/cloud/transfers/gdrive_to_local.py +2 -2
- airflow/providers/google/cloud/transfers/http_to_gcs.py +193 -0
- airflow/providers/google/cloud/transfers/local_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/oracle_to_gcs.py +36 -11
- airflow/providers/google/cloud/transfers/postgres_to_gcs.py +44 -12
- airflow/providers/google/cloud/transfers/s3_to_gcs.py +12 -6
- airflow/providers/google/cloud/transfers/salesforce_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +36 -14
- airflow/providers/google/cloud/transfers/sheets_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +10 -10
- airflow/providers/google/cloud/triggers/bigquery.py +75 -34
- airflow/providers/google/cloud/triggers/bigquery_dts.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_batch.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_build.py +3 -2
- airflow/providers/google/cloud/triggers/cloud_composer.py +303 -47
- airflow/providers/google/cloud/triggers/cloud_run.py +2 -2
- airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +96 -5
- airflow/providers/google/cloud/triggers/dataflow.py +125 -2
- airflow/providers/google/cloud/triggers/datafusion.py +1 -1
- airflow/providers/google/cloud/triggers/dataplex.py +16 -3
- airflow/providers/google/cloud/triggers/dataproc.py +124 -53
- airflow/providers/google/cloud/triggers/kubernetes_engine.py +46 -28
- airflow/providers/google/cloud/triggers/mlengine.py +1 -1
- airflow/providers/google/cloud/triggers/pubsub.py +17 -20
- airflow/providers/google/cloud/triggers/vertex_ai.py +8 -7
- airflow/providers/google/cloud/utils/bigquery.py +5 -7
- airflow/providers/google/cloud/utils/bigquery_get_data.py +1 -1
- airflow/providers/google/cloud/utils/credentials_provider.py +4 -3
- airflow/providers/google/cloud/utils/dataform.py +1 -1
- airflow/providers/google/cloud/utils/external_token_supplier.py +0 -1
- airflow/providers/google/cloud/utils/field_validator.py +1 -2
- airflow/providers/google/cloud/utils/validators.py +43 -0
- airflow/providers/google/common/auth_backend/google_openid.py +26 -9
- airflow/providers/google/common/consts.py +2 -1
- airflow/providers/google/common/deprecated.py +2 -1
- airflow/providers/google/common/hooks/base_google.py +40 -43
- airflow/providers/google/common/hooks/operation_helpers.py +78 -0
- airflow/providers/google/common/links/storage.py +0 -22
- airflow/providers/google/common/utils/get_secret.py +31 -0
- airflow/providers/google/common/utils/id_token_credentials.py +4 -5
- airflow/providers/google/firebase/operators/firestore.py +2 -2
- airflow/providers/google/get_provider_info.py +61 -216
- airflow/providers/google/go_module_utils.py +35 -3
- airflow/providers/google/leveldb/hooks/leveldb.py +30 -6
- airflow/providers/google/leveldb/operators/leveldb.py +2 -2
- airflow/providers/google/marketing_platform/hooks/analytics_admin.py +3 -2
- airflow/providers/google/marketing_platform/hooks/display_video.py +3 -109
- airflow/providers/google/marketing_platform/hooks/search_ads.py +1 -1
- airflow/providers/google/marketing_platform/links/analytics_admin.py +4 -5
- airflow/providers/google/marketing_platform/operators/analytics_admin.py +7 -6
- airflow/providers/google/marketing_platform/operators/campaign_manager.py +5 -5
- airflow/providers/google/marketing_platform/operators/display_video.py +28 -489
- airflow/providers/google/marketing_platform/operators/search_ads.py +2 -2
- airflow/providers/google/marketing_platform/sensors/campaign_manager.py +2 -2
- airflow/providers/google/marketing_platform/sensors/display_video.py +4 -64
- airflow/providers/google/suite/hooks/calendar.py +1 -1
- airflow/providers/google/suite/hooks/drive.py +2 -2
- airflow/providers/google/suite/hooks/sheets.py +15 -1
- airflow/providers/google/suite/operators/sheets.py +8 -3
- airflow/providers/google/suite/sensors/drive.py +2 -2
- airflow/providers/google/suite/transfers/gcs_to_gdrive.py +2 -2
- airflow/providers/google/suite/transfers/gcs_to_sheets.py +1 -1
- airflow/providers/google/suite/transfers/local_to_drive.py +3 -3
- airflow/providers/google/suite/transfers/sql_to_sheets.py +5 -4
- airflow/providers/google/version_compat.py +15 -1
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/METADATA +117 -72
- apache_airflow_providers_google-19.1.0rc1.dist-info/RECORD +331 -0
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/WHEEL +1 -1
- apache_airflow_providers_google-19.1.0rc1.dist-info/licenses/NOTICE +5 -0
- airflow/providers/google/cloud/example_dags/example_cloud_task.py +0 -54
- airflow/providers/google/cloud/hooks/automl.py +0 -679
- airflow/providers/google/cloud/hooks/life_sciences.py +0 -159
- airflow/providers/google/cloud/links/automl.py +0 -193
- airflow/providers/google/cloud/operators/automl.py +0 -1360
- airflow/providers/google/cloud/operators/life_sciences.py +0 -119
- airflow/providers/google/cloud/operators/mlengine.py +0 -1515
- airflow/providers/google/cloud/utils/mlengine_operator_utils.py +0 -273
- apache_airflow_providers_google-14.0.0.dist-info/RECORD +0 -318
- /airflow/providers/google/cloud/{example_dags → bundles}/__init__.py +0 -0
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/entry_points.txt +0 -0
- {airflow/providers/google → apache_airflow_providers_google-19.1.0rc1.dist-info/licenses}/LICENSE +0 -0
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
BSD 3-Clause "New" or "Revised" License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2015, Open Data Services Coop
|
|
4
|
+
All rights reserved.
|
|
5
|
+
|
|
6
|
+
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
|
7
|
+
|
|
8
|
+
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
|
|
9
|
+
|
|
10
|
+
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
|
|
11
|
+
|
|
12
|
+
3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
|
|
13
|
+
|
|
14
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
@@ -29,11 +29,11 @@ from airflow import __version__ as airflow_version
|
|
|
29
29
|
|
|
30
30
|
__all__ = ["__version__"]
|
|
31
31
|
|
|
32
|
-
__version__ = "
|
|
32
|
+
__version__ = "19.1.0"
|
|
33
33
|
|
|
34
34
|
if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
|
|
35
|
-
"2.
|
|
35
|
+
"2.11.0"
|
|
36
36
|
):
|
|
37
37
|
raise RuntimeError(
|
|
38
|
-
f"The package `apache-airflow-providers-google:{__version__}` needs Apache Airflow 2.
|
|
38
|
+
f"The package `apache-airflow-providers-google:{__version__}` needs Apache Airflow 2.11.0+"
|
|
39
39
|
)
|
|
File without changes
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# BSD 3-Clause "New" or "Revised" License
|
|
2
|
+
#
|
|
3
|
+
# Copyright (c) 2015, Open Data Services Coop
|
|
4
|
+
# All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
|
7
|
+
#
|
|
8
|
+
# 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
|
|
9
|
+
#
|
|
10
|
+
# 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
|
|
11
|
+
#
|
|
12
|
+
# 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
|
|
13
|
+
#
|
|
14
|
+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
15
|
+
|
|
16
|
+
from collections import OrderedDict
|
|
17
|
+
import sys
|
|
18
|
+
|
|
19
|
+
def merge(*objs, **kw):
|
|
20
|
+
result = objs[0]
|
|
21
|
+
for obj in objs[1:]:
|
|
22
|
+
result = merge_obj(result, obj, kw.get('position'))
|
|
23
|
+
return result
|
|
24
|
+
|
|
25
|
+
def move_to_start(result, key):
|
|
26
|
+
result_copy = result.copy()
|
|
27
|
+
result.clear()
|
|
28
|
+
result[key] = result_copy.pop(key)
|
|
29
|
+
result.update(result_copy)
|
|
30
|
+
|
|
31
|
+
def merge_obj(result, obj, position=None):
|
|
32
|
+
if not isinstance(result, dict):
|
|
33
|
+
result = OrderedDict() if position else {}
|
|
34
|
+
|
|
35
|
+
if not isinstance(obj, dict):
|
|
36
|
+
return obj
|
|
37
|
+
|
|
38
|
+
if position:
|
|
39
|
+
if position not in ('first', 'last'):
|
|
40
|
+
raise ValueError("position can either be first or last")
|
|
41
|
+
if not isinstance(result, OrderedDict) or not isinstance(obj, OrderedDict):
|
|
42
|
+
raise ValueError("If using position all dicts need to be OrderedDicts")
|
|
43
|
+
|
|
44
|
+
for key, value in obj.items():
|
|
45
|
+
if isinstance(value, dict):
|
|
46
|
+
target = result.get(key)
|
|
47
|
+
if isinstance(target, dict):
|
|
48
|
+
merge_obj(target, value, position)
|
|
49
|
+
continue
|
|
50
|
+
result[key] = OrderedDict() if position else {}
|
|
51
|
+
if position and position == 'first':
|
|
52
|
+
if sys.version_info >= (3, 2):
|
|
53
|
+
result.move_to_end(key, False)
|
|
54
|
+
else:
|
|
55
|
+
move_to_start(result, key)
|
|
56
|
+
merge_obj(result[key], value, position)
|
|
57
|
+
continue
|
|
58
|
+
if value is None:
|
|
59
|
+
result.pop(key, None)
|
|
60
|
+
continue
|
|
61
|
+
if key not in result and position == 'first':
|
|
62
|
+
result[key] = value
|
|
63
|
+
if sys.version_info >= (3, 2):
|
|
64
|
+
result.move_to_end(key, False)
|
|
65
|
+
else:
|
|
66
|
+
move_to_start(result, key)
|
|
67
|
+
else:
|
|
68
|
+
result[key] = value
|
|
69
|
+
|
|
70
|
+
return result
|
|
71
|
+
|
|
72
|
+
def create_patch(source, target):
|
|
73
|
+
return create_patch_obj(source, target)
|
|
74
|
+
|
|
75
|
+
def create_patch_obj(source, target):
|
|
76
|
+
if not isinstance(target, dict) or not isinstance(source, dict):
|
|
77
|
+
return target
|
|
78
|
+
|
|
79
|
+
result = {}
|
|
80
|
+
|
|
81
|
+
for key in set(source.keys()) - set(target.keys()):
|
|
82
|
+
result[key] = None
|
|
83
|
+
|
|
84
|
+
for key, value in target.items():
|
|
85
|
+
if key not in source:
|
|
86
|
+
result[key] = value
|
|
87
|
+
continue
|
|
88
|
+
if value == source[key]:
|
|
89
|
+
continue
|
|
90
|
+
result[key] = create_patch_obj(source[key], value)
|
|
91
|
+
return result
|
|
@@ -19,23 +19,23 @@
|
|
|
19
19
|
|
|
20
20
|
from __future__ import annotations
|
|
21
21
|
|
|
22
|
-
import warnings
|
|
23
22
|
from functools import cached_property
|
|
24
23
|
from tempfile import NamedTemporaryFile
|
|
25
24
|
from typing import IO, TYPE_CHECKING, Any, Literal
|
|
26
25
|
|
|
27
|
-
from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
|
|
28
|
-
from airflow.hooks.base import BaseHook
|
|
29
|
-
from airflow.providers.google.common.hooks.base_google import get_field
|
|
30
26
|
from google.ads.googleads.client import GoogleAdsClient
|
|
31
27
|
from google.ads.googleads.errors import GoogleAdsException
|
|
32
28
|
from google.auth.exceptions import GoogleAuthError
|
|
33
29
|
|
|
30
|
+
from airflow.exceptions import AirflowException
|
|
31
|
+
from airflow.providers.common.compat.sdk import BaseHook
|
|
32
|
+
from airflow.providers.google.common.hooks.base_google import get_field
|
|
33
|
+
|
|
34
34
|
if TYPE_CHECKING:
|
|
35
|
-
from google.ads.googleads.
|
|
36
|
-
from google.ads.googleads.
|
|
37
|
-
from google.ads.googleads.
|
|
38
|
-
from google.
|
|
35
|
+
from google.ads.googleads.v21.services.services.customer_service import CustomerServiceClient
|
|
36
|
+
from google.ads.googleads.v21.services.services.google_ads_service import GoogleAdsServiceClient
|
|
37
|
+
from google.ads.googleads.v21.services.services.google_ads_service.pagers import SearchPager
|
|
38
|
+
from google.ads.googleads.v21.services.types.google_ads_service import GoogleAdsRow
|
|
39
39
|
|
|
40
40
|
|
|
41
41
|
class GoogleAdsHook(BaseHook):
|
|
@@ -101,6 +101,40 @@ class GoogleAdsHook(BaseHook):
|
|
|
101
101
|
:param api_version: The Google Ads API version to use.
|
|
102
102
|
"""
|
|
103
103
|
|
|
104
|
+
conn_name_attr = "google_ads_conn_id"
|
|
105
|
+
default_conn_name = "google_ads_default"
|
|
106
|
+
conn_type = "google_ads"
|
|
107
|
+
hook_name = "Google Ads"
|
|
108
|
+
|
|
109
|
+
@classmethod
|
|
110
|
+
def get_connection_form_widgets(cls) -> dict[str, Any]:
|
|
111
|
+
"""Return connection widgets to add to Google Ads connection form."""
|
|
112
|
+
from flask_appbuilder.fieldwidgets import BS3PasswordFieldWidget, BS3TextFieldWidget
|
|
113
|
+
from flask_babel import lazy_gettext
|
|
114
|
+
from wtforms import PasswordField, StringField
|
|
115
|
+
|
|
116
|
+
return {
|
|
117
|
+
"developer_token": StringField(lazy_gettext("Developer token"), widget=BS3TextFieldWidget()),
|
|
118
|
+
"client_id": StringField(lazy_gettext("OAuth2 Client ID"), widget=BS3TextFieldWidget()),
|
|
119
|
+
"client_secret": PasswordField(
|
|
120
|
+
lazy_gettext("OAuth2 Client Secret"), widget=BS3PasswordFieldWidget()
|
|
121
|
+
),
|
|
122
|
+
"refresh_token": PasswordField(
|
|
123
|
+
lazy_gettext("OAuth2 Refresh Token"), widget=BS3PasswordFieldWidget()
|
|
124
|
+
),
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
@classmethod
|
|
128
|
+
def get_ui_field_behaviour(cls) -> dict[str, Any]:
|
|
129
|
+
"""Return custom UI field behaviour for Google Ads connection."""
|
|
130
|
+
return {
|
|
131
|
+
"hidden_fields": ["host", "login", "schema", "port"],
|
|
132
|
+
"relabeling": {},
|
|
133
|
+
"placeholders": {
|
|
134
|
+
"password": "Leave blank (optional)",
|
|
135
|
+
},
|
|
136
|
+
}
|
|
137
|
+
|
|
104
138
|
def __init__(
|
|
105
139
|
self,
|
|
106
140
|
api_version: str | None = None,
|
|
@@ -115,9 +149,7 @@ class GoogleAdsHook(BaseHook):
|
|
|
115
149
|
self.google_ads_config: dict[str, Any] = {}
|
|
116
150
|
self.authentication_method: Literal["service_account", "developer_token"] = "service_account"
|
|
117
151
|
|
|
118
|
-
def search(
|
|
119
|
-
self, client_ids: list[str], query: str, page_size: int | None = None, **kwargs
|
|
120
|
-
) -> list[GoogleAdsRow]:
|
|
152
|
+
def search(self, client_ids: list[str], query: str, **kwargs) -> list[GoogleAdsRow]:
|
|
121
153
|
"""
|
|
122
154
|
Pull data from the Google Ads API.
|
|
123
155
|
|
|
@@ -133,18 +165,14 @@ class GoogleAdsHook(BaseHook):
|
|
|
133
165
|
|
|
134
166
|
:param client_ids: Google Ads client ID(s) to query the API for.
|
|
135
167
|
:param query: Google Ads Query Language query.
|
|
136
|
-
:param page_size: Number of results to return per page. Max 10000 (for version 16 and 16.1)
|
|
137
|
-
This parameter deprecated. After February 05, 2025, it will be removed.
|
|
138
168
|
:return: Google Ads API response, converted to Google Ads Row objects.
|
|
139
169
|
"""
|
|
140
|
-
data_proto_plus = self._search(client_ids, query,
|
|
170
|
+
data_proto_plus = self._search(client_ids, query, **kwargs)
|
|
141
171
|
data_native_pb = [row._pb for row in data_proto_plus]
|
|
142
172
|
|
|
143
173
|
return data_native_pb
|
|
144
174
|
|
|
145
|
-
def search_proto_plus(
|
|
146
|
-
self, client_ids: list[str], query: str, page_size: int | None = None, **kwargs
|
|
147
|
-
) -> list[GoogleAdsRow]:
|
|
175
|
+
def search_proto_plus(self, client_ids: list[str], query: str, **kwargs) -> list[GoogleAdsRow]:
|
|
148
176
|
"""
|
|
149
177
|
Pull data from the Google Ads API.
|
|
150
178
|
|
|
@@ -153,11 +181,9 @@ class GoogleAdsHook(BaseHook):
|
|
|
153
181
|
|
|
154
182
|
:param client_ids: Google Ads client ID(s) to query the API for.
|
|
155
183
|
:param query: Google Ads Query Language query.
|
|
156
|
-
:param page_size: Number of results to return per page. Max 10000 (for version 16 and 16.1)
|
|
157
|
-
This parameter is deprecated. After February 05, 2025, it will be removed.
|
|
158
184
|
:return: Google Ads API response, converted to Google Ads Row objects
|
|
159
185
|
"""
|
|
160
|
-
return self._search(client_ids, query,
|
|
186
|
+
return self._search(client_ids, query, **kwargs)
|
|
161
187
|
|
|
162
188
|
def list_accessible_customers(self) -> list[str]:
|
|
163
189
|
"""
|
|
@@ -177,7 +203,7 @@ class GoogleAdsHook(BaseHook):
|
|
|
177
203
|
"""
|
|
178
204
|
try:
|
|
179
205
|
accessible_customers = self._get_customer_service.list_accessible_customers()
|
|
180
|
-
return accessible_customers.resource_names
|
|
206
|
+
return list(accessible_customers.resource_names)
|
|
181
207
|
except GoogleAdsException as ex:
|
|
182
208
|
for error in ex.failure.errors:
|
|
183
209
|
self.log.error('\tError with message "%s".', error.message)
|
|
@@ -268,48 +294,31 @@ class GoogleAdsHook(BaseHook):
|
|
|
268
294
|
|
|
269
295
|
self.google_ads_config["json_key_file_path"] = secrets_temp.name
|
|
270
296
|
|
|
271
|
-
def _search(
|
|
272
|
-
self, client_ids: list[str], query: str, page_size: int | None = None, **kwargs
|
|
273
|
-
) -> list[GoogleAdsRow]:
|
|
297
|
+
def _search(self, client_ids: list[str], query: str, **kwargs) -> list[GoogleAdsRow]:
|
|
274
298
|
"""
|
|
275
299
|
Pull data from the Google Ads API.
|
|
276
300
|
|
|
277
301
|
:param client_ids: Google Ads client ID(s) to query the API for.
|
|
278
302
|
:param query: Google Ads Query Language query.
|
|
279
|
-
:param page_size: Number of results to return per page. Max 10000 (for version 16 and 16.1)
|
|
280
|
-
This parameter is deprecated. After February 05, 2025, it will be removed.
|
|
281
303
|
|
|
282
304
|
:return: Google Ads API response, converted to Google Ads Row objects
|
|
283
305
|
"""
|
|
284
306
|
service = self._get_service
|
|
285
307
|
|
|
286
|
-
extra_req_params = {}
|
|
287
|
-
if self.api_version == "v16": # TODO: remove this after deprecation removal for page_size parameter
|
|
288
|
-
extra_req_params["page_size"] = page_size or 10000
|
|
289
|
-
else:
|
|
290
|
-
if page_size:
|
|
291
|
-
warnings.warn(
|
|
292
|
-
"page_size parameter for the GoogleAdsHook.search and "
|
|
293
|
-
"GoogleAdsHook.search_proto_plus method is deprecated and will be removed "
|
|
294
|
-
"after February 05, 2025.",
|
|
295
|
-
AirflowProviderDeprecationWarning,
|
|
296
|
-
stacklevel=2,
|
|
297
|
-
)
|
|
298
|
-
|
|
299
308
|
iterators = []
|
|
300
309
|
for client_id in client_ids:
|
|
301
|
-
iterator = service.search(request={"customer_id": client_id, "query": query
|
|
310
|
+
iterator = service.search(request={"customer_id": client_id, "query": query})
|
|
302
311
|
iterators.append(iterator)
|
|
303
312
|
|
|
304
313
|
self.log.info("Fetched Google Ads Iterators")
|
|
305
314
|
|
|
306
315
|
return self._extract_rows(iterators)
|
|
307
316
|
|
|
308
|
-
def _extract_rows(self, iterators: list[
|
|
317
|
+
def _extract_rows(self, iterators: list[SearchPager]) -> list[GoogleAdsRow]:
|
|
309
318
|
"""
|
|
310
|
-
Convert Google Page Iterator (
|
|
319
|
+
Convert Google Page Iterator (SearchPager) objects to Google Ads Rows.
|
|
311
320
|
|
|
312
|
-
:param iterators: List of Google Page Iterator (
|
|
321
|
+
:param iterators: List of Google Page Iterator (SearchPager) objects
|
|
313
322
|
:return: API response for all clients in the form of Google Ads Row object(s)
|
|
314
323
|
"""
|
|
315
324
|
try:
|
|
@@ -24,12 +24,12 @@ from collections.abc import Sequence
|
|
|
24
24
|
from tempfile import NamedTemporaryFile
|
|
25
25
|
from typing import TYPE_CHECKING
|
|
26
26
|
|
|
27
|
-
from airflow.models import BaseOperator
|
|
28
27
|
from airflow.providers.google.ads.hooks.ads import GoogleAdsHook
|
|
29
28
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
|
29
|
+
from airflow.providers.google.version_compat import BaseOperator
|
|
30
30
|
|
|
31
31
|
if TYPE_CHECKING:
|
|
32
|
-
from airflow.
|
|
32
|
+
from airflow.providers.common.compat.sdk import Context
|
|
33
33
|
|
|
34
34
|
|
|
35
35
|
class GoogleAdsListAccountsOperator(BaseOperator):
|
|
@@ -17,19 +17,17 @@
|
|
|
17
17
|
from __future__ import annotations
|
|
18
18
|
|
|
19
19
|
import csv
|
|
20
|
-
import warnings
|
|
21
20
|
from collections.abc import Sequence
|
|
22
21
|
from operator import attrgetter
|
|
23
22
|
from tempfile import NamedTemporaryFile
|
|
24
23
|
from typing import TYPE_CHECKING
|
|
25
24
|
|
|
26
|
-
from airflow.exceptions import AirflowProviderDeprecationWarning
|
|
27
|
-
from airflow.models import BaseOperator
|
|
28
25
|
from airflow.providers.google.ads.hooks.ads import GoogleAdsHook
|
|
29
26
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
|
27
|
+
from airflow.providers.google.version_compat import BaseOperator
|
|
30
28
|
|
|
31
29
|
if TYPE_CHECKING:
|
|
32
|
-
from airflow.
|
|
30
|
+
from airflow.providers.common.compat.sdk import Context
|
|
33
31
|
|
|
34
32
|
|
|
35
33
|
class GoogleAdsToGcsOperator(BaseOperator):
|
|
@@ -54,8 +52,6 @@ class GoogleAdsToGcsOperator(BaseOperator):
|
|
|
54
52
|
:param obj: GCS path to save the object. Must be the full file path (ex. `path/to/file.txt`)
|
|
55
53
|
:param gcp_conn_id: Airflow Google Cloud connection ID
|
|
56
54
|
:param google_ads_conn_id: Airflow Google Ads connection ID
|
|
57
|
-
:param page_size: The number of results per API page request. Max 10,000 (for version 16 and 16.1)
|
|
58
|
-
This parameter deprecated. After March 01, 2025, it will be removed.
|
|
59
55
|
:param gzip: Option to compress local file or file data for upload
|
|
60
56
|
:param impersonation_chain: Optional service account to impersonate using short-term
|
|
61
57
|
credentials, or chained list of accounts required to get the access_token
|
|
@@ -87,7 +83,6 @@ class GoogleAdsToGcsOperator(BaseOperator):
|
|
|
87
83
|
obj: str,
|
|
88
84
|
gcp_conn_id: str = "google_cloud_default",
|
|
89
85
|
google_ads_conn_id: str = "google_ads_default",
|
|
90
|
-
page_size: int | None = None,
|
|
91
86
|
gzip: bool = False,
|
|
92
87
|
impersonation_chain: str | Sequence[str] | None = None,
|
|
93
88
|
api_version: str | None = None,
|
|
@@ -101,8 +96,6 @@ class GoogleAdsToGcsOperator(BaseOperator):
|
|
|
101
96
|
self.obj = obj
|
|
102
97
|
self.gcp_conn_id = gcp_conn_id
|
|
103
98
|
self.google_ads_conn_id = google_ads_conn_id
|
|
104
|
-
# TODO: remove this after deprecation removal for page_size parameter
|
|
105
|
-
self.page_size = page_size or 10000 if api_version == "v16" else None
|
|
106
99
|
self.gzip = gzip
|
|
107
100
|
self.impersonation_chain = impersonation_chain
|
|
108
101
|
self.api_version = api_version
|
|
@@ -114,16 +107,7 @@ class GoogleAdsToGcsOperator(BaseOperator):
|
|
|
114
107
|
api_version=self.api_version,
|
|
115
108
|
)
|
|
116
109
|
|
|
117
|
-
|
|
118
|
-
warnings.warn(
|
|
119
|
-
"page_size parameter for the GoogleAdsToGcsOperator is deprecated and will be removed "
|
|
120
|
-
"after March 01, 2025.",
|
|
121
|
-
AirflowProviderDeprecationWarning,
|
|
122
|
-
stacklevel=2,
|
|
123
|
-
)
|
|
124
|
-
rows = service.search(client_ids=self.client_ids, query=self.query)
|
|
125
|
-
else:
|
|
126
|
-
rows = service.search(client_ids=self.client_ids, query=self.query, page_size=self.page_size)
|
|
110
|
+
rows = service.search(client_ids=self.client_ids, query=self.query)
|
|
127
111
|
|
|
128
112
|
try:
|
|
129
113
|
getter = attrgetter(*self.attributes)
|
|
@@ -18,23 +18,13 @@ from __future__ import annotations
|
|
|
18
18
|
|
|
19
19
|
from typing import TYPE_CHECKING
|
|
20
20
|
|
|
21
|
+
from airflow.providers.common.compat.sdk import Asset
|
|
21
22
|
from airflow.providers.google.cloud.hooks.gcs import _parse_gcs_url
|
|
22
|
-
from airflow.providers.google.version_compat import AIRFLOW_V_3_0_PLUS
|
|
23
23
|
|
|
24
24
|
if TYPE_CHECKING:
|
|
25
25
|
from urllib.parse import SplitResult
|
|
26
26
|
|
|
27
|
-
from airflow.providers.common.compat.assets import Asset
|
|
28
27
|
from airflow.providers.common.compat.openlineage.facet import Dataset as OpenLineageDataset
|
|
29
|
-
else:
|
|
30
|
-
try:
|
|
31
|
-
from airflow.providers.common.compat.assets import Asset
|
|
32
|
-
except ImportError:
|
|
33
|
-
if AIRFLOW_V_3_0_PLUS:
|
|
34
|
-
from airflow.sdk.definitions.asset import Asset
|
|
35
|
-
else:
|
|
36
|
-
# dataset is renamed to asset since Airflow 3.0
|
|
37
|
-
from airflow.datasets import Dataset as Asset
|
|
38
28
|
|
|
39
29
|
|
|
40
30
|
def create_asset(*, bucket: str, key: str, extra: dict | None = None) -> Asset:
|
|
@@ -20,11 +20,12 @@ import re
|
|
|
20
20
|
from functools import cached_property
|
|
21
21
|
from typing import TYPE_CHECKING
|
|
22
22
|
|
|
23
|
-
from airflow.providers.google.common.consts import CLIENT_INFO
|
|
24
|
-
from airflow.utils.log.logging_mixin import LoggingMixin
|
|
25
23
|
from google.api_core.exceptions import InvalidArgument, NotFound, PermissionDenied
|
|
26
24
|
from google.cloud.secretmanager_v1 import SecretManagerServiceClient
|
|
27
25
|
|
|
26
|
+
from airflow.providers.google.common.consts import CLIENT_INFO
|
|
27
|
+
from airflow.utils.log.logging_mixin import LoggingMixin
|
|
28
|
+
|
|
28
29
|
if TYPE_CHECKING:
|
|
29
30
|
import google
|
|
30
31
|
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
|
3
|
+
# distributed with this work for additional information
|
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
|
6
|
+
# "License"); you may not use this file except in compliance
|
|
7
|
+
# with the License. You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
|
12
|
+
# software distributed under the License is distributed on an
|
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
+
# KIND, either express or implied. See the License for the
|
|
15
|
+
# specific language governing permissions and limitations
|
|
16
|
+
# under the License.
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import os
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
import structlog
|
|
23
|
+
from google.api_core.exceptions import NotFound
|
|
24
|
+
|
|
25
|
+
from airflow.dag_processing.bundles.base import BaseDagBundle
|
|
26
|
+
from airflow.exceptions import AirflowException
|
|
27
|
+
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
|
28
|
+
from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class GCSDagBundle(BaseDagBundle):
|
|
32
|
+
"""
|
|
33
|
+
GCS Dag bundle - exposes a directory in GCS as a Dag bundle.
|
|
34
|
+
|
|
35
|
+
This allows Airflow to load Dags directly from a GCS bucket.
|
|
36
|
+
|
|
37
|
+
:param gcp_conn_id: Airflow connection ID for GCS. Defaults to GoogleBaseHook.default_conn_name.
|
|
38
|
+
:param bucket_name: The name of the GCS bucket containing the Dag files.
|
|
39
|
+
:param prefix: Optional subdirectory within the GCS bucket where the Dags are stored.
|
|
40
|
+
If None, Dags are assumed to be at the root of the bucket (Optional).
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
supports_versioning = False
|
|
44
|
+
|
|
45
|
+
def __init__(
|
|
46
|
+
self,
|
|
47
|
+
*,
|
|
48
|
+
gcp_conn_id: str = GoogleBaseHook.default_conn_name,
|
|
49
|
+
bucket_name: str,
|
|
50
|
+
prefix: str = "",
|
|
51
|
+
**kwargs,
|
|
52
|
+
) -> None:
|
|
53
|
+
super().__init__(**kwargs)
|
|
54
|
+
self.gcp_conn_id = gcp_conn_id
|
|
55
|
+
self.bucket_name = bucket_name
|
|
56
|
+
self.prefix = prefix
|
|
57
|
+
# Local path where GCS Dags are downloaded
|
|
58
|
+
self.gcs_dags_dir: Path = self.base_dir
|
|
59
|
+
|
|
60
|
+
log = structlog.get_logger(__name__)
|
|
61
|
+
self._log = log.bind(
|
|
62
|
+
bundle_name=self.name,
|
|
63
|
+
version=self.version,
|
|
64
|
+
bucket_name=self.bucket_name,
|
|
65
|
+
prefix=self.prefix,
|
|
66
|
+
gcp_conn_id=self.gcp_conn_id,
|
|
67
|
+
)
|
|
68
|
+
self._gcs_hook: GCSHook | None = None
|
|
69
|
+
|
|
70
|
+
def _initialize(self):
|
|
71
|
+
with self.lock():
|
|
72
|
+
if not self.gcs_dags_dir.exists():
|
|
73
|
+
self._log.info("Creating local Dags directory: %s", self.gcs_dags_dir)
|
|
74
|
+
os.makedirs(self.gcs_dags_dir)
|
|
75
|
+
|
|
76
|
+
if not self.gcs_dags_dir.is_dir():
|
|
77
|
+
raise NotADirectoryError(f"Local Dags path: {self.gcs_dags_dir} is not a directory.")
|
|
78
|
+
|
|
79
|
+
try:
|
|
80
|
+
self.gcs_hook.get_bucket(bucket_name=self.bucket_name)
|
|
81
|
+
except NotFound:
|
|
82
|
+
raise ValueError(f"GCS bucket '{self.bucket_name}' does not exist.")
|
|
83
|
+
|
|
84
|
+
if self.prefix:
|
|
85
|
+
# don't check when prefix is ""
|
|
86
|
+
if not self.gcs_hook.list(bucket_name=self.bucket_name, prefix=self.prefix):
|
|
87
|
+
raise ValueError(f"GCS prefix 'gs://{self.bucket_name}/{self.prefix}' does not exist.")
|
|
88
|
+
self.refresh()
|
|
89
|
+
|
|
90
|
+
def initialize(self) -> None:
|
|
91
|
+
self._initialize()
|
|
92
|
+
super().initialize()
|
|
93
|
+
|
|
94
|
+
@property
|
|
95
|
+
def gcs_hook(self):
|
|
96
|
+
if self._gcs_hook is None:
|
|
97
|
+
try:
|
|
98
|
+
self._gcs_hook: GCSHook = GCSHook(gcp_conn_id=self.gcp_conn_id) # Initialize GCS hook.
|
|
99
|
+
except AirflowException as e:
|
|
100
|
+
self._log.warning("Could not create GCSHook for connection %s: %s", self.gcp_conn_id, e)
|
|
101
|
+
return self._gcs_hook
|
|
102
|
+
|
|
103
|
+
def __repr__(self):
|
|
104
|
+
return (
|
|
105
|
+
f"<GCSDagBundle("
|
|
106
|
+
f"name={self.name!r}, "
|
|
107
|
+
f"bucket_name={self.bucket_name!r}, "
|
|
108
|
+
f"prefix={self.prefix!r}, "
|
|
109
|
+
f"version={self.version!r}"
|
|
110
|
+
f")>"
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
def get_current_version(self) -> str | None:
|
|
114
|
+
"""Return the current version of the Dag bundle. Currently not supported."""
|
|
115
|
+
return None
|
|
116
|
+
|
|
117
|
+
@property
|
|
118
|
+
def path(self) -> Path:
|
|
119
|
+
"""Return the local path to the Dag files."""
|
|
120
|
+
return self.gcs_dags_dir # Path where Dags are downloaded.
|
|
121
|
+
|
|
122
|
+
def refresh(self) -> None:
|
|
123
|
+
"""Refresh the Dag bundle by re-downloading the Dags from GCS."""
|
|
124
|
+
if self.version:
|
|
125
|
+
raise ValueError("Refreshing a specific version is not supported")
|
|
126
|
+
|
|
127
|
+
with self.lock():
|
|
128
|
+
self._log.debug(
|
|
129
|
+
"Downloading Dags from gs://%s/%s to %s", self.bucket_name, self.prefix, self.gcs_dags_dir
|
|
130
|
+
)
|
|
131
|
+
self.gcs_hook.sync_to_local_dir(
|
|
132
|
+
bucket_name=self.bucket_name,
|
|
133
|
+
prefix=self.prefix,
|
|
134
|
+
local_dir=self.gcs_dags_dir,
|
|
135
|
+
delete_stale=True,
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
def view_url(self, version: str | None = None) -> str | None:
|
|
139
|
+
"""
|
|
140
|
+
Return a URL for viewing the Dags in GCS. Currently, versioning is not supported.
|
|
141
|
+
|
|
142
|
+
This method is deprecated and will be removed when the minimum supported Airflow version is 3.1.
|
|
143
|
+
Use `view_url_template` instead.
|
|
144
|
+
"""
|
|
145
|
+
return self.view_url_template()
|
|
146
|
+
|
|
147
|
+
def view_url_template(self) -> str | None:
|
|
148
|
+
"""Return a URL for viewing the Dags in GCS. Currently, versioning is not supported."""
|
|
149
|
+
if self.version:
|
|
150
|
+
raise ValueError("GCS url with version is not supported")
|
|
151
|
+
if hasattr(self, "_view_url_template") and self._view_url_template:
|
|
152
|
+
# Because we use this method in the view_url method, we need to handle
|
|
153
|
+
# backward compatibility for Airflow versions that doesn't have the
|
|
154
|
+
# _view_url_template attribute. Should be removed when we drop support for Airflow 3.0
|
|
155
|
+
return self._view_url_template
|
|
156
|
+
# https://console.cloud.google.com/storage/browser/<bucket-name>/<prefix>
|
|
157
|
+
url = f"https://console.cloud.google.com/storage/browser/{self.bucket_name}"
|
|
158
|
+
if self.prefix:
|
|
159
|
+
url += f"/{self.prefix}"
|
|
160
|
+
|
|
161
|
+
return url
|
|
@@ -24,16 +24,15 @@ from copy import deepcopy
|
|
|
24
24
|
from typing import TYPE_CHECKING
|
|
25
25
|
|
|
26
26
|
import tenacity
|
|
27
|
+
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
28
|
+
from google.cloud import alloydb_v1
|
|
27
29
|
|
|
28
30
|
from airflow.exceptions import AirflowException
|
|
29
31
|
from airflow.providers.google.common.consts import CLIENT_INFO
|
|
30
32
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
|
|
31
|
-
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
32
|
-
from google.cloud import alloydb_v1
|
|
33
33
|
|
|
34
34
|
if TYPE_CHECKING:
|
|
35
35
|
import proto
|
|
36
|
-
|
|
37
36
|
from google.api_core.operation import Operation
|
|
38
37
|
from google.api_core.retry import Retry
|
|
39
38
|
from google.protobuf.field_mask_pb2 import FieldMask
|