apache-airflow-providers-google 10.26.0rc1__py3-none-any.whl → 11.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/__init__.py +1 -1
- airflow/providers/google/ads/hooks/ads.py +2 -1
- airflow/providers/google/ads/operators/ads.py +2 -1
- airflow/providers/google/ads/transfers/ads_to_gcs.py +2 -1
- airflow/providers/google/assets/gcs.py +17 -1
- airflow/providers/google/cloud/hooks/automl.py +3 -6
- airflow/providers/google/cloud/hooks/bigquery.py +41 -1486
- airflow/providers/google/cloud/hooks/bigquery_dts.py +4 -11
- airflow/providers/google/cloud/hooks/bigtable.py +3 -6
- airflow/providers/google/cloud/hooks/cloud_batch.py +6 -3
- airflow/providers/google/cloud/hooks/cloud_build.py +3 -15
- airflow/providers/google/cloud/hooks/cloud_composer.py +2 -17
- airflow/providers/google/cloud/hooks/cloud_memorystore.py +5 -6
- airflow/providers/google/cloud/hooks/cloud_run.py +10 -5
- airflow/providers/google/cloud/hooks/cloud_sql.py +5 -7
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +3 -7
- airflow/providers/google/cloud/hooks/compute.py +3 -6
- airflow/providers/google/cloud/hooks/compute_ssh.py +0 -5
- airflow/providers/google/cloud/hooks/datacatalog.py +3 -6
- airflow/providers/google/cloud/hooks/dataflow.py +3 -14
- airflow/providers/google/cloud/hooks/dataform.py +2 -9
- airflow/providers/google/cloud/hooks/datafusion.py +4 -15
- airflow/providers/google/cloud/hooks/dataplex.py +4 -7
- airflow/providers/google/cloud/hooks/dataprep.py +2 -2
- airflow/providers/google/cloud/hooks/dataproc.py +77 -22
- airflow/providers/google/cloud/hooks/dataproc_metastore.py +2 -9
- airflow/providers/google/cloud/hooks/datastore.py +3 -6
- airflow/providers/google/cloud/hooks/dlp.py +3 -6
- airflow/providers/google/cloud/hooks/functions.py +2 -6
- airflow/providers/google/cloud/hooks/gcs.py +2 -18
- airflow/providers/google/cloud/hooks/gdm.py +1 -17
- airflow/providers/google/cloud/hooks/kms.py +3 -6
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +7 -97
- airflow/providers/google/cloud/hooks/life_sciences.py +2 -6
- airflow/providers/google/cloud/hooks/looker.py +2 -1
- airflow/providers/google/cloud/hooks/mlengine.py +0 -8
- airflow/providers/google/cloud/hooks/natural_language.py +3 -6
- airflow/providers/google/cloud/hooks/os_login.py +3 -6
- airflow/providers/google/cloud/hooks/pubsub.py +3 -6
- airflow/providers/google/cloud/hooks/secret_manager.py +3 -73
- airflow/providers/google/cloud/hooks/spanner.py +3 -6
- airflow/providers/google/cloud/hooks/speech_to_text.py +3 -6
- airflow/providers/google/cloud/hooks/stackdriver.py +3 -6
- airflow/providers/google/cloud/hooks/tasks.py +3 -6
- airflow/providers/google/cloud/hooks/text_to_speech.py +3 -6
- airflow/providers/google/cloud/hooks/translate.py +455 -9
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +3 -6
- airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +3 -6
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +3 -6
- airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +2 -9
- airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +2 -9
- airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +1 -14
- airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +3 -6
- airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +2 -9
- airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +3 -1
- airflow/providers/google/cloud/hooks/vertex_ai/prediction_service.py +2 -1
- airflow/providers/google/cloud/hooks/video_intelligence.py +3 -6
- airflow/providers/google/cloud/hooks/vision.py +3 -6
- airflow/providers/google/cloud/hooks/workflows.py +2 -9
- airflow/providers/google/cloud/links/dataproc.py +0 -1
- airflow/providers/google/cloud/links/translate.py +91 -0
- airflow/providers/google/cloud/log/gcs_task_handler.py +2 -1
- airflow/providers/google/cloud/log/stackdriver_task_handler.py +11 -3
- airflow/providers/google/cloud/openlineage/utils.py +54 -21
- airflow/providers/google/cloud/operators/automl.py +5 -4
- airflow/providers/google/cloud/operators/bigquery.py +2 -341
- airflow/providers/google/cloud/operators/bigquery_dts.py +2 -1
- airflow/providers/google/cloud/operators/bigtable.py +2 -1
- airflow/providers/google/cloud/operators/cloud_batch.py +2 -1
- airflow/providers/google/cloud/operators/cloud_build.py +2 -1
- airflow/providers/google/cloud/operators/cloud_composer.py +2 -1
- airflow/providers/google/cloud/operators/cloud_memorystore.py +2 -1
- airflow/providers/google/cloud/operators/cloud_run.py +2 -1
- airflow/providers/google/cloud/operators/cloud_sql.py +2 -1
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +2 -1
- airflow/providers/google/cloud/operators/compute.py +2 -1
- airflow/providers/google/cloud/operators/datacatalog.py +2 -1
- airflow/providers/google/cloud/operators/dataflow.py +2 -517
- airflow/providers/google/cloud/operators/dataform.py +2 -1
- airflow/providers/google/cloud/operators/datafusion.py +2 -1
- airflow/providers/google/cloud/operators/dataplex.py +37 -31
- airflow/providers/google/cloud/operators/dataprep.py +2 -1
- airflow/providers/google/cloud/operators/dataproc.py +3 -633
- airflow/providers/google/cloud/operators/dataproc_metastore.py +2 -1
- airflow/providers/google/cloud/operators/datastore.py +2 -1
- airflow/providers/google/cloud/operators/dlp.py +2 -1
- airflow/providers/google/cloud/operators/functions.py +2 -1
- airflow/providers/google/cloud/operators/gcs.py +5 -4
- airflow/providers/google/cloud/operators/kubernetes_engine.py +2 -11
- airflow/providers/google/cloud/operators/life_sciences.py +2 -1
- airflow/providers/google/cloud/operators/mlengine.py +2 -1
- airflow/providers/google/cloud/operators/natural_language.py +3 -2
- airflow/providers/google/cloud/operators/pubsub.py +2 -1
- airflow/providers/google/cloud/operators/spanner.py +2 -1
- airflow/providers/google/cloud/operators/speech_to_text.py +2 -1
- airflow/providers/google/cloud/operators/stackdriver.py +2 -1
- airflow/providers/google/cloud/operators/tasks.py +3 -2
- airflow/providers/google/cloud/operators/text_to_speech.py +2 -1
- airflow/providers/google/cloud/operators/translate.py +622 -32
- airflow/providers/google/cloud/operators/translate_speech.py +2 -1
- airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +2 -93
- airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +3 -13
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +2 -17
- airflow/providers/google/cloud/operators/vertex_ai/dataset.py +2 -1
- airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +2 -1
- airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +2 -1
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +3 -13
- airflow/providers/google/cloud/operators/vertex_ai/model_service.py +2 -1
- airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +2 -1
- airflow/providers/google/cloud/operators/video_intelligence.py +2 -1
- airflow/providers/google/cloud/operators/vision.py +3 -2
- airflow/providers/google/cloud/operators/workflows.py +3 -2
- airflow/providers/google/cloud/secrets/secret_manager.py +2 -19
- airflow/providers/google/cloud/sensors/bigquery.py +2 -81
- airflow/providers/google/cloud/sensors/bigquery_dts.py +2 -1
- airflow/providers/google/cloud/sensors/bigtable.py +2 -1
- airflow/providers/google/cloud/sensors/cloud_composer.py +8 -94
- airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +2 -1
- airflow/providers/google/cloud/sensors/dataflow.py +2 -1
- airflow/providers/google/cloud/sensors/dataform.py +2 -1
- airflow/providers/google/cloud/sensors/datafusion.py +2 -1
- airflow/providers/google/cloud/sensors/dataplex.py +2 -1
- airflow/providers/google/cloud/sensors/dataprep.py +2 -1
- airflow/providers/google/cloud/sensors/dataproc.py +2 -1
- airflow/providers/google/cloud/sensors/dataproc_metastore.py +2 -1
- airflow/providers/google/cloud/sensors/gcs.py +4 -36
- airflow/providers/google/cloud/sensors/pubsub.py +2 -1
- airflow/providers/google/cloud/sensors/tasks.py +2 -1
- airflow/providers/google/cloud/sensors/workflows.py +2 -1
- airflow/providers/google/cloud/transfers/adls_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +75 -18
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +9 -7
- airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +2 -1
- airflow/providers/google/cloud/transfers/bigquery_to_mysql.py +1 -1
- airflow/providers/google/cloud/transfers/bigquery_to_sql.py +2 -1
- airflow/providers/google/cloud/transfers/calendar_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +13 -9
- airflow/providers/google/cloud/transfers/gcs_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/gcs_to_local.py +2 -1
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +2 -1
- airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/gdrive_to_local.py +2 -1
- airflow/providers/google/cloud/transfers/local_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/s3_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/salesforce_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/sheets_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +2 -1
- airflow/providers/google/cloud/triggers/bigquery.py +2 -1
- airflow/providers/google/cloud/triggers/bigquery_dts.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_batch.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_build.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_composer.py +3 -2
- airflow/providers/google/cloud/triggers/cloud_run.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_sql.py +1 -1
- airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +2 -1
- airflow/providers/google/cloud/triggers/dataflow.py +2 -1
- airflow/providers/google/cloud/triggers/datafusion.py +2 -1
- airflow/providers/google/cloud/triggers/dataplex.py +1 -1
- airflow/providers/google/cloud/triggers/dataproc.py +2 -1
- airflow/providers/google/cloud/triggers/gcs.py +3 -2
- airflow/providers/google/cloud/triggers/kubernetes_engine.py +2 -1
- airflow/providers/google/cloud/triggers/mlengine.py +2 -1
- airflow/providers/google/cloud/triggers/pubsub.py +2 -1
- airflow/providers/google/cloud/triggers/vertex_ai.py +2 -1
- airflow/providers/google/cloud/utils/credentials_provider.py +1 -1
- airflow/providers/google/cloud/utils/dataform.py +1 -1
- airflow/providers/google/cloud/utils/field_validator.py +2 -1
- airflow/providers/google/cloud/utils/mlengine_operator_utils.py +2 -1
- airflow/providers/google/common/hooks/base_google.py +4 -11
- airflow/providers/google/common/hooks/discovery_api.py +1 -6
- airflow/providers/google/firebase/hooks/firestore.py +1 -1
- airflow/providers/google/firebase/operators/firestore.py +2 -1
- airflow/providers/google/get_provider_info.py +7 -22
- airflow/providers/google/marketing_platform/hooks/analytics_admin.py +2 -1
- airflow/providers/google/marketing_platform/hooks/campaign_manager.py +2 -3
- airflow/providers/google/marketing_platform/hooks/display_video.py +4 -3
- airflow/providers/google/marketing_platform/hooks/search_ads.py +6 -6
- airflow/providers/google/marketing_platform/operators/analytics_admin.py +2 -1
- airflow/providers/google/marketing_platform/operators/campaign_manager.py +2 -42
- airflow/providers/google/marketing_platform/operators/display_video.py +2 -47
- airflow/providers/google/marketing_platform/operators/search_ads.py +2 -1
- airflow/providers/google/marketing_platform/sensors/campaign_manager.py +2 -7
- airflow/providers/google/marketing_platform/sensors/display_video.py +2 -13
- airflow/providers/google/suite/hooks/calendar.py +2 -8
- airflow/providers/google/suite/hooks/drive.py +2 -6
- airflow/providers/google/suite/hooks/sheets.py +2 -7
- airflow/providers/google/suite/operators/sheets.py +2 -7
- airflow/providers/google/suite/sensors/drive.py +2 -7
- airflow/providers/google/suite/transfers/gcs_to_gdrive.py +2 -7
- airflow/providers/google/suite/transfers/gcs_to_sheets.py +2 -7
- airflow/providers/google/suite/transfers/local_to_drive.py +2 -7
- airflow/providers/google/suite/transfers/sql_to_sheets.py +2 -7
- {apache_airflow_providers_google-10.26.0rc1.dist-info → apache_airflow_providers_google-11.0.0.dist-info}/METADATA +17 -17
- apache_airflow_providers_google-11.0.0.dist-info/RECORD +315 -0
- airflow/providers/google/marketing_platform/hooks/analytics.py +0 -211
- airflow/providers/google/marketing_platform/operators/analytics.py +0 -551
- apache_airflow_providers_google-10.26.0rc1.dist-info/RECORD +0 -317
- {apache_airflow_providers_google-10.26.0rc1.dist-info → apache_airflow_providers_google-11.0.0.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_google-10.26.0rc1.dist-info → apache_airflow_providers_google-11.0.0.dist-info}/entry_points.txt +0 -0
@@ -19,7 +19,8 @@
|
|
19
19
|
|
20
20
|
from __future__ import annotations
|
21
21
|
|
22
|
-
from
|
22
|
+
from collections.abc import Sequence
|
23
|
+
from typing import TYPE_CHECKING, Any
|
23
24
|
|
24
25
|
from googleapiclient.discovery import build
|
25
26
|
|
@@ -38,9 +39,6 @@ class GoogleCalendarHook(GoogleBaseHook):
|
|
38
39
|
|
39
40
|
:param gcp_conn_id: The connection ID to use when fetching connection info.
|
40
41
|
:param api_version: API Version. For example v3
|
41
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
42
|
-
if any. For this to work, the service account making the request must have
|
43
|
-
domain-wide delegation enabled.
|
44
42
|
:param impersonation_chain: Optional service account to impersonate using short-term
|
45
43
|
credentials, or chained list of accounts required to get the access_token
|
46
44
|
of the last account in the list, which will be impersonated in the request.
|
@@ -55,17 +53,13 @@ class GoogleCalendarHook(GoogleBaseHook):
|
|
55
53
|
self,
|
56
54
|
api_version: str,
|
57
55
|
gcp_conn_id: str = "google_cloud_default",
|
58
|
-
delegate_to: str | None = None,
|
59
56
|
impersonation_chain: str | Sequence[str] | None = None,
|
60
57
|
) -> None:
|
61
58
|
super().__init__(
|
62
59
|
gcp_conn_id=gcp_conn_id,
|
63
|
-
delegate_to=delegate_to,
|
64
60
|
impersonation_chain=impersonation_chain,
|
65
61
|
)
|
66
|
-
self.gcp_conn_id = gcp_conn_id
|
67
62
|
self.api_version = api_version
|
68
|
-
self.delegate_to = delegate_to
|
69
63
|
self._conn = None
|
70
64
|
|
71
65
|
def get_conn(self) -> Any:
|
@@ -19,7 +19,8 @@
|
|
19
19
|
|
20
20
|
from __future__ import annotations
|
21
21
|
|
22
|
-
from
|
22
|
+
from collections.abc import Sequence
|
23
|
+
from typing import IO, Any
|
23
24
|
|
24
25
|
from googleapiclient.discovery import Resource, build
|
25
26
|
from googleapiclient.errors import Error as GoogleApiClientError
|
@@ -34,9 +35,6 @@ class GoogleDriveHook(GoogleBaseHook):
|
|
34
35
|
|
35
36
|
:param api_version: API version used (for example v3).
|
36
37
|
:param gcp_conn_id: The connection ID to use when fetching connection info.
|
37
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
38
|
-
if any. For this to work, the service account making the request must have
|
39
|
-
domain-wide delegation enabled.
|
40
38
|
:param impersonation_chain: Optional service account to impersonate using short-term
|
41
39
|
credentials, or chained list of accounts required to get the access_token
|
42
40
|
of the last account in the list, which will be impersonated in the request.
|
@@ -53,12 +51,10 @@ class GoogleDriveHook(GoogleBaseHook):
|
|
53
51
|
self,
|
54
52
|
api_version: str = "v3",
|
55
53
|
gcp_conn_id: str = "google_cloud_default",
|
56
|
-
delegate_to: str | None = None,
|
57
54
|
impersonation_chain: str | Sequence[str] | None = None,
|
58
55
|
) -> None:
|
59
56
|
super().__init__(
|
60
57
|
gcp_conn_id=gcp_conn_id,
|
61
|
-
delegate_to=delegate_to,
|
62
58
|
impersonation_chain=impersonation_chain,
|
63
59
|
)
|
64
60
|
self.api_version = api_version
|
@@ -19,7 +19,8 @@
|
|
19
19
|
|
20
20
|
from __future__ import annotations
|
21
21
|
|
22
|
-
from
|
22
|
+
from collections.abc import Sequence
|
23
|
+
from typing import Any
|
23
24
|
|
24
25
|
from googleapiclient.discovery import build
|
25
26
|
|
@@ -35,9 +36,6 @@ class GSheetsHook(GoogleBaseHook):
|
|
35
36
|
|
36
37
|
:param gcp_conn_id: The connection ID to use when fetching connection info.
|
37
38
|
:param api_version: API Version
|
38
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
39
|
-
if any. For this to work, the service account making the request must have
|
40
|
-
domain-wide delegation enabled.
|
41
39
|
:param impersonation_chain: Optional service account to impersonate using short-term
|
42
40
|
credentials, or chained list of accounts required to get the access_token
|
43
41
|
of the last account in the list, which will be impersonated in the request.
|
@@ -52,17 +50,14 @@ class GSheetsHook(GoogleBaseHook):
|
|
52
50
|
self,
|
53
51
|
gcp_conn_id: str = "google_cloud_default",
|
54
52
|
api_version: str = "v4",
|
55
|
-
delegate_to: str | None = None,
|
56
53
|
impersonation_chain: str | Sequence[str] | None = None,
|
57
54
|
) -> None:
|
58
55
|
super().__init__(
|
59
56
|
gcp_conn_id=gcp_conn_id,
|
60
|
-
delegate_to=delegate_to,
|
61
57
|
impersonation_chain=impersonation_chain,
|
62
58
|
)
|
63
59
|
self.gcp_conn_id = gcp_conn_id
|
64
60
|
self.api_version = api_version
|
65
|
-
self.delegate_to = delegate_to
|
66
61
|
self._conn = None
|
67
62
|
|
68
63
|
def get_conn(self) -> Any:
|
@@ -16,7 +16,8 @@
|
|
16
16
|
# under the License.
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
|
-
from
|
19
|
+
from collections.abc import Sequence
|
20
|
+
from typing import Any
|
20
21
|
|
21
22
|
from airflow.models import BaseOperator
|
22
23
|
from airflow.providers.google.suite.hooks.sheets import GSheetsHook
|
@@ -33,9 +34,6 @@ class GoogleSheetsCreateSpreadsheetOperator(BaseOperator):
|
|
33
34
|
:param spreadsheet: an instance of Spreadsheet
|
34
35
|
https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets#Spreadsheet
|
35
36
|
:param gcp_conn_id: The connection ID to use when fetching connection info.
|
36
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
37
|
-
if any. For this to work, the service account making the request must have
|
38
|
-
domain-wide delegation enabled.
|
39
37
|
:param impersonation_chain: Optional service account to impersonate using short-term
|
40
38
|
credentials, or chained list of accounts required to get the access_token
|
41
39
|
of the last account in the list, which will be impersonated in the request.
|
@@ -56,20 +54,17 @@ class GoogleSheetsCreateSpreadsheetOperator(BaseOperator):
|
|
56
54
|
*,
|
57
55
|
spreadsheet: dict[str, Any],
|
58
56
|
gcp_conn_id: str = "google_cloud_default",
|
59
|
-
delegate_to: str | None = None,
|
60
57
|
impersonation_chain: str | Sequence[str] | None = None,
|
61
58
|
**kwargs,
|
62
59
|
) -> None:
|
63
60
|
super().__init__(**kwargs)
|
64
61
|
self.gcp_conn_id = gcp_conn_id
|
65
62
|
self.spreadsheet = spreadsheet
|
66
|
-
self.delegate_to = delegate_to
|
67
63
|
self.impersonation_chain = impersonation_chain
|
68
64
|
|
69
65
|
def execute(self, context: Any) -> dict[str, Any]:
|
70
66
|
hook = GSheetsHook(
|
71
67
|
gcp_conn_id=self.gcp_conn_id,
|
72
|
-
delegate_to=self.delegate_to,
|
73
68
|
impersonation_chain=self.impersonation_chain,
|
74
69
|
)
|
75
70
|
spreadsheet = hook.create_spreadsheet(spreadsheet=self.spreadsheet)
|
@@ -19,7 +19,8 @@
|
|
19
19
|
|
20
20
|
from __future__ import annotations
|
21
21
|
|
22
|
-
from
|
22
|
+
from collections.abc import Sequence
|
23
|
+
from typing import TYPE_CHECKING
|
23
24
|
|
24
25
|
from airflow.providers.google.suite.hooks.drive import GoogleDriveHook
|
25
26
|
from airflow.sensors.base import BaseSensorOperator
|
@@ -37,9 +38,6 @@ class GoogleDriveFileExistenceSensor(BaseSensorOperator):
|
|
37
38
|
:param drive_id: Optional. The id of the shared Google Drive in which the file resides.
|
38
39
|
:param gcp_conn_id: The connection ID to use when
|
39
40
|
connecting to Google Cloud Storage.
|
40
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
41
|
-
if any. For this to work, the service account making the request must have
|
42
|
-
domain-wide delegation enabled.
|
43
41
|
:param impersonation_chain: Optional service account to impersonate using short-term
|
44
42
|
credentials, or chained list of accounts required to get the access_token
|
45
43
|
of the last account in the list, which will be impersonated in the request.
|
@@ -65,7 +63,6 @@ class GoogleDriveFileExistenceSensor(BaseSensorOperator):
|
|
65
63
|
file_name: str,
|
66
64
|
drive_id: str | None = None,
|
67
65
|
gcp_conn_id: str = "google_cloud_default",
|
68
|
-
delegate_to: str | None = None,
|
69
66
|
impersonation_chain: str | Sequence[str] | None = None,
|
70
67
|
**kwargs,
|
71
68
|
) -> None:
|
@@ -74,14 +71,12 @@ class GoogleDriveFileExistenceSensor(BaseSensorOperator):
|
|
74
71
|
self.file_name = file_name
|
75
72
|
self.drive_id = drive_id
|
76
73
|
self.gcp_conn_id = gcp_conn_id
|
77
|
-
self.delegate_to = delegate_to
|
78
74
|
self.impersonation_chain = impersonation_chain
|
79
75
|
|
80
76
|
def poke(self, context: Context) -> bool:
|
81
77
|
self.log.info("Sensor is checking for the file %s in the folder %s", self.file_name, self.folder_id)
|
82
78
|
hook = GoogleDriveHook(
|
83
79
|
gcp_conn_id=self.gcp_conn_id,
|
84
|
-
delegate_to=self.delegate_to,
|
85
80
|
impersonation_chain=self.impersonation_chain,
|
86
81
|
)
|
87
82
|
return hook.exists(folder_id=self.folder_id, file_name=self.file_name, drive_id=self.drive_id)
|
@@ -20,7 +20,8 @@
|
|
20
20
|
from __future__ import annotations
|
21
21
|
|
22
22
|
import tempfile
|
23
|
-
from
|
23
|
+
from collections.abc import Sequence
|
24
|
+
from typing import TYPE_CHECKING
|
24
25
|
|
25
26
|
from airflow.exceptions import AirflowException
|
26
27
|
from airflow.models import BaseOperator
|
@@ -80,9 +81,6 @@ class GCSToGoogleDriveOperator(BaseOperator):
|
|
80
81
|
If set as a sequence, the identities from the list must grant
|
81
82
|
Service Account Token Creator IAM role to the directly preceding identity, with first
|
82
83
|
account from the list granting this role to the originating account (templated).
|
83
|
-
:param delegate_to: (Optional) The account to impersonate using domain-wide delegation
|
84
|
-
of authority, if any. For this to work, the service account making the
|
85
|
-
request must have domain-wide delegation enabled. This only applies to the Google Drive connection.
|
86
84
|
"""
|
87
85
|
|
88
86
|
template_fields: Sequence[str] = (
|
@@ -103,7 +101,6 @@ class GCSToGoogleDriveOperator(BaseOperator):
|
|
103
101
|
move_object: bool = False,
|
104
102
|
gcp_conn_id: str = "google_cloud_default",
|
105
103
|
impersonation_chain: str | Sequence[str] | None = None,
|
106
|
-
delegate_to: str | None = None,
|
107
104
|
**kwargs,
|
108
105
|
) -> None:
|
109
106
|
super().__init__(**kwargs)
|
@@ -115,7 +112,6 @@ class GCSToGoogleDriveOperator(BaseOperator):
|
|
115
112
|
self.move_object = move_object
|
116
113
|
self.gcp_conn_id = gcp_conn_id
|
117
114
|
self.impersonation_chain = impersonation_chain
|
118
|
-
self.delegate_to = delegate_to
|
119
115
|
self.gcs_hook: GCSHook | None = None
|
120
116
|
self.gdrive_hook: GoogleDriveHook | None = None
|
121
117
|
|
@@ -127,7 +123,6 @@ class GCSToGoogleDriveOperator(BaseOperator):
|
|
127
123
|
self.gdrive_hook = GoogleDriveHook(
|
128
124
|
gcp_conn_id=self.gcp_conn_id,
|
129
125
|
impersonation_chain=self.impersonation_chain,
|
130
|
-
delegate_to=self.delegate_to,
|
131
126
|
)
|
132
127
|
|
133
128
|
if WILDCARD in self.source_object:
|
@@ -17,8 +17,9 @@
|
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
19
|
import csv
|
20
|
+
from collections.abc import Sequence
|
20
21
|
from tempfile import NamedTemporaryFile
|
21
|
-
from typing import Any
|
22
|
+
from typing import Any
|
22
23
|
|
23
24
|
from airflow.models import BaseOperator
|
24
25
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
@@ -38,9 +39,6 @@ class GCSToGoogleSheetsOperator(BaseOperator):
|
|
38
39
|
:param object_name: Path to the .csv file on the GCS bucket.
|
39
40
|
:param spreadsheet_range: The A1 notation of the values to retrieve.
|
40
41
|
:param gcp_conn_id: The connection ID to use when fetching connection info.
|
41
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
42
|
-
if any. For this to work, the service account making the request must have
|
43
|
-
domain-wide delegation enabled. This only applies to the Google Sheet Connection
|
44
42
|
:param impersonation_chain: Optional service account to impersonate using short-term
|
45
43
|
credentials, or chained list of accounts required to get the access_token
|
46
44
|
of the last account in the list, which will be impersonated in the request.
|
@@ -68,7 +66,6 @@ class GCSToGoogleSheetsOperator(BaseOperator):
|
|
68
66
|
spreadsheet_range: str = "Sheet1",
|
69
67
|
gcp_conn_id: str = "google_cloud_default",
|
70
68
|
impersonation_chain: str | Sequence[str] | None = None,
|
71
|
-
delegate_to: str | None = None,
|
72
69
|
**kwargs,
|
73
70
|
) -> None:
|
74
71
|
super().__init__(**kwargs)
|
@@ -79,13 +76,11 @@ class GCSToGoogleSheetsOperator(BaseOperator):
|
|
79
76
|
self.bucket_name = bucket_name
|
80
77
|
self.object_name = object_name
|
81
78
|
self.impersonation_chain = impersonation_chain
|
82
|
-
self.delegate_to = delegate_to
|
83
79
|
|
84
80
|
def execute(self, context: Any) -> None:
|
85
81
|
sheet_hook = GSheetsHook(
|
86
82
|
gcp_conn_id=self.gcp_conn_id,
|
87
83
|
impersonation_chain=self.impersonation_chain,
|
88
|
-
delegate_to=self.delegate_to,
|
89
84
|
)
|
90
85
|
gcs_hook = GCSHook(
|
91
86
|
gcp_conn_id=self.gcp_conn_id,
|
@@ -19,8 +19,9 @@
|
|
19
19
|
from __future__ import annotations
|
20
20
|
|
21
21
|
import os
|
22
|
+
from collections.abc import Sequence
|
22
23
|
from pathlib import Path
|
23
|
-
from typing import TYPE_CHECKING
|
24
|
+
from typing import TYPE_CHECKING
|
24
25
|
|
25
26
|
from airflow.exceptions import AirflowFailException
|
26
27
|
from airflow.models import BaseOperator
|
@@ -55,9 +56,6 @@ class LocalFilesystemToGoogleDriveOperator(BaseOperator):
|
|
55
56
|
larger than 5MB, or to -1.
|
56
57
|
:param resumable: True if this is a resumable upload. False means upload
|
57
58
|
in a single request.
|
58
|
-
:param delegate_to: The account to impersonate using domain-wide delegation
|
59
|
-
of authority, if any. For this to work, the service account making the
|
60
|
-
request must have domain-wide delegation enabled.
|
61
59
|
:param impersonation_chain: Optional service account to impersonate using
|
62
60
|
short-term credentials, or chained list of accounts required to get the
|
63
61
|
access token of the last account in the list, which will be impersonated
|
@@ -87,7 +85,6 @@ class LocalFilesystemToGoogleDriveOperator(BaseOperator):
|
|
87
85
|
ignore_if_missing: bool = False,
|
88
86
|
chunk_size: int = 100 * 1024 * 1024,
|
89
87
|
resumable: bool = False,
|
90
|
-
delegate_to: str | None = None,
|
91
88
|
impersonation_chain: str | Sequence[str] | None = None,
|
92
89
|
folder_id: str = "root",
|
93
90
|
show_full_target_path: bool = True,
|
@@ -101,7 +98,6 @@ class LocalFilesystemToGoogleDriveOperator(BaseOperator):
|
|
101
98
|
self.ignore_if_missing = ignore_if_missing
|
102
99
|
self.chunk_size = chunk_size
|
103
100
|
self.resumable = resumable
|
104
|
-
self.delegate_to = delegate_to
|
105
101
|
self.impersonation_chain = impersonation_chain
|
106
102
|
self.folder_id = folder_id
|
107
103
|
self.show_full_target_path = show_full_target_path
|
@@ -109,7 +105,6 @@ class LocalFilesystemToGoogleDriveOperator(BaseOperator):
|
|
109
105
|
def execute(self, context: Context) -> list[str]:
|
110
106
|
hook = GoogleDriveHook(
|
111
107
|
gcp_conn_id=self.gcp_conn_id,
|
112
|
-
delegate_to=self.delegate_to,
|
113
108
|
impersonation_chain=self.impersonation_chain,
|
114
109
|
)
|
115
110
|
|
@@ -19,8 +19,9 @@ from __future__ import annotations
|
|
19
19
|
import datetime
|
20
20
|
import logging
|
21
21
|
import numbers
|
22
|
+
from collections.abc import Iterable, Mapping, Sequence
|
22
23
|
from contextlib import closing
|
23
|
-
from typing import Any
|
24
|
+
from typing import Any
|
24
25
|
|
25
26
|
from airflow.providers.common.sql.operators.sql import BaseSQLOperator
|
26
27
|
from airflow.providers.google.suite.hooks.sheets import GSheetsHook
|
@@ -37,9 +38,6 @@ class SQLToGoogleSheetsOperator(BaseSQLOperator):
|
|
37
38
|
:param database: name of database which overwrite the defined one in connection
|
38
39
|
:param spreadsheet_range: The A1 notation of the values to retrieve.
|
39
40
|
:param gcp_conn_id: The connection ID to use when fetching connection info.
|
40
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
41
|
-
if any. For this to work, the service account making the request must have
|
42
|
-
domain-wide delegation enabled.
|
43
41
|
:param impersonation_chain: Optional service account to impersonate using short-term
|
44
42
|
credentials, or chained list of accounts required to get the access_token
|
45
43
|
of the last account in the list, which will be impersonated in the request.
|
@@ -72,7 +70,6 @@ class SQLToGoogleSheetsOperator(BaseSQLOperator):
|
|
72
70
|
database: str | None = None,
|
73
71
|
spreadsheet_range: str = "Sheet1",
|
74
72
|
gcp_conn_id: str = "google_cloud_default",
|
75
|
-
delegate_to: str | None = None,
|
76
73
|
impersonation_chain: str | Sequence[str] | None = None,
|
77
74
|
**kwargs,
|
78
75
|
) -> None:
|
@@ -85,7 +82,6 @@ class SQLToGoogleSheetsOperator(BaseSQLOperator):
|
|
85
82
|
self.gcp_conn_id = gcp_conn_id
|
86
83
|
self.spreadsheet_id = spreadsheet_id
|
87
84
|
self.spreadsheet_range = spreadsheet_range
|
88
|
-
self.delegate_to = delegate_to
|
89
85
|
self.impersonation_chain = impersonation_chain
|
90
86
|
|
91
87
|
def _data_prep(self, data):
|
@@ -117,7 +113,6 @@ class SQLToGoogleSheetsOperator(BaseSQLOperator):
|
|
117
113
|
self.log.info("Connecting to Google")
|
118
114
|
sheet_hook = GSheetsHook(
|
119
115
|
gcp_conn_id=self.gcp_conn_id,
|
120
|
-
delegate_to=self.delegate_to,
|
121
116
|
impersonation_chain=self.impersonation_chain,
|
122
117
|
)
|
123
118
|
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.3
|
2
2
|
Name: apache-airflow-providers-google
|
3
|
-
Version:
|
3
|
+
Version: 11.0.0
|
4
4
|
Summary: Provider package apache-airflow-providers-google for Apache Airflow
|
5
5
|
Keywords: airflow-provider,google,airflow,integration
|
6
6
|
Author-email: Apache Software Foundation <dev@airflow.apache.org>
|
@@ -21,9 +21,9 @@ Classifier: Programming Language :: Python :: 3.11
|
|
21
21
|
Classifier: Programming Language :: Python :: 3.12
|
22
22
|
Classifier: Topic :: System :: Monitoring
|
23
23
|
Requires-Dist: PyOpenSSL>=23.0.0
|
24
|
-
Requires-Dist: apache-airflow-providers-common-compat>=1.2.
|
25
|
-
Requires-Dist: apache-airflow-providers-common-sql>=1.20.
|
26
|
-
Requires-Dist: apache-airflow>=2.8.
|
24
|
+
Requires-Dist: apache-airflow-providers-common-compat>=1.2.1
|
25
|
+
Requires-Dist: apache-airflow-providers-common-sql>=1.20.0
|
26
|
+
Requires-Dist: apache-airflow>=2.8.0
|
27
27
|
Requires-Dist: asgiref>=3.5.2
|
28
28
|
Requires-Dist: dill>=0.2.3
|
29
29
|
Requires-Dist: gcloud-aio-auth>=5.2.0
|
@@ -45,7 +45,7 @@ Requires-Dist: google-cloud-bigtable>=2.17.0
|
|
45
45
|
Requires-Dist: google-cloud-build>=3.22.0
|
46
46
|
Requires-Dist: google-cloud-compute>=1.10.0
|
47
47
|
Requires-Dist: google-cloud-container>=2.17.4
|
48
|
-
Requires-Dist: google-cloud-datacatalog>=3.
|
48
|
+
Requires-Dist: google-cloud-datacatalog>=3.23.0
|
49
49
|
Requires-Dist: google-cloud-dataflow-client>=0.8.6
|
50
50
|
Requires-Dist: google-cloud-dataform>=0.5.0
|
51
51
|
Requires-Dist: google-cloud-dataplex>=1.10.0
|
@@ -69,7 +69,7 @@ Requires-Dist: google-cloud-storage-transfer>=1.4.1
|
|
69
69
|
Requires-Dist: google-cloud-storage>=2.7.0
|
70
70
|
Requires-Dist: google-cloud-tasks>=2.13.0
|
71
71
|
Requires-Dist: google-cloud-texttospeech>=2.14.1
|
72
|
-
Requires-Dist: google-cloud-translate>=3.
|
72
|
+
Requires-Dist: google-cloud-translate>=3.16.0
|
73
73
|
Requires-Dist: google-cloud-videointelligence>=2.11.0
|
74
74
|
Requires-Dist: google-cloud-vision>=3.4.0
|
75
75
|
Requires-Dist: google-cloud-workflows>=1.10.0
|
@@ -87,20 +87,20 @@ Requires-Dist: python-slugify>=7.0.0
|
|
87
87
|
Requires-Dist: sqlalchemy-bigquery>=1.2.1
|
88
88
|
Requires-Dist: sqlalchemy-spanner>=1.6.2
|
89
89
|
Requires-Dist: tenacity>=8.1.0
|
90
|
-
Requires-Dist: apache-airflow-providers-amazon>=2.6.
|
90
|
+
Requires-Dist: apache-airflow-providers-amazon>=2.6.0 ; extra == "amazon"
|
91
91
|
Requires-Dist: apache-airflow-providers-apache-beam ; extra == "apache-beam"
|
92
92
|
Requires-Dist: apache-beam[gcp] ; extra == "apache-beam"
|
93
93
|
Requires-Dist: apache-airflow-providers-apache-cassandra ; extra == "apache-cassandra"
|
94
|
-
Requires-Dist: apache-airflow-providers-cncf-kubernetes>=7.2.
|
94
|
+
Requires-Dist: apache-airflow-providers-cncf-kubernetes>=7.2.0 ; extra == "cncf-kubernetes"
|
95
95
|
Requires-Dist: apache-airflow-providers-common-compat ; extra == "common-compat"
|
96
96
|
Requires-Dist: apache-airflow-providers-common-sql ; extra == "common-sql"
|
97
|
-
Requires-Dist: apache-airflow-providers-facebook>=2.2.
|
97
|
+
Requires-Dist: apache-airflow-providers-facebook>=2.2.0 ; extra == "facebook"
|
98
98
|
Requires-Dist: plyvel>=1.5.1 ; extra == "leveldb"
|
99
99
|
Requires-Dist: apache-airflow-providers-microsoft-azure ; extra == "microsoft-azure"
|
100
100
|
Requires-Dist: apache-airflow-providers-microsoft-mssql ; extra == "microsoft-mssql"
|
101
101
|
Requires-Dist: apache-airflow-providers-mysql ; extra == "mysql"
|
102
102
|
Requires-Dist: apache-airflow-providers-openlineage ; extra == "openlineage"
|
103
|
-
Requires-Dist: apache-airflow-providers-oracle>=3.1.
|
103
|
+
Requires-Dist: apache-airflow-providers-oracle>=3.1.0 ; extra == "oracle"
|
104
104
|
Requires-Dist: apache-airflow-providers-postgres ; extra == "postgres"
|
105
105
|
Requires-Dist: apache-airflow-providers-presto ; extra == "presto"
|
106
106
|
Requires-Dist: apache-airflow-providers-salesforce ; extra == "salesforce"
|
@@ -108,8 +108,8 @@ Requires-Dist: apache-airflow-providers-sftp ; extra == "sftp"
|
|
108
108
|
Requires-Dist: apache-airflow-providers-ssh ; extra == "ssh"
|
109
109
|
Requires-Dist: apache-airflow-providers-trino ; extra == "trino"
|
110
110
|
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
111
|
-
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-google/
|
112
|
-
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-google/
|
111
|
+
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-google/11.0.0/changelog.html
|
112
|
+
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-google/11.0.0
|
113
113
|
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
114
114
|
Project-URL: Source Code, https://github.com/apache/airflow
|
115
115
|
Project-URL: Twitter, https://twitter.com/ApacheAirflow
|
@@ -178,7 +178,7 @@ Provides-Extra: trino
|
|
178
178
|
|
179
179
|
Package ``apache-airflow-providers-google``
|
180
180
|
|
181
|
-
Release: ``
|
181
|
+
Release: ``11.0.0``
|
182
182
|
|
183
183
|
|
184
184
|
Google services including:
|
@@ -198,7 +198,7 @@ This is a provider package for ``google`` provider. All classes for this provide
|
|
198
198
|
are in ``airflow.providers.google`` python package.
|
199
199
|
|
200
200
|
You can find package information and changelog for the provider
|
201
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-google/
|
201
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-google/11.0.0/>`_.
|
202
202
|
|
203
203
|
Installation
|
204
204
|
------------
|
@@ -238,7 +238,7 @@ PIP package Version required
|
|
238
238
|
``google-cloud-build`` ``>=3.22.0``
|
239
239
|
``google-cloud-compute`` ``>=1.10.0``
|
240
240
|
``google-cloud-container`` ``>=2.17.4``
|
241
|
-
``google-cloud-datacatalog`` ``>=3.
|
241
|
+
``google-cloud-datacatalog`` ``>=3.23.0``
|
242
242
|
``google-cloud-dataflow-client`` ``>=0.8.6``
|
243
243
|
``google-cloud-dataform`` ``>=0.5.0``
|
244
244
|
``google-cloud-dataplex`` ``>=1.10.0``
|
@@ -261,7 +261,7 @@ PIP package Version required
|
|
261
261
|
``google-cloud-storage-transfer`` ``>=1.4.1``
|
262
262
|
``google-cloud-tasks`` ``>=2.13.0``
|
263
263
|
``google-cloud-texttospeech`` ``>=2.14.1``
|
264
|
-
``google-cloud-translate`` ``>=3.
|
264
|
+
``google-cloud-translate`` ``>=3.16.0``
|
265
265
|
``google-cloud-videointelligence`` ``>=2.11.0``
|
266
266
|
``google-cloud-vision`` ``>=3.4.0``
|
267
267
|
``google-cloud-workflows`` ``>=1.10.0``
|
@@ -321,4 +321,4 @@ Dependent package
|
|
321
321
|
======================================================================================================================== ====================
|
322
322
|
|
323
323
|
The changelog for the provider package can be found in the
|
324
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-google/
|
324
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-google/11.0.0/changelog.html>`_.
|