apache-airflow-providers-google 16.1.0__py3-none-any.whl → 17.0.0rc1__py3-none-any.whl

airflow/providers/google/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "16.1.0"
+__version__ = "17.0.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.10.0"

airflow/providers/google/ads/hooks/ads.py

@@ -29,11 +29,7 @@ from google.auth.exceptions import GoogleAuthError
 
 from airflow.exceptions import AirflowException
 from airflow.providers.google.common.hooks.base_google import get_field
-
-try:
-    from airflow.sdk import BaseHook
-except ImportError:
-    from airflow.hooks.base import BaseHook  # type: ignore[attr-defined,no-redef]
+from airflow.providers.google.version_compat import BaseHook
 
 if TYPE_CHECKING:
     from google.ads.googleads.v20.services.services.customer_service import CustomerServiceClient

airflow/providers/google/cloud/hooks/bigquery.py

@@ -400,135 +400,6 @@ class BigQueryHook(GoogleBaseHook, DbApiHook):
         except NotFound:
             return False
 
-    @deprecated(
-        planned_removal_date="July 30, 2025",
-        use_instead="airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.create_table",
-        category=AirflowProviderDeprecationWarning,
-    )
-    @GoogleBaseHook.fallback_to_default_project_id
-    def create_empty_table(
-        self,
-        project_id: str = PROVIDE_PROJECT_ID,
-        dataset_id: str | None = None,
-        table_id: str | None = None,
-        table_resource: dict[str, Any] | None = None,
-        schema_fields: list | None = None,
-        time_partitioning: dict | None = None,
-        cluster_fields: list[str] | None = None,
-        labels: dict | None = None,
-        view: dict | None = None,
-        materialized_view: dict | None = None,
-        encryption_configuration: dict | None = None,
-        retry: Retry = DEFAULT_RETRY,
-        location: str | None = None,
-        exists_ok: bool = True,
-    ) -> Table:
-        """
-        Create a new, empty table in the dataset.
-
-        To create a view, which is defined by a SQL query, parse a dictionary to
-        the *view* argument.
-
-        :param project_id: The project to create the table into.
-        :param dataset_id: The dataset to create the table into.
-        :param table_id: The Name of the table to be created.
-        :param table_resource: Table resource as described in documentation:
-            https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#Table
-            If provided all other parameters are ignored.
-        :param schema_fields: If set, the schema field list as defined here:
-            https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.load.schema
-
-            .. code-block:: python
-
-                schema_fields = [
-                    {"name": "emp_name", "type": "STRING", "mode": "REQUIRED"},
-                    {"name": "salary", "type": "INTEGER", "mode": "NULLABLE"},
-                ]
-
-        :param labels: a dictionary containing labels for the table, passed to BigQuery
-        :param retry: Optional. How to retry the RPC.
-        :param time_partitioning: configure optional time partitioning fields i.e.
-            partition by field, type and expiration as per API specifications.
-
-            .. seealso::
-                https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#timePartitioning
-        :param cluster_fields: [Optional] The fields used for clustering.
-            BigQuery supports clustering for both partitioned and
-            non-partitioned tables.
-            https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#clustering.fields
-        :param view: [Optional] A dictionary containing definition for the view.
-            If set, it will create a view instead of a table:
-            https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ViewDefinition
-
-            .. code-block:: python
-
-                view = {
-                    "query": "SELECT * FROM `test-project-id.test_dataset_id.test_table_prefix*` LIMIT 1000",
-                    "useLegacySql": False,
-                }
-
-        :param materialized_view: [Optional] The materialized view definition.
-        :param encryption_configuration: [Optional] Custom encryption configuration (e.g., Cloud KMS keys).
-
-            .. code-block:: python
-
-                encryption_configuration = {
-                    "kmsKeyName": "projects/testp/locations/us/keyRings/test-kr/cryptoKeys/test-key",
-                }
-
-        :param num_retries: Maximum number of retries in case of connection problems.
-        :param location: (Optional) The geographic location where the table should reside.
-        :param exists_ok: If ``True``, ignore "already exists" errors when creating the table.
-        :return: Created table
-        """
-        _table_resource: dict[str, Any] = {}
-
-        if self.location:
-            _table_resource["location"] = self.location
-
-        if schema_fields:
-            _table_resource["schema"] = {"fields": schema_fields}
-
-        if time_partitioning:
-            _table_resource["timePartitioning"] = time_partitioning
-
-        if cluster_fields:
-            _table_resource["clustering"] = {"fields": cluster_fields}
-
-        if labels:
-            _table_resource["labels"] = labels
-
-        if view:
-            _table_resource["view"] = view
-
-        if materialized_view:
-            _table_resource["materializedView"] = materialized_view
-
-        if encryption_configuration:
-            _table_resource["encryptionConfiguration"] = encryption_configuration
-
-        table_resource = table_resource or _table_resource
-        table_resource = self._resolve_table_reference(
-            table_resource=table_resource,
-            project_id=project_id,
-            dataset_id=dataset_id,
-            table_id=table_id,
-        )
-        table = Table.from_api_repr(table_resource)
-        result = self.get_client(project_id=project_id, location=location).create_table(
-            table=table, exists_ok=exists_ok, retry=retry
-        )
-        get_hook_lineage_collector().add_output_asset(
-            context=self,
-            scheme="bigquery",
-            asset_kwargs={
-                "project_id": result.project,
-                "dataset_id": result.dataset_id,
-                "table_id": result.table_id,
-            },
-        )
-        return result
-
     @GoogleBaseHook.fallback_to_default_project_id
     def create_table(
         self,
@@ -2248,7 +2119,7 @@ class BigQueryAsyncHook(GoogleBaseAsyncHook):
         self,
         sql: str,
         pass_value: Any,
-        records: list[Any],
+        records: list[Any] | None = None,
         tolerance: float | None = None,
     ) -> None:
         """

airflow/providers/google/cloud/hooks/cloud_logging.py

@@ -0,0 +1,109 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import TYPE_CHECKING
+
+from google.cloud.logging_v2.services.config_service_v2 import ConfigServiceV2Client
+from google.cloud.logging_v2.types import (
+    CreateSinkRequest,
+    DeleteSinkRequest,
+    GetSinkRequest,
+    ListSinksRequest,
+    LogSink,
+    UpdateSinkRequest,
+)
+
+from airflow.providers.google.common.consts import CLIENT_INFO
+from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
+
+if TYPE_CHECKING:
+    from google.protobuf.field_mask_pb2 import FieldMask
+
+
+class CloudLoggingHook(GoogleBaseHook):
+    """
+    Hook for Google Cloud Logging Log Sinks API.
+
+    :param gcp_conn_id: The connection ID to use when fetching connection info.
+    :param impersonation_chain: Optional service account to impersonate.
+    """
+
+    def __init__(
+        self,
+        gcp_conn_id: str = "google_cloud_default",
+        impersonation_chain: str | Sequence[str] | None = None,
+        **kwargs,
+    ) -> None:
+        super().__init__(gcp_conn_id=gcp_conn_id, impersonation_chain=impersonation_chain, **kwargs)
+        self._client: ConfigServiceV2Client | None = None
+
+    def get_conn(self) -> ConfigServiceV2Client:
+        """Return the Google Cloud Logging Config client."""
+        if not self._client:
+            self._client = ConfigServiceV2Client(credentials=self.get_credentials(), client_info=CLIENT_INFO)
+        return self._client
+
+    def get_parent(self, project_id):
+        return f"projects/{project_id}"
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def create_sink(
+        self, sink: LogSink | dict, unique_writer_identity: bool = True, project_id: str = PROVIDE_PROJECT_ID
+    ) -> LogSink:
+        if isinstance(sink, dict):
+            sink = LogSink(**sink)
+        request = CreateSinkRequest(
+            parent=self.get_parent(project_id), sink=sink, unique_writer_identity=unique_writer_identity
+        )
+        return self.get_conn().create_sink(request=request)
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def get_sink(self, sink_name: str, project_id: str = PROVIDE_PROJECT_ID) -> LogSink:
+        request = GetSinkRequest(sink_name=f"projects/{project_id}/sinks/{sink_name}")
+        return self.get_conn().get_sink(request=request)
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def list_sinks(self, page_size: int | None = None, project_id: str = PROVIDE_PROJECT_ID) -> list[LogSink]:
+        request = ListSinksRequest(parent=self.get_parent(project_id), page_size=page_size)
+        return list(self.get_conn().list_sinks(request=request))
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def delete_sink(self, sink_name: str, project_id: str = PROVIDE_PROJECT_ID) -> None:
+        request = DeleteSinkRequest(sink_name=f"projects/{project_id}/sinks/{sink_name}")
+        self.get_conn().delete_sink(request=request)
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def update_sink(
+        self,
+        sink_name: str,
+        sink: LogSink | dict,
+        unique_writer_identity: bool,
+        update_mask: FieldMask | dict,
+        project_id: str = PROVIDE_PROJECT_ID,
+    ) -> LogSink:
+        if isinstance(sink, dict):
+            sink = LogSink(**sink)
+        request = UpdateSinkRequest(
+            sink_name=f"projects/{project_id}/sinks/{sink_name}",
+            sink=sink,
+            unique_writer_identity=unique_writer_identity,
+            update_mask=update_mask,
+        )
+        return self.get_conn().update_sink(request=request)

airflow/providers/google/cloud/hooks/cloud_run.py

@@ -38,7 +38,7 @@ from google.cloud.run_v2 import (
     ServicesClient,
     UpdateJobRequest,
 )
-from google.longrunning import operations_pb2  # type: ignore[attr-defined]
+from google.longrunning import operations_pb2
 
 from airflow.exceptions import AirflowException
 from airflow.providers.google.common.consts import CLIENT_INFO

airflow/providers/google/cloud/hooks/cloud_sql.py

@@ -60,11 +60,7 @@ from airflow.providers.google.common.hooks.base_google import (
     GoogleBaseHook,
     get_field,
 )
-
-try:
-    from airflow.sdk import BaseHook
-except ImportError:
-    from airflow.hooks.base import BaseHook  # type: ignore[attr-defined,no-redef]
+from airflow.providers.google.version_compat import BaseHook
 from airflow.utils.log.logging_mixin import LoggingMixin
 
 if TYPE_CHECKING:
@@ -1107,6 +1103,8 @@ class CloudSQLDatabaseHook(BaseHook):
         return connection_uri
 
     def _get_instance_socket_name(self) -> str:
+        if self.project_id is None:
+            raise ValueError("The project_id should not be none")
         return self.project_id + ":" + self.location + ":" + self.instance
 
     def _get_sqlproxy_instance_specification(self) -> str:
@@ -1139,6 +1137,8 @@ class CloudSQLDatabaseHook(BaseHook):
             raise ValueError("Proxy runner can only be retrieved in case of use_proxy = True")
         if not self.sql_proxy_unique_path:
             raise ValueError("The sql_proxy_unique_path should be set")
+        if self.project_id is None:
+            raise ValueError("The project_id should not be None")
         return CloudSqlProxyRunner(
             path_prefix=self.sql_proxy_unique_path,
             instance_specification=self._get_sqlproxy_instance_specification(),

airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py

@@ -58,7 +58,7 @@ if TYPE_CHECKING:
     from google.cloud.storage_transfer_v1.services.storage_transfer_service.pagers import (
         ListTransferJobsAsyncPager,
     )
-    from google.longrunning import operations_pb2  # type: ignore[attr-defined]
+    from google.longrunning import operations_pb2
     from proto import Message
 
 log = logging.getLogger(__name__)

airflow/providers/google/cloud/hooks/dataflow.py

@@ -51,7 +51,6 @@ from googleapiclient.discovery import Resource, build
 
 from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
 from airflow.providers.apache.beam.hooks.beam import BeamHook, BeamRunnerType, beam_options_to_args
-from airflow.providers.google.common.deprecated import deprecated
 from airflow.providers.google.common.hooks.base_google import (
     PROVIDE_PROJECT_ID,
     GoogleBaseAsyncHook,
@@ -1126,90 +1125,6 @@ class DataflowHook(GoogleBaseHook):
         )
         jobs_controller.cancel()
 
-    @deprecated(
-        planned_removal_date="July 01, 2025",
-        use_instead="airflow.providers.google.cloud.hooks.dataflow.DataflowHook.launch_beam_yaml_job",
-        category=AirflowProviderDeprecationWarning,
-    )
-    @GoogleBaseHook.fallback_to_default_project_id
-    def start_sql_job(
-        self,
-        job_name: str,
-        query: str,
-        options: dict[str, Any],
-        project_id: str,
-        location: str = DEFAULT_DATAFLOW_LOCATION,
-        on_new_job_id_callback: Callable[[str], None] | None = None,
-        on_new_job_callback: Callable[[dict], None] | None = None,
-    ):
-        """
-        Start Dataflow SQL query.
-
-        :param job_name: The unique name to assign to the Cloud Dataflow job.
-        :param query: The SQL query to execute.
-        :param options: Job parameters to be executed.
-            For more information, look at:
-            `https://cloud.google.com/sdk/gcloud/reference/beta/dataflow/sql/query
-            <gcloud beta dataflow sql query>`__
-            command reference
-        :param location: The location of the Dataflow job (for example europe-west1)
-        :param project_id: The ID of the GCP project that owns the job.
-            If set to ``None`` or missing, the default project_id from the GCP connection is used.
-        :param on_new_job_id_callback: (Deprecated) Callback called when the job ID is known.
-        :param on_new_job_callback: Callback called when the job is known.
-        :return: the new job object
-        """
-        gcp_options = {
-            "project": project_id,
-            "format": "value(job.id)",
-            "job-name": job_name,
-            "region": location,
-        }
-        cmd = self._build_gcloud_command(
-            command=["gcloud", "dataflow", "sql", "query", query], parameters={**gcp_options, **options}
-        )
-        self.log.info("Executing command: %s", " ".join(shlex.quote(c) for c in cmd))
-        with self.provide_authorized_gcloud():
-            proc = subprocess.run(cmd, capture_output=True)
-        self.log.info("Output: %s", proc.stdout.decode())
-        self.log.warning("Stderr: %s", proc.stderr.decode())
-        self.log.info("Exit code %d", proc.returncode)
-        stderr_last_20_lines = "\n".join(proc.stderr.decode().strip().splitlines()[-20:])
-        if proc.returncode != 0:
-            raise AirflowException(
-                f"Process exit with non-zero exit code. Exit code: {proc.returncode} Error Details : "
-                f"{stderr_last_20_lines}"
-            )
-        job_id = proc.stdout.decode().strip()
-
-        self.log.info("Created job ID: %s", job_id)
-
-        jobs_controller = _DataflowJobsController(
-            dataflow=self.get_conn(),
-            project_number=project_id,
-            job_id=job_id,
-            location=location,
-            poll_sleep=self.poll_sleep,
-            num_retries=self.num_retries,
-            drain_pipeline=self.drain_pipeline,
-            wait_until_finished=self.wait_until_finished,
-        )
-        job = jobs_controller.get_jobs(refresh=True)[0]
-
-        if on_new_job_id_callback:
-            warnings.warn(
-                "on_new_job_id_callback is Deprecated. Please start using on_new_job_callback",
-                AirflowProviderDeprecationWarning,
-                stacklevel=3,
-            )
-            on_new_job_id_callback(cast("str", job.get("id")))
-
-        if on_new_job_callback:
-            on_new_job_callback(job)
-
-        jobs_controller.wait_for_done()
-        return jobs_controller.get_jobs(refresh=True)[0]
-
     @GoogleBaseHook.fallback_to_default_project_id
     def get_job(
         self,

airflow/providers/google/cloud/hooks/datafusion.py

@@ -434,7 +434,7 @@ class DataFusionHook(GoogleBaseHook):
         pipeline_id: str,
         pipeline_type: DataFusionPipelineType = DataFusionPipelineType.BATCH,
         namespace: str = "default",
-    ) -> Any:
+    ) -> dict:
         url = os.path.join(
             self._base_url(instance_url, namespace),
             quote(pipeline_name),

airflow/providers/google/cloud/hooks/dataprep.py

@@ -28,10 +28,7 @@ import requests
 from requests import HTTPError
 from tenacity import retry, stop_after_attempt, wait_exponential
 
-try:
-    from airflow.sdk import BaseHook
-except ImportError:
-    from airflow.hooks.base import BaseHook  # type: ignore[attr-defined,no-redef]
+from airflow.providers.google.version_compat import BaseHook
 
 
 def _get_field(extras: dict, field_name: str) -> str | None: