apache-airflow-providers-google 10.18.0rc1__tar.gz → 10.18.0rc2__tar.gz

{apache_airflow_providers_google-10.18.0rc1 → apache_airflow_providers_google-10.18.0rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-google
-Version: 10.18.0rc1
+Version: 10.18.0rc2
 Summary: Provider package apache-airflow-providers-google for Apache Airflow
 Keywords: airflow-provider,google,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -31,7 +31,7 @@ Requires-Dist: gcloud-aio-storage>=9.0.0
 Requires-Dist: gcsfs>=2023.10.0
 Requires-Dist: google-ads>=23.1.0
 Requires-Dist: google-analytics-admin
-Requires-Dist: google-api-core>=2.11.0,!=2.16.0
+Requires-Dist: google-api-core>=2.11.0,!=2.16.0,!=2.18.0
 Requires-Dist: google-api-python-client>=1.6.0
 Requires-Dist: google-auth-httplib2>=0.0.1
 Requires-Dist: google-auth>=1.0.0
@@ -171,7 +171,7 @@ Provides-Extra: trino
 
 Package ``apache-airflow-providers-google``
 
-Release: ``10.18.0.rc1``
+Release: ``10.18.0.rc2``
 
 
 Google services including:
@@ -205,9 +205,9 @@ The package supports the following python versions: 3.8,3.9,3.10,3.11,3.12
 Requirements
 ------------
 
-=======================================  =====================
+=======================================  ==============================
 PIP package                              Version required
-=======================================  =====================
+=======================================  ==============================
 ``apache-airflow``                       ``>=2.7.0``
 ``apache-airflow-providers-common-sql``  ``>=1.7.2``
 ``asgiref``                              ``>=3.5.2``
@@ -217,7 +217,7 @@ PIP package                              Version required
 ``gcsfs``                                ``>=2023.10.0``
 ``google-ads``                           ``>=23.1.0``
 ``google-analytics-admin``
-``google-api-core``                      ``>=2.11.0,!=2.16.0``
+``google-api-core``                      ``>=2.11.0,!=2.16.0,!=2.18.0``
 ``google-api-python-client``             ``>=1.6.0``
 ``google-auth``                          ``>=1.0.0``
 ``google-auth-httplib2``                 ``>=0.0.1``
@@ -269,7 +269,7 @@ PIP package                              Version required
 ``sqlalchemy-bigquery``                  ``>=1.2.1``
 ``sqlalchemy-spanner``                   ``>=1.6.2``
 ``python-slugify``                       ``>=5.0``
-=======================================  =====================
+=======================================  ==============================
 
 Cross provider package dependencies
 -----------------------------------

{apache_airflow_providers_google-10.18.0rc1 → apache_airflow_providers_google-10.18.0rc2}/README.rst

@@ -42,7 +42,7 @@
 
 Package ``apache-airflow-providers-google``
 
-Release: ``10.18.0.rc1``
+Release: ``10.18.0.rc2``
 
 
 Google services including:
@@ -76,9 +76,9 @@ The package supports the following python versions: 3.8,3.9,3.10,3.11,3.12
 Requirements
 ------------
 
-=======================================  =====================
+=======================================  ==============================
 PIP package                              Version required
-=======================================  =====================
+=======================================  ==============================
 ``apache-airflow``                       ``>=2.7.0``
 ``apache-airflow-providers-common-sql``  ``>=1.7.2``
 ``asgiref``                              ``>=3.5.2``
@@ -88,7 +88,7 @@ PIP package                              Version required
 ``gcsfs``                                ``>=2023.10.0``
 ``google-ads``                           ``>=23.1.0``
 ``google-analytics-admin``
-``google-api-core``                      ``>=2.11.0,!=2.16.0``
+``google-api-core``                      ``>=2.11.0,!=2.16.0,!=2.18.0``
 ``google-api-python-client``             ``>=1.6.0``
 ``google-auth``                          ``>=1.0.0``
 ``google-auth-httplib2``                 ``>=0.0.1``
@@ -140,7 +140,7 @@ PIP package                              Version required
 ``sqlalchemy-bigquery``                  ``>=1.2.1``
 ``sqlalchemy-spanner``                   ``>=1.6.2``
 ``python-slugify``                       ``>=5.0``
-=======================================  =====================
+=======================================  ==============================
 
 Cross provider package dependencies
 -----------------------------------

{apache_airflow_providers_google-10.18.0rc1 → apache_airflow_providers_google-10.18.0rc2}/airflow/providers/google/__init__.py

@@ -25,15 +25,12 @@ from __future__ import annotations
 
 import packaging.version
 
+from airflow import __version__ as airflow_version
+
 __all__ = ["__version__"]
 
 __version__ = "10.18.0"
 
-try:
-    from airflow import __version__ as airflow_version
-except ImportError:
-    from airflow.version import version as airflow_version
-
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.7.0"
 ):

{apache_airflow_providers_google-10.18.0rc1 → apache_airflow_providers_google-10.18.0rc2}/airflow/providers/google/cloud/hooks/automl.py

@@ -640,3 +640,37 @@ class CloudAutoMLHook(GoogleBaseHook):
             metadata=metadata,
         )
         return result
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def get_dataset(
+        self,
+        dataset_id: str,
+        location: str,
+        project_id: str,
+        retry: Retry | _MethodDefault = DEFAULT,
+        timeout: float | None = None,
+        metadata: Sequence[tuple[str, str]] = (),
+    ) -> Dataset:
+        """
+        Retrieve the dataset for the given dataset_id.
+
+        :param dataset_id: ID of dataset to be retrieved.
+        :param location: The location of the project.
+        :param project_id: ID of the Google Cloud project where dataset is located if None then
+            default project_id is used.
+        :param retry: A retry object used to retry requests. If `None` is specified, requests will not be
+            retried.
+        :param timeout: The amount of time, in seconds, to wait for the request to complete. Note that if
+            `retry` is specified, the timeout applies to each individual attempt.
+        :param metadata: Additional metadata that is provided to the method.
+
+        :return: `google.cloud.automl_v1beta1.types.dataset.Dataset` instance.
+        """
+        client = self.get_conn()
+        name = f"projects/{project_id}/locations/{location}/datasets/{dataset_id}"
+        return client.get_dataset(
+            request={"name": name},
+            retry=retry,
+            timeout=timeout,
+            metadata=metadata,
+        )

{apache_airflow_providers_google-10.18.0rc1 → apache_airflow_providers_google-10.18.0rc2}/airflow/providers/google/cloud/hooks/bigquery.py

@@ -46,7 +46,14 @@ from google.cloud.bigquery import (
     UnknownJob,
 )
 from google.cloud.bigquery.dataset import AccessEntry, Dataset, DatasetListItem, DatasetReference
-from google.cloud.bigquery.table import EncryptionConfiguration, Row, RowIterator, Table, TableReference
+from google.cloud.bigquery.retry import DEFAULT_JOB_RETRY
+from google.cloud.bigquery.table import (
+    EncryptionConfiguration,
+    Row,
+    RowIterator,
+    Table,
+    TableReference,
+)
 from google.cloud.exceptions import NotFound
 from googleapiclient.discovery import Resource, build
 from pandas_gbq import read_gbq
@@ -65,12 +72,7 @@ from airflow.providers.google.common.hooks.base_google import (
     GoogleBaseHook,
     get_field,
 )
-
-try:
-    from airflow.utils.hashlib_wrapper import md5
-except ModuleNotFoundError:
-    # Remove when Airflow providers min Airflow version is "2.7.0"
-    from hashlib import md5
+from airflow.utils.hashlib_wrapper import md5
 from airflow.utils.helpers import convert_camel_to_snake
 from airflow.utils.log.logging_mixin import LoggingMixin
 
@@ -2390,6 +2392,48 @@ class BigQueryHook(GoogleBaseHook, DbApiHook):
 
         return project_id, dataset_id, table_id
 
+    @GoogleBaseHook.fallback_to_default_project_id
+    def get_query_results(
+        self,
+        job_id: str,
+        location: str,
+        max_results: int | None = None,
+        selected_fields: list[str] | str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
+        retry: Retry = DEFAULT_RETRY,
+        job_retry: Retry = DEFAULT_JOB_RETRY,
+    ) -> list[dict[str, Any]]:
+        """
+        Get query results given a job_id.
+
+        :param job_id: The ID of the job.
+            The ID must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), or
+            dashes (-). The maximum length is 1,024 characters.
+        :param location: The location used for the operation.
+        :param selected_fields: List of fields to return (comma-separated). If
+            unspecified, all fields are returned.
+        :param max_results: The maximum number of records (rows) to be fetched
+            from the table.
+        :param project_id: Google Cloud Project where the job ran.
+        :param retry: How to retry the RPC.
+        :param job_retry: How to retry failed jobs.
+
+        :return: List of rows where columns are filtered by selected fields, when given
+
+        :raises: AirflowException
+        """
+        if isinstance(selected_fields, str):
+            selected_fields = selected_fields.split(",")
+        job = self.get_job(job_id=job_id, project_id=project_id, location=location)
+        if not isinstance(job, QueryJob):
+            raise AirflowException(f"Job '{job_id}' is not a query job")
+
+        if job.state != "DONE":
+            raise AirflowException(f"Job '{job_id}' is not in DONE state")
+
+        rows = [dict(row) for row in job.result(max_results=max_results, retry=retry, job_retry=job_retry)]
+        return [{k: row[k] for k in row if k in selected_fields} for row in rows] if selected_fields else rows
+
     @property
     def scopes(self) -> Sequence[str]:
         """
@@ -3421,15 +3465,25 @@ class BigQueryAsyncHook(GoogleBaseAsyncHook):
                 self.log.error("Failed to cancel BigQuery job %s: %s", job_id, str(e))
                 raise
 
-    def get_records(self, query_results: dict[str, Any], as_dict: bool = False) -> list[Any]:
+    # TODO: Convert get_records into an async method
+    def get_records(
+        self,
+        query_results: dict[str, Any],
+        as_dict: bool = False,
+        selected_fields: str | list[str] | None = None,
+    ) -> list[Any]:
         """Convert a response from BigQuery to records.
 
         :param query_results: the results from a SQL query
         :param as_dict: if True returns the result as a list of dictionaries, otherwise as list of lists.
+        :param selected_fields:
         """
+        if isinstance(selected_fields, str):
+            selected_fields = selected_fields.split(",")
         buffer: list[Any] = []
         if rows := query_results.get("rows"):
             fields = query_results["schema"]["fields"]
+            fields = [field for field in fields if not selected_fields or field["name"] in selected_fields]
             fields_names = [field["name"] for field in fields]
             col_types = [field["type"] for field in fields]
             for dict_row in rows:

apache_airflow_providers_google-10.18.0rc2/airflow/providers/google/cloud/hooks/vertex_ai/prediction_service.py

@@ -0,0 +1,91 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Sequence
+
+from google.api_core.client_options import ClientOptions
+from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
+from google.cloud.aiplatform_v1 import PredictionServiceClient
+
+from airflow.providers.google.common.consts import CLIENT_INFO
+from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
+
+if TYPE_CHECKING:
+    from google.api_core.retry import Retry
+    from google.cloud.aiplatform_v1.types import PredictResponse
+
+
+class PredictionServiceHook(GoogleBaseHook):
+    """Hook for Google Cloud Vertex AI Prediction API."""
+
+    def get_prediction_service_client(self, region: str | None = None) -> PredictionServiceClient:
+        """
+        Return PredictionServiceClient object.
+
+        :param region: The ID of the Google Cloud region that the service belongs to. Default is None.
+
+        :return: `google.cloud.aiplatform_v1.services.prediction_service.client.PredictionServiceClient` instance.
+        """
+        if region and region != "global":
+            client_options = ClientOptions(api_endpoint=f"{region}-aiplatform.googleapis.com:443")
+        else:
+            client_options = ClientOptions()
+
+        return PredictionServiceClient(
+            credentials=self.get_credentials(), client_info=CLIENT_INFO, client_options=client_options
+        )
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def predict(
+        self,
+        endpoint_id: str,
+        instances: list[str],
+        location: str,
+        project_id: str = PROVIDE_PROJECT_ID,
+        parameters: dict[str, str] | None = None,
+        retry: Retry | _MethodDefault = DEFAULT,
+        timeout: float | None = None,
+        metadata: Sequence[tuple[str, str]] = (),
+    ) -> PredictResponse:
+        """
+        Perform an online prediction and returns the prediction result in the response.
+
+        :param endpoint_id: Name of the endpoint_id requested to serve the prediction.
+        :param instances: Required. The instances that are the input to the prediction call. A DeployedModel
+            may have an upper limit on the number of instances it supports per request, and when it is
+            exceeded the prediction call errors in case of AutoML Models, or, in case of customer created
+            Models, the behaviour is as documented by that Model.
+        :param parameters: Additional domain-specific parameters, any string must be up to 25000 characters long.
+        :param project_id: ID of the Google Cloud project where model is located if None then
+            default project_id is used.
+        :param location: The location of the project.
+        :param retry: A retry object used to retry requests. If `None` is specified, requests will not be
+            retried.
+        :param timeout: The amount of time, in seconds, to wait for the request to complete. Note that if
+            `retry` is specified, the timeout applies to each individual attempt.
+        :param metadata: Additional metadata that is provided to the method.
+        """
+        client = self.get_prediction_service_client(location)
+        endpoint = f"projects/{project_id}/locations/{location}/endpoints/{endpoint_id}"
+        return client.predict(
+            request={"endpoint": endpoint, "instances": instances, "parameters": parameters},
+            retry=retry,
+            timeout=timeout,
+            metadata=metadata,
+        )