apache-airflow-providers-google 14.1.0__py3-none-any.whl → 15.0.0rc1__py3-none-any.whl

airflow/providers/google/cloud/operators/spanner.py

@@ -403,8 +403,7 @@ class SpannerDeployDatabaseInstanceOperator(GoogleCloudBaseOperator):
             )
         else:
             self.log.info(
-                "The database '%s' in project '%s' and instance '%s'"
-                " already exists. Nothing to do. Exiting.",
+                "The database '%s' in project '%s' and instance '%s' already exists. Nothing to do. Exiting.",
                 self.database_id,
                 self.project_id,
                 self.instance_id,

airflow/providers/google/cloud/operators/translate.py

@@ -1145,7 +1145,7 @@ class TranslateDocumentOperator(GoogleCloudBaseOperator):
                 project_id=self.project_id or hook.project_id,
                 output_config=self.document_output_config,
             )
-        return cast(dict, type(doc_translation_result).to_dict(doc_translation_result))
+        return cast("dict", type(doc_translation_result).to_dict(doc_translation_result))
 
 
 class TranslateDocumentBatchOperator(GoogleCloudBaseOperator):
@@ -1310,7 +1310,7 @@ class TranslateDocumentBatchOperator(GoogleCloudBaseOperator):
         )
         result = hook.wait_for_operation_result(batch_document_translate_operation)
         self.log.info("Batch document translation job finished")
-        return cast(dict, type(result).to_dict(result))
+        return cast("dict", type(result).to_dict(result))
 
 
 class TranslateCreateGlossaryOperator(GoogleCloudBaseOperator):

airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py

@@ -29,7 +29,6 @@ from google.cloud.aiplatform import datasets
 from google.cloud.aiplatform.models import Model
 from google.cloud.aiplatform_v1.types.training_pipeline import TrainingPipeline
 
-from airflow.exceptions import AirflowProviderDeprecationWarning
 from airflow.providers.google.cloud.hooks.vertex_ai.auto_ml import AutoMLHook
 from airflow.providers.google.cloud.links.vertex_ai import (
     VertexAIModelLink,
@@ -37,7 +36,6 @@ from airflow.providers.google.cloud.links.vertex_ai import (
     VertexAITrainingPipelinesLink,
 )
 from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
-from airflow.providers.google.common.deprecated import deprecated
 
 if TYPE_CHECKING:
     from google.api_core.retry import Retry
@@ -575,16 +573,6 @@ class DeleteAutoMLTrainingJobOperator(GoogleCloudBaseOperator):
         self.gcp_conn_id = gcp_conn_id
         self.impersonation_chain = impersonation_chain
 
-    @property
-    @deprecated(
-        planned_removal_date="March 01, 2025",
-        use_instead="training_pipeline_id",
-        category=AirflowProviderDeprecationWarning,
-    )
-    def training_pipeline(self):
-        """Alias for ``training_pipeline_id``, used for compatibility (deprecated)."""
-        return self.training_pipeline_id
-
     def execute(self, context: Context):
         hook = AutoMLHook(
             gcp_conn_id=self.gcp_conn_id,

airflow/providers/google/cloud/operators/vertex_ai/custom_job.py

@@ -30,7 +30,7 @@ from google.cloud.aiplatform_v1.types.dataset import Dataset
 from google.cloud.aiplatform_v1.types.training_pipeline import TrainingPipeline
 
 from airflow.configuration import conf
-from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
+from airflow.exceptions import AirflowException
 from airflow.providers.google.cloud.hooks.vertex_ai.custom_job import CustomJobHook
 from airflow.providers.google.cloud.links.vertex_ai import (
     VertexAIModelLink,
@@ -43,7 +43,6 @@ from airflow.providers.google.cloud.triggers.vertex_ai import (
     CustomPythonPackageTrainingJobTrigger,
     CustomTrainingJobTrigger,
 )
-from airflow.providers.google.common.deprecated import deprecated
 
 if TYPE_CHECKING:
     from google.api_core.retry import Retry
@@ -1634,26 +1633,6 @@ class DeleteCustomTrainingJobOperator(GoogleCloudBaseOperator):
         self.gcp_conn_id = gcp_conn_id
         self.impersonation_chain = impersonation_chain
 
-    @property
-    @deprecated(
-        planned_removal_date="March 01, 2025",
-        use_instead="training_pipeline_id",
-        category=AirflowProviderDeprecationWarning,
-    )
-    def training_pipeline(self):
-        """Alias for ``training_pipeline_id``, used for compatibility (deprecated)."""
-        return self.training_pipeline_id
-
-    @property
-    @deprecated(
-        planned_removal_date="March 01, 2025",
-        use_instead="custom_job_id",
-        category=AirflowProviderDeprecationWarning,
-    )
-    def custom_job(self):
-        """Alias for ``custom_job_id``, used for compatibility (deprecated)."""
-        return self.custom_job_id
-
     def execute(self, context: Context):
         hook = CustomJobHook(
             gcp_conn_id=self.gcp_conn_id,

airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py

@@ -98,6 +98,7 @@ class CloudDataTransferServiceJobStatusSensor(BaseSensorOperator):
         self.deferrable = deferrable
 
     def poke(self, context: Context) -> bool:
+        ti = context["ti"]
         hook = CloudDataTransferServiceHook(
             gcp_conn_id=self.gcp_cloud_conn_id,
             impersonation_chain=self.impersonation_chain,
@@ -113,13 +114,12 @@ class CloudDataTransferServiceJobStatusSensor(BaseSensorOperator):
             operations=operations, expected_statuses=self.expected_statuses
         )
         if check:
-            self.xcom_push(key="sensed_operations", value=operations, context=context)
+            ti.xcom_push(key="sensed_operations", value=operations)
 
         project_id = self.project_id or hook.project_id
         if project_id:
             CloudStorageTransferJobLink.persist(
                 context=context,
-                task_instance=self,
                 project_id=project_id,
                 job_name=self.job_name,
             )
@@ -154,4 +154,5 @@ class CloudDataTransferServiceJobStatusSensor(BaseSensorOperator):
         if event["status"] == "error":
             raise AirflowException(event["message"])
 
-        self.xcom_push(key="sensed_operations", value=event["operations"], context=context)
+        ti = context["ti"]
+        ti.xcom_push(key="sensed_operations", value=event["operations"])

airflow/providers/google/cloud/sensors/dataproc_metastore.py

@@ -112,7 +112,7 @@ class MetastoreHivePartitionSensor(BaseSensorOperator):
 
         # Extract actual query results
         result_base_uri = result_manifest_uri.rsplit("/", 1)[0]
-        results = (f"{result_base_uri}//{filename}" for filename in manifest.get("filenames", []))
+        results = (f"{result_base_uri}/{filename}" for filename in manifest.get("filenames", []))
         found_partitions = sum(
             len(
                 parse_json_from_gcs(

airflow/providers/google/cloud/transfers/gcs_to_bigquery.py

@@ -486,8 +486,7 @@ class GCSToBigQueryOperator(BaseOperator):
         if self.max_id_key:
             self.log.info("Selecting the MAX value from BigQuery column %r...", self.max_id_key)
             select_command = (
-                f"SELECT MAX({self.max_id_key}) AS max_value "
-                f"FROM {self.destination_project_dataset_table}"
+                f"SELECT MAX({self.max_id_key}) AS max_value FROM {self.destination_project_dataset_table}"
             )
             self.configuration = {
                 "query": {

airflow/providers/google/cloud/transfers/sftp_to_gcs.py

@@ -73,6 +73,11 @@ class SFTPToGCSOperator(BaseOperator):
         Service Account Token Creator IAM role to the directly preceding identity, with first
         account from the list granting this role to the originating account (templated).
     :param sftp_prefetch: Whether to enable SFTP prefetch, the default is True.
+    :param use_stream: Determines the transfer method from SFTP to GCS.
+        When ``False`` (default), the file downloads locally
+        then uploads (may require significant disk space).
+        When ``True``, the file streams directly without using local disk.
+        Defaults to ``False``.
     """
 
     template_fields: Sequence[str] = (
@@ -95,6 +100,7 @@ class SFTPToGCSOperator(BaseOperator):
         move_object: bool = False,
         impersonation_chain: str | Sequence[str] | None = None,
         sftp_prefetch: bool = True,
+        use_stream: bool = False,
         **kwargs,
     ) -> None:
         super().__init__(**kwargs)
@@ -109,6 +115,7 @@ class SFTPToGCSOperator(BaseOperator):
         self.move_object = move_object
         self.impersonation_chain = impersonation_chain
         self.sftp_prefetch = sftp_prefetch
+        self.use_stream = use_stream
 
     @cached_property
     def sftp_hook(self):
@@ -166,16 +173,22 @@ class SFTPToGCSOperator(BaseOperator):
             destination_object,
         )
 
-        with NamedTemporaryFile("w") as tmp:
-            sftp_hook.retrieve_file(source_path, tmp.name, prefetch=self.sftp_prefetch)
-
-            gcs_hook.upload(
-                bucket_name=self.destination_bucket,
-                object_name=destination_object,
-                filename=tmp.name,
-                mime_type=self.mime_type,
-                gzip=self.gzip,
-            )
+        if self.use_stream:
+            dest_bucket = gcs_hook.get_bucket(self.destination_bucket)
+            dest_blob = dest_bucket.blob(destination_object)
+            with dest_blob.open("wb") as write_stream:
+                sftp_hook.retrieve_file(source_path, write_stream, prefetch=self.sftp_prefetch)
+        else:
+            with NamedTemporaryFile("w") as tmp:
+                sftp_hook.retrieve_file(source_path, tmp.name, prefetch=self.sftp_prefetch)
+
+                gcs_hook.upload(
+                    bucket_name=self.destination_bucket,
+                    object_name=destination_object,
+                    filename=tmp.name,
+                    mime_type=self.mime_type,
+                    gzip=self.gzip,
+                )
 
         if self.move_object:
             self.log.info("Executing delete of %s", source_path)

airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py

@@ -59,7 +59,7 @@ class CloudStorageTransferServiceCreateJobsTrigger(BaseTrigger):
     def serialize(self) -> tuple[str, dict[str, Any]]:
         """Serialize StorageTransferJobsTrigger arguments and classpath."""
         return (
-            f"{self.__class__.__module__ }.{self.__class__.__qualname__}",
+            f"{self.__class__.__module__}.{self.__class__.__qualname__}",
             {
                 "project_id": self.project_id,
                 "job_names": self.job_names,
@@ -176,7 +176,7 @@ class CloudStorageTransferServiceCheckJobStatusTrigger(BaseTrigger):
     def serialize(self) -> tuple[str, dict[str, Any]]:
         """Serialize CloudStorageTransferServiceCheckJobStatusTrigger arguments and classpath."""
         return (
-            f"{self.__class__.__module__ }.{self.__class__.__qualname__}",
+            f"{self.__class__.__module__}.{self.__class__.__qualname__}",
             {
                 "job_name": self.job_name,
                 "expected_statuses": self.expected_statuses,

airflow/providers/google/common/auth_backend/google_openid.py

@@ -146,4 +146,4 @@ def requires_authentication(function: T):
 
         return function(*args, **kwargs)
 
-    return cast(T, decorated)
+    return cast("T", decorated)

airflow/providers/google/common/hooks/base_google.py

@@ -49,20 +49,17 @@ from googleapiclient.http import MediaIoBaseDownload, build_http, set_user_agent
 from requests import Session
 
 from airflow import version
-from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
+from airflow.exceptions import AirflowException
 from airflow.hooks.base import BaseHook
 from airflow.providers.google.cloud.utils.credentials_provider import (
     _get_scopes,
     _get_target_principal_and_delegates,
     get_credentials_and_project_id,
 )
-from airflow.providers.google.common.consts import CLIENT_INFO
-from airflow.providers.google.common.deprecated import deprecated
 from airflow.utils.process_utils import patch_environ
 
 if TYPE_CHECKING:
     from aiohttp import ClientSession
-    from google.api_core.gapic_v1.client_info import ClientInfo
     from google.auth.credentials import Credentials
 
 log = logging.getLogger(__name__)
@@ -153,7 +150,7 @@ class retry_if_temporary_refresh_credentials(tenacity.retry_if_exception):
 # This allows the 'project_id' argument to be of type str instead of str | None,
 # making it easier to type hint the function body without dealing with the None
 # case that can never happen at runtime.
-PROVIDE_PROJECT_ID: str = cast(str, None)
+PROVIDE_PROJECT_ID: str = cast("str", None)
 
 T = TypeVar("T", bound=Callable)
 RT = TypeVar("RT")
@@ -442,24 +439,6 @@ class GoogleBaseHook(BaseHook):
                 f"Please check the connection configuration."
             )
 
-    @property
-    @deprecated(
-        planned_removal_date="March 01, 2025",
-        use_instead="airflow.providers.google.common.consts.CLIENT_INFO",
-        category=AirflowProviderDeprecationWarning,
-    )
-    def client_info(self) -> ClientInfo:
-        """
-        Return client information used to generate a user-agent for API calls.
-
-        It allows for better errors tracking.
-
-        This object is only used by the google-cloud-* libraries that are built specifically for
-        the Google Cloud. It is not supported by The Google APIs Python Client that use Discovery
-        based APIs.
-        """
-        return CLIENT_INFO
-
     @property
     def scopes(self) -> Sequence[str]:
         """
@@ -499,7 +478,7 @@ class GoogleBaseHook(BaseHook):
                 "after": tenacity.after_log(log, logging.DEBUG),
             }
             default_kwargs.update(**kwargs)
-            return cast(T, tenacity.retry(*args, **default_kwargs)(func))
+            return cast("T", tenacity.retry(*args, **default_kwargs)(func))
 
         return decorator
 
@@ -517,7 +496,7 @@ class GoogleBaseHook(BaseHook):
                 "after": tenacity.after_log(log, logging.DEBUG),
             }
             default_kwargs.update(**kwargs)
-            return cast(T, tenacity.retry(*args, **default_kwargs)(func))
+            return cast("T", tenacity.retry(*args, **default_kwargs)(func))
 
         return decorator
 
@@ -569,7 +548,7 @@ class GoogleBaseHook(BaseHook):
             with self.provide_gcp_credential_file_as_context():
                 return func(self, *args, **kwargs)
 
-        return cast(T, wrapper)
+        return cast("T", wrapper)
 
     @contextmanager
     def provide_gcp_credential_file_as_context(self) -> Generator[str | None, None, None]:
@@ -718,7 +697,7 @@ class _CredentialsToken(Token):
         scopes: Sequence[str] | None = None,
     ) -> None:
         _scopes: list[str] | None = list(scopes) if scopes else None
-        super().__init__(session=cast(Session, session), scopes=_scopes)
+        super().__init__(session=cast("Session", session), scopes=_scopes)
         self.credentials = credentials
         self.project = project
 
@@ -743,7 +722,7 @@ class _CredentialsToken(Token):
     async def refresh(self, *, timeout: int) -> TokenResponse:
         await sync_to_async(self.credentials.refresh)(google.auth.transport.requests.Request())
 
-        self.access_token = cast(str, self.credentials.token)
+        self.access_token = cast("str", self.credentials.token)
         self.access_token_duration = 3600
         self.access_token_acquired_at = self._now()
         return TokenResponse(value=self.access_token, expires_in=self.access_token_duration)

airflow/providers/google/get_provider_info.py

@@ -27,8 +27,9 @@ def get_provider_info():
         "name": "Google",
         "description": "Google services including:\n\n  - `Google Ads <https://ads.google.com/>`__\n  - `Google Cloud (GCP) <https://cloud.google.com/>`__\n  - `Google Firebase <https://firebase.google.com/>`__\n  - `Google LevelDB <https://github.com/google/leveldb/>`__\n  - `Google Marketing Platform <https://marketingplatform.google.com/>`__\n  - `Google Workspace <https://workspace.google.com/>`__ (formerly Google Suite)\n",
         "state": "ready",
-        "source-date-epoch": 1742980830,
+        "source-date-epoch": 1743836162,
         "versions": [
+            "15.0.0",
             "14.1.0",
             "14.0.0",
             "12.0.0",
@@ -1652,6 +1653,7 @@ def get_provider_info():
             "sqlalchemy-spanner>=1.6.2",
             "tenacity>=8.1.0",
             "immutabledict>=4.2.0",
+            "types-protobuf!=5.29.1.20250402",
         ],
         "optional-dependencies": {
             "apache.beam": [

airflow/providers/google/marketing_platform/sensors/display_video.py

@@ -83,7 +83,7 @@ class GoogleDisplayVideo360GetSDFDownloadOperationSensor(BaseSensorOperator):
         )
         operation = hook.get_sdf_download_operation(operation_name=self.operation_name)
         if "error" in operation:
-            message = f'The operation finished in error with {operation["error"]}'
+            message = f"The operation finished in error with {operation['error']}"
             raise AirflowException(message)
         if operation and operation.get("done"):
             return True

airflow/providers/google/suite/hooks/drive.py

@@ -192,9 +192,9 @@ class GoogleDriveHook(GoogleBaseHook):
             # current_file_id can be file or directory id, Google API treats them the same way.
             file_info = self._get_file_info(current_file_id)
             if current_file_id == file_id:
-                path = f'{file_info["name"]}'
+                path = f"{file_info['name']}"
             else:
-                path = f'{file_info["name"]}/{path}'
+                path = f"{file_info['name']}/{path}"
 
             # Google API returns parents array if there is at least one object inside
             if "parents" in file_info and len(file_info["parents"]) == 1:

{apache_airflow_providers_google-14.1.0.dist-info → apache_airflow_providers_google-15.0.0rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apache-airflow-providers-google
-Version: 14.1.0
+Version: 15.0.0rc1
 Summary: Provider package apache-airflow-providers-google for Apache Airflow
 Keywords: airflow-provider,google,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -20,9 +20,9 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
-Requires-Dist: apache-airflow>=2.9.0
-Requires-Dist: apache-airflow-providers-common-compat>=1.4.0
-Requires-Dist: apache-airflow-providers-common-sql>=1.20.0
+Requires-Dist: apache-airflow>=2.9.0rc0
+Requires-Dist: apache-airflow-providers-common-compat>=1.4.0rc0
+Requires-Dist: apache-airflow-providers-common-sql>=1.20.0rc0
 Requires-Dist: asgiref>=3.5.2
 Requires-Dist: dill>=0.2.3
 Requires-Dist: gcloud-aio-auth>=5.2.0
@@ -87,6 +87,7 @@ Requires-Dist: sqlalchemy-bigquery>=1.2.1
 Requires-Dist: sqlalchemy-spanner>=1.6.2
 Requires-Dist: tenacity>=8.1.0
 Requires-Dist: immutabledict>=4.2.0
+Requires-Dist: types-protobuf!=5.29.1.20250402
 Requires-Dist: apache-airflow-providers-amazon>=2.6.0 ; extra == "amazon"
 Requires-Dist: apache-beam[gcp]>=2.53.0 ; extra == "apache-beam" and ( python_version < "3.12")
 Requires-Dist: apache-beam[gcp]>=2.57.0 ; extra == "apache-beam" and ( python_version >= "3.12")
@@ -106,8 +107,8 @@ Requires-Dist: apache-airflow-providers-sftp ; extra == "sftp"
 Requires-Dist: apache-airflow-providers-ssh ; extra == "ssh"
 Requires-Dist: apache-airflow-providers-trino ; extra == "trino"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-google/14.1.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-google/14.1.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-google/15.0.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-google/15.0.0
 Project-URL: Mastodon, https://fosstodon.org/@airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
@@ -155,7 +156,7 @@ Provides-Extra: trino
 
 Package ``apache-airflow-providers-google``
 
-Release: ``14.1.0``
+Release: ``15.0.0``
 
 
 Google services including:
@@ -175,7 +176,7 @@ This is a provider package for ``google`` provider. All classes for this provide
 are in ``airflow.providers.google`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-google/14.1.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-google/15.0.0/>`_.
 
 Installation
 ------------
@@ -259,6 +260,7 @@ PIP package                                 Version required
 ``sqlalchemy-spanner``                      ``>=1.6.2``
 ``tenacity``                                ``>=8.1.0``
 ``immutabledict``                           ``>=4.2.0``
+``types-protobuf``                          ``!=5.29.1.20250402``
 ==========================================  ======================================
 
 Cross provider package dependencies
@@ -298,5 +300,5 @@ Dependent package
 ========================================================================================================================  ====================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-google/14.1.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-google/15.0.0/changelog.html>`_.