apache-airflow-providers-microsoft-azure 10.3.0__tar.gz → 10.4.0__tar.gz

{apache_airflow_providers_microsoft_azure-10.3.0 → apache_airflow_providers_microsoft_azure-10.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-microsoft-azure
-Version: 10.3.0
+Version: 10.4.0
 Summary: Provider package apache-airflow-providers-microsoft-azure for Apache Airflow
 Keywords: airflow-provider,microsoft.azure,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -23,7 +23,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
 Requires-Dist: adal>=1.2.7
 Requires-Dist: adlfs>=2023.10.0
-Requires-Dist: apache-airflow>=2.7.0
+Requires-Dist: apache-airflow>=2.8.0
 Requires-Dist: azure-batch>=8.0.0
 Requires-Dist: azure-cosmos>=4.6.0
 Requires-Dist: azure-datalake-store>=0.0.45
@@ -49,8 +49,8 @@ Requires-Dist: apache-airflow-providers-google ; extra == "google"
 Requires-Dist: apache-airflow-providers-oracle ; extra == "oracle"
 Requires-Dist: apache-airflow-providers-sftp ; extra == "sftp"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.3.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.3.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.4.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.4.0
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -104,7 +104,7 @@ Provides-Extra: sftp
 
 Package ``apache-airflow-providers-microsoft-azure``
 
-Release: ``10.3.0``
+Release: ``10.4.0``
 
 
 `Microsoft Azure <https://azure.microsoft.com/>`__
@@ -117,7 +117,7 @@ This is a provider package for ``microsoft.azure`` provider. All classes for thi
 are in ``airflow.providers.microsoft.azure`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.3.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.4.0/>`_.
 
 Installation
 ------------
@@ -134,7 +134,7 @@ Requirements
 ================================  ==================
 PIP package                       Version required
 ================================  ==================
-``apache-airflow``                ``>=2.7.0``
+``apache-airflow``                ``>=2.8.0``
 ``adlfs``                         ``>=2023.10.0``
 ``azure-batch``                   ``>=8.0.0``
 ``azure-cosmos``                  ``>=4.6.0``
@@ -182,4 +182,4 @@ Dependent package
 ====================================================================================================  ==========
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.3.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.4.0/changelog.html>`_.

{apache_airflow_providers_microsoft_azure-10.3.0 → apache_airflow_providers_microsoft_azure-10.4.0}/README.rst

@@ -42,7 +42,7 @@
 
 Package ``apache-airflow-providers-microsoft-azure``
 
-Release: ``10.3.0``
+Release: ``10.4.0``
 
 
 `Microsoft Azure <https://azure.microsoft.com/>`__
@@ -55,7 +55,7 @@ This is a provider package for ``microsoft.azure`` provider. All classes for thi
 are in ``airflow.providers.microsoft.azure`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.3.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.4.0/>`_.
 
 Installation
 ------------
@@ -72,7 +72,7 @@ Requirements
 ================================  ==================
 PIP package                       Version required
 ================================  ==================
-``apache-airflow``                ``>=2.7.0``
+``apache-airflow``                ``>=2.8.0``
 ``adlfs``                         ``>=2023.10.0``
 ``azure-batch``                   ``>=8.0.0``
 ``azure-cosmos``                  ``>=4.6.0``
@@ -120,4 +120,4 @@ Dependent package
 ====================================================================================================  ==========
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.3.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.4.0/changelog.html>`_.

{apache_airflow_providers_microsoft_azure-10.3.0 → apache_airflow_providers_microsoft_azure-10.4.0}/airflow/providers/microsoft/azure/__init__.py

@@ -29,11 +29,11 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "10.3.0"
+__version__ = "10.4.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
-    "2.7.0"
+    "2.8.0"
 ):
     raise RuntimeError(
-        f"The package `apache-airflow-providers-microsoft-azure:{__version__}` needs Apache Airflow 2.7.0+"
+        f"The package `apache-airflow-providers-microsoft-azure:{__version__}` needs Apache Airflow 2.8.0+"
     )

{apache_airflow_providers_microsoft_azure-10.3.0 → apache_airflow_providers_microsoft_azure-10.4.0}/airflow/providers/microsoft/azure/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "Microsoft Azure",
         "description": "`Microsoft Azure <https://azure.microsoft.com/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1722664136,
+        "source-date-epoch": 1723970344,
         "versions": [
+            "10.4.0",
             "10.3.0",
             "10.2.0",
             "10.1.2",
@@ -88,7 +89,7 @@ def get_provider_info():
             "1.0.0",
         ],
         "dependencies": [
-            "apache-airflow>=2.7.0",
+            "apache-airflow>=2.8.0",
             "adlfs>=2023.10.0",
             "azure-batch>=8.0.0",
             "azure-cosmos>=4.6.0",
@@ -207,6 +208,11 @@ def get_provider_info():
                 "how-to-guide": ["/docs/apache-airflow-providers-microsoft-azure/operators/msgraph.rst"],
                 "tags": ["azure"],
             },
+            {
+                "integration-name": "Microsoft Power BI",
+                "external-doc-url": "https://learn.microsoft.com/en-us/rest/api/power-bi/",
+                "tags": ["azure"],
+            },
         ],
         "operators": [
             {
@@ -249,6 +255,10 @@ def get_provider_info():
                 "integration-name": "Microsoft Graph API",
                 "python-modules": ["airflow.providers.microsoft.azure.operators.msgraph"],
             },
+            {
+                "integration-name": "Microsoft Power BI",
+                "python-modules": ["airflow.providers.microsoft.azure.operators.powerbi"],
+            },
         ],
         "sensors": [
             {
@@ -326,6 +336,10 @@ def get_provider_info():
                 "integration-name": "Microsoft Graph API",
                 "python-modules": ["airflow.providers.microsoft.azure.hooks.msgraph"],
             },
+            {
+                "integration-name": "Microsoft Power BI",
+                "python-modules": ["airflow.providers.microsoft.azure.hooks.powerbi"],
+            },
         ],
         "triggers": [
             {
@@ -340,6 +354,10 @@ def get_provider_info():
                 "integration-name": "Microsoft Graph API",
                 "python-modules": ["airflow.providers.microsoft.azure.triggers.msgraph"],
             },
+            {
+                "integration-name": "Microsoft Power BI",
+                "python-modules": ["airflow.providers.microsoft.azure.triggers.powerbi"],
+            },
         ],
         "transfers": [
             {
@@ -435,12 +453,17 @@ def get_provider_info():
                 "hook-class-name": "airflow.providers.microsoft.azure.hooks.data_lake.AzureDataLakeStorageV2Hook",
                 "connection-type": "adls",
             },
+            {
+                "hook-class-name": "airflow.providers.microsoft.azure.hooks.powerbi.PowerBIHook",
+                "connection-type": "powerbi",
+            },
         ],
         "secrets-backends": ["airflow.providers.microsoft.azure.secrets.key_vault.AzureKeyVaultBackend"],
         "logging": ["airflow.providers.microsoft.azure.log.wasb_task_handler.WasbTaskHandler"],
         "extra-links": [
             "airflow.providers.microsoft.azure.operators.data_factory.AzureDataFactoryPipelineRunLink",
             "airflow.providers.microsoft.azure.operators.synapse.AzureSynapsePipelineRunLink",
+            "airflow.providers.microsoft.azure.operators.powerbi.PowerBILink",
         ],
         "config": {
             "azure_remote_logging": {

{apache_airflow_providers_microsoft_azure-10.3.0 → apache_airflow_providers_microsoft_azure-10.4.0}/airflow/providers/microsoft/azure/hooks/msgraph.py

@@ -110,12 +110,16 @@ class KiotaRequestAdapterHook(BaseHook):
         conn_id: str = default_conn_name,
         timeout: float | None = None,
         proxies: dict | None = None,
+        host: str = NationalClouds.Global.value,
+        scopes: list[str] | None = None,
         api_version: APIVersion | str | None = None,
     ):
         super().__init__()
         self.conn_id = conn_id
         self.timeout = timeout
         self.proxies = proxies
+        self.host = host
+        self.scopes = scopes or ["https://graph.microsoft.com/.default"]
         self._api_version = self.resolve_api_version_from_value(api_version)
 
     @property
@@ -141,11 +145,10 @@ class KiotaRequestAdapterHook(BaseHook):
             )
         return self._api_version
 
-    @staticmethod
-    def get_host(connection: Connection) -> str:
+    def get_host(self, connection: Connection) -> str:
         if connection.schema and connection.host:
             return f"{connection.schema}://{connection.host}"
-        return NationalClouds.Global.value
+        return self.host
 
     @staticmethod
     def format_no_proxy_url(url: str) -> str:
@@ -198,7 +201,7 @@ class KiotaRequestAdapterHook(BaseHook):
             proxies = self.proxies or config.get("proxies", {})
             msal_proxies = self.to_msal_proxies(authority=authority, proxies=proxies)
             httpx_proxies = self.to_httpx_proxies(proxies=proxies)
-            scopes = config.get("scopes", ["https://graph.microsoft.com/.default"])
+            scopes = config.get("scopes", self.scopes)
             verify = config.get("verify", True)
             trust_env = config.get("trust_env", False)
             disable_instance_discovery = config.get("disable_instance_discovery", False)

apache_airflow_providers_microsoft_azure-10.4.0/airflow/providers/microsoft/azure/hooks/powerbi.py

@@ -0,0 +1,218 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import TYPE_CHECKING, Any
+
+from airflow.exceptions import AirflowException
+from airflow.providers.microsoft.azure.hooks.msgraph import KiotaRequestAdapterHook
+
+if TYPE_CHECKING:
+    from msgraph_core import APIVersion
+
+
+class PowerBIDatasetRefreshFields(Enum):
+    """Power BI refresh dataset details."""
+
+    REQUEST_ID = "request_id"
+    STATUS = "status"
+    ERROR = "error"
+
+
+class PowerBIDatasetRefreshStatus:
+    """Power BI refresh dataset statuses."""
+
+    IN_PROGRESS = "In Progress"
+    FAILED = "Failed"
+    COMPLETED = "Completed"
+    DISABLED = "Disabled"
+
+    TERMINAL_STATUSES = {FAILED, COMPLETED}
+
+
+class PowerBIDatasetRefreshException(AirflowException):
+    """An exception that indicates a dataset refresh failed to complete."""
+
+
+class PowerBIHook(KiotaRequestAdapterHook):
+    """
+    A async hook to interact with Power BI.
+
+    :param conn_id: The Power BI connection id.
+    """
+
+    conn_type: str = "powerbi"
+    conn_name_attr: str = "conn_id"
+    default_conn_name: str = "powerbi_default"
+    hook_name: str = "Power BI"
+
+    def __init__(
+        self,
+        conn_id: str = default_conn_name,
+        proxies: dict | None = None,
+        timeout: float = 60 * 60 * 24 * 7,
+        api_version: APIVersion | str | None = None,
+    ):
+        super().__init__(
+            conn_id=conn_id,
+            proxies=proxies,
+            timeout=timeout,
+            host="https://api.powerbi.com",
+            scopes=["https://analysis.windows.net/powerbi/api/.default"],
+            api_version=api_version,
+        )
+
+    @classmethod
+    def get_connection_form_widgets(cls) -> dict[str, Any]:
+        """Return connection widgets to add to connection form."""
+        from flask_appbuilder.fieldwidgets import BS3TextFieldWidget
+        from flask_babel import lazy_gettext
+        from wtforms import StringField
+
+        return {
+            "tenant_id": StringField(lazy_gettext("Tenant ID"), widget=BS3TextFieldWidget()),
+        }
+
+    @classmethod
+    def get_ui_field_behaviour(cls) -> dict[str, Any]:
+        """Return custom field behaviour."""
+        return {
+            "hidden_fields": ["schema", "port", "host", "extra"],
+            "relabeling": {
+                "login": "Client ID",
+                "password": "Client Secret",
+            },
+        }
+
+    async def get_refresh_history(
+        self,
+        dataset_id: str,
+        group_id: str,
+    ) -> list[dict[str, str]]:
+        """
+        Retrieve the refresh history of the specified dataset from the given group ID.
+
+        :param dataset_id: The dataset ID.
+        :param group_id: The workspace ID.
+
+        :return: Dictionary containing all the refresh histories of the dataset.
+        """
+        try:
+            response = await self.run(
+                url="myorg/groups/{group_id}/datasets/{dataset_id}/refreshes",
+                path_parameters={
+                    "group_id": group_id,
+                    "dataset_id": dataset_id,
+                },
+            )
+
+            refresh_histories = response.get("value")
+            return [self.raw_to_refresh_details(refresh_history) for refresh_history in refresh_histories]
+
+        except AirflowException:
+            raise PowerBIDatasetRefreshException("Failed to retrieve refresh history")
+
+    @classmethod
+    def raw_to_refresh_details(cls, refresh_details: dict) -> dict[str, str]:
+        """
+        Convert raw refresh details into a dictionary containing required fields.
+
+        :param refresh_details: Raw object of refresh details.
+        """
+        return {
+            PowerBIDatasetRefreshFields.REQUEST_ID.value: str(refresh_details.get("requestId")),
+            PowerBIDatasetRefreshFields.STATUS.value: (
+                "In Progress"
+                if str(refresh_details.get("status")) == "Unknown"
+                else str(refresh_details.get("status"))
+            ),
+            PowerBIDatasetRefreshFields.ERROR.value: str(refresh_details.get("serviceExceptionJson")),
+        }
+
+    async def get_refresh_details_by_refresh_id(
+        self, dataset_id: str, group_id: str, refresh_id: str
+    ) -> dict[str, str]:
+        """
+        Get the refresh details of the given request Id.
+
+        :param refresh_id: Request Id of the Dataset refresh.
+        """
+        refresh_histories = await self.get_refresh_history(dataset_id=dataset_id, group_id=group_id)
+
+        if len(refresh_histories) == 0:
+            raise PowerBIDatasetRefreshException(
+                f"Unable to fetch the details of dataset refresh with Request Id: {refresh_id}"
+            )
+
+        refresh_ids = [
+            refresh_history.get(PowerBIDatasetRefreshFields.REQUEST_ID.value)
+            for refresh_history in refresh_histories
+        ]
+
+        if refresh_id not in refresh_ids:
+            raise PowerBIDatasetRefreshException(
+                f"Unable to fetch the details of dataset refresh with Request Id: {refresh_id}"
+            )
+
+        refresh_details = refresh_histories[refresh_ids.index(refresh_id)]
+
+        return refresh_details
+
+    async def trigger_dataset_refresh(self, *, dataset_id: str, group_id: str) -> str:
+        """
+        Triggers a refresh for the specified dataset from the given group id.
+
+        :param dataset_id: The dataset id.
+        :param group_id: The workspace id.
+
+        :return: Request id of the dataset refresh request.
+        """
+        try:
+            response = await self.run(
+                url="myorg/groups/{group_id}/datasets/{dataset_id}/refreshes",
+                method="POST",
+                path_parameters={
+                    "group_id": group_id,
+                    "dataset_id": dataset_id,
+                },
+            )
+
+            request_id = response.get("requestid")
+            return request_id
+        except AirflowException:
+            raise PowerBIDatasetRefreshException("Failed to trigger dataset refresh.")
+
+    async def cancel_dataset_refresh(self, dataset_id: str, group_id: str, dataset_refresh_id: str) -> None:
+        """
+        Cancel the dataset refresh.
+
+        :param dataset_id: The dataset Id.
+        :param group_id: The workspace Id.
+        :param dataset_refresh_id: The dataset refresh Id.
+        """
+        await self.run(
+            url="myorg/groups/{group_id}/datasets/{dataset_id}/refreshes/{dataset_refresh_id}",
+            response_type=None,
+            path_parameters={
+                "group_id": group_id,
+                "dataset_id": dataset_id,
+                "dataset_refresh_id": dataset_refresh_id,
+            },
+            method="DELETE",
+        )

{apache_airflow_providers_microsoft_azure-10.3.0 → apache_airflow_providers_microsoft_azure-10.4.0}/airflow/providers/microsoft/azure/hooks/wasb.py

@@ -213,7 +213,8 @@ class WasbHook(BaseHook):
             **extra,
         )
 
-    def _get_container_client(self, container_name: str) -> ContainerClient:
+    # TODO: rework the interface as it might also return AsyncContainerClient
+    def _get_container_client(self, container_name: str) -> ContainerClient:  # type: ignore[override]
         """
         Instantiate a container client.
 
@@ -222,7 +223,7 @@ class WasbHook(BaseHook):
         """
         return self.blob_service_client.get_container_client(container_name)
 
-    def _get_blob_client(self, container_name: str, blob_name: str) -> BlobClient:
+    def _get_blob_client(self, container_name: str, blob_name: str) -> BlobClient | AsyncBlobClient:
         """
         Instantiate a blob client.
 
@@ -415,7 +416,8 @@ class WasbHook(BaseHook):
             self.create_container(container_name)
 
         blob_client = self._get_blob_client(container_name, blob_name)
-        return blob_client.upload_blob(data, blob_type, length=length, **kwargs)
+        # TODO: rework the interface as it might also return Awaitable
+        return blob_client.upload_blob(data, blob_type, length=length, **kwargs)  # type: ignore[return-value]
 
     def download(
         self, container_name, blob_name, offset: int | None = None, length: int | None = None, **kwargs
@@ -430,7 +432,8 @@ class WasbHook(BaseHook):
         :param length: Number of bytes to read from the stream.
         """
         blob_client = self._get_blob_client(container_name, blob_name)
-        return blob_client.download_blob(offset=offset, length=length, **kwargs)
+        # TODO: rework the interface as it might also return Awaitable
+        return blob_client.download_blob(offset=offset, length=length, **kwargs)  # type: ignore[return-value]
 
     def create_container(self, container_name: str) -> None:
         """
@@ -656,7 +659,8 @@ class WasbAsyncHook(WasbHook):
             return False
         return True
 
-    def _get_container_client(self, container_name: str) -> AsyncContainerClient:
+    # TODO: rework the interface as in parent Hook it returns ContainerClient
+    def _get_container_client(self, container_name: str) -> AsyncContainerClient:  # type: ignore[override]
         """
         Instantiate a container client.
 

{apache_airflow_providers_microsoft_azure-10.3.0 → apache_airflow_providers_microsoft_azure-10.4.0}/airflow/providers/microsoft/azure/operators/container_instances.py

@@ -86,6 +86,12 @@ class AzureContainerInstancesOperator(BaseOperator):
     :param container_timeout: max time allowed for the execution of
         the container instance.
     :param tags: azure tags as dict of str:str
+    :param xcom_all: Control if logs are pushed to XCOM similarly to how DockerOperator does.
+        Possible values include: 'None', 'True', 'False'. Defaults to 'None', meaning no logs
+        are pushed to XCOM which is the historical behaviour. 'True' means push all logs to XCOM
+        which may run the risk of hitting XCOM size limits. 'False' means push only the last line
+        of the logs to XCOM. However, the logs are pushed into XCOM under "logs", not return_value
+        to avoid breaking the existing behaviour.
     :param os_type: The operating system type required by the containers
         in the container group. Possible values include: 'Windows', 'Linux'
     :param restart_policy: Restart policy for all containers within the container group.
@@ -158,6 +164,7 @@ class AzureContainerInstancesOperator(BaseOperator):
         remove_on_error: bool = True,
         fail_if_exists: bool = True,
         tags: dict[str, str] | None = None,
+        xcom_all: bool | None = None,
         os_type: str = "Linux",
         restart_policy: str = "Never",
         ip_address: IpAddress | None = None,
@@ -187,6 +194,7 @@ class AzureContainerInstancesOperator(BaseOperator):
         self.fail_if_exists = fail_if_exists
         self._ci_hook: Any = None
         self.tags = tags
+        self.xcom_all = xcom_all
         self.os_type = os_type
         if self.os_type not in ["Linux", "Windows"]:
             raise AirflowException(
@@ -296,6 +304,16 @@ class AzureContainerInstancesOperator(BaseOperator):
             self.log.info("Container group started %s/%s", self.resource_group, self.name)
 
             exit_code = self._monitor_logging(self.resource_group, self.name)
+            if self.xcom_all is not None:
+                logs = self._ci_hook.get_logs(self.resource_group, self.name)
+                if logs is None:
+                    context["ti"].xcom_push(key="logs", value=[])
+                else:
+                    if self.xcom_all:
+                        context["ti"].xcom_push(key="logs", value=logs)
+                    else:
+                        # slice off the last entry in the list logs and return it as a list
+                        context["ti"].xcom_push(key="logs", value=logs[-1:])
 
             self.log.info("Container had exit code: %s", exit_code)
             if exit_code != 0:

apache_airflow_providers_microsoft_azure-10.4.0/airflow/providers/microsoft/azure/operators/powerbi.py

@@ -0,0 +1,120 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Sequence
+
+from airflow.exceptions import AirflowException
+from airflow.models import BaseOperator, BaseOperatorLink
+from airflow.providers.microsoft.azure.hooks.powerbi import (
+    PowerBIHook,
+)
+from airflow.providers.microsoft.azure.triggers.powerbi import PowerBITrigger
+
+if TYPE_CHECKING:
+    from msgraph_core import APIVersion
+
+    from airflow.models.taskinstancekey import TaskInstanceKey
+    from airflow.utils.context import Context
+
+
+class PowerBILink(BaseOperatorLink):
+    """Construct a link to monitor a dataset in Power BI."""
+
+    name = "Monitor PowerBI Dataset"
+
+    def get_link(self, operator: BaseOperator, *, ti_key: TaskInstanceKey):
+        url = (
+            "https://app.powerbi.com"  # type: ignore[attr-defined]
+            f"/groups/{operator.group_id}/datasets/{operator.dataset_id}"  # type: ignore[attr-defined]
+            "/details?experience=power-bi"
+        )
+
+        return url
+
+
+class PowerBIDatasetRefreshOperator(BaseOperator):
+    """
+    Refreshes a Power BI dataset.
+
+    :param dataset_id: The dataset id.
+    :param group_id: The workspace id.
+    :param conn_id: Airflow Connection ID that contains the connection information for the Power BI account used for authentication.
+    :param timeout: Time in seconds to wait for a dataset to reach a terminal status for asynchronous waits. Used only if ``wait_for_termination`` is True.
+    :param check_interval: Number of seconds to wait before rechecking the
+        refresh status.
+    """
+
+    template_fields: Sequence[str] = (
+        "dataset_id",
+        "group_id",
+    )
+    template_fields_renderers = {"parameters": "json"}
+
+    operator_extra_links = (PowerBILink(),)
+
+    def __init__(
+        self,
+        *,
+        dataset_id: str,
+        group_id: str,
+        conn_id: str = PowerBIHook.default_conn_name,
+        timeout: float = 60 * 60 * 24 * 7,
+        proxies: dict | None = None,
+        api_version: APIVersion | None = None,
+        check_interval: int = 60,
+        **kwargs,
+    ) -> None:
+        super().__init__(**kwargs)
+        self.hook = PowerBIHook(conn_id=conn_id, proxies=proxies, api_version=api_version, timeout=timeout)
+        self.dataset_id = dataset_id
+        self.group_id = group_id
+        self.wait_for_termination = True
+        self.conn_id = conn_id
+        self.timeout = timeout
+        self.check_interval = check_interval
+
+    def execute(self, context: Context):
+        """Refresh the Power BI Dataset."""
+        if self.wait_for_termination:
+            self.defer(
+                trigger=PowerBITrigger(
+                    conn_id=self.conn_id,
+                    group_id=self.group_id,
+                    dataset_id=self.dataset_id,
+                    timeout=self.timeout,
+                    check_interval=self.check_interval,
+                    wait_for_termination=self.wait_for_termination,
+                ),
+                method_name=self.execute_complete.__name__,
+            )
+
+    def execute_complete(self, context: Context, event: dict[str, str]) -> Any:
+        """
+        Return immediately - callback for when the trigger fires.
+
+        Relies on trigger to throw an exception, otherwise it assumes execution was successful.
+        """
+        if event:
+            if event["status"] == "error":
+                raise AirflowException(event["message"])
+
+            self.xcom_push(
+                context=context, key="powerbi_dataset_refresh_Id", value=event["dataset_refresh_id"]
+            )
+            self.xcom_push(context=context, key="powerbi_dataset_refresh_status", value=event["status"])

apache_airflow_providers_microsoft_azure-10.4.0/airflow/providers/microsoft/azure/triggers/powerbi.py

@@ -0,0 +1,181 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+import asyncio
+import time
+from typing import TYPE_CHECKING, AsyncIterator
+
+from airflow.providers.microsoft.azure.hooks.powerbi import (
+    PowerBIDatasetRefreshStatus,
+    PowerBIHook,
+)
+from airflow.triggers.base import BaseTrigger, TriggerEvent
+
+if TYPE_CHECKING:
+    from msgraph_core import APIVersion
+
+
+class PowerBITrigger(BaseTrigger):
+    """
+    Triggers when Power BI dataset refresh is completed.
+
+    Wait for termination will always be True.
+
+    :param conn_id: The connection Id to connect to PowerBI.
+    :param timeout: The HTTP timeout being used by the `KiotaRequestAdapter` (default is None).
+        When no timeout is specified or set to None then there is no HTTP timeout on each request.
+    :param proxies: A dict defining the HTTP proxies to be used (default is None).
+    :param api_version: The API version of the Microsoft Graph API to be used (default is v1).
+        You can pass an enum named APIVersion which has 2 possible members v1 and beta,
+        or you can pass a string as `v1.0` or `beta`.
+    :param dataset_id: The dataset Id to refresh.
+    :param group_id: The workspace Id where dataset is located.
+    :param end_time: Time in seconds when trigger should stop polling.
+    :param check_interval: Time in seconds to wait between each poll.
+    :param wait_for_termination: Wait for the dataset refresh to complete or fail.
+    """
+
+    def __init__(
+        self,
+        conn_id: str,
+        dataset_id: str,
+        group_id: str,
+        timeout: float = 60 * 60 * 24 * 7,
+        proxies: dict | None = None,
+        api_version: APIVersion | None = None,
+        check_interval: int = 60,
+        wait_for_termination: bool = True,
+    ):
+        super().__init__()
+        self.hook = PowerBIHook(conn_id=conn_id, proxies=proxies, api_version=api_version, timeout=timeout)
+        self.dataset_id = dataset_id
+        self.timeout = timeout
+        self.group_id = group_id
+        self.check_interval = check_interval
+        self.wait_for_termination = wait_for_termination
+
+    def serialize(self):
+        """Serialize the trigger instance."""
+        api_version = self.api_version.value if self.api_version else None
+        return (
+            "airflow.providers.microsoft.azure.triggers.powerbi.PowerBITrigger",
+            {
+                "conn_id": self.conn_id,
+                "proxies": self.proxies,
+                "api_version": api_version,
+                "dataset_id": self.dataset_id,
+                "group_id": self.group_id,
+                "timeout": self.timeout,
+                "check_interval": self.check_interval,
+                "wait_for_termination": self.wait_for_termination,
+            },
+        )
+
+    @property
+    def conn_id(self) -> str:
+        return self.hook.conn_id
+
+    @property
+    def proxies(self) -> dict | None:
+        return self.hook.proxies
+
+    @property
+    def api_version(self) -> APIVersion:
+        return self.hook.api_version
+
+    async def run(self) -> AsyncIterator[TriggerEvent]:
+        """Make async connection to the PowerBI and polls for the dataset refresh status."""
+        self.dataset_refresh_id = await self.hook.trigger_dataset_refresh(
+            dataset_id=self.dataset_id,
+            group_id=self.group_id,
+        )
+        try:
+            dataset_refresh_status = None
+            start_time = time.monotonic()
+            while start_time + self.timeout > time.monotonic():
+                refresh_details = await self.hook.get_refresh_details_by_refresh_id(
+                    dataset_id=self.dataset_id,
+                    group_id=self.group_id,
+                    refresh_id=self.dataset_refresh_id,
+                )
+                dataset_refresh_status = refresh_details.get("status")
+
+                if dataset_refresh_status == PowerBIDatasetRefreshStatus.COMPLETED:
+                    yield TriggerEvent(
+                        {
+                            "status": dataset_refresh_status,
+                            "message": f"The dataset refresh {self.dataset_refresh_id} has {dataset_refresh_status}.",
+                            "dataset_refresh_id": self.dataset_refresh_id,
+                        }
+                    )
+                    return
+                elif dataset_refresh_status == PowerBIDatasetRefreshStatus.FAILED:
+                    yield TriggerEvent(
+                        {
+                            "status": dataset_refresh_status,
+                            "message": f"The dataset refresh {self.dataset_refresh_id} has {dataset_refresh_status}.",
+                            "dataset_refresh_id": self.dataset_refresh_id,
+                        }
+                    )
+                    return
+
+                self.log.info(
+                    "Sleeping for %s. The dataset refresh status is %s.",
+                    self.check_interval,
+                    dataset_refresh_status,
+                )
+                await asyncio.sleep(self.check_interval)
+
+            yield TriggerEvent(
+                {
+                    "status": "error",
+                    "message": f"Timeout occurred while waiting for dataset refresh to complete: The dataset refresh {self.dataset_refresh_id} has status {dataset_refresh_status}.",
+                    "dataset_refresh_id": self.dataset_refresh_id,
+                }
+            )
+            return
+        except Exception as error:
+            if self.dataset_refresh_id:
+                try:
+                    self.log.info(
+                        "Unexpected error %s caught. Canceling dataset refresh %s",
+                        error,
+                        self.dataset_refresh_id,
+                    )
+                    await self.hook.cancel_dataset_refresh(
+                        dataset_id=self.dataset_id,
+                        group_id=self.group_id,
+                        dataset_refresh_id=self.dataset_refresh_id,
+                    )
+                except Exception as e:
+                    yield TriggerEvent(
+                        {
+                            "status": "error",
+                            "message": f"An error occurred while canceling dataset: {e}",
+                            "dataset_refresh_id": self.dataset_refresh_id,
+                        }
+                    )
+                    return
+            yield TriggerEvent(
+                {
+                    "status": "error",
+                    "message": f"An error occurred: {error}",
+                    "dataset_refresh_id": self.dataset_refresh_id,
+                }
+            )

{apache_airflow_providers_microsoft_azure-10.3.0 → apache_airflow_providers_microsoft_azure-10.4.0}/pyproject.toml

@@ -28,7 +28,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "apache-airflow-providers-microsoft-azure"
-version = "10.3.0"
+version = "10.4.0"
 description = "Provider package apache-airflow-providers-microsoft-azure for Apache Airflow"
 readme = "README.rst"
 authors = [
@@ -58,7 +58,7 @@ requires-python = "~=3.8"
 dependencies = [
     "adal>=1.2.7",
     "adlfs>=2023.10.0",
-    "apache-airflow>=2.7.0",
+    "apache-airflow>=2.8.0",
     "azure-batch>=8.0.0",
     "azure-cosmos>=4.6.0",
     "azure-datalake-store>=0.0.45",
@@ -82,8 +82,8 @@ dependencies = [
 ]
 
 [project.urls]
-"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.3.0"
-"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.3.0/changelog.html"
+"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.4.0"
+"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.4.0/changelog.html"
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
 "Source Code" = "https://github.com/apache/airflow"
 "Slack Chat" = "https://s.apache.org/airflow-slack"