apache-airflow-providers-http 4.9.0rc1__tar.gz → 4.9.1rc1__tar.gz

{apache_airflow_providers_http-4.9.0rc1 → apache_airflow_providers_http-4.9.1rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-http
-Version: 4.9.0rc1
+Version: 4.9.1rc1
 Summary: Provider package apache-airflow-providers-http for Apache Airflow
 Keywords: airflow-provider,http,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -20,14 +20,14 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Topic :: System :: Monitoring
-Requires-Dist: aiohttp
+Requires-Dist: aiohttp>=3.9.2
 Requires-Dist: apache-airflow>=2.6.0.dev0
 Requires-Dist: asgiref
-Requires-Dist: requests>=2.26.0
+Requires-Dist: requests>=2.27.0,<3
 Requires-Dist: requests_toolbelt
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-http/4.9.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-http/4.9.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-http/4.9.1/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-http/4.9.1
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -77,7 +77,7 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 
 Package ``apache-airflow-providers-http``
 
-Release: ``4.9.0.rc1``
+Release: ``4.9.1.rc1``
 
 
 `Hypertext Transfer Protocol (HTTP) <https://www.w3.org/Protocols/>`__
@@ -90,7 +90,7 @@ This is a provider package for ``http`` provider. All classes for this provider
 are in ``airflow.providers.http`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-http/4.9.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-http/4.9.1/>`_.
 
 Installation
 ------------
@@ -108,11 +108,11 @@ Requirements
 PIP package            Version required
 =====================  ==================
 ``apache-airflow``     ``>=2.6.0``
-``requests``           ``>=2.26.0``
+``requests``           ``>=2.27.0,<3``
 ``requests_toolbelt``
-``aiohttp``
+``aiohttp``            ``>=3.9.2``
 ``asgiref``
 =====================  ==================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-http/4.9.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-http/4.9.1/changelog.html>`_.

{apache_airflow_providers_http-4.9.0rc1 → apache_airflow_providers_http-4.9.1rc1}/README.rst

@@ -42,7 +42,7 @@
 
 Package ``apache-airflow-providers-http``
 
-Release: ``4.9.0.rc1``
+Release: ``4.9.1.rc1``
 
 
 `Hypertext Transfer Protocol (HTTP) <https://www.w3.org/Protocols/>`__
@@ -55,7 +55,7 @@ This is a provider package for ``http`` provider. All classes for this provider
 are in ``airflow.providers.http`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-http/4.9.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-http/4.9.1/>`_.
 
 Installation
 ------------
@@ -73,11 +73,11 @@ Requirements
 PIP package            Version required
 =====================  ==================
 ``apache-airflow``     ``>=2.6.0``
-``requests``           ``>=2.26.0``
+``requests``           ``>=2.27.0,<3``
 ``requests_toolbelt``
-``aiohttp``
+``aiohttp``            ``>=3.9.2``
 ``asgiref``
 =====================  ==================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-http/4.9.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-http/4.9.1/changelog.html>`_.

{apache_airflow_providers_http-4.9.0rc1 → apache_airflow_providers_http-4.9.1rc1}/airflow/providers/http/__init__.py

@@ -27,7 +27,7 @@ import packaging.version
 
 __all__ = ["__version__"]
 
-__version__ = "4.9.0"
+__version__ = "4.9.1"
 
 try:
     from airflow import __version__ as airflow_version

{apache_airflow_providers_http-4.9.0rc1 → apache_airflow_providers_http-4.9.1rc1}/airflow/providers/http/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "Hypertext Transfer Protocol (HTTP)",
         "description": "`Hypertext Transfer Protocol (HTTP) <https://www.w3.org/Protocols/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1705912133,
+        "source-date-epoch": 1707636407,
         "versions": [
+            "4.9.1",
             "4.9.0",
             "4.8.0",
             "4.7.0",
@@ -59,9 +60,9 @@ def get_provider_info():
         ],
         "dependencies": [
             "apache-airflow>=2.6.0",
-            "requests>=2.26.0",
+            "requests>=2.27.0,<3",
             "requests_toolbelt",
-            "aiohttp",
+            "aiohttp>=3.9.2",
             "asgiref",
         ],
         "integrations": [

{apache_airflow_providers_http-4.9.0rc1 → apache_airflow_providers_http-4.9.1rc1}/airflow/providers/http/hooks/http.py

@@ -68,9 +68,8 @@ class HttpHook(BaseHook):
         tcp_keep_alive_idle: int = 120,
         tcp_keep_alive_count: int = 20,
         tcp_keep_alive_interval: int = 30,
-        **kwargs,
     ) -> None:
-        super().__init__(**kwargs)
+        super().__init__()
         self.http_conn_id = http_conn_id
         self.method = method.upper()
         self.base_url: str = ""
@@ -259,7 +258,8 @@ class HttpHook(BaseHook):
         """
         self._retry_obj = tenacity.Retrying(**_retry_args)
 
-        return self._retry_obj(self.run, *args, **kwargs)
+        # TODO: remove ignore type when https://github.com/jd/tenacity/issues/428 is resolved
+        return self._retry_obj(self.run, *args, **kwargs)  # type: ignore
 
     def url_from_endpoint(self, endpoint: str | None) -> str:
         """Combine base url with endpoint."""
@@ -298,9 +298,7 @@ class HttpAsyncHook(BaseHook):
         auth_type: Any = aiohttp.BasicAuth,
         retry_limit: int = 3,
         retry_delay: float = 1.0,
-        **kwargs,
     ) -> None:
-        super().__init__(**kwargs)
         self.http_conn_id = http_conn_id
         self.method = method.upper()
         self.base_url: str = ""

{apache_airflow_providers_http-4.9.0rc1 → apache_airflow_providers_http-4.9.1rc1}/airflow/providers/http/operators/http.py

@@ -19,9 +19,9 @@ from __future__ import annotations
 
 import base64
 import pickle
-import warnings
 from typing import TYPE_CHECKING, Any, Callable, Sequence
 
+from deprecated import deprecated
 from requests import Response
 
 from airflow.configuration import conf
@@ -233,7 +233,7 @@ class HttpOperator(BaseOperator):
         self, context: Context, event: dict, paginated_responses: None | list[Response] = None
     ):
         """
-        Callback for when the trigger fires - returns immediately.
+        Execute callback when the trigger fires; returns immediately.
 
         Relies on trigger to throw an exception, otherwise it assumes execution was successful.
         """
@@ -291,6 +291,13 @@ class HttpOperator(BaseOperator):
         )
 
 
+@deprecated(
+    reason=(
+        "Class `SimpleHttpOperator` is deprecated and "
+        "will be removed in a future release. Please use `HttpOperator` instead."
+    ),
+    category=AirflowProviderDeprecationWarning,
+)
 class SimpleHttpOperator(HttpOperator):
     """
     Calls an endpoint on an HTTP system to execute an action.
@@ -345,10 +352,4 @@ class SimpleHttpOperator(HttpOperator):
     """
 
     def __init__(self, **kwargs: Any):
-        warnings.warn(
-            "Class `SimpleHttpOperator` is deprecated and "
-            "will be removed in a future release. Please use `HttpOperator` instead.",
-            AirflowProviderDeprecationWarning,
-            stacklevel=2,
-        )
         super().__init__(**kwargs)

{apache_airflow_providers_http-4.9.0rc1 → apache_airflow_providers_http-4.9.1rc1}/airflow/providers/http/sensors/http.py

@@ -17,10 +17,13 @@
 # under the License.
 from __future__ import annotations
 
+from datetime import timedelta
 from typing import TYPE_CHECKING, Any, Callable, Sequence
 
+from airflow.configuration import conf
 from airflow.exceptions import AirflowException, AirflowSkipException
 from airflow.providers.http.hooks.http import HttpHook
+from airflow.providers.http.triggers.http import HttpSensorTrigger
 from airflow.sensors.base import BaseSensorOperator
 
 if TYPE_CHECKING:
@@ -78,6 +81,8 @@ class HttpSensor(BaseSensorOperator):
     :param tcp_keep_alive_count: The TCP Keep Alive count parameter (corresponds to ``socket.TCP_KEEPCNT``)
     :param tcp_keep_alive_interval: The TCP Keep Alive interval parameter (corresponds to
         ``socket.TCP_KEEPINTVL``)
+    :param deferrable: If waiting for completion, whether to defer the task until done,
+        default is ``False``
     """
 
     template_fields: Sequence[str] = ("endpoint", "request_params", "headers")
@@ -97,6 +102,7 @@ class HttpSensor(BaseSensorOperator):
         tcp_keep_alive_idle: int = 120,
         tcp_keep_alive_count: int = 20,
         tcp_keep_alive_interval: int = 30,
+        deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
         **kwargs: Any,
     ) -> None:
         super().__init__(**kwargs)
@@ -114,6 +120,7 @@ class HttpSensor(BaseSensorOperator):
         self.tcp_keep_alive_idle = tcp_keep_alive_idle
         self.tcp_keep_alive_count = tcp_keep_alive_count
         self.tcp_keep_alive_interval = tcp_keep_alive_interval
+        self.deferrable = deferrable
 
     def poke(self, context: Context) -> bool:
         from airflow.utils.operator_helpers import determine_kwargs
@@ -135,9 +142,12 @@ class HttpSensor(BaseSensorOperator):
                 headers=self.headers,
                 extra_options=self.extra_options,
             )
+
             if self.response_check:
                 kwargs = determine_kwargs(self.response_check, [response], context)
+
                 return self.response_check(response, **kwargs)
+
         except AirflowException as exc:
             if str(exc).startswith(self.response_error_codes_allowlist):
                 return False
@@ -148,3 +158,24 @@ class HttpSensor(BaseSensorOperator):
             raise exc
 
         return True
+
+    def execute(self, context: Context) -> None:
+        if not self.deferrable or self.response_check:
+            super().execute(context=context)
+        elif not self.poke(context):
+            self.defer(
+                timeout=timedelta(seconds=self.timeout),
+                trigger=HttpSensorTrigger(
+                    endpoint=self.endpoint,
+                    http_conn_id=self.http_conn_id,
+                    data=self.request_params,
+                    headers=self.headers,
+                    method=self.method,
+                    extra_options=self.extra_options,
+                    poke_interval=self.poke_interval,
+                ),
+                method_name="execute_complete",
+            )
+
+    def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
+        self.log.info("%s completed successfully.", self.task_id)

{apache_airflow_providers_http-4.9.0rc1 → apache_airflow_providers_http-4.9.1rc1}/airflow/providers/http/triggers/http.py

@@ -16,6 +16,7 @@
 # under the License.
 from __future__ import annotations
 
+import asyncio
 import base64
 import pickle
 from typing import TYPE_CHECKING, Any, AsyncIterator
@@ -24,6 +25,7 @@ import requests
 from requests.cookies import RequestsCookieJar
 from requests.structures import CaseInsensitiveDict
 
+from airflow.exceptions import AirflowException
 from airflow.providers.http.hooks.http import HttpAsyncHook
 from airflow.triggers.base import BaseTrigger, TriggerEvent
 
@@ -69,7 +71,7 @@ class HttpTrigger(BaseTrigger):
         self.extra_options = extra_options
 
     def serialize(self) -> tuple[str, dict[str, Any]]:
-        """Serializes HttpTrigger arguments and classpath."""
+        """Serialize HttpTrigger arguments and classpath."""
         return (
             "airflow.providers.http.triggers.http.HttpTrigger",
             {
@@ -84,7 +86,7 @@ class HttpTrigger(BaseTrigger):
         )
 
     async def run(self) -> AsyncIterator[TriggerEvent]:
-        """Makes a series of asynchronous http calls via an http hook."""
+        """Make a series of asynchronous http calls via a http hook."""
         hook = HttpAsyncHook(
             method=self.method,
             http_conn_id=self.http_conn_id,
@@ -124,3 +126,73 @@ class HttpTrigger(BaseTrigger):
             cookies.set(k, v)
         response.cookies = cookies
         return response
+
+
+class HttpSensorTrigger(BaseTrigger):
+    """
+    A trigger that fires when the request to a URL returns a non-404 status code.
+
+    :param endpoint: The relative part of the full url
+    :param http_conn_id: The HTTP Connection ID to run the sensor against
+    :param method: The HTTP request method to use
+    :param data: payload to be uploaded or aiohttp parameters
+    :param headers: The HTTP headers to be added to the GET request
+    :param extra_options: Additional kwargs to pass when creating a request.
+        For example, ``run(json=obj)`` is passed as ``aiohttp.ClientSession().get(json=obj)``
+    :param poke_interval: Time to sleep using asyncio
+    """
+
+    def __init__(
+        self,
+        endpoint: str | None = None,
+        http_conn_id: str = "http_default",
+        method: str = "GET",
+        data: dict[str, Any] | str | None = None,
+        headers: dict[str, str] | None = None,
+        extra_options: dict[str, Any] | None = None,
+        poke_interval: float = 5.0,
+    ):
+        super().__init__()
+        self.endpoint = endpoint
+        self.method = method
+        self.data = data
+        self.headers = headers
+        self.extra_options = extra_options or {}
+        self.http_conn_id = http_conn_id
+        self.poke_interval = poke_interval
+
+    def serialize(self) -> tuple[str, dict[str, Any]]:
+        """Serialize HttpTrigger arguments and classpath."""
+        return (
+            "airflow.providers.http.triggers.http.HttpSensorTrigger",
+            {
+                "endpoint": self.endpoint,
+                "data": self.data,
+                "headers": self.headers,
+                "extra_options": self.extra_options,
+                "http_conn_id": self.http_conn_id,
+                "poke_interval": self.poke_interval,
+            },
+        )
+
+    async def run(self) -> AsyncIterator[TriggerEvent]:
+        """Make a series of asynchronous http calls via an http hook."""
+        hook = self._get_async_hook()
+        while True:
+            try:
+                await hook.run(
+                    endpoint=self.endpoint,
+                    data=self.data,
+                    headers=self.headers,
+                    extra_options=self.extra_options,
+                )
+                yield TriggerEvent(True)
+            except AirflowException as exc:
+                if str(exc).startswith("404"):
+                    await asyncio.sleep(self.poke_interval)
+
+    def _get_async_hook(self) -> HttpAsyncHook:
+        return HttpAsyncHook(
+            method=self.method,
+            http_conn_id=self.http_conn_id,
+        )

{apache_airflow_providers_http-4.9.0rc1 → apache_airflow_providers_http-4.9.1rc1}/pyproject.toml

@@ -28,7 +28,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "apache-airflow-providers-http"
-version = "4.9.0.rc1"
+version = "4.9.1.rc1"
 description = "Provider package apache-airflow-providers-http for Apache Airflow"
 readme = "README.rst"
 authors = [
@@ -55,16 +55,16 @@ classifiers = [
 ]
 requires-python = "~=3.8"
 dependencies = [
-    "aiohttp",
+    "aiohttp>=3.9.2",
     "apache-airflow>=2.6.0.dev0",
     "asgiref",
-    "requests>=2.26.0",
+    "requests>=2.27.0,<3",
     "requests_toolbelt",
 ]
 
 [project.urls]
-"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-http/4.9.0"
-"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-http/4.9.0/changelog.html"
+"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-http/4.9.1"
+"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-http/4.9.1/changelog.html"
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
 "Source Code" = "https://github.com/apache/airflow"
 "Slack Chat" = "https://s.apache.org/airflow-slack"