port-ocean 0.12.2.dev14__py3-none-any.whl → 0.12.2.dev17__py3-none-any.whl

port_ocean/core/defaults/initialize.py

@@ -1,7 +1,7 @@
 import asyncio
 from typing import Type, Any
 
-import aiohttp
+import httpx
 from loguru import logger
 
 from port_ocean.clients.port.client import PortClient
@@ -21,7 +21,7 @@ from port_ocean.exceptions.port_defaults import (
 
 
 def deconstruct_blueprints_to_creation_steps(
-        raw_blueprints: list[dict[str, Any]],
+    raw_blueprints: list[dict[str, Any]],
 ) -> tuple[list[dict[str, Any]], ...]:
     """
     Deconstructing the blueprint into stages so the api wont fail to create a blueprint if there is a conflict
@@ -53,9 +53,9 @@ def deconstruct_blueprints_to_creation_steps(
 
 
 async def _initialize_required_integration_settings(
-        port_client: PortClient,
-        default_mapping: PortAppConfig,
-        integration_config: IntegrationConfiguration,
+    port_client: PortClient,
+    default_mapping: PortAppConfig,
+    integration_config: IntegrationConfiguration,
 ) -> None:
     try:
         logger.info("Initializing integration at port")
@@ -80,8 +80,8 @@ async def _initialize_required_integration_settings(
                 integration_config.event_listener.to_request(),
                 port_app_config=default_mapping,
             )
-    except aiohttp.ClientResponseError as err:
-        logger.error(f"Failed to apply default mapping: {err.message}.")
+    except httpx.HTTPStatusError as err:
+        logger.error(f"Failed to apply default mapping: {err.response.text}.")
         raise err
 
     logger.info("Checking for diff in integration configuration")
@@ -89,9 +89,9 @@ async def _initialize_required_integration_settings(
         "changelog_destination"
     )
     if (
-            integration.get("changelogDestination") != changelog_destination
-            or integration.get("installationAppType") != integration_config.integration.type
-            or integration.get("version") != port_client.integration_version
+        integration.get("changelogDestination") != changelog_destination
+        or integration.get("installationAppType") != integration_config.integration.type
+        or integration.get("version") != port_client.integration_version
     ):
         await port_client.patch_integration(
             integration_config.integration.type, changelog_destination
@@ -99,8 +99,8 @@ async def _initialize_required_integration_settings(
 
 
 async def _create_resources(
-        port_client: PortClient,
-        defaults: Defaults,
+    port_client: PortClient,
+    defaults: Defaults,
 ) -> None:
     creation_stage, *blueprint_patches = deconstruct_blueprints_to_creation_steps(
         defaults.blueprints
@@ -133,9 +133,9 @@ async def _create_resources(
 
     if blueprint_errors:
         for error in blueprint_errors:
-            if isinstance(error, aiohttp.ClientResponseError):
+            if isinstance(error, httpx.HTTPStatusError):
                 logger.warning(
-                    f"Failed to create resources: {error.message}. Rolling back changes..."
+                    f"Failed to create resources: {error.response.text}. Rolling back changes..."
                 )
 
         raise AbortDefaultCreationError(
@@ -155,8 +155,8 @@ async def _create_resources(
                 )
             )
 
-    except aiohttp.ClientResponseError as err:
-        logger.error(f"Failed to create resources: {err.message}. continuing...")
+    except httpx.HTTPStatusError as err:
+        logger.error(f"Failed to create resources: {err.response.text}. continuing...")
         raise AbortDefaultCreationError(created_blueprints_identifiers, [err])
     try:
         created_actions, actions_errors = await gather_and_split_errors_from_results(
@@ -185,9 +185,9 @@ async def _create_resources(
         errors = actions_errors + scorecards_errors + pages_errors
         if errors:
             for error in errors:
-                if isinstance(error, aiohttp.ClientResponseError):
+                if isinstance(error, httpx.HTTPStatusError):
                     logger.warning(
-                        f"Failed to create resource: {error.message}. continuing..."
+                        f"Failed to create resource: {error.response.text}. continuing..."
                     )
 
     except Exception as err:
@@ -195,45 +195,44 @@ async def _create_resources(
 
 
 async def _initialize_defaults(
-        config_class: Type[PortAppConfig], integration_config: IntegrationConfiguration
+    config_class: Type[PortAppConfig], integration_config: IntegrationConfiguration
 ) -> None:
     port_client = ocean.port_client
-    async with ocean.port_client.client:
-        defaults = get_port_integration_defaults(config_class)
-        if not defaults:
-            logger.warning("No defaults found. Skipping initialization...")
-            return None
-
-        if defaults.port_app_config:
-            await _initialize_required_integration_settings(
-                port_client, defaults.port_app_config, integration_config
-            )
+    defaults = get_port_integration_defaults(config_class)
+    if not defaults:
+        logger.warning("No defaults found. Skipping initialization...")
+        return None
+
+    if defaults.port_app_config:
+        await _initialize_required_integration_settings(
+            port_client, defaults.port_app_config, integration_config
+        )
 
-        if not integration_config.initialize_port_resources:
-            return
+    if not integration_config.initialize_port_resources:
+        return
 
-        try:
-            logger.info("Found default resources, starting creation process")
-            await _create_resources(port_client, defaults)
-        except AbortDefaultCreationError as e:
-            logger.warning(
-                f"Failed to create resources. Rolling back blueprints : {e.blueprints_to_rollback}"
-            )
-            await asyncio.gather(
-                *(
-                    port_client.delete_blueprint(
-                        identifier,
-                        should_raise=False,
-                        user_agent_type=UserAgentType.exporter,
-                    )
-                    for identifier in e.blueprints_to_rollback
+    try:
+        logger.info("Found default resources, starting creation process")
+        await _create_resources(port_client, defaults)
+    except AbortDefaultCreationError as e:
+        logger.warning(
+            f"Failed to create resources. Rolling back blueprints : {e.blueprints_to_rollback}"
+        )
+        await asyncio.gather(
+            *(
+                port_client.delete_blueprint(
+                    identifier,
+                    should_raise=False,
+                    user_agent_type=UserAgentType.exporter,
                 )
+                for identifier in e.blueprints_to_rollback
             )
-            raise ExceptionGroup(str(e), e.errors)
+        )
+        raise ExceptionGroup(str(e), e.errors)
 
 
 def initialize_defaults(
-        config_class: Type[PortAppConfig], integration_config: IntegrationConfiguration
+    config_class: Type[PortAppConfig], integration_config: IntegrationConfiguration
 ) -> None:
     asyncio.new_event_loop().run_until_complete(
         _initialize_defaults(config_class, integration_config)

port_ocean/core/integrations/mixins/sync_raw.py

@@ -431,13 +431,13 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
             #     entities_at_port = await ocean.port_client.search_entities(
             #         user_agent_type
             #     )
-            # except aiohttp.ClientError as e:
+            # except httpx.HTTPError as e:
             #     logger.warning(
             #         "Failed to fetch the current state of entities at Port. "
             #         "Skipping delete phase due to unknown initial state. "
             #         f"Error: {e}\n"
-            #         f"Response status code: {e.status if isinstance(e, aiohttp.ClientResponseError) else None}\n"
-            #         f"Response content: {e.message if isinstance(e, aiohttp.ClientResponseError) else None}\n"
+            #         f"Response status code: {e.response.status_code if isinstance(e, httpx.HTTPStatusError) else None}\n"
+            #         f"Response content: {e.response.text if isinstance(e, httpx.HTTPStatusError) else None}\n"
             #     )
             #     did_fetched_current_state = False
 

port_ocean/core/utils.py

@@ -39,10 +39,9 @@ async def validate_integration_runtime(
     requested_runtime: Runtime,
 ) -> None:
     logger.debug("Validating integration runtime")
-    async with port_client.client:
-        current_integration = await port_client.get_current_integration(
-            should_raise=False, should_log=False
-        )
+    current_integration = await port_client.get_current_integration(
+        should_raise=False, should_log=False
+    )
     current_runtime = current_integration.get("installationType", "OnPrem")
     if current_integration and current_runtime != requested_runtime.value:
         raise IntegrationRuntimeException(

port_ocean/helpers/async_client.py

@@ -0,0 +1,53 @@
+from typing import Any, Callable, Type
+
+import httpx
+from loguru import logger
+
+from port_ocean.helpers.retry import RetryTransport
+
+
+class OceanAsyncClient(httpx.AsyncClient):
+    """
+    This class is a wrapper around httpx.AsyncClient that uses a custom transport class.
+    This is done to allow passing our custom transport class to the AsyncClient constructor while still allowing
+    all the default AsyncClient behavior that is changed when passing a custom transport instance.
+    """
+
+    def __init__(
+        self,
+        transport_class: Type[RetryTransport] = RetryTransport,
+        transport_kwargs: dict[str, Any] | None = None,
+        **kwargs: Any,
+    ):
+        self._transport_kwargs = transport_kwargs
+        self._transport_class = transport_class
+        super().__init__(**kwargs)
+
+    def _init_transport(  # type: ignore[override]
+        self,
+        transport: httpx.AsyncBaseTransport | None = None,
+        app: Callable[..., Any] | None = None,
+        **kwargs: Any,
+    ) -> httpx.AsyncBaseTransport:
+        if transport is not None or app is not None:
+            return super()._init_transport(transport=transport, app=app, **kwargs)
+
+        return self._transport_class(
+            wrapped_transport=httpx.AsyncHTTPTransport(
+                **kwargs,
+            ),
+            logger=logger,
+            **(self._transport_kwargs or {}),
+        )
+
+    def _init_proxy_transport(  # type: ignore[override]
+        self, proxy: httpx.Proxy, **kwargs: Any
+    ) -> httpx.AsyncBaseTransport:
+        return self._transport_class(
+            wrapped_transport=httpx.AsyncHTTPTransport(
+                proxy=proxy,
+                **kwargs,
+            ),
+            logger=logger,
+            **(self._transport_kwargs or {}),
+        )

port_ocean/helpers/retry.py

@@ -1,18 +1,51 @@
 import asyncio
-import functools
 import random
+import time
 from datetime import datetime
+from functools import partial
 from http import HTTPStatus
-from typing import Any, Callable, Coroutine, Iterable, Mapping
+from typing import Any, Callable, Coroutine, Iterable, Mapping, Union
 
-import aiohttp
-from aiohttp import ClientResponse
+import httpx
 from dateutil.parser import isoparse
-from loguru import logger
 
 
-# Adapted from https://github.com/encode/httpx/issues/108#issuecomment-1434439481n
-class RetryRequestClass(aiohttp.ClientRequest):
+# Adapted from https://github.com/encode/httpx/issues/108#issuecomment-1434439481
+class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
+    """
+    A custom HTTP transport that automatically retries requests using an exponential backoff strategy
+    for specific HTTP status codes and request methods.
+
+    Args:
+        wrapped_transport (Union[httpx.BaseTransport, httpx.AsyncBaseTransport]): The underlying HTTP transport
+            to wrap and use for making requests.
+        max_attempts (int, optional): The maximum number of times to retry a request before giving up. Defaults to 10.
+        max_backoff_wait (float, optional): The maximum time to wait between retries in seconds. Defaults to 60.
+        backoff_factor (float, optional): The factor by which the wait time increases with each retry attempt.
+            Defaults to 0.1.
+        jitter_ratio (float, optional): The amount of jitter to add to the backoff time. Jitter is a random
+            value added to the backoff time to avoid a "thundering herd" effect. The value should be between 0 and 0.5.
+            Defaults to 0.1.
+        respect_retry_after_header (bool, optional): Whether to respect the Retry-After header in HTTP responses
+            when deciding how long to wait before retrying. Defaults to True.
+        retryable_methods (Iterable[str], optional): The HTTP methods that can be retried. Defaults to
+            ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"].
+        retry_status_codes (Iterable[int], optional): The HTTP status codes that can be retried. Defaults to
+            [429, 502, 503, 504].
+
+    Attributes:
+        _wrapped_transport (Union[httpx.BaseTransport, httpx.AsyncBaseTransport]): The underlying HTTP transport
+            being wrapped.
+        _max_attempts (int): The maximum number of times to retry a request.
+        _backoff_factor (float): The factor by which the wait time increases with each retry attempt.
+        _respect_retry_after_header (bool): Whether to respect the Retry-After header in HTTP responses.
+        _retryable_methods (frozenset): The HTTP methods that can be retried.
+        _retry_status_codes (frozenset): The HTTP status codes that can be retried.
+        _jitter_ratio (float): The amount of jitter to add to the backoff time.
+        _max_backoff_wait (float): The maximum time to wait between retries in seconds.
+
+    """
+
     RETRYABLE_METHODS = frozenset(["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"])
     RETRYABLE_STATUS_CODES = frozenset(
         [
@@ -25,23 +58,23 @@ class RetryRequestClass(aiohttp.ClientRequest):
     MAX_BACKOFF_WAIT = 60
 
     def __init__(
-            self,
-            method,
-            url,
-            max_attempts: int = 10,
-            max_backoff_wait: float = MAX_BACKOFF_WAIT,
-            backoff_factor: float = 0.1,
-            jitter_ratio: float = 0.1,
-            respect_retry_after_header: bool = True,
-            retryable_methods: Iterable[str] | None = None,
-            retry_status_codes: Iterable[int] | None = None,
-            *args,
-            **kwargs
+        self,
+        wrapped_transport: Union[httpx.BaseTransport, httpx.AsyncBaseTransport],
+        max_attempts: int = 10,
+        max_backoff_wait: float = MAX_BACKOFF_WAIT,
+        backoff_factor: float = 0.1,
+        jitter_ratio: float = 0.1,
+        respect_retry_after_header: bool = True,
+        retryable_methods: Iterable[str] | None = None,
+        retry_status_codes: Iterable[int] | None = None,
+        logger: Any | None = None,
     ) -> None:
         """
         Initializes the instance of RetryTransport class with the given parameters.
 
         Args:
+            wrapped_transport (Union[httpx.BaseTransport, httpx.AsyncBaseTransport]):
+                The transport layer that will be wrapped and retried upon failure.
             max_attempts (int, optional):
                 The maximum number of times the request can be retried in case of failure.
                 Defaults to 10.
@@ -65,6 +98,7 @@ class RetryRequestClass(aiohttp.ClientRequest):
                 Defaults to [429, 502, 503, 504].
             logger (Any): The logger to use for logging retries.
         """
+        self._wrapped_transport = wrapped_transport
         if jitter_ratio < 0 or jitter_ratio > 0.5:
             raise ValueError(
                 f"Jitter ratio should be between 0 and 0.5, actual {jitter_ratio}"
@@ -85,34 +119,117 @@ class RetryRequestClass(aiohttp.ClientRequest):
         )
         self._jitter_ratio = jitter_ratio
         self._max_backoff_wait = max_backoff_wait
+        self._logger = logger
+
+    def handle_request(self, request: httpx.Request) -> httpx.Response:
+        """
+        Sends an HTTP request, possibly with retries.
+
+        Args:
+            request (httpx.Request): The request to send.
+
+        Returns:
+            httpx.Response: The response received.
+
+        """
+        transport: httpx.BaseTransport = self._wrapped_transport  # type: ignore
+        if request.method in self._retryable_methods:
+            send_method = partial(transport.handle_request)
+            response = self._retry_operation(request, send_method)
+        else:
+            response = transport.handle_request(request)
+        return response
+
+    async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
+        """Sends an HTTP request, possibly with retries.
+
+        Args:
+            request: The request to perform.
+
+        Returns:
+            The response.
+
+        """
+        transport: httpx.AsyncBaseTransport = self._wrapped_transport  # type: ignore
+        if self._is_retryable_method(request):
+            send_method = partial(transport.handle_async_request)
+            response = await self._retry_operation_async(request, send_method)
+        else:
+            response = await transport.handle_async_request(request)
+        return response
+
+    async def aclose(self) -> None:
+        """
+        Closes the underlying HTTP transport, terminating all outstanding connections and rejecting any further
+        requests.
+
+        This should be called before the object is dereferenced, to ensure that connections are properly cleaned up.
+        """
+        transport: httpx.AsyncBaseTransport = self._wrapped_transport  # type: ignore
+        await transport.aclose()
 
-        super().__init__(method, url, *args, **kwargs)
+    def close(self) -> None:
+        """
+        Closes the underlying HTTP transport, terminating all outstanding connections and rejecting any further
+        requests.
 
-    def _is_retryable(self) -> bool:
-        return self.method in self._retryable_methods
+        This should be called before the object is dereferenced, to ensure that connections are properly cleaned up.
+        """
+        transport: httpx.BaseTransport = self._wrapped_transport  # type: ignore
+        transport.close()
+
+    def _is_retryable_method(self, request: httpx.Request) -> bool:
+        return request.method in self._retryable_methods or request.extensions.get(
+            "retryable", False
+        )
+
+    def _should_retry(self, response: httpx.Response) -> bool:
+        return response.status_code in self._retry_status_codes
 
     def _log_error(
-            self,
-            error: Exception | None,
+        self,
+        request: httpx.Request,
+        error: Exception | None,
     ) -> None:
-        if isinstance(error, aiohttp.ServerConnectionError):
-            logger.error(
-                f"Request {self.method} {self.url} failed to connect: {str(error)}"
+        if not self._logger:
+            return
+
+        if isinstance(error, httpx.ConnectTimeout):
+            self._logger.error(
+                f"Request {request.method} {request.url} failed to connect: {str(error)}"
             )
-        elif isinstance(error, aiohttp.ConnectionTimeoutError):
-            logger.error(
-                f"Request {self.method} {self.url} failed with a timeout exception: {str(error)}"
+        elif isinstance(error, httpx.TimeoutException):
+            self._logger.error(
+                f"Request {request.method} {request.url} failed with a timeout exception: {str(error)}"
             )
-        elif isinstance(error, aiohttp.ClientError):
-            logger.error(
-                f"Request {self.method} {self.url} failed with an HTTP error: {str(error)}"
+        elif isinstance(error, httpx.HTTPError):
+            self._logger.error(
+                f"Request {request.method} {request.url} failed with an HTTP error: {str(error)}"
             )
 
-    async def _should_retry_async(self, response: ClientResponse) -> bool:
-        return response.status in self._retry_status_codes
+    def _log_before_retry(
+        self,
+        request: httpx.Request,
+        sleep_time: float,
+        response: httpx.Response | None,
+        error: Exception | None,
+    ) -> None:
+        if self._logger and response:
+            self._logger.warning(
+                f"Request {request.method} {request.url} failed with status code:"
+                f" {response.status_code}, retrying in {sleep_time} seconds."  # noqa: F821
+            )
+        elif self._logger and error:
+            self._logger.warning(
+                f"Request {request.method} {request.url} failed with exception:"
+                f" {type(error).__name__} - {str(error) or 'No error message'}, retrying in {sleep_time} seconds."
+            )
+
+    async def _should_retry_async(self, response: httpx.Response) -> bool:
+        return response.status_code in self._retry_status_codes
 
     def _calculate_sleep(
-            self, attempts_made: int, headers: Mapping[str, str]
+        self, attempts_made: int, headers: Union[httpx.Headers, Mapping[str, str]]
     ) -> float:
         # Retry-After
         # The Retry-After response HTTP header indicates how long the user agent should wait before
@@ -143,58 +260,91 @@ class RetryRequestClass(aiohttp.ClientRequest):
         total_backoff = backoff + jitter
         return min(total_backoff, self._max_backoff_wait)
 
-    def _log_before_retry(
-            self,
-            sleep_time: float,
-            response: ClientResponse | None,
-            error: Exception | None,
-    ) -> None:
-        if response:
-            logger.warning(
-                f"Request {self.method} {self.url} failed with status code:"
-                f" {response.status}, retrying in {sleep_time} seconds."  # noqa: F821
-            )
-        elif error:
-            logger.warning(
-                f"Request {self.method} {self.url} failed with exception:"
-                f" {type(error).__name__} - {str(error) or 'No error message'}, retrying in {sleep_time} seconds."
-            )
-
     async def _retry_operation_async(
-            self, request: Callable[..., Coroutine[Any, Any, ClientResponse]]
-    ) -> ClientResponse:
+        self,
+        request: httpx.Request,
+        send_method: Callable[..., Coroutine[Any, Any, httpx.Response]],
+    ) -> httpx.Response:
         remaining_attempts = self._max_attempts
         attempts_made = 0
-        response: ClientResponse | None = None
+        response: httpx.Response | None = None
         error: Exception | None = None
         while True:
             if attempts_made > 0:
-                sleep_time = self._calculate_sleep(attempts_made, response.headers if response else {})
-                self._log_before_retry(sleep_time, response, error)
+                sleep_time = self._calculate_sleep(attempts_made, {})
+                self._log_before_retry(request, sleep_time, response, error)
                 await asyncio.sleep(sleep_time)
 
             error = None
             response = None
             try:
-                response = await request()
+                response = await send_method(request)
+                response.request = request
                 if remaining_attempts < 1 or not (
-                        await self._should_retry_async(response)
+                    await self._should_retry_async(response)
                 ):
                     return response
-            except (aiohttp.ServerConnectionError, aiohttp.ConnectionTimeoutError, aiohttp.ClientError) as e:
+                await response.aclose()
+            except httpx.ConnectTimeout as e:
                 error = e
                 if remaining_attempts < 1:
-                    self._log_error(error)
+                    self._log_error(request, error)
+                    raise
+            except httpx.ReadTimeout as e:
+                error = e
+                if remaining_attempts < 1:
+                    self._log_error(request, error)
+                    raise
+            except httpx.TimeoutException as e:
+                error = e
+                if remaining_attempts < 1:
+                    self._log_error(request, error)
+                    raise
+            except httpx.HTTPError as e:
+                error = e
+                if remaining_attempts < 1:
+                    self._log_error(request, error)
                     raise
             attempts_made += 1
             remaining_attempts -= 1
 
-    async def send(self, conn: "Connection") -> "ClientResponse":
-        request = functools.partial(
-            super().send, conn
-        )
-        if self._is_retryable():
-            response = await self._retry_operation_async(request)
-        else:
-            response = await request()
-        return response
+    def _retry_operation(
+        self,
+        request: httpx.Request,
+        send_method: Callable[..., httpx.Response],
+    ) -> httpx.Response:
+        remaining_attempts = self._max_attempts
+        attempts_made = 0
+        response: httpx.Response | None = None
+        error: Exception | None = None
+        while True:
+            if attempts_made > 0:
+                sleep_time = self._calculate_sleep(attempts_made, {})
+                self._log_before_retry(request, sleep_time, response, error)
+                time.sleep(sleep_time)
+
+            error = None
+            response = None
+            try:
+                response = send_method(request)
+                response.request = request
+                if remaining_attempts < 1 or not self._should_retry(response):
+                    return response
+                response.close()
+            except httpx.ConnectTimeout as e:
+                error = e
+                if remaining_attempts < 1:
+                    self._log_error(request, error)
+                    raise
+            except httpx.TimeoutException as e:
+                error = e
+                if remaining_attempts < 1:
+                    self._log_error(request, error)
+                    raise
+            except httpx.HTTPError as e:
+                error = e
+                if remaining_attempts < 1:
+                    self._log_error(request, error)
+                    raise
+            attempts_made += 1
+            remaining_attempts -= 1