port-ocean 0.12.2.dev11__py3-none-any.whl → 0.12.2.dev12__py3-none-any.whl

port_ocean/core/defaults/initialize.py

@@ -1,7 +1,7 @@
 import asyncio
 from typing import Type, Any
 
-import httpx
+import aiohttp
 from loguru import logger
 
 from port_ocean.clients.port.client import PortClient
@@ -21,7 +21,7 @@ from port_ocean.exceptions.port_defaults import (
 
 
 def deconstruct_blueprints_to_creation_steps(
-    raw_blueprints: list[dict[str, Any]],
+        raw_blueprints: list[dict[str, Any]],
 ) -> tuple[list[dict[str, Any]], ...]:
     """
     Deconstructing the blueprint into stages so the api wont fail to create a blueprint if there is a conflict
@@ -53,9 +53,9 @@ def deconstruct_blueprints_to_creation_steps(
 
 
 async def _initialize_required_integration_settings(
-    port_client: PortClient,
-    default_mapping: PortAppConfig,
-    integration_config: IntegrationConfiguration,
+        port_client: PortClient,
+        default_mapping: PortAppConfig,
+        integration_config: IntegrationConfiguration,
 ) -> None:
     try:
         logger.info("Initializing integration at port")
@@ -80,8 +80,8 @@ async def _initialize_required_integration_settings(
                 integration_config.event_listener.to_request(),
                 port_app_config=default_mapping,
             )
-    except httpx.HTTPStatusError as err:
-        logger.error(f"Failed to apply default mapping: {err.response.text}.")
+    except aiohttp.ClientResponseError as err:
+        logger.error(f"Failed to apply default mapping: {err.message}.")
         raise err
 
     logger.info("Checking for diff in integration configuration")
@@ -89,9 +89,9 @@ async def _initialize_required_integration_settings(
         "changelog_destination"
     )
     if (
-        integration.get("changelogDestination") != changelog_destination
-        or integration.get("installationAppType") != integration_config.integration.type
-        or integration.get("version") != port_client.integration_version
+            integration.get("changelogDestination") != changelog_destination
+            or integration.get("installationAppType") != integration_config.integration.type
+            or integration.get("version") != port_client.integration_version
     ):
         await port_client.patch_integration(
             integration_config.integration.type, changelog_destination
@@ -99,8 +99,8 @@ async def _initialize_required_integration_settings(
 
 
 async def _create_resources(
-    port_client: PortClient,
-    defaults: Defaults,
+        port_client: PortClient,
+        defaults: Defaults,
 ) -> None:
     creation_stage, *blueprint_patches = deconstruct_blueprints_to_creation_steps(
         defaults.blueprints
@@ -133,9 +133,9 @@ async def _create_resources(
 
     if blueprint_errors:
         for error in blueprint_errors:
-            if isinstance(error, httpx.HTTPStatusError):
+            if isinstance(error, aiohttp.ClientResponseError):
                 logger.warning(
-                    f"Failed to create resources: {error.response.text}. Rolling back changes..."
+                    f"Failed to create resources: {error.message}. Rolling back changes..."
                 )
 
         raise AbortDefaultCreationError(
@@ -155,8 +155,8 @@ async def _create_resources(
                 )
             )
 
-    except httpx.HTTPStatusError as err:
-        logger.error(f"Failed to create resources: {err.response.text}. continuing...")
+    except aiohttp.ClientResponseError as err:
+        logger.error(f"Failed to create resources: {err.message}. continuing...")
         raise AbortDefaultCreationError(created_blueprints_identifiers, [err])
     try:
         created_actions, actions_errors = await gather_and_split_errors_from_results(
@@ -185,9 +185,9 @@ async def _create_resources(
         errors = actions_errors + scorecards_errors + pages_errors
         if errors:
             for error in errors:
-                if isinstance(error, httpx.HTTPStatusError):
+                if isinstance(error, aiohttp.ClientResponseError):
                     logger.warning(
-                        f"Failed to create resource: {error.response.text}. continuing..."
+                        f"Failed to create resource: {error.message}. continuing..."
                     )
 
     except Exception as err:
@@ -195,44 +195,45 @@ async def _create_resources(
 
 
 async def _initialize_defaults(
-    config_class: Type[PortAppConfig], integration_config: IntegrationConfiguration
+        config_class: Type[PortAppConfig], integration_config: IntegrationConfiguration
 ) -> None:
     port_client = ocean.port_client
-    defaults = get_port_integration_defaults(config_class)
-    if not defaults:
-        logger.warning("No defaults found. Skipping initialization...")
-        return None
-
-    if defaults.port_app_config:
-        await _initialize_required_integration_settings(
-            port_client, defaults.port_app_config, integration_config
-        )
+    async with ocean.port_client.client:
+        defaults = get_port_integration_defaults(config_class)
+        if not defaults:
+            logger.warning("No defaults found. Skipping initialization...")
+            return None
+
+        if defaults.port_app_config:
+            await _initialize_required_integration_settings(
+                port_client, defaults.port_app_config, integration_config
+            )
 
-    if not integration_config.initialize_port_resources:
-        return
+        if not integration_config.initialize_port_resources:
+            return
 
-    try:
-        logger.info("Found default resources, starting creation process")
-        await _create_resources(port_client, defaults)
-    except AbortDefaultCreationError as e:
-        logger.warning(
-            f"Failed to create resources. Rolling back blueprints : {e.blueprints_to_rollback}"
-        )
-        await asyncio.gather(
-            *(
-                port_client.delete_blueprint(
-                    identifier,
-                    should_raise=False,
-                    user_agent_type=UserAgentType.exporter,
+        try:
+            logger.info("Found default resources, starting creation process")
+            await _create_resources(port_client, defaults)
+        except AbortDefaultCreationError as e:
+            logger.warning(
+                f"Failed to create resources. Rolling back blueprints : {e.blueprints_to_rollback}"
+            )
+            await asyncio.gather(
+                *(
+                    port_client.delete_blueprint(
+                        identifier,
+                        should_raise=False,
+                        user_agent_type=UserAgentType.exporter,
+                    )
+                    for identifier in e.blueprints_to_rollback
                 )
-                for identifier in e.blueprints_to_rollback
             )
-        )
-        raise ExceptionGroup(str(e), e.errors)
+            raise ExceptionGroup(str(e), e.errors)
 
 
 def initialize_defaults(
-    config_class: Type[PortAppConfig], integration_config: IntegrationConfiguration
+        config_class: Type[PortAppConfig], integration_config: IntegrationConfiguration
 ) -> None:
     asyncio.new_event_loop().run_until_complete(
         _initialize_defaults(config_class, integration_config)

port_ocean/core/integrations/mixins/sync_raw.py

@@ -431,13 +431,13 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
             #     entities_at_port = await ocean.port_client.search_entities(
             #         user_agent_type
             #     )
-            # except httpx.HTTPError as e:
+            # except aiohttp.ClientError as e:
             #     logger.warning(
             #         "Failed to fetch the current state of entities at Port. "
             #         "Skipping delete phase due to unknown initial state. "
             #         f"Error: {e}\n"
-            #         f"Response status code: {e.response.status_code if isinstance(e, httpx.HTTPStatusError) else None}\n"
-            #         f"Response content: {e.response.text if isinstance(e, httpx.HTTPStatusError) else None}\n"
+            #         f"Response status code: {e.status if isinstance(e, aiohttp.ClientResponseError) else None}\n"
+            #         f"Response content: {e.message if isinstance(e, aiohttp.ClientResponseError) else None}\n"
             #     )
             #     did_fetched_current_state = False
 

port_ocean/core/utils.py

@@ -39,9 +39,10 @@ async def validate_integration_runtime(
     requested_runtime: Runtime,
 ) -> None:
     logger.debug("Validating integration runtime")
-    current_integration = await port_client.get_current_integration(
-        should_raise=False, should_log=False
-    )
+    async with port_client.client:
+        current_integration = await port_client.get_current_integration(
+            should_raise=False, should_log=False
+        )
     current_runtime = current_integration.get("installationType", "OnPrem")
     if current_integration and current_runtime != requested_runtime.value:
         raise IntegrationRuntimeException(

port_ocean/helpers/retry.py

@@ -1,51 +1,18 @@
 import asyncio
+import functools
 import random
-import time
 from datetime import datetime
-from functools import partial
 from http import HTTPStatus
-from typing import Any, Callable, Coroutine, Iterable, Mapping, Union
+from typing import Any, Callable, Coroutine, Iterable, Mapping
 
-import httpx
+import aiohttp
+from aiohttp import ClientResponse
 from dateutil.parser import isoparse
+from loguru import logger
 
 
-# Adapted from https://github.com/encode/httpx/issues/108#issuecomment-1434439481
-class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
-    """
-    A custom HTTP transport that automatically retries requests using an exponential backoff strategy
-    for specific HTTP status codes and request methods.
-
-    Args:
-        wrapped_transport (Union[httpx.BaseTransport, httpx.AsyncBaseTransport]): The underlying HTTP transport
-            to wrap and use for making requests.
-        max_attempts (int, optional): The maximum number of times to retry a request before giving up. Defaults to 10.
-        max_backoff_wait (float, optional): The maximum time to wait between retries in seconds. Defaults to 60.
-        backoff_factor (float, optional): The factor by which the wait time increases with each retry attempt.
-            Defaults to 0.1.
-        jitter_ratio (float, optional): The amount of jitter to add to the backoff time. Jitter is a random
-            value added to the backoff time to avoid a "thundering herd" effect. The value should be between 0 and 0.5.
-            Defaults to 0.1.
-        respect_retry_after_header (bool, optional): Whether to respect the Retry-After header in HTTP responses
-            when deciding how long to wait before retrying. Defaults to True.
-        retryable_methods (Iterable[str], optional): The HTTP methods that can be retried. Defaults to
-            ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"].
-        retry_status_codes (Iterable[int], optional): The HTTP status codes that can be retried. Defaults to
-            [429, 502, 503, 504].
-
-    Attributes:
-        _wrapped_transport (Union[httpx.BaseTransport, httpx.AsyncBaseTransport]): The underlying HTTP transport
-            being wrapped.
-        _max_attempts (int): The maximum number of times to retry a request.
-        _backoff_factor (float): The factor by which the wait time increases with each retry attempt.
-        _respect_retry_after_header (bool): Whether to respect the Retry-After header in HTTP responses.
-        _retryable_methods (frozenset): The HTTP methods that can be retried.
-        _retry_status_codes (frozenset): The HTTP status codes that can be retried.
-        _jitter_ratio (float): The amount of jitter to add to the backoff time.
-        _max_backoff_wait (float): The maximum time to wait between retries in seconds.
-
-    """
-
+# Adapted from https://github.com/encode/httpx/issues/108#issuecomment-1434439481n
+class RetryRequestClass(aiohttp.ClientRequest):
     RETRYABLE_METHODS = frozenset(["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"])
     RETRYABLE_STATUS_CODES = frozenset(
         [
@@ -58,23 +25,23 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
     MAX_BACKOFF_WAIT = 60
 
     def __init__(
-        self,
-        wrapped_transport: Union[httpx.BaseTransport, httpx.AsyncBaseTransport],
-        max_attempts: int = 10,
-        max_backoff_wait: float = MAX_BACKOFF_WAIT,
-        backoff_factor: float = 0.1,
-        jitter_ratio: float = 0.1,
-        respect_retry_after_header: bool = True,
-        retryable_methods: Iterable[str] | None = None,
-        retry_status_codes: Iterable[int] | None = None,
-        logger: Any | None = None,
+            self,
+            method,
+            url,
+            max_attempts: int = 10,
+            max_backoff_wait: float = MAX_BACKOFF_WAIT,
+            backoff_factor: float = 0.1,
+            jitter_ratio: float = 0.1,
+            respect_retry_after_header: bool = True,
+            retryable_methods: Iterable[str] | None = None,
+            retry_status_codes: Iterable[int] | None = None,
+            *args,
+            **kwargs
     ) -> None:
         """
         Initializes the instance of RetryTransport class with the given parameters.
 
         Args:
-            wrapped_transport (Union[httpx.BaseTransport, httpx.AsyncBaseTransport]):
-                The transport layer that will be wrapped and retried upon failure.
             max_attempts (int, optional):
                 The maximum number of times the request can be retried in case of failure.
                 Defaults to 10.
@@ -98,7 +65,6 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
                 Defaults to [429, 502, 503, 504].
             logger (Any): The logger to use for logging retries.
         """
-        self._wrapped_transport = wrapped_transport
         if jitter_ratio < 0 or jitter_ratio > 0.5:
             raise ValueError(
                 f"Jitter ratio should be between 0 and 0.5, actual {jitter_ratio}"
@@ -119,117 +85,34 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
         )
         self._jitter_ratio = jitter_ratio
         self._max_backoff_wait = max_backoff_wait
-        self._logger = logger
-
-    def handle_request(self, request: httpx.Request) -> httpx.Response:
-        """
-        Sends an HTTP request, possibly with retries.
-
-        Args:
-            request (httpx.Request): The request to send.
-
-        Returns:
-            httpx.Response: The response received.
-
-        """
-        transport: httpx.BaseTransport = self._wrapped_transport  # type: ignore
-        if request.method in self._retryable_methods:
-            send_method = partial(transport.handle_request)
-            response = self._retry_operation(request, send_method)
-        else:
-            response = transport.handle_request(request)
-        return response
-
-    async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
-        """Sends an HTTP request, possibly with retries.
-
-        Args:
-            request: The request to perform.
-
-        Returns:
-            The response.
-
-        """
-        transport: httpx.AsyncBaseTransport = self._wrapped_transport  # type: ignore
-        if self._is_retryable_method(request):
-            send_method = partial(transport.handle_async_request)
-            response = await self._retry_operation_async(request, send_method)
-        else:
-            response = await transport.handle_async_request(request)
-        return response
-
-    async def aclose(self) -> None:
-        """
-        Closes the underlying HTTP transport, terminating all outstanding connections and rejecting any further
-        requests.
-
-        This should be called before the object is dereferenced, to ensure that connections are properly cleaned up.
-        """
-        transport: httpx.AsyncBaseTransport = self._wrapped_transport  # type: ignore
-        await transport.aclose()
 
-    def close(self) -> None:
-        """
-        Closes the underlying HTTP transport, terminating all outstanding connections and rejecting any further
-        requests.
+        super().__init__(method, url, *args, **kwargs)
 
-        This should be called before the object is dereferenced, to ensure that connections are properly cleaned up.
-        """
-        transport: httpx.BaseTransport = self._wrapped_transport  # type: ignore
-        transport.close()
-
-    def _is_retryable_method(self, request: httpx.Request) -> bool:
-        return request.method in self._retryable_methods or request.extensions.get(
-            "retryable", False
-        )
-
-    def _should_retry(self, response: httpx.Response) -> bool:
-        return response.status_code in self._retry_status_codes
+    def _is_retryable(self) -> bool:
+        return self.method in self._retryable_methods
 
     def _log_error(
-        self,
-        request: httpx.Request,
-        error: Exception | None,
+            self,
+            error: Exception | None,
     ) -> None:
-        if not self._logger:
-            return
-
-        if isinstance(error, httpx.ConnectTimeout):
-            self._logger.error(
-                f"Request {request.method} {request.url} failed to connect: {str(error)}"
+        if isinstance(error, aiohttp.ServerConnectionError):
+            logger.error(
+                f"Request {self.method} {self.url} failed to connect: {str(error)}"
             )
-        elif isinstance(error, httpx.TimeoutException):
-            self._logger.error(
-                f"Request {request.method} {request.url} failed with a timeout exception: {str(error)}"
+        elif isinstance(error, aiohttp.ConnectionTimeoutError):
+            logger.error(
+                f"Request {self.method} {self.url} failed with a timeout exception: {str(error)}"
             )
-        elif isinstance(error, httpx.HTTPError):
-            self._logger.error(
-                f"Request {request.method} {request.url} failed with an HTTP error: {str(error)}"
+        elif isinstance(error, aiohttp.ClientError):
+            logger.error(
+                f"Request {self.method} {self.url} failed with an HTTP error: {str(error)}"
             )
 
-    def _log_before_retry(
-        self,
-        request: httpx.Request,
-        sleep_time: float,
-        response: httpx.Response | None,
-        error: Exception | None,
-    ) -> None:
-        if self._logger and response:
-            self._logger.warning(
-                f"Request {request.method} {request.url} failed with status code:"
-                f" {response.status_code}, retrying in {sleep_time} seconds."  # noqa: F821
-            )
-        elif self._logger and error:
-            self._logger.warning(
-                f"Request {request.method} {request.url} failed with exception:"
-                f" {type(error).__name__} - {str(error) or 'No error message'}, retrying in {sleep_time} seconds."
-            )
-
-    async def _should_retry_async(self, response: httpx.Response) -> bool:
-        return response.status_code in self._retry_status_codes
+    async def _should_retry_async(self, response: ClientResponse) -> bool:
+        return response.status in self._retry_status_codes
 
     def _calculate_sleep(
-        self, attempts_made: int, headers: Union[httpx.Headers, Mapping[str, str]]
+            self, attempts_made: int, headers: Mapping[str, str]
     ) -> float:
         # Retry-After
         # The Retry-After response HTTP header indicates how long the user agent should wait before
@@ -260,91 +143,58 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
         total_backoff = backoff + jitter
         return min(total_backoff, self._max_backoff_wait)
 
+    def _log_before_retry(
+            self,
+            sleep_time: float,
+            response: ClientResponse | None,
+            error: Exception | None,
+    ) -> None:
+        if response:
+            logger.warning(
+                f"Request {self.method} {self.url} failed with status code:"
+                f" {response.status}, retrying in {sleep_time} seconds."  # noqa: F821
+            )
+        elif error:
+            logger.warning(
+                f"Request {self.method} {self.url} failed with exception:"
+                f" {type(error).__name__} - {str(error) or 'No error message'}, retrying in {sleep_time} seconds."
+            )
+
     async def _retry_operation_async(
-        self,
-        request: httpx.Request,
-        send_method: Callable[..., Coroutine[Any, Any, httpx.Response]],
-    ) -> httpx.Response:
+            self, request: Callable[..., Coroutine[Any, Any, ClientResponse]]
+    ) -> ClientResponse:
         remaining_attempts = self._max_attempts
         attempts_made = 0
-        response: httpx.Response | None = None
+        response: ClientResponse | None = None
         error: Exception | None = None
         while True:
             if attempts_made > 0:
-                sleep_time = self._calculate_sleep(attempts_made, {})
-                self._log_before_retry(request, sleep_time, response, error)
+                sleep_time = self._calculate_sleep(attempts_made, response.headers if response else {})
+                self._log_before_retry(sleep_time, response, error)
                 await asyncio.sleep(sleep_time)
 
             error = None
             response = None
             try:
-                response = await send_method(request)
-                response.request = request
+                response = await request()
                 if remaining_attempts < 1 or not (
-                    await self._should_retry_async(response)
+                        await self._should_retry_async(response)
                 ):
                     return response
-                await response.aclose()
-            except httpx.ConnectTimeout as e:
+            except (aiohttp.ServerConnectionError, aiohttp.ConnectionTimeoutError, aiohttp.ClientError) as e:
                 error = e
                 if remaining_attempts < 1:
-                    self._log_error(request, error)
-                    raise
-            except httpx.ReadTimeout as e:
-                error = e
-                if remaining_attempts < 1:
-                    self._log_error(request, error)
-                    raise
-            except httpx.TimeoutException as e:
-                error = e
-                if remaining_attempts < 1:
-                    self._log_error(request, error)
-                    raise
-            except httpx.HTTPError as e:
-                error = e
-                if remaining_attempts < 1:
-                    self._log_error(request, error)
+                    self._log_error(error)
                     raise
             attempts_made += 1
             remaining_attempts -= 1
 
-    def _retry_operation(
-        self,
-        request: httpx.Request,
-        send_method: Callable[..., httpx.Response],
-    ) -> httpx.Response:
-        remaining_attempts = self._max_attempts
-        attempts_made = 0
-        response: httpx.Response | None = None
-        error: Exception | None = None
-        while True:
-            if attempts_made > 0:
-                sleep_time = self._calculate_sleep(attempts_made, {})
-                self._log_before_retry(request, sleep_time, response, error)
-                time.sleep(sleep_time)
-
-            error = None
-            response = None
-            try:
-                response = send_method(request)
-                response.request = request
-                if remaining_attempts < 1 or not self._should_retry(response):
-                    return response
-                response.close()
-            except httpx.ConnectTimeout as e:
-                error = e
-                if remaining_attempts < 1:
-                    self._log_error(request, error)
-                    raise
-            except httpx.TimeoutException as e:
-                error = e
-                if remaining_attempts < 1:
-                    self._log_error(request, error)
-                    raise
-            except httpx.HTTPError as e:
-                error = e
-                if remaining_attempts < 1:
-                    self._log_error(request, error)
-                    raise
-            attempts_made += 1
-            remaining_attempts -= 1
+    async def send(self, conn: "Connection") -> "ClientResponse":
+        request = functools.partial(
+            super().send, conn
+        )
+        if self._is_retryable():
+            response = await self._retry_operation_async(request)
+        else:
+            response = await request()
+        return response

port_ocean/ocean.py

@@ -1,6 +1,5 @@
 import asyncio
 import sys
-import threading
 from contextlib import asynccontextmanager
 from typing import Callable, Any, Dict, AsyncIterator, Type
 
@@ -30,12 +29,12 @@ from port_ocean.version import __integration_version__
 
 class Ocean:
     def __init__(
-        self,
-        app: FastAPI | None = None,
-        integration_class: Callable[[PortOceanContext], BaseIntegration] | None = None,
-        integration_router: APIRouter | None = None,
-        config_factory: Type[BaseModel] | None = None,
-        config_override: Dict[str, Any] | None = None,
+            self,
+            app: FastAPI | None = None,
+            integration_class: Callable[[PortOceanContext], BaseIntegration] | None = None,
+            integration_router: APIRouter | None = None,
+            config_factory: Type[BaseModel] | None = None,
+            config_override: Dict[str, Any] | None = None,
     ):
         initialize_port_ocean_context(self)
         self.fast_api_app = app or FastAPI()
@@ -73,7 +72,7 @@ class Ocean:
         return self.config.runtime == Runtime.Saas
 
     async def _setup_scheduled_resync(
-        self,
+            self,
     ) -> None:
         async def execute_resync_all() -> None:
             # await self.resync_state_updater.update_before_resync()
@@ -103,11 +102,7 @@ class Ocean:
                 # Not running the resync immediately because the event listener should run resync on startup
                 wait_first=False,
             )(
-                lambda: threading.Thread(
-                    target=lambda: asyncio.run_coroutine_threadsafe(
-                        execute_resync_all(), loop
-                    )
-                ).start()
+                execute_resync_all
             )
             await repeated_function()
 

port_ocean/tests/clients/port/mixins/test_entities.py

@@ -1,12 +1,11 @@
 from typing import Any
 from unittest.mock import MagicMock
 
+import aiohttp
 import pytest
 
 from port_ocean.clients.port.mixins.entities import EntityClientMixin
 from port_ocean.core.models import Entity
-from httpx import ReadTimeout
-
 
 errored_entity_identifier: str = "a"
 expected_result_entities = [
@@ -20,7 +19,7 @@ all_entities = [
 
 async def mock_upsert_entity(entity: Entity, *args: Any, **kwargs: Any) -> Entity:
     if entity.identifier == errored_entity_identifier:
-        raise ReadTimeout("")
+        raise aiohttp.ConnectionTimeoutError("")
     else:
         return entity
 
@@ -47,7 +46,7 @@ async def test_batch_upsert_entities_read_timeout_should_raise_false(
 async def test_batch_upsert_entities_read_timeout_should_raise_true(
     entity_client: EntityClientMixin,
 ) -> None:
-    with pytest.raises(ReadTimeout):
+    with pytest.raises(aiohttp.ConnectionTimeoutError):
         await entity_client.batch_upsert_entities(
             entities=all_entities, request_options=MagicMock(), should_raise=True
         )