port-ocean 0.4.9__tar.gz → 0.4.10__tar.gz

{port_ocean-0.4.9 → port_ocean-0.4.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.4.9
+Version: 0.4.10
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.4.9 → port_ocean-0.4.10}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/Makefile

@@ -4,7 +4,6 @@ define run_checks
 	exit_code=0; \
 	cd $1; \
 	poetry check || exit_code=$$?;\
-  	poetry lock --check || exit_code=$$?;\
 	mypy . || exit_code=$$?; \
 	ruff . || exit_code=$$?; \
 	black --check . || exit_code=$$?; \

{port_ocean-0.4.9 → port_ocean-0.4.10}/port_ocean/clients/port/mixins/entities.py

@@ -1,3 +1,4 @@
+import asyncio
 from urllib.parse import quote_plus
 
 import httpx
@@ -5,7 +6,10 @@ from loguru import logger
 
 from port_ocean.clients.port.authentication import PortAuthentication
 from port_ocean.clients.port.types import RequestOptions, UserAgentType
-from port_ocean.clients.port.utils import handle_status_code
+from port_ocean.clients.port.utils import (
+    handle_status_code,
+    PORT_HTTP_MAX_CONNECTIONS_LIMIT,
+)
 from port_ocean.core.models import Entity
 
 
@@ -13,6 +17,10 @@ class EntityClientMixin:
     def __init__(self, auth: PortAuthentication, client: httpx.AsyncClient):
         self.auth = auth
         self.client = client
+        # Semaphore is used to limit the number of concurrent requests to port, to avoid overloading it.
+        # The number of concurrent requests is set to 90% of the max connections limit, to leave some room for other
+        # requests that are not related to entities.
+        self.semaphore = asyncio.Semaphore(round(0.9 * PORT_HTTP_MAX_CONNECTIONS_LIMIT))
 
     async def upsert_entity(
         self,
@@ -22,24 +30,24 @@ class EntityClientMixin:
         should_raise: bool = True,
     ) -> None:
         validation_only = request_options["validation_only"]
-        logger.info(
-            f"{'Validating' if validation_only else 'Upserting'} entity: {entity.identifier} of blueprint: {entity.blueprint}"
-        )
-        headers = await self.auth.headers(user_agent_type)
-
-        response = await self.client.post(
-            f"{self.auth.api_url}/blueprints/{entity.blueprint}/entities",
-            json=entity.dict(exclude_unset=True, by_alias=True),
-            headers=headers,
-            params={
-                "upsert": "true",
-                "merge": str(request_options["merge"]).lower(),
-                "create_missing_related_entities": str(
-                    request_options["create_missing_related_entities"]
-                ).lower(),
-                "validation_only": str(validation_only).lower(),
-            },
-        )
+        async with self.semaphore:
+            logger.info(
+                f"{'Validating' if validation_only else 'Upserting'} entity: {entity.identifier} of blueprint: {entity.blueprint}"
+            )
+            headers = await self.auth.headers(user_agent_type)
+            response = await self.client.post(
+                f"{self.auth.api_url}/blueprints/{entity.blueprint}/entities",
+                json=entity.dict(exclude_unset=True, by_alias=True),
+                headers=headers,
+                params={
+                    "upsert": "true",
+                    "merge": str(request_options["merge"]).lower(),
+                    "create_missing_related_entities": str(
+                        request_options["create_missing_related_entities"]
+                    ).lower(),
+                    "validation_only": str(validation_only).lower(),
+                },
+            )
 
         if response.is_error:
             logger.error(
@@ -49,34 +57,81 @@ class EntityClientMixin:
             )
         handle_status_code(response, should_raise)
 
-    async def delete_entity(
+    async def batch_upsert_entities(
         self,
-        entity: Entity,
+        entities: list[Entity],
         request_options: RequestOptions,
         user_agent_type: UserAgentType | None = None,
         should_raise: bool = True,
     ) -> None:
-        logger.info(
-            f"Delete entity: {entity.identifier} of blueprint: {entity.blueprint}"
-        )
-        response = await self.client.delete(
-            f"{self.auth.api_url}/blueprints/{entity.blueprint}/entities/{quote_plus(entity.identifier)}",
-            headers=await self.auth.headers(user_agent_type),
-            params={
-                "delete_dependents": str(
-                    request_options["delete_dependent_entities"]
-                ).lower()
-            },
+        await asyncio.gather(
+            *(
+                self.upsert_entity(
+                    entity,
+                    request_options,
+                    user_agent_type,
+                    should_raise=should_raise,
+                )
+                for entity in entities
+            ),
+            return_exceptions=True,
         )
 
-        if response.is_error:
-            logger.error(
-                f"Error deleting "
-                f"entity: {entity.identifier} of "
-                f"blueprint: {entity.blueprint}"
+    async def delete_entity(
+        self,
+        entity: Entity,
+        request_options: RequestOptions,
+        user_agent_type: UserAgentType | None = None,
+        should_raise: bool = True,
+    ) -> None:
+        async with self.semaphore:
+            logger.info(
+                f"Delete entity: {entity.identifier} of blueprint: {entity.blueprint}"
+            )
+            response = await self.client.delete(
+                f"{self.auth.api_url}/blueprints/{entity.blueprint}/entities/{quote_plus(entity.identifier)}",
+                headers=await self.auth.headers(user_agent_type),
+                params={
+                    "delete_dependents": str(
+                        request_options["delete_dependent_entities"]
+                    ).lower()
+                },
             )
 
-        handle_status_code(response, should_raise)
+            if response.is_error:
+                if response.status_code == 404:
+                    logger.info(
+                        f"Weren't able to delete entity: {entity.identifier} of blueprint: {entity.blueprint},"
+                        f" as it was already deleted from port"
+                    )
+                    return
+                logger.error(
+                    f"Error deleting "
+                    f"entity: {entity.identifier} of "
+                    f"blueprint: {entity.blueprint}"
+                )
+
+            handle_status_code(response, should_raise)
+
+    async def batch_delete_entities(
+        self,
+        entities: list[Entity],
+        request_options: RequestOptions,
+        user_agent_type: UserAgentType | None = None,
+        should_raise: bool = True,
+    ) -> None:
+        await asyncio.gather(
+            *(
+                self.delete_entity(
+                    entity,
+                    request_options,
+                    user_agent_type,
+                    should_raise=should_raise,
+                )
+                for entity in entities
+            ),
+            return_exceptions=True,
+        )
 
     async def validate_entity_exist(self, identifier: str, blueprint: str) -> None:
         logger.info(f"Validating entity {identifier} of blueprint {blueprint} exists")

{port_ocean-0.4.9 → port_ocean-0.4.10}/port_ocean/clients/port/retry_transport.py

@@ -11,8 +11,8 @@ if TYPE_CHECKING:
 
 
 class TokenRetryTransport(RetryTransport):
-    def __init__(self, port_client: "PortClient", *args: Any, **kwargs: Any) -> None:
-        super().__init__(*args, **kwargs)
+    def __init__(self, port_client: "PortClient", **kwargs: Any) -> None:
+        super().__init__(**kwargs)
         self.port_client = port_client
 
     def _is_retryable_method(self, request: httpx.Request) -> bool:

{port_ocean-0.4.9 → port_ocean-0.4.10}/port_ocean/clients/port/utils.py

@@ -5,13 +5,11 @@ from loguru import logger
 from werkzeug.local import LocalStack, LocalProxy
 
 from port_ocean.clients.port.retry_transport import TokenRetryTransport
+from port_ocean.helpers.async_client import OceanAsyncClient
 
 if TYPE_CHECKING:
     from port_ocean.clients.port.client import PortClient
 
-_http_client: LocalStack[httpx.AsyncClient] = LocalStack()
-
-
 # In case the framework sends more requests to port in parallel then allowed by the limits, a PoolTimeout exception will
 # be raised.
 # Raising defaults for the timeout, in addition to the limits, will allow request to wait for a connection for a longer
@@ -19,23 +17,25 @@ _http_client: LocalStack[httpx.AsyncClient] = LocalStack()
 # We don't want to set the max_connections too highly, as it will cause the application to run out of memory.
 # We also don't want to set the max_keepalive_connections too highly, as it will cause the application to run out of
 # available connections.
-PORT_HTTP_CLIENT_TIMEOUT = httpx.Timeout(10.0)
-PORT_HTTP_CLIENT_CONNECTIONS_LIMIT = httpx.Limits(
-    max_connections=200, max_keepalive_connections=50
-)
+PORT_HTTP_MAX_CONNECTIONS_LIMIT = 200
+PORT_HTTP_MAX_KEEP_ALIVE_CONNECTIONS = 50
+PORT_HTTP_TIMEOUT = 10.0
+
+
+_http_client: LocalStack[httpx.AsyncClient] = LocalStack()
 
 
 def _get_http_client_context(port_client: "PortClient") -> httpx.AsyncClient:
     client = _http_client.top
     if client is None:
-        client = httpx.AsyncClient(
-            transport=TokenRetryTransport(
-                port_client,
-                httpx.AsyncHTTPTransport(),
-                logger=logger,
+        client = OceanAsyncClient(
+            TokenRetryTransport,
+            transport_kwargs={"port_client": port_client},
+            timeout=httpx.Timeout(PORT_HTTP_TIMEOUT),
+            limits=httpx.Limits(
+                max_connections=PORT_HTTP_MAX_CONNECTIONS_LIMIT,
+                max_keepalive_connections=PORT_HTTP_MAX_KEEP_ALIVE_CONNECTIONS,
             ),
-            timeout=PORT_HTTP_CLIENT_TIMEOUT,
-            limits=PORT_HTTP_CLIENT_CONNECTIONS_LIMIT,
         )
         _http_client.push(client)
 

{port_ocean-0.4.9 → port_ocean-0.4.10}/port_ocean/core/handlers/entities_state_applier/port/applier.py

@@ -158,17 +158,11 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
     ) -> None:
         logger.info(f"Upserting {len(entities)} entities")
         if event.port_app_config.create_missing_related_entities:
-            await asyncio.gather(
-                *(
-                    self.context.port_client.upsert_entity(
-                        entity,
-                        event.port_app_config.get_port_request_options(),
-                        user_agent_type,
-                        should_raise=False,
-                    )
-                    for entity in entities
-                ),
-                return_exceptions=True,
+            await self.context.port_client.batch_upsert_entities(
+                entities,
+                event.port_app_config.get_port_request_options(),
+                user_agent_type,
+                should_raise=False,
             )
         else:
             ordered_created_entities = reversed(
@@ -188,16 +182,11 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
     ) -> None:
         logger.info(f"Deleting {len(entities)} entities")
         if event.port_app_config.delete_dependent_entities:
-            await asyncio.gather(
-                *(
-                    self.context.port_client.delete_entity(
-                        entity,
-                        event.port_app_config.get_port_request_options(),
-                        user_agent_type,
-                        should_raise=False,
-                    )
-                    for entity in entities
-                )
+            await self.context.port_client.batch_delete_entities(
+                entities,
+                event.port_app_config.get_port_request_options(),
+                user_agent_type,
+                should_raise=False,
             )
         else:
             ordered_deleted_entities = order_by_entities_dependencies(entities)

port_ocean-0.4.10/port_ocean/helpers/async_client.py

@@ -0,0 +1,53 @@
+from typing import Any, Callable, Type
+
+import httpx
+from loguru import logger
+
+from port_ocean.helpers.retry import RetryTransport
+
+
+class OceanAsyncClient(httpx.AsyncClient):
+    """
+    This class is a wrapper around httpx.AsyncClient that uses a custom transport class.
+    This is done to allow passing our custom transport class to the AsyncClient constructor while still allowing
+    all the default AsyncClient behavior that is changed when passing a custom transport instance.
+    """
+
+    def __init__(
+        self,
+        transport_class: Type[RetryTransport] = RetryTransport,
+        transport_kwargs: dict[str, Any] | None = None,
+        **kwargs: Any,
+    ):
+        self._transport_kwargs = transport_kwargs
+        self._transport_class = transport_class
+        super().__init__(**kwargs)
+
+    def _init_transport(  # type: ignore[override]
+        self,
+        transport: httpx.AsyncBaseTransport | None = None,
+        app: Callable[..., Any] | None = None,
+        **kwargs: Any,
+    ) -> httpx.AsyncBaseTransport:
+        if transport is not None or app is not None:
+            return super()._init_transport(transport=transport, app=app, **kwargs)
+
+        return self._transport_class(
+            wrapped_transport=httpx.AsyncHTTPTransport(
+                **kwargs,
+            ),
+            logger=logger,
+            **(self._transport_kwargs or {}),
+        )
+
+    def _init_proxy_transport(  # type: ignore[override]
+        self, proxy: httpx.Proxy, **kwargs: Any
+    ) -> httpx.AsyncBaseTransport:
+        return self._transport_class(
+            wrapped_transport=httpx.AsyncHTTPTransport(
+                proxy=proxy,
+                **kwargs,
+            ),
+            logger=logger,
+            **(self._transport_kwargs or {}),
+        )

{port_ocean-0.4.9 → port_ocean-0.4.10}/port_ocean/utils.py

@@ -3,6 +3,7 @@ import inspect
 from asyncio import ensure_future
 from functools import wraps
 from importlib.util import module_from_spec, spec_from_file_location
+from pathlib import Path
 from time import time
 from traceback import format_exception
 from types import ModuleType
@@ -13,10 +14,10 @@ import httpx
 import tomli
 import yaml
 from loguru import logger
-from pathlib import Path
 from starlette.concurrency import run_in_threadpool
 from werkzeug.local import LocalStack, LocalProxy
 
+from port_ocean.helpers.async_client import OceanAsyncClient
 from port_ocean.helpers.retry import RetryTransport
 
 _http_client: LocalStack[httpx.AsyncClient] = LocalStack()
@@ -25,12 +26,7 @@ _http_client: LocalStack[httpx.AsyncClient] = LocalStack()
 def _get_http_client_context() -> httpx.AsyncClient:
     client = _http_client.top
     if client is None:
-        client = httpx.AsyncClient(
-            transport=RetryTransport(
-                httpx.AsyncHTTPTransport(),
-                logger=logger,
-            )
-        )
+        client = OceanAsyncClient(RetryTransport)
         _http_client.push(client)
 
     return client

{port_ocean-0.4.9 → port_ocean-0.4.10}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "port-ocean"
-version = "0.4.9"
+version = "0.4.10"
 description = "Port Ocean is a CLI tool for managing your Port projects."
 readme = "README.md"
 homepage = "https://app.getport.io"