port-ocean 0.4.9__py3-none-any.whl → 0.4.10__py3-none-any.whl

port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/Makefile

@@ -4,7 +4,6 @@ define run_checks
 	exit_code=0; \
 	cd $1; \
 	poetry check || exit_code=$$?;\
-  	poetry lock --check || exit_code=$$?;\
 	mypy . || exit_code=$$?; \
 	ruff . || exit_code=$$?; \
 	black --check . || exit_code=$$?; \

port_ocean/clients/port/mixins/entities.py

@@ -1,3 +1,4 @@
+import asyncio
 from urllib.parse import quote_plus
 
 import httpx
@@ -5,7 +6,10 @@ from loguru import logger
 
 from port_ocean.clients.port.authentication import PortAuthentication
 from port_ocean.clients.port.types import RequestOptions, UserAgentType
-from port_ocean.clients.port.utils import handle_status_code
+from port_ocean.clients.port.utils import (
+    handle_status_code,
+    PORT_HTTP_MAX_CONNECTIONS_LIMIT,
+)
 from port_ocean.core.models import Entity
 
 
@@ -13,6 +17,10 @@ class EntityClientMixin:
     def __init__(self, auth: PortAuthentication, client: httpx.AsyncClient):
         self.auth = auth
         self.client = client
+        # Semaphore is used to limit the number of concurrent requests to port, to avoid overloading it.
+        # The number of concurrent requests is set to 90% of the max connections limit, to leave some room for other
+        # requests that are not related to entities.
+        self.semaphore = asyncio.Semaphore(round(0.9 * PORT_HTTP_MAX_CONNECTIONS_LIMIT))
 
     async def upsert_entity(
         self,
@@ -22,24 +30,24 @@ class EntityClientMixin:
         should_raise: bool = True,
     ) -> None:
         validation_only = request_options["validation_only"]
-        logger.info(
-            f"{'Validating' if validation_only else 'Upserting'} entity: {entity.identifier} of blueprint: {entity.blueprint}"
-        )
-        headers = await self.auth.headers(user_agent_type)
-
-        response = await self.client.post(
-            f"{self.auth.api_url}/blueprints/{entity.blueprint}/entities",
-            json=entity.dict(exclude_unset=True, by_alias=True),
-            headers=headers,
-            params={
-                "upsert": "true",
-                "merge": str(request_options["merge"]).lower(),
-                "create_missing_related_entities": str(
-                    request_options["create_missing_related_entities"]
-                ).lower(),
-                "validation_only": str(validation_only).lower(),
-            },
-        )
+        async with self.semaphore:
+            logger.info(
+                f"{'Validating' if validation_only else 'Upserting'} entity: {entity.identifier} of blueprint: {entity.blueprint}"
+            )
+            headers = await self.auth.headers(user_agent_type)
+            response = await self.client.post(
+                f"{self.auth.api_url}/blueprints/{entity.blueprint}/entities",
+                json=entity.dict(exclude_unset=True, by_alias=True),
+                headers=headers,
+                params={
+                    "upsert": "true",
+                    "merge": str(request_options["merge"]).lower(),
+                    "create_missing_related_entities": str(
+                        request_options["create_missing_related_entities"]
+                    ).lower(),
+                    "validation_only": str(validation_only).lower(),
+                },
+            )
 
         if response.is_error:
             logger.error(
@@ -49,34 +57,81 @@ class EntityClientMixin:
             )
         handle_status_code(response, should_raise)
 
-    async def delete_entity(
+    async def batch_upsert_entities(
         self,
-        entity: Entity,
+        entities: list[Entity],
         request_options: RequestOptions,
         user_agent_type: UserAgentType | None = None,
         should_raise: bool = True,
     ) -> None:
-        logger.info(
-            f"Delete entity: {entity.identifier} of blueprint: {entity.blueprint}"
-        )
-        response = await self.client.delete(
-            f"{self.auth.api_url}/blueprints/{entity.blueprint}/entities/{quote_plus(entity.identifier)}",
-            headers=await self.auth.headers(user_agent_type),
-            params={
-                "delete_dependents": str(
-                    request_options["delete_dependent_entities"]
-                ).lower()
-            },
+        await asyncio.gather(
+            *(
+                self.upsert_entity(
+                    entity,
+                    request_options,
+                    user_agent_type,
+                    should_raise=should_raise,
+                )
+                for entity in entities
+            ),
+            return_exceptions=True,
         )
 
-        if response.is_error:
-            logger.error(
-                f"Error deleting "
-                f"entity: {entity.identifier} of "
-                f"blueprint: {entity.blueprint}"
+    async def delete_entity(
+        self,
+        entity: Entity,
+        request_options: RequestOptions,
+        user_agent_type: UserAgentType | None = None,
+        should_raise: bool = True,
+    ) -> None:
+        async with self.semaphore:
+            logger.info(
+                f"Delete entity: {entity.identifier} of blueprint: {entity.blueprint}"
+            )
+            response = await self.client.delete(
+                f"{self.auth.api_url}/blueprints/{entity.blueprint}/entities/{quote_plus(entity.identifier)}",
+                headers=await self.auth.headers(user_agent_type),
+                params={
+                    "delete_dependents": str(
+                        request_options["delete_dependent_entities"]
+                    ).lower()
+                },
             )
 
-        handle_status_code(response, should_raise)
+            if response.is_error:
+                if response.status_code == 404:
+                    logger.info(
+                        f"Weren't able to delete entity: {entity.identifier} of blueprint: {entity.blueprint},"
+                        f" as it was already deleted from port"
+                    )
+                    return
+                logger.error(
+                    f"Error deleting "
+                    f"entity: {entity.identifier} of "
+                    f"blueprint: {entity.blueprint}"
+                )
+
+            handle_status_code(response, should_raise)
+
+    async def batch_delete_entities(
+        self,
+        entities: list[Entity],
+        request_options: RequestOptions,
+        user_agent_type: UserAgentType | None = None,
+        should_raise: bool = True,
+    ) -> None:
+        await asyncio.gather(
+            *(
+                self.delete_entity(
+                    entity,
+                    request_options,
+                    user_agent_type,
+                    should_raise=should_raise,
+                )
+                for entity in entities
+            ),
+            return_exceptions=True,
+        )
 
     async def validate_entity_exist(self, identifier: str, blueprint: str) -> None:
         logger.info(f"Validating entity {identifier} of blueprint {blueprint} exists")

port_ocean/clients/port/retry_transport.py

@@ -11,8 +11,8 @@ if TYPE_CHECKING:
 
 
 class TokenRetryTransport(RetryTransport):
-    def __init__(self, port_client: "PortClient", *args: Any, **kwargs: Any) -> None:
-        super().__init__(*args, **kwargs)
+    def __init__(self, port_client: "PortClient", **kwargs: Any) -> None:
+        super().__init__(**kwargs)
         self.port_client = port_client
 
     def _is_retryable_method(self, request: httpx.Request) -> bool:

port_ocean/clients/port/utils.py

@@ -5,13 +5,11 @@ from loguru import logger
 from werkzeug.local import LocalStack, LocalProxy
 
 from port_ocean.clients.port.retry_transport import TokenRetryTransport
+from port_ocean.helpers.async_client import OceanAsyncClient
 
 if TYPE_CHECKING:
     from port_ocean.clients.port.client import PortClient
 
-_http_client: LocalStack[httpx.AsyncClient] = LocalStack()
-
-
 # In case the framework sends more requests to port in parallel then allowed by the limits, a PoolTimeout exception will
 # be raised.
 # Raising defaults for the timeout, in addition to the limits, will allow request to wait for a connection for a longer
@@ -19,23 +17,25 @@ _http_client: LocalStack[httpx.AsyncClient] = LocalStack()
 # We don't want to set the max_connections too highly, as it will cause the application to run out of memory.
 # We also don't want to set the max_keepalive_connections too highly, as it will cause the application to run out of
 # available connections.
-PORT_HTTP_CLIENT_TIMEOUT = httpx.Timeout(10.0)
-PORT_HTTP_CLIENT_CONNECTIONS_LIMIT = httpx.Limits(
-    max_connections=200, max_keepalive_connections=50
-)
+PORT_HTTP_MAX_CONNECTIONS_LIMIT = 200
+PORT_HTTP_MAX_KEEP_ALIVE_CONNECTIONS = 50
+PORT_HTTP_TIMEOUT = 10.0
+
+
+_http_client: LocalStack[httpx.AsyncClient] = LocalStack()
 
 
 def _get_http_client_context(port_client: "PortClient") -> httpx.AsyncClient:
     client = _http_client.top
     if client is None:
-        client = httpx.AsyncClient(
-            transport=TokenRetryTransport(
-                port_client,
-                httpx.AsyncHTTPTransport(),
-                logger=logger,
+        client = OceanAsyncClient(
+            TokenRetryTransport,
+            transport_kwargs={"port_client": port_client},
+            timeout=httpx.Timeout(PORT_HTTP_TIMEOUT),
+            limits=httpx.Limits(
+                max_connections=PORT_HTTP_MAX_CONNECTIONS_LIMIT,
+                max_keepalive_connections=PORT_HTTP_MAX_KEEP_ALIVE_CONNECTIONS,
             ),
-            timeout=PORT_HTTP_CLIENT_TIMEOUT,
-            limits=PORT_HTTP_CLIENT_CONNECTIONS_LIMIT,
         )
         _http_client.push(client)
 

port_ocean/core/handlers/entities_state_applier/port/applier.py

@@ -158,17 +158,11 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
     ) -> None:
         logger.info(f"Upserting {len(entities)} entities")
         if event.port_app_config.create_missing_related_entities:
-            await asyncio.gather(
-                *(
-                    self.context.port_client.upsert_entity(
-                        entity,
-                        event.port_app_config.get_port_request_options(),
-                        user_agent_type,
-                        should_raise=False,
-                    )
-                    for entity in entities
-                ),
-                return_exceptions=True,
+            await self.context.port_client.batch_upsert_entities(
+                entities,
+                event.port_app_config.get_port_request_options(),
+                user_agent_type,
+                should_raise=False,
             )
         else:
             ordered_created_entities = reversed(
@@ -188,16 +182,11 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
     ) -> None:
         logger.info(f"Deleting {len(entities)} entities")
         if event.port_app_config.delete_dependent_entities:
-            await asyncio.gather(
-                *(
-                    self.context.port_client.delete_entity(
-                        entity,
-                        event.port_app_config.get_port_request_options(),
-                        user_agent_type,
-                        should_raise=False,
-                    )
-                    for entity in entities
-                )
+            await self.context.port_client.batch_delete_entities(
+                entities,
+                event.port_app_config.get_port_request_options(),
+                user_agent_type,
+                should_raise=False,
             )
         else:
             ordered_deleted_entities = order_by_entities_dependencies(entities)

port_ocean/helpers/async_client.py

@@ -0,0 +1,53 @@
+from typing import Any, Callable, Type
+
+import httpx
+from loguru import logger
+
+from port_ocean.helpers.retry import RetryTransport
+
+
+class OceanAsyncClient(httpx.AsyncClient):
+    """
+    This class is a wrapper around httpx.AsyncClient that uses a custom transport class.
+    This is done to allow passing our custom transport class to the AsyncClient constructor while still allowing
+    all the default AsyncClient behavior that is changed when passing a custom transport instance.
+    """
+
+    def __init__(
+        self,
+        transport_class: Type[RetryTransport] = RetryTransport,
+        transport_kwargs: dict[str, Any] | None = None,
+        **kwargs: Any,
+    ):
+        self._transport_kwargs = transport_kwargs
+        self._transport_class = transport_class
+        super().__init__(**kwargs)
+
+    def _init_transport(  # type: ignore[override]
+        self,
+        transport: httpx.AsyncBaseTransport | None = None,
+        app: Callable[..., Any] | None = None,
+        **kwargs: Any,
+    ) -> httpx.AsyncBaseTransport:
+        if transport is not None or app is not None:
+            return super()._init_transport(transport=transport, app=app, **kwargs)
+
+        return self._transport_class(
+            wrapped_transport=httpx.AsyncHTTPTransport(
+                **kwargs,
+            ),
+            logger=logger,
+            **(self._transport_kwargs or {}),
+        )
+
+    def _init_proxy_transport(  # type: ignore[override]
+        self, proxy: httpx.Proxy, **kwargs: Any
+    ) -> httpx.AsyncBaseTransport:
+        return self._transport_class(
+            wrapped_transport=httpx.AsyncHTTPTransport(
+                proxy=proxy,
+                **kwargs,
+            ),
+            logger=logger,
+            **(self._transport_kwargs or {}),
+        )

port_ocean/utils.py

@@ -3,6 +3,7 @@ import inspect
 from asyncio import ensure_future
 from functools import wraps
 from importlib.util import module_from_spec, spec_from_file_location
+from pathlib import Path
 from time import time
 from traceback import format_exception
 from types import ModuleType
@@ -13,10 +14,10 @@ import httpx
 import tomli
 import yaml
 from loguru import logger
-from pathlib import Path
 from starlette.concurrency import run_in_threadpool
 from werkzeug.local import LocalStack, LocalProxy
 
+from port_ocean.helpers.async_client import OceanAsyncClient
 from port_ocean.helpers.retry import RetryTransport
 
 _http_client: LocalStack[httpx.AsyncClient] = LocalStack()
@@ -25,12 +26,7 @@ _http_client: LocalStack[httpx.AsyncClient] = LocalStack()
 def _get_http_client_context() -> httpx.AsyncClient:
     client = _http_client.top
     if client is None:
-        client = httpx.AsyncClient(
-            transport=RetryTransport(
-                httpx.AsyncHTTPTransport(),
-                logger=logger,
-            )
-        )
+        client = OceanAsyncClient(RetryTransport)
         _http_client.push(client)
 
     return client

{port_ocean-0.4.9.dist-info → port_ocean-0.4.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.4.9
+Version: 0.4.10
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.4.9.dist-info → port_ocean-0.4.10.dist-info}/RECORD

@@ -23,7 +23,7 @@ port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/resources/.g
 port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/spec.yaml,sha256=Yq06gYoC6jFWES5mxlvJGFTGXbfD7E9R8j_PDbVIM3M,497
 port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/CHANGELOG.md,sha256=nzAmB0Bjnd2eZo79OjrlyVOdpTBHTmTxvO7c2C8Q-VQ,292
 port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/Dockerfile,sha256=Hh1dBnL959V2n28pmqFpXSrNvSMQjX6fDCUos8ITiu0,326
-port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/Makefile,sha256=uTm8iPDPwf7bmhNkIi8cyHxoTnlBREDfiXNqOEPjgoo,1815
+port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/Makefile,sha256=HdDAoHiL0unYoMubrfngUA9GrweJny8jdTDqYFcNnTY,1773
 port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/README.md,sha256=5VZmgDRW9gO4d8UuzkujslOIDfIDBiAGL2Hd74HK770,468
 port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/changelog/.gitignore,sha256=kCpRPdl3S_jqYYZaOrc0-xa6-l3KqVjNRXc6jCkd_-Q,12
 port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/config.yaml,sha256=zKaBQNCbWEE3MFxDHaMn9NIeBGVPiZUV6cBjH35f2kw,906
@@ -39,12 +39,12 @@ port_ocean/clients/port/authentication.py,sha256=DXqZQaYUb2mhWRZXoTivteEyQPTsHtJ
 port_ocean/clients/port/client.py,sha256=3GYCM0ZkX3pB6sNoOb-7_6dm0Jr5_vqhflD9iltf_As,2640
 port_ocean/clients/port/mixins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/clients/port/mixins/blueprints.py,sha256=vp7kVzr704fCcp6Cvuwv4LR1GeM_Jh6FpaycLe9mv20,3794
-port_ocean/clients/port/mixins/entities.py,sha256=VA-pVLYBPP933EivK-YhOmCMTG0UhEP85SZta4ifWjA,5810
+port_ocean/clients/port/mixins/entities.py,sha256=j1E3I_opymxUisP1FMpj-sbOcMV0Mhc7_YnfyGpcYzU,7908
 port_ocean/clients/port/mixins/integrations.py,sha256=FuDoQfQnfJvhK-nKpftQ8k7VsB9odNRlkMkGCmxwaWY,4280
 port_ocean/clients/port/mixins/migrations.py,sha256=M93i_aryfDazRHjkpTS3-sV3UshLCxmsv6yAIOziFl8,1463
-port_ocean/clients/port/retry_transport.py,sha256=6p64Ek8vpe0hHVcrNIXij8KQ9K2zwPuMUnJp2R8ZYUI,2075
+port_ocean/clients/port/retry_transport.py,sha256=J25q5QpSYSVsVa24Sr1IvXu-KpYL88Pd9xe3xvYWYsg,2056
 port_ocean/clients/port/types.py,sha256=nvlgiAq4WH5_F7wQbz_GAWl-faob84LVgIjZ2Ww5mTk,451
-port_ocean/clients/port/utils.py,sha256=x20rtGgnYodklU8AcndF93fI4R58ggExxc9gvpiVeq0,2273
+port_ocean/clients/port/utils.py,sha256=G4xvkjZYCPWaxh96ylVmZzDkfULkcw9K9ebxUQhgLMA,2360
 port_ocean/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/config/base.py,sha256=e1oW66ynjo7Sp2PcLBp9aCzj59JpTaiKUXVJK8VB8TA,5540
 port_ocean/config/dynamic.py,sha256=CIRDnqFUPSnNMLZ-emRCMVAjEQNBlIujhZ7OGwi_aKs,1816
@@ -74,7 +74,7 @@ port_ocean/core/handlers/base.py,sha256=cTarblazu8yh8xz2FpB-dzDKuXxtoi143XJgPbV_
 port_ocean/core/handlers/entities_state_applier/__init__.py,sha256=kgLZDCeCEzi4r-0nzW9k78haOZNf6PX7mJOUr34A4c8,173
 port_ocean/core/handlers/entities_state_applier/base.py,sha256=FMsrBOVgaO4o7B1klLDY8fobTUDvyrerCKCICyYtkXs,2193
 port_ocean/core/handlers/entities_state_applier/port/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-port_ocean/core/handlers/entities_state_applier/port/applier.py,sha256=SFdj1elXDMnoxqfGQDu5osiTX40k6vxhcBF9-ljklow,8197
+port_ocean/core/handlers/entities_state_applier/port/applier.py,sha256=PyBtze8Xvhu2994B8YAxoB0ci-FPGAuYblvw2Wnxrok,7836
 port_ocean/core/handlers/entities_state_applier/port/get_related_entities.py,sha256=1zncwCbE-Gej0xaWKlzZgoXxOBe9bgs_YxlZ8QW3NdI,1751
 port_ocean/core/handlers/entities_state_applier/port/order_by_entities_dependencies.py,sha256=82BvU8t5w9uhsxX8hbnwuRPuWhW3cMeuT_5sVIkip1I,1550
 port_ocean/core/handlers/entities_state_applier/port/validate_entity_relations.py,sha256=nKuQ-RlalGG07olxm6l5NHeOuQT9dEZLoMpD-AN5nq0,1392
@@ -104,16 +104,17 @@ port_ocean/exceptions/context.py,sha256=qC1OLppiv6XZuXSHRO2P_VPFSzrc_uUQNvfSYLeA
 port_ocean/exceptions/core.py,sha256=ygxtPQ9IG8NzIrzZok5OqkefVrqcC4bjZ-2Vf9IPZuA,790
 port_ocean/exceptions/port_defaults.py,sha256=R3ufJcfllb7NZSwHOpBs8kbjsIZVQLM6vKO6dz4w-EE,407
 port_ocean/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+port_ocean/helpers/async_client.py,sha256=SRlP6o7_FCSY3UHnRlZdezppePVxxOzZ0z861vE3K40,1783
 port_ocean/helpers/retry.py,sha256=xrkXeVBHk1yIM6yVse5DpDZJMUj310ZpXqHFZFzCFRM,13171
 port_ocean/logger_setup.py,sha256=dOA58ZBAfe9dcAoBybXFWgDfsJG-uIX1ABI0r2CyUG8,1160
 port_ocean/middlewares.py,sha256=8rGu9XSKvbNCQGzWvfaijDrp-0ATJrWAQfBji2CnSck,2475
 port_ocean/ocean.py,sha256=9dFBf46N6fht6ibQUjnDDizY-p0qa0Uw-415JTDQmus,3626
 port_ocean/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/run.py,sha256=xvYZlfi-J-IcqsAg8tNVnvl1mEUr6wPSya_-Bbf6jAU,1811
-port_ocean/utils.py,sha256=ZQCdOk1DzAA3hwUxFAzmbxQurnNYGpl1bDsPocJSkko,6129
+port_ocean/utils.py,sha256=VcDhnqFwUYCPr9EZFtg5M2CIUC5Y4nN1Bv7gmCF4VYc,6067
 port_ocean/version.py,sha256=2ugCk8TXPsRIuFviZ8j3RPaszSw2HE-KuaW8vhgWJVM,172
-port_ocean-0.4.9.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-port_ocean-0.4.9.dist-info/METADATA,sha256=r0Wm43wriHPQmVlQVnafGaTrP7p-8oMQynYV6WY8s7M,6490
-port_ocean-0.4.9.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-port_ocean-0.4.9.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
-port_ocean-0.4.9.dist-info/RECORD,,
+port_ocean-0.4.10.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+port_ocean-0.4.10.dist-info/METADATA,sha256=KZ3bZbJt_-KBrbM6NHjppTX0r2wpbhaMuXEJ8W-xFiI,6491
+port_ocean-0.4.10.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+port_ocean-0.4.10.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
+port_ocean-0.4.10.dist-info/RECORD,,