port-ocean 0.5.10__py3-none-any.whl → 0.5.14__py3-none-any.whl

port_ocean/clients/port/authentication.py

@@ -49,7 +49,9 @@ class PortAuthentication:
 
         credentials = {"clientId": client_id, "clientSecret": client_secret}
         response = await self.client.post(
-            f"{self.api_url}/auth/access_token", json=credentials
+            f"{self.api_url}/auth/access_token",
+            json=credentials,
+            extensions={"retryable": True},
         )
         handle_status_code(response)
         return TokenResponse(**response.json())

port_ocean/clients/port/mixins/entities.py

@@ -1,4 +1,5 @@
 import asyncio
+from typing import Any
 from urllib.parse import quote_plus
 
 import httpx
@@ -133,23 +134,10 @@ class EntityClientMixin:
             return_exceptions=True,
         )
 
-    async def validate_entity_exist(self, identifier: str, blueprint: str) -> None:
-        logger.info(f"Validating entity {identifier} of blueprint {blueprint} exists")
-
-        response = await self.client.get(
-            f"{self.auth.api_url}/blueprints/{blueprint}/entities/{identifier}",
-            headers=await self.auth.headers(),
-        )
-        if response.is_error:
-            logger.error(
-                f"Error validating "
-                f"entity: {identifier} of "
-                f"blueprint: {blueprint}"
-            )
-        handle_status_code(response)
-
-    async def search_entities(self, user_agent_type: UserAgentType) -> list[Entity]:
-        query = {
+    async def search_entities(
+        self, user_agent_type: UserAgentType, query: dict[Any, Any] | None = None
+    ) -> list[Entity]:
+        default_query = {
             "combinator": "and",
             "rules": [
                 {
@@ -165,6 +153,11 @@ class EntityClientMixin:
             ],
         }
 
+        if query is None:
+            query = default_query
+        elif query.get("rules"):
+            query["rules"].append(default_query)
+
         logger.info(f"Searching entities with query {query}")
         response = await self.client.post(
             f"{self.auth.api_url}/entities/search",
@@ -174,43 +167,39 @@ class EntityClientMixin:
                 "exclude_calculated_properties": "true",
                 "include": ["blueprint", "identifier"],
             },
+            extensions={"retryable": True},
+            timeout=30,
         )
         handle_status_code(response)
         return [Entity.parse_obj(result) for result in response.json()["entities"]]
 
-    async def search_dependent_entities(self, entity: Entity) -> list[Entity]:
-        body = {
-            "combinator": "and",
-            "rules": [
+    async def search_batch_entities(
+        self, user_agent_type: UserAgentType, entities_to_search: list[Entity]
+    ) -> list[Entity]:
+        search_rules = []
+        for entity in entities_to_search:
+            search_rules.append(
                 {
-                    "operator": "relatedTo",
-                    "blueprint": entity.blueprint,
-                    "value": entity.identifier,
-                    "direction": "downstream",
+                    "combinator": "and",
+                    "rules": [
+                        {
+                            "property": "$identifier",
+                            "operator": "=",
+                            "value": entity.identifier,
+                        },
+                        {
+                            "property": "$blueprint",
+                            "operator": "=",
+                            "value": entity.blueprint,
+                        },
+                    ],
                 }
-            ],
-        }
-
-        logger.info(f"Searching dependent entity with body {body}")
-        response = await self.client.post(
-            f"{self.auth.api_url}/entities/search",
-            headers=await self.auth.headers(),
-            json=body,
-        )
-        handle_status_code(response)
-
-        return [Entity.parse_obj(result) for result in response.json()["entities"]]
+            )
 
-    async def validate_entity_payload(
-        self, entity: Entity, merge: bool, create_missing_related_entities: bool
-    ) -> None:
-        logger.info(f"Validating entity {entity.identifier}")
-        await self.upsert_entity(
-            entity,
+        return await self.search_entities(
+            user_agent_type,
             {
-                "merge": merge,
-                "create_missing_related_entities": create_missing_related_entities,
-                "delete_dependent_entities": False,
-                "validation_only": True,
+                "combinator": "and",
+                "rules": [{"combinator": "or", "rules": search_rules}],
             },
         )

port_ocean/clients/port/retry_transport.py

@@ -15,11 +15,6 @@ class TokenRetryTransport(RetryTransport):
         super().__init__(**kwargs)
         self.port_client = port_client
 
-    def _is_retryable_method(self, request: httpx.Request) -> bool:
-        return super()._is_retryable_method(request) or request.url.path.endswith(
-            "/auth/access_token"
-        )
-
     async def _handle_unauthorized(self, response: httpx.Response) -> None:
         token = await self.port_client.auth.token
         response.headers["Authorization"] = f"Bearer {token}"

port_ocean/core/handlers/entities_state_applier/port/applier.py

@@ -1,6 +1,3 @@
-import asyncio
-from itertools import chain
-
 from loguru import logger
 
 from port_ocean.clients.port.types import UserAgentType
@@ -14,14 +11,9 @@ from port_ocean.core.handlers.entities_state_applier.port.get_related_entities i
 from port_ocean.core.handlers.entities_state_applier.port.order_by_entities_dependencies import (
     order_by_entities_dependencies,
 )
-from port_ocean.core.handlers.entities_state_applier.port.validate_entity_relations import (
-    validate_entity_relations,
-)
-from port_ocean.core.handlers.entity_processor.base import EntityPortDiff
 from port_ocean.core.models import Entity
 from port_ocean.core.ocean_types import EntityDiff
-from port_ocean.core.utils import is_same_entity, get_unique, get_port_diff
-from port_ocean.exceptions.core import RelationValidationException
+from port_ocean.core.utils import is_same_entity, get_port_diff
 
 
 class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
@@ -32,63 +24,17 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
     through HTTP requests.
     """
 
-    async def _validate_delete_dependent_entities(self, entities: list[Entity]) -> None:
-        logger.info("Validated deleted entities")
-        if not event.port_app_config.delete_dependent_entities:
-            dependent_entities = await asyncio.gather(
-                *(
-                    self.context.port_client.search_dependent_entities(entity)
-                    for entity in entities
-                )
-            )
-            new_dependent_entities = get_unique(
-                [
-                    entity
-                    for entity in chain.from_iterable(dependent_entities)
-                    if not any(is_same_entity(item, entity) for item in entities)
-                ]
-            )
-
-            if new_dependent_entities:
-                raise RelationValidationException(
-                    f"Must enable delete_dependent_entities flag or delete all dependent entities: "
-                    f" {[(dep.blueprint, dep.identifier) for dep in new_dependent_entities]}"
-                )
-
-    async def _validate_entity_diff(self, diff: EntityPortDiff) -> None:
-        config = event.port_app_config
-        await self._validate_delete_dependent_entities(diff.deleted)
-        modified_or_created_entities = diff.modified + diff.created
-
-        if modified_or_created_entities and not config.create_missing_related_entities:
-            logger.info("Validating modified or created entities")
-
-            await asyncio.gather(
-                *(
-                    self.context.port_client.validate_entity_payload(
-                        entity,
-                        config.enable_merge_entity,
-                        create_missing_related_entities=config.create_missing_related_entities,
-                    )
-                    for entity in modified_or_created_entities
-                )
-            )
-
-        if not event.port_app_config.delete_dependent_entities:
-            logger.info("Validating no relation blocks the operation")
-            await validate_entity_relations(diff, self.context.port_client)
-
-    async def _delete_diff(
+    async def _safe_delete(
         self,
         entities_to_delete: list[Entity],
-        created_entities: list[Entity],
+        entities_to_protect: list[Entity],
         user_agent_type: UserAgentType,
     ) -> None:
         if not entities_to_delete:
             return
 
         related_entities = await get_related_entities(
-            created_entities, self.context.port_client
+            entities_to_protect, self.context.port_client
         )
 
         allowed_entities_to_delete = []
@@ -98,7 +44,8 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
                 is_same_entity(entity, entity_to_delete) for entity in related_entities
             )
             is_part_of_created = any(
-                is_same_entity(entity, entity_to_delete) for entity in created_entities
+                is_same_entity(entity, entity_to_delete)
+                for entity in entities_to_protect
             )
             if is_part_of_related:
                 if event.port_app_config.create_missing_related_entities:
@@ -119,21 +66,14 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
         user_agent_type: UserAgentType,
     ) -> None:
         diff = get_port_diff(entities["before"], entities["after"])
+        kept_entities = diff.created + diff.modified
 
         logger.info(
             f"Updating entity diff (created: {len(diff.created)}, deleted: {len(diff.deleted)}, modified: {len(diff.modified)})"
         )
-        await self._validate_entity_diff(diff)
+        await self.upsert(kept_entities, user_agent_type)
 
-        logger.info("Upserting new entities")
-        await self.upsert(diff.created, user_agent_type)
-        logger.info("Upserting modified entities")
-        await self.upsert(diff.modified, user_agent_type)
-
-        logger.info("Deleting diff entities")
-        await self._delete_diff(
-            diff.deleted, diff.created + diff.modified, user_agent_type
-        )
+        await self._safe_delete(diff.deleted, kept_entities, user_agent_type)
 
     async def delete_diff(
         self,
@@ -145,15 +85,13 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
         if not diff.deleted:
             return
 
+        kept_entities = diff.created + diff.modified
+
         logger.info(
-            f"Updating entity diff (created: {len(diff.created)}, deleted: {len(diff.deleted)}, modified: {len(diff.modified)})"
+            f"Determining entities to delete ({len(diff.deleted)}/{len(kept_entities)})"
         )
-        await self._validate_entity_diff(diff)
 
-        logger.info("Deleting diff entities")
-        await self._delete_diff(
-            diff.deleted, diff.created + diff.modified, user_agent_type
-        )
+        await self._safe_delete(diff.deleted, kept_entities, user_agent_type)
 
     async def upsert(
         self, entities: list[Entity], user_agent_type: UserAgentType

port_ocean/core/handlers/entity_processor/base.py

@@ -6,7 +6,10 @@ from loguru import logger
 from port_ocean.core.handlers.base import BaseHandler
 from port_ocean.core.handlers.port_app_config.models import ResourceConfig
 from port_ocean.core.models import Entity
-from port_ocean.core.ocean_types import RawEntityDiff, EntityDiff
+from port_ocean.core.ocean_types import (
+    RAW_ITEM,
+    EntitySelectorDiff,
+)
 
 
 @dataclass
@@ -33,21 +36,28 @@ class BaseEntityProcessor(BaseHandler):
 
     @abstractmethod
     async def _parse_items(
-        self, mapping: ResourceConfig, raw_data: RawEntityDiff
-    ) -> EntityDiff:
+        self,
+        mapping: ResourceConfig,
+        raw_data: list[RAW_ITEM],
+        parse_all: bool = False,
+    ) -> EntitySelectorDiff:
         pass
 
     async def parse_items(
-        self, mapping: ResourceConfig, raw_data: RawEntityDiff
-    ) -> EntityDiff:
+        self,
+        mapping: ResourceConfig,
+        raw_data: list[RAW_ITEM],
+        parse_all: bool = False,
+    ) -> EntitySelectorDiff:
         """Public method to parse raw entity data and map it to an EntityDiff.
 
         Args:
             mapping (ResourceConfig): The configuration for entity mapping.
-            raw_data (RawEntityDiff): The raw data to be parsed.
+            raw_data (list[RawEntity]): The raw data to be parsed.
+            parse_all (bool): Whether to parse all data or just data that passed the selector.
 
         Returns:
             EntityDiff: The parsed entity differences.
         """
         with logger.contextualize(kind=mapping.kind):
-            return await self._parse_items(mapping, raw_data)
+            return await self._parse_items(mapping, raw_data, parse_all)

port_ocean/core/handlers/entity_processor/jq_entity_processor.py

@@ -9,7 +9,10 @@ import pyjq as jq  # type: ignore
 from port_ocean.core.handlers.entity_processor.base import BaseEntityProcessor
 from port_ocean.core.handlers.port_app_config.models import ResourceConfig
 from port_ocean.core.models import Entity
-from port_ocean.core.ocean_types import RawEntityDiff, EntityDiff
+from port_ocean.core.ocean_types import (
+    RAW_ITEM,
+    EntitySelectorDiff,
+)
 from port_ocean.exceptions.core import EntityProcessorException
 
 
@@ -68,16 +71,19 @@ class JQEntityProcessor(BaseEntityProcessor):
 
         return result
 
-    async def _get_entity_if_passed_selector(
+    async def _get_mapped_entity(
         self,
         data: dict[str, Any],
         raw_entity_mappings: dict[str, Any],
         selector_query: str,
-    ) -> dict[str, Any]:
+        parse_all: bool = False,
+    ) -> tuple[dict[str, Any], bool]:
         should_run = await self._search_as_bool(data, selector_query)
-        if should_run:
-            return await self._search_as_object(data, raw_entity_mappings)
-        return {}
+        if parse_all or should_run:
+            mapped_entity = await self._search_as_object(data, raw_entity_mappings)
+            return mapped_entity, should_run
+
+        return {}, False
 
     async def _calculate_entity(
         self,
@@ -85,16 +91,18 @@ class JQEntityProcessor(BaseEntityProcessor):
         raw_entity_mappings: dict[str, Any],
         items_to_parse: str | None,
         selector_query: str,
-    ) -> list[dict[str, Any]]:
+        parse_all: bool = False,
+    ) -> list[tuple[dict[str, Any], bool]]:
         if items_to_parse:
             items = await self._search(data, items_to_parse)
             if isinstance(items, list):
                 return await asyncio.gather(
                     *[
-                        self._get_entity_if_passed_selector(
+                        self._get_mapped_entity(
                             {"item": item, **data},
                             raw_entity_mappings,
                             selector_query,
+                            parse_all,
                         )
                         for item in items
                     ]
@@ -105,51 +113,44 @@ class JQEntityProcessor(BaseEntityProcessor):
             )
         else:
             return [
-                await self._get_entity_if_passed_selector(
-                    data, raw_entity_mappings, selector_query
+                await self._get_mapped_entity(
+                    data, raw_entity_mappings, selector_query, parse_all
                 )
             ]
-        return [{}]
+        return [({}, False)]
 
-    async def _calculate_entities(
-        self, mapping: ResourceConfig, raw_data: list[dict[str, Any]]
-    ) -> list[Entity]:
+    async def _parse_items(
+        self,
+        mapping: ResourceConfig,
+        raw_results: list[RAW_ITEM],
+        parse_all: bool = False,
+    ) -> EntitySelectorDiff:
         raw_entity_mappings: dict[str, Any] = mapping.port.entity.mappings.dict(
             exclude_unset=True
         )
-        entities_tasks = [
+        calculate_entities_tasks = [
             asyncio.create_task(
                 self._calculate_entity(
                     data,
                     raw_entity_mappings,
                     mapping.port.items_to_parse,
                     mapping.selector.query,
+                    parse_all,
                 )
             )
-            for data in raw_data
-        ]
-        entities = await asyncio.gather(*entities_tasks)
-
-        return [
-            Entity.parse_obj(entity_data)
-            for flatten in entities
-            for entity_data in filter(
-                lambda entity: entity.get("identifier") and entity.get("blueprint"),
-                flatten,
-            )
+            for data in raw_results
         ]
-
-    async def _parse_items(
-        self, mapping: ResourceConfig, raw_results: RawEntityDiff
-    ) -> EntityDiff:
-        entities_before: list[Entity] = await self._calculate_entities(
-            mapping, raw_results["before"]
-        )
-        entities_after: list[Entity] = await self._calculate_entities(
-            mapping, raw_results["after"]
-        )
-
-        return {
-            "before": entities_before,
-            "after": entities_after,
-        }
+        calculate_entities_results = await asyncio.gather(*calculate_entities_tasks)
+
+        passed_entities = []
+        failed_entities = []
+        for entities_results in calculate_entities_results:
+            for entity, did_entity_pass_selector in entities_results:
+                if entity.get("identifier") and entity.get("blueprint"):
+                    parsed_entity = Entity.parse_obj(entity)
+                    if did_entity_pass_selector:
+                        passed_entities.append(parsed_entity)
+                    else:
+                        failed_entities.append(parsed_entity)
+
+        return EntitySelectorDiff(passed=passed_entities, failed=failed_entities)

port_ocean/core/integrations/mixins/sync_raw.py

@@ -22,8 +22,9 @@ from port_ocean.core.ocean_types import (
     RAW_RESULT,
     RESYNC_RESULT,
     RawEntityDiff,
-    EntityDiff,
     ASYNC_GENERATOR_RESYNC_TYPE,
+    RAW_ITEM,
+    EntitySelectorDiff,
 )
 from port_ocean.core.utils import zip_and_sum
 from port_ocean.exceptions.core import OceanAbortException
@@ -120,12 +121,13 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         return results, errors
 
     async def _calculate_raw(
-        self, raw_diff: list[tuple[ResourceConfig, RawEntityDiff]]
-    ) -> list[EntityDiff]:
-        logger.info("Calculating diff in entities between states")
+        self,
+        raw_diff: list[tuple[ResourceConfig, list[RAW_ITEM]]],
+        parse_all: bool = False,
+    ) -> list[EntitySelectorDiff]:
         return await asyncio.gather(
             *(
-                self.entity_processor.parse_items(mapping, results)
+                self.entity_processor.parse_items(mapping, results, parse_all)
                 for mapping, results in raw_diff
             )
         )
@@ -135,42 +137,24 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         resource: ResourceConfig,
         results: list[dict[Any, Any]],
         user_agent_type: UserAgentType,
-    ) -> list[Entity]:
-        objects_diff = await self._calculate_raw(
-            [
-                (
-                    resource,
-                    {
-                        "before": [],
-                        "after": results,
-                    },
-                )
-            ]
+        parse_all: bool = False,
+    ) -> EntitySelectorDiff:
+        objects_diff = await self._calculate_raw([(resource, results)], parse_all)
+        await self.entities_state_applier.upsert(
+            objects_diff[0].passed, user_agent_type
         )
 
-        entities_after: list[Entity] = objects_diff[0]["after"]
-        await self.entities_state_applier.upsert(entities_after, user_agent_type)
-        return entities_after
+        return objects_diff[0]
 
     async def _unregister_resource_raw(
         self,
         resource: ResourceConfig,
-        results: list[dict[Any, Any]],
+        results: list[RAW_ITEM],
         user_agent_type: UserAgentType,
     ) -> list[Entity]:
-        objects_diff = await self._calculate_raw(
-            [
-                (
-                    resource,
-                    {
-                        "before": results,
-                        "after": [],
-                    },
-                )
-            ]
-        )
+        objects_diff = await self._calculate_raw([(resource, results)])
 
-        entities_after: list[Entity] = objects_diff[0]["before"]
+        entities_after: list[Entity] = objects_diff[0].passed
         await self.entities_state_applier.delete(entities_after, user_agent_type)
         logger.info("Finished unregistering change")
         return entities_after
@@ -187,17 +171,21 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
             else:
                 async_generators.append(result)
 
-        entities = await self._register_resource_raw(
-            resource_config, raw_results, user_agent_type
-        )
+        entities = (
+            await self._register_resource_raw(
+                resource_config, raw_results, user_agent_type
+            )
+        ).passed
 
         for generator in async_generators:
             try:
                 async for items in generator:
                     entities.extend(
-                        await self._register_resource_raw(
-                            resource_config, items, user_agent_type
-                        )
+                        (
+                            await self._register_resource_raw(
+                                resource_config, items, user_agent_type
+                            )
+                        ).passed
                     )
             except* OceanAbortException as error:
                 errors.append(error)
@@ -231,13 +219,44 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
             resource for resource in config.resources if resource.kind == kind
         ]
 
-        return await asyncio.gather(
+        diffs: list[EntitySelectorDiff] = await asyncio.gather(
             *(
-                self._register_resource_raw(resource, results, user_agent_type)
+                self._register_resource_raw(resource, results, user_agent_type, True)
                 for resource in resource_mappings
             )
         )
 
+        registered_entities, entities_to_delete = zip_and_sum(
+            (entities_diff.passed, entities_diff.failed) for entities_diff in diffs
+        )
+
+        registered_entities_attributes = {
+            (entity.identifier, entity.blueprint) for entity in registered_entities
+        }
+
+        filtered_entities_to_delete: list[Entity] = (
+            await ocean.port_client.search_batch_entities(
+                user_agent_type,
+                [
+                    entity
+                    for entity in entities_to_delete
+                    if (entity.identifier, entity.blueprint)
+                    not in registered_entities_attributes
+                ],
+            )
+        )
+
+        if filtered_entities_to_delete:
+            logger.info(
+                f"Deleting {len(filtered_entities_to_delete)} entities that didn't pass any of the selectors"
+            )
+
+            await self.entities_state_applier.delete(
+                filtered_entities_to_delete, user_agent_type
+            )
+
+        return registered_entities
+
     async def unregister_raw(
         self,
         kind: str,
@@ -293,19 +312,28 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         with logger.contextualize(kind=kind):
             logger.info(f"Found {len(resource_mappings)} resources for {kind}")
 
-            objects_diff = await self._calculate_raw(
-                [(mapping, raw_desired_state) for mapping in resource_mappings]
+            entities_before = await self._calculate_raw(
+                [
+                    (mapping, raw_desired_state["before"])
+                    for mapping in resource_mappings
+                ]
+            )
+
+            entities_after = await self._calculate_raw(
+                [(mapping, raw_desired_state["after"]) for mapping in resource_mappings]
             )
 
-            entities_before, entities_after = zip_and_sum(
-                (
-                    (entities_change["before"], entities_change["after"])
-                    for entities_change in objects_diff
-                )
+            entities_before_flatten: list[Entity] = sum(
+                (entities_diff.passed for entities_diff in entities_before), []
+            )
+
+            entities_after_flatten: list[Entity] = sum(
+                (entities_diff.passed for entities_diff in entities_after), []
             )
 
             await self.entities_state_applier.apply_diff(
-                {"before": entities_before, "after": entities_after}, user_agent_type
+                {"before": entities_before_flatten, "after": entities_after_flatten},
+                user_agent_type,
             )
 
     async def sync_raw_all(
@@ -333,8 +361,6 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         ):
             app_config = await self.port_app_config_handler.get_port_app_config()
 
-            entities_at_port = await ocean.port_client.search_entities(user_agent_type)
-
             creation_results: list[tuple[list[Entity], list[Exception]]] = []
 
             try:
@@ -369,8 +395,11 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
 
                     logger.error(message, exc_info=error_group)
                 else:
+                    entities_at_port = await ocean.port_client.search_entities(
+                        user_agent_type
+                    )
                     logger.info(
-                        f"Running resync diff calculation, number of entities at Port before resync: {len(entities_at_port)}, number of entities created during sync: {len(flat_created_entities)}"
+                        f"Running resync diff calculation, number of entities found at Port: {len(entities_at_port)}, number of entities found during sync: {len(flat_created_entities)}"
                     )
                     await self.entities_state_applier.delete_diff(
                         {"before": entities_at_port, "after": flat_created_entities},

port_ocean/core/ocean_types.py

@@ -1,18 +1,8 @@
-from typing import TypedDict, Any, AsyncIterator, Callable, Awaitable
+from typing import TypedDict, Any, AsyncIterator, Callable, Awaitable, NamedTuple
 
 from port_ocean.core.models import Entity
 
 
-class RawEntityDiff(TypedDict):
-    before: list[dict[Any, Any]]
-    after: list[dict[Any, Any]]
-
-
-class EntityDiff(TypedDict):
-    before: list[Entity]
-    after: list[Entity]
-
-
 RAW_ITEM = dict[Any, Any]
 RAW_RESULT = list[RAW_ITEM]
 ASYNC_GENERATOR_RESYNC_TYPE = AsyncIterator[RAW_RESULT]
@@ -23,6 +13,21 @@ RESYNC_EVENT_LISTENER = Callable[[str], LISTENER_RESULT]
 START_EVENT_LISTENER = Callable[[], Awaitable[None]]
 
 
+class RawEntityDiff(TypedDict):
+    before: list[RAW_ITEM]
+    after: list[RAW_ITEM]
+
+
+class EntityDiff(TypedDict):
+    before: list[Entity]
+    after: list[Entity]
+
+
+class EntitySelectorDiff(NamedTuple):
+    passed: list[Entity]
+    failed: list[Entity]
+
+
 class IntegrationEventsCallbacks(TypedDict):
     start: list[START_EVENT_LISTENER]
     resync: dict[str | None, list[RESYNC_EVENT_LISTENER]]

port_ocean/helpers/retry.py

@@ -179,12 +179,35 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
         transport.close()
 
     def _is_retryable_method(self, request: httpx.Request) -> bool:
-        return request.method in self._retryable_methods
+        return request.method in self._retryable_methods or request.extensions.get(
+            "retryable", False
+        )
 
     def _should_retry(self, response: httpx.Response) -> bool:
         return response.status_code in self._retry_status_codes
 
-    def _log_failure(
+    def _log_error(
+        self,
+        request: httpx.Request,
+        error: Exception | None,
+    ) -> None:
+        if not self._logger:
+            return
+
+        if isinstance(error, httpx.ConnectTimeout):
+            self._logger.error(
+                f"Request {request.method} {request.url} failed to connect: {str(error)}"
+            )
+        elif isinstance(error, httpx.TimeoutException):
+            self._logger.error(
+                f"Request {request.method} {request.url} failed with a timeout exception: {str(error)}"
+            )
+        elif isinstance(error, httpx.HTTPError):
+            self._logger.error(
+                f"Request {request.method} {request.url} failed with an HTTP error: {str(error)}"
+            )
+
+    def _log_before_retry(
         self,
         request: httpx.Request,
         sleep_time: float,
@@ -249,7 +272,7 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
         while True:
             if attempts_made > 0:
                 sleep_time = self._calculate_sleep(attempts_made, {})
-                self._log_failure(request, sleep_time, response, error)
+                self._log_before_retry(request, sleep_time, response, error)
                 await asyncio.sleep(sleep_time)
 
             error = None
@@ -262,9 +285,20 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
                 ):
                     return response
                 await response.aclose()
+            except httpx.ConnectTimeout as e:
+                error = e
+                if remaining_attempts < 1:
+                    self._log_error(request, error)
+                    raise
+            except httpx.TimeoutException as e:
+                error = e
+                if remaining_attempts < 1:
+                    self._log_error(request, error)
+                    raise
             except httpx.HTTPError as e:
                 error = e
                 if remaining_attempts < 1:
+                    self._log_error(request, error)
                     raise
             attempts_made += 1
             remaining_attempts -= 1
@@ -281,7 +315,7 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
         while True:
             if attempts_made > 0:
                 sleep_time = self._calculate_sleep(attempts_made, {})
-                self._log_failure(request, sleep_time, response, error)
+                self._log_before_retry(request, sleep_time, response, error)
                 time.sleep(sleep_time)
 
             error = None
@@ -292,9 +326,20 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
                 if remaining_attempts < 1 or not self._should_retry(response):
                     return response
                 response.close()
+            except httpx.ConnectTimeout as e:
+                error = e
+                if remaining_attempts < 1:
+                    self._log_error(request, error)
+                    raise
+            except httpx.TimeoutException as e:
+                error = e
+                if remaining_attempts < 1:
+                    self._log_error(request, error)
+                    raise
             except httpx.HTTPError as e:
                 error = e
                 if remaining_attempts < 1:
+                    self._log_error(request, error)
                     raise
             attempts_made += 1
             remaining_attempts -= 1

port_ocean/utils/async_iterators.py

@@ -0,0 +1,49 @@
+import typing
+
+import aiostream
+
+
+async def stream_async_iterators_tasks(
+    *tasks: typing.AsyncIterable[typing.Any],
+) -> typing.AsyncIterable[typing.Any]:
+    """
+    This function takes a list of async iterators and streams the results of each iterator as they are available.
+    By using this function you can combine multiple async iterators into a single stream of results, instead of waiting
+    for each iterator to finish before starting the next one.
+
+    Usage:
+    ```python
+    async def async_iterator1():
+        for i in range(10):
+            yield i
+            await asyncio.sleep(1)
+
+    async def async_iterator2():
+        for i in range(10, 20):
+            yield i
+            await asyncio.sleep(1)
+
+    async def main():
+        async for result in stream_async_iterators_tasks([async_iterator1(), async_iterator2()]):
+            print(result)
+    ```
+
+    Caution - Before using this function, make sure that the third-party API you are calling allows the number of
+    concurrent requests you are making. If the API has a rate limit, you may need to adjust the number of concurrent
+    requests to avoid hitting the rate limit.
+
+    :param tasks: A list of async iterators
+    :return: A stream of results
+    """
+    if not tasks:
+        raise StopAsyncIteration("No tasks provided")
+
+    if len(tasks) == 1:
+        async for batch_items in tasks[0]:
+            yield batch_items
+        return
+
+    combine = aiostream.stream.merge(tasks[0], *tasks[1:])
+    async with combine.stream() as streamer:
+        async for batch_items in streamer:
+            yield batch_items

{port_ocean-0.5.10.dist-info → port_ocean-0.5.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.5.10
+Version: 0.5.14
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port
@@ -21,11 +21,12 @@ Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: Utilities
 Provides-Extra: cli
+Requires-Dist: aiostream (>=0.5.2,<0.6.0)
 Requires-Dist: click (>=8.1.3,<9.0.0) ; extra == "cli"
 Requires-Dist: confluent-kafka (>=2.1.1,<3.0.0)
 Requires-Dist: cookiecutter (>=2.1.1,<3.0.0) ; extra == "cli"
-Requires-Dist: fastapi (>=0.100,<0.110)
-Requires-Dist: httpx (>=0.24.1,<0.25.0)
+Requires-Dist: fastapi (>=0.100,<0.111)
+Requires-Dist: httpx (>=0.24.1,<0.28.0)
 Requires-Dist: jinja2-time (>=0.2.0,<0.3.0) ; extra == "cli"
 Requires-Dist: loguru (>=0.7.0,<0.8.0)
 Requires-Dist: pydantic (>=1.10.8,<2.0.0)
@@ -36,8 +37,8 @@ Requires-Dist: pyyaml (>=6.0,<7.0)
 Requires-Dist: rich (>=13.4.1,<14.0.0) ; extra == "cli"
 Requires-Dist: six (>=1.16.0,<2.0.0)
 Requires-Dist: tomli (>=2.0.1,<3.0.0)
-Requires-Dist: urllib3 (>=1.26.16,<2.0.0)
-Requires-Dist: uvicorn (>=0.22.0,<0.23.0)
+Requires-Dist: urllib3 (>=1.26.16,<3.0.0)
+Requires-Dist: uvicorn (>=0.22,<0.30)
 Requires-Dist: werkzeug (>=2.3.4,<4.0.0)
 Project-URL: Repository, https://github.com/port-labs/Port-Ocean
 Description-Content-Type: text/markdown

{port_ocean-0.5.10.dist-info → port_ocean-0.5.14.dist-info}/RECORD

@@ -36,14 +36,14 @@ port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/tests/__init__.py,
 port_ocean/cli/utils.py,sha256=IUK2UbWqjci-lrcDdynZXqVP5B5TcjF0w5CpEVUks-k,54
 port_ocean/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/clients/port/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-port_ocean/clients/port/authentication.py,sha256=1SPb12ieHLYvDHfaMiSM23DQSg7yGpk7TpsrXsvosGU,2892
+port_ocean/clients/port/authentication.py,sha256=t3z6h4vld-Tzkpth15sstaMJg0rccX-pXXjNtOa-nCY,2949
 port_ocean/clients/port/client.py,sha256=3GYCM0ZkX3pB6sNoOb-7_6dm0Jr5_vqhflD9iltf_As,2640
 port_ocean/clients/port/mixins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/clients/port/mixins/blueprints.py,sha256=BiqkhvDFdkySWgL1NHI-LAQ9ieZWazZAojPo9E8d7U4,4575
-port_ocean/clients/port/mixins/entities.py,sha256=BLEfyPn1YdGeK_0RtRHguLRxD_XPdhSGpTq8CXrDZ3o,7902
+port_ocean/clients/port/mixins/entities.py,sha256=Lg5Sa6jQuhDTQKLURVavqXlBQt4-XPUUigB9JqQ1X0k,7364
 port_ocean/clients/port/mixins/integrations.py,sha256=GJRweeibfKhukOkWB07pb09G6pp06FOiXPJU-pOV7M8,5155
 port_ocean/clients/port/mixins/migrations.py,sha256=A6896oJF6WbFL2WroyTkMzr12yhVyWqGoq9dtLNSKBY,1457
-port_ocean/clients/port/retry_transport.py,sha256=J25q5QpSYSVsVa24Sr1IvXu-KpYL88Pd9xe3xvYWYsg,2056
+port_ocean/clients/port/retry_transport.py,sha256=PtIZOAZ6V-ncpVysRUsPOgt8Sf01QLnTKB5YeKBxkJk,1861
 port_ocean/clients/port/types.py,sha256=nvlgiAq4WH5_F7wQbz_GAWl-faob84LVgIjZ2Ww5mTk,451
 port_ocean/clients/port/utils.py,sha256=O9mBu6zp4TfpS4SQ3qCPpn9ZVyYF8GKnji4UnYhM3yg,2373
 port_ocean/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -73,13 +73,12 @@ port_ocean/core/handlers/base.py,sha256=cTarblazu8yh8xz2FpB-dzDKuXxtoi143XJgPbV_
 port_ocean/core/handlers/entities_state_applier/__init__.py,sha256=kgLZDCeCEzi4r-0nzW9k78haOZNf6PX7mJOUr34A4c8,173
 port_ocean/core/handlers/entities_state_applier/base.py,sha256=FMsrBOVgaO4o7B1klLDY8fobTUDvyrerCKCICyYtkXs,2193
 port_ocean/core/handlers/entities_state_applier/port/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-port_ocean/core/handlers/entities_state_applier/port/applier.py,sha256=6Zg93oPJyShMGU62FAuLbN7LQB7xFkkg4GPUZ5JFTw4,7928
+port_ocean/core/handlers/entities_state_applier/port/applier.py,sha256=G6RyPgBAyUcP41ySKaiXhKQhdrz8NixEEwsnU5q_-0c,5225
 port_ocean/core/handlers/entities_state_applier/port/get_related_entities.py,sha256=1zncwCbE-Gej0xaWKlzZgoXxOBe9bgs_YxlZ8QW3NdI,1751
 port_ocean/core/handlers/entities_state_applier/port/order_by_entities_dependencies.py,sha256=82BvU8t5w9uhsxX8hbnwuRPuWhW3cMeuT_5sVIkip1I,1550
-port_ocean/core/handlers/entities_state_applier/port/validate_entity_relations.py,sha256=nKuQ-RlalGG07olxm6l5NHeOuQT9dEZLoMpD-AN5nq0,1392
 port_ocean/core/handlers/entity_processor/__init__.py,sha256=FvFCunFg44wNQoqlybem9MthOs7p1Wawac87uSXz9U8,156
-port_ocean/core/handlers/entity_processor/base.py,sha256=3ucPc-esMI5M6gFl_l2j_ncEqOIe7m1bCH4rAXtGDzg,1710
-port_ocean/core/handlers/entity_processor/jq_entity_processor.py,sha256=Xn_BfsddY_NR2Ssbiw6TT6xAm_7gKlZSFVlhlmb2TyA,5425
+port_ocean/core/handlers/entity_processor/base.py,sha256=4JVCAAohEKtl8FdlnuyIxJ1afSXk3o2-e_m4LSy7vmw,1952
+port_ocean/core/handlers/entity_processor/jq_entity_processor.py,sha256=R5PwTcsJYRVU-lwoYZa1Af8kserp2BXIpyzVtRdEqA8,5617
 port_ocean/core/handlers/port_app_config/__init__.py,sha256=8AAT5OthiVM7KCcM34iEgEeXtn2pRMrT4Dze5r1Ixbk,134
 port_ocean/core/handlers/port_app_config/api.py,sha256=6VbKPwFzsWG0IYsVD81hxSmfqtHUFqrfUuj1DBX5g4w,853
 port_ocean/core/handlers/port_app_config/base.py,sha256=nnMZ4jH6a-4Of9Cn-apMsU0CgNLD9avd5q0gRmc7nZ8,1495
@@ -90,10 +89,10 @@ port_ocean/core/integrations/mixins/__init__.py,sha256=FA1FEKMM6P-L2_m7Q4L20mFa4
 port_ocean/core/integrations/mixins/events.py,sha256=Ddfx2L4FpghV38waF8OfVeOV0bHBxNIgjU-q5ffillI,2341
 port_ocean/core/integrations/mixins/handler.py,sha256=mZ7-0UlG3LcrwJttFbMe-R4xcOU2H_g33tZar7PwTv8,3771
 port_ocean/core/integrations/mixins/sync.py,sha256=TKqRytxXONVhuCo3CB3rDvWNbITnZz33TYTDs3SWWVk,3880
-port_ocean/core/integrations/mixins/sync_raw.py,sha256=MxhCcvtFZCD77a2pR7vRFBQs6LPAg4OuLPg6FGzNIjA,13810
+port_ocean/core/integrations/mixins/sync_raw.py,sha256=-JhaWt7N4R4vYe73cmJ00PEeIHUYJq-x8Pf89_-WgZ0,14992
 port_ocean/core/integrations/mixins/utils.py,sha256=7y1rGETZIjOQadyIjFJXIHKkQFKx_SwiP-TrAIsyyLY,2303
 port_ocean/core/models.py,sha256=bDO_I4Yd33TEZIh2QSV8UwXQIuwE7IgrINkYDHI0dkc,714
-port_ocean/core/ocean_types.py,sha256=egpOUJCZujjcj2favI7XK5Iq-7kBUk9cGKRrQzbRMas,789
+port_ocean/core/ocean_types.py,sha256=ltnn22eRuDMFW02kIgmIAu6S06-i9jJV2NJ-MZcwwj0,879
 port_ocean/core/utils.py,sha256=B040Wokk28g9tQj_06qk_uvm85RIXc6XGXysZV6gtQw,1957
 port_ocean/exceptions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/exceptions/api.py,sha256=TLmTMqn4uHGaHgZK8PMIJ0TVJlPB4iP7xl9rx7GtCyY,426
@@ -105,7 +104,7 @@ port_ocean/exceptions/port_defaults.py,sha256=pJ5im06ziDX2IVstSlYIjVHhs4JW-N0Vr0
 port_ocean/exceptions/utils.py,sha256=gjOqpi-HpY1l4WlMFsGA9yzhxDhajhoGGdDDyGbLnqI,197
 port_ocean/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/helpers/async_client.py,sha256=SRlP6o7_FCSY3UHnRlZdezppePVxxOzZ0z861vE3K40,1783
-port_ocean/helpers/retry.py,sha256=qZAiNCdhZ9pg55Ol70uJef2PAxHBriBCBg4kbT4naqo,13155
+port_ocean/helpers/retry.py,sha256=hcfVmt2Fp53EQViYiOWA-FTWD80jvS-mWKhwMHiBM7Q,14869
 port_ocean/log/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/log/handlers.py,sha256=k9G_Mb4ga2-Jke9irpdlYqj6EYiwv0gEsh4TgyqqOmI,2853
 port_ocean/log/logger_setup.py,sha256=NGU9EdRTPLloiHdgSw0JYaYGr9mx1hYJUEPFHP6BA14,2175
@@ -117,13 +116,14 @@ port_ocean/run.py,sha256=vyShtqg_jEiE6M4SJpci6c4oRD9k2ztesAXEx_6Sc9M,1906
 port_ocean/sonar-project.properties,sha256=X_wLzDOkEVmpGLRMb2fg9Rb0DxWwUFSvESId8qpvrPI,73
 port_ocean/utils/__init__.py,sha256=KMGnCPXZJbNwtgxtyMycapkDz8tpSyw23MSYT3iVeHs,91
 port_ocean/utils/async_http.py,sha256=arnH458TExn2Dju_Sy6pHas_vF5RMWnOp-jBz5WAAcE,1226
+port_ocean/utils/async_iterators.py,sha256=buFBiPdsqkNMCk91h6ZG8hJa181j7RjgHajbfgeB8A8,1608
 port_ocean/utils/cache.py,sha256=3KItZDE2yVrbVDr-hoM8lNna8s2dlpxhP4ICdLjH4LQ,2231
 port_ocean/utils/misc.py,sha256=2XmO8W0SgPjV0rd9HZvrHhoMlHprIwmMFsINxlAmgyw,1723
 port_ocean/utils/repeat.py,sha256=0EFWM9d8lLXAhZmAyczY20LAnijw6UbIECf5lpGbOas,3231
 port_ocean/utils/signal.py,sha256=Fab0049Cjs69TPTQgvEvilaVZKACQr6tGkRdySjNCi8,1515
 port_ocean/version.py,sha256=UsuJdvdQlazzKGD3Hd5-U7N69STh8Dq9ggJzQFnu9fU,177
-port_ocean-0.5.10.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-port_ocean-0.5.10.dist-info/METADATA,sha256=LhzUQd5z1OGFly2eIkUwciYjPG9XUaGTlGiuRDqqEJI,6516
-port_ocean-0.5.10.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-port_ocean-0.5.10.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
-port_ocean-0.5.10.dist-info/RECORD,,
+port_ocean-0.5.14.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+port_ocean-0.5.14.dist-info/METADATA,sha256=5DHXN2mLFMECLS5hooI0bnSMhqLa0FiTTyA65V9ozO8,6554
+port_ocean-0.5.14.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+port_ocean-0.5.14.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
+port_ocean-0.5.14.dist-info/RECORD,,

port_ocean/core/handlers/entities_state_applier/port/validate_entity_relations.py

@@ -1,40 +0,0 @@
-import asyncio
-
-from port_ocean.clients.port.client import PortClient
-from port_ocean.core.handlers.entities_state_applier.port.get_related_entities import (
-    get_related_entities,
-)
-from port_ocean.core.handlers.entity_processor.base import EntityPortDiff
-from port_ocean.core.utils import is_same_entity
-from port_ocean.exceptions.core import RelationValidationException
-
-
-async def validate_entity_relations(
-    diff: EntityPortDiff, port_client: PortClient
-) -> None:
-    modified_or_created_entities = diff.modified + diff.created
-    related_entities = await get_related_entities(
-        modified_or_created_entities, port_client
-    )
-
-    required_entities = []
-
-    for entity in related_entities:
-        if any(is_same_entity(item, entity) for item in diff.deleted):
-            raise RelationValidationException(
-                f"Cant delete entity {entity} of blueprint {entity.blueprint} "
-                f"because it was specified as relation target of entity {entity} "
-                f"of blueprint {entity.blueprint}"
-            )
-
-        if not any(
-            is_same_entity(item, entity) for item in modified_or_created_entities
-        ):
-            required_entities.append(entity)
-
-    await asyncio.gather(
-        *(
-            port_client.validate_entity_exist(item.identifier, item.blueprint)
-            for item in required_entities
-        )
-    )