port-ocean 0.5.5__py3-none-any.whl → 0.17.8__py3-none-any.whl

port_ocean/core/integrations/mixins/sync_raw.py

@@ -1,8 +1,10 @@
 import asyncio
+from graphlib import CycleError
 import inspect
 import typing
 from typing import Callable, Awaitable, Any
 
+import httpx
 from loguru import logger
 
 from port_ocean.clients.port.types import UserAgentType
@@ -22,12 +24,15 @@ from port_ocean.core.ocean_types import (
     RAW_RESULT,
     RESYNC_RESULT,
     RawEntityDiff,
-    EntityDiff,
     ASYNC_GENERATOR_RESYNC_TYPE,
+    RAW_ITEM,
+    CalculationResult,
 )
-from port_ocean.core.utils import zip_and_sum
+from port_ocean.core.utils.utils import zip_and_sum, gather_and_split_errors_from_results
 from port_ocean.exceptions.core import OceanAbortException
 
+SEND_RAW_DATA_EXAMPLES_AMOUNT = 5
+
 
 class SyncRawMixin(HandlerMixin, EventsMixin):
     """Mixin class for synchronization of raw constructed entities.
@@ -86,7 +91,6 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
     ) -> tuple[RESYNC_RESULT, list[RAW_RESULT | Exception]]:
         tasks = []
         results = []
-
         for task in fns:
             if inspect.isasyncgenfunction(task):
                 results.append(resync_generator_wrapper(task, resource_config.kind))
@@ -97,21 +101,13 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         logger.info(
             f"Found {len(tasks) + len(results)} resync tasks for {resource_config.kind}"
         )
-
-        results_with_error = await asyncio.gather(*tasks, return_exceptions=True)
+        successful_results, errors = await gather_and_split_errors_from_results(tasks)
         results.extend(
             sum(
-                [
-                    result
-                    for result in results_with_error
-                    if not isinstance(result, Exception)
-                ],
+                successful_results,
                 [],
             )
         )
-        errors = [
-            result for result in results_with_error if isinstance(result, Exception)
-        ]
 
         logger.info(
             f"Triggered {len(tasks)} tasks for {resource_config.kind}, failed: {len(errors)}"
@@ -120,12 +116,16 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         return results, errors
 
     async def _calculate_raw(
-        self, raw_diff: list[tuple[ResourceConfig, RawEntityDiff]]
-    ) -> list[EntityDiff]:
-        logger.info("Calculating diff in entities between states")
+        self,
+        raw_diff: list[tuple[ResourceConfig, list[RAW_ITEM]]],
+        parse_all: bool = False,
+        send_raw_data_examples_amount: int = 0,
+    ) -> list[CalculationResult]:
         return await asyncio.gather(
             *(
-                self.entity_processor.parse_items(mapping, results)
+                self.entity_processor.parse_items(
+                    mapping, results, parse_all, send_raw_data_examples_amount
+                )
                 for mapping, results in raw_diff
             )
         )
@@ -135,45 +135,41 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         resource: ResourceConfig,
         results: list[dict[Any, Any]],
         user_agent_type: UserAgentType,
-    ) -> list[Entity]:
+        parse_all: bool = False,
+        send_raw_data_examples_amount: int = 0,
+    ) -> CalculationResult:
         objects_diff = await self._calculate_raw(
-            [
-                (
-                    resource,
-                    {
-                        "before": [],
-                        "after": results,
-                    },
-                )
-            ]
+            [(resource, results)], parse_all, send_raw_data_examples_amount
+        )
+        modified_objects = await self.entities_state_applier.upsert(
+            objects_diff[0].entity_selector_diff.passed, user_agent_type
+        )
+        return CalculationResult(
+            objects_diff[0].entity_selector_diff._replace(passed=modified_objects),
+            errors=objects_diff[0].errors,
+            misonfigured_entity_keys=objects_diff[0].misonfigured_entity_keys,
         )
-
-        entities_after: list[Entity] = objects_diff[0]["after"]
-        await self.entities_state_applier.upsert(entities_after, user_agent_type)
-        return entities_after
 
     async def _unregister_resource_raw(
         self,
         resource: ResourceConfig,
-        results: list[dict[Any, Any]],
+        results: list[RAW_ITEM],
         user_agent_type: UserAgentType,
-    ) -> list[Entity]:
-        objects_diff = await self._calculate_raw(
-            [
-                (
-                    resource,
-                    {
-                        "before": results,
-                        "after": [],
-                    },
-                )
-            ]
-        )
+    ) -> tuple[list[Entity], list[Exception]]:
+        if resource.port.entity.mappings.is_using_search_identifier:
+            logger.info(
+                f"Skip unregistering resource of kind {resource.kind}, as mapping defined with search identifier"
+            )
+            return [], []
 
-        entities_after: list[Entity] = objects_diff[0]["before"]
-        await self.entities_state_applier.delete(entities_after, user_agent_type)
+        objects_diff = await self._calculate_raw([(resource, results)])
+        entities_selector_diff, errors, _ = objects_diff[0]
+
+        await self.entities_state_applier.delete(
+            entities_selector_diff.passed, user_agent_type
+        )
         logger.info("Finished unregistering change")
-        return entities_after
+        return entities_selector_diff.passed, errors
 
     async def _register_in_batches(
         self, resource_config: ResourceConfig, user_agent_type: UserAgentType
@@ -187,25 +183,41 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
             else:
                 async_generators.append(result)
 
-        entities = await self._register_resource_raw(
-            resource_config, raw_results, user_agent_type
+        send_raw_data_examples_amount = (
+            SEND_RAW_DATA_EXAMPLES_AMOUNT if ocean.config.send_raw_data_examples else 0
         )
+        all_entities, register_errors,_ = await self._register_resource_raw(
+            resource_config,
+            raw_results,
+            user_agent_type,
+            send_raw_data_examples_amount=send_raw_data_examples_amount,
+        )
+        errors.extend(register_errors)
+        passed_entities = list(all_entities.passed)
 
         for generator in async_generators:
             try:
                 async for items in generator:
-                    entities.extend(
-                        await self._register_resource_raw(
-                            resource_config, items, user_agent_type
+                    if send_raw_data_examples_amount > 0:
+                        send_raw_data_examples_amount = max(
+                            0, send_raw_data_examples_amount - len(passed_entities)
                         )
+
+                    entities, register_errors,_ = await self._register_resource_raw(
+                        resource_config,
+                        items,
+                        user_agent_type,
+                        send_raw_data_examples_amount=send_raw_data_examples_amount,
                     )
+                    errors.extend(register_errors)
+                    passed_entities.extend(entities.passed)
             except* OceanAbortException as error:
                 errors.append(error)
 
         logger.info(
-            f"Finished registering change for {len(results)} raw results for kind: {resource_config.kind}. {len(entities)} entities were affected"
+            f"Finished registering change for {len(results)} raw results for kind: {resource_config.kind}. {len(passed_entities)} entities were affected"
         )
-        return entities, errors
+        return passed_entities, errors
 
     async def register_raw(
         self,
@@ -231,13 +243,63 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
             resource for resource in config.resources if resource.kind == kind
         ]
 
-        return await asyncio.gather(
-            *(
-                self._register_resource_raw(resource, results, user_agent_type)
-                for resource in resource_mappings
+        if not resource_mappings:
+            return []
+
+        diffs, errors, misconfigured_entity_keys = zip(
+            *await asyncio.gather(
+                *(
+                    self._register_resource_raw(
+                        resource, results, user_agent_type, True
+                    )
+                    for resource in resource_mappings
+                )
+            )
+        )
+
+        diffs = list(diffs)
+        errors = sum(errors, [])
+        misconfigured_entity_keys = list(misconfigured_entity_keys)
+
+
+        if errors:
+            message = f"Failed to register {len(errors)} entities. Skipping delete phase due to incomplete state"
+            logger.error(message, exc_info=errors)
+            raise ExceptionGroup(
+                message,
+                errors,
+            )
+
+        registered_entities, entities_to_delete = zip_and_sum(diffs)
+
+        registered_entities_attributes = {
+            (entity.identifier, entity.blueprint) for entity in registered_entities
+        }
+
+        filtered_entities_to_delete: list[Entity] = (
+            await ocean.port_client.search_batch_entities(
+                user_agent_type,
+                [
+                    entity
+                    for entity in entities_to_delete
+                    if not entity.is_using_search_identifier
+                    and (entity.identifier, entity.blueprint)
+                    not in registered_entities_attributes
+                ],
             )
         )
 
+        if filtered_entities_to_delete:
+            logger.info(
+                f"Deleting {len(filtered_entities_to_delete)} entities that didn't pass any of the selectors"
+            )
+
+            await self.entities_state_applier.delete(
+                filtered_entities_to_delete, user_agent_type
+            )
+
+        return registered_entities
+
     async def unregister_raw(
         self,
         kind: str,
@@ -262,13 +324,25 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
             resource for resource in config.resources if resource.kind == kind
         ]
 
-        return await asyncio.gather(
-            *(
-                self._unregister_resource_raw(resource, results, user_agent_type)
-                for resource in resource_mappings
+        entities, errors = zip_and_sum(
+            await asyncio.gather(
+                *(
+                    self._unregister_resource_raw(resource, results, user_agent_type)
+                    for resource in resource_mappings
+                )
             )
         )
 
+        if errors:
+            message = f"Failed to unregister all entities with {len(errors)} errors"
+            logger.error(message, exc_info=errors)
+            raise ExceptionGroup(
+                message,
+                errors,
+            )
+
+        return entities
+
     async def update_raw_diff(
         self,
         kind: str,
@@ -293,21 +367,53 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         with logger.contextualize(kind=kind):
             logger.info(f"Found {len(resource_mappings)} resources for {kind}")
 
-            objects_diff = await self._calculate_raw(
-                [(mapping, raw_desired_state) for mapping in resource_mappings]
+            entities_before, _ = zip(
+                await self._calculate_raw(
+                    [
+                        (mapping, raw_desired_state["before"])
+                        for mapping in resource_mappings
+                    ]
+                )
             )
 
-            entities_before, entities_after = zip_and_sum(
-                (
-                    (entities_change["before"], entities_change["after"])
-                    for entities_change in objects_diff
-                )
+            entities_after, after_errors = await self._calculate_raw(
+                [(mapping, raw_desired_state["after"]) for mapping in resource_mappings]
             )
 
-            await self.entities_state_applier.apply_diff(
-                {"before": entities_before, "after": entities_after}, user_agent_type
+            entities_before_flatten: list[Entity] = sum(
+                (entities_diff.passed for entities_diff in entities_before), []
             )
 
+            entities_after_flatten: list[Entity] = sum(
+                (entities_diff.passed for entities_diff in entities_after), []
+            )
+
+            if after_errors:
+                message = (
+                    f"Failed to calculate diff for entities with {len(after_errors)} errors. "
+                    f"Skipping delete phase due to incomplete state"
+                )
+                logger.error(message, exc_info=after_errors)
+                entities_before_flatten = []
+
+            await self.entities_state_applier.apply_diff(
+                {"before": entities_before_flatten, "after": entities_after_flatten},
+                user_agent_type,
+            )
+    async def sort_and_upsert_failed_entities(self,user_agent_type: UserAgentType)->None:
+        try:
+            if not event.entity_topological_sorter.should_execute():
+                return None
+            logger.info(f"Executings topological sort of {event.entity_topological_sorter.get_entities_count()} entities failed to upsert.",failed_toupsert_entities_count=event.entity_topological_sorter.get_entities_count())
+
+            for entity in event.entity_topological_sorter.get_entities():
+                await self.entities_state_applier.context.port_client.upsert_entity(entity,event.port_app_config.get_port_request_options(),user_agent_type,should_raise=False)
+
+        except OceanAbortException as ocean_abort:
+            logger.info(f"Failed topological sort of failed to upsert entites - trying to upsert unordered {event.entity_topological_sorter.get_entities_count()} entities.",failed_topological_sort_entities_count=event.entity_topological_sorter.get_entities_count() )
+            if isinstance(ocean_abort.__cause__,CycleError):
+                for entity in event.entity_topological_sorter.get_entities(False):
+                    await self.entities_state_applier.context.port_client.upsert_entity(entity,event.port_app_config.get_port_request_options(),user_agent_type,should_raise=False)
     async def sync_raw_all(
         self,
         _: dict[Any, Any] | None = None,
@@ -331,9 +437,27 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
             EventType.RESYNC,
             trigger_type=trigger_type,
         ):
-            app_config = await self.port_app_config_handler.get_port_app_config()
+            # If a resync is triggered due to a mappings change, we want to make sure that we have the updated version
+            # rather than the old cache
+            app_config = await self.port_app_config_handler.get_port_app_config(
+                use_cache=False
+            )
+            logger.info(f"Resync will use the following mappings: {app_config.dict()}")
 
-            entities_at_port = await ocean.port_client.search_entities(user_agent_type)
+            try:
+                did_fetched_current_state = True
+                entities_at_port = await ocean.port_client.search_entities(
+                    user_agent_type
+                )
+            except httpx.HTTPError as e:
+                logger.warning(
+                    "Failed to fetch the current state of entities at Port. "
+                    "Skipping delete phase due to unknown initial state. "
+                    f"Error: {e}\n"
+                    f"Response status code: {e.response.status_code if isinstance(e, httpx.HTTPStatusError) else None}\n"
+                    f"Response content: {e.response.text if isinstance(e, httpx.HTTPStatusError) else None}\n"
+                )
+                did_fetched_current_state = False
 
             creation_results: list[tuple[list[Entity], list[Exception]]] = []
 
@@ -349,9 +473,19 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                         event.on_abort(lambda: task.cancel())
 
                         creation_results.append(await task)
+
+                await self.sort_and_upsert_failed_entities(user_agent_type)
             except asyncio.CancelledError as e:
-                logger.warning("Resync aborted successfully")
+                logger.warning("Resync aborted successfully, skipping delete phase. This leads to an incomplete state")
+                raise
             else:
+                if not did_fetched_current_state:
+                    logger.warning(
+                        "Due to an error before the resync, the previous state of entities at Port is unknown."
+                        " Skipping delete phase due to unknown initial state."
+                    )
+                    return
+
                 logger.info("Starting resync diff calculation")
                 flat_created_entities, errors = zip_and_sum(creation_results) or [
                     [],
@@ -376,3 +510,4 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                         {"before": entities_at_port, "after": flat_created_entities},
                         user_agent_type,
                     )
+                    logger.info("Resync finished successfully")

port_ocean/core/integrations/mixins/utils.py

@@ -9,7 +9,7 @@ from port_ocean.core.ocean_types import (
     RESYNC_EVENT_LISTENER,
     RESYNC_RESULT,
 )
-from port_ocean.core.utils import validate_result
+from port_ocean.core.utils.utils import validate_result
 from port_ocean.exceptions.core import (
     RawObjectValidationException,
     OceanAbortException,

port_ocean/core/models.py

@@ -1,9 +1,36 @@
+from dataclasses import dataclass, field
+from enum import Enum
 from typing import Any
 
 from pydantic import BaseModel
 from pydantic.fields import Field
 
 
+class Runtime(Enum):
+    Saas = "Saas"
+    OnPrem = "OnPrem"
+
+    @property
+    def is_saas_runtime(self) -> bool:
+        return self in [Runtime.Saas]
+
+    def is_installation_type_compatible(self, installation_type: str) -> bool:
+        """
+        Check if the installation type is compatible with the runtime
+
+        if the runtime is Saas, the installation type should start with Saas
+        else the installation type should be OnPrem
+        """
+        return (
+            self.value == Runtime.Saas.value
+            and installation_type.startswith(self.value)
+        ) or installation_type == self.value
+
+
+class PortAPIErrorMessage(Enum):
+    NOT_FOUND = "not_found"
+
+
 class Entity(BaseModel):
     identifier: Any
     blueprint: Any
@@ -12,6 +39,10 @@ class Entity(BaseModel):
     properties: dict[str, Any] = {}
     relations: dict[str, Any] = {}
 
+    @property
+    def is_using_search_identifier(self) -> bool:
+        return isinstance(self.identifier, dict)
+
 
 class BlueprintRelation(BaseModel):
     many: bool
@@ -34,3 +65,16 @@ class Migration(BaseModel):
     sourceBlueprint: str
     mapping: dict[str, Any]
     status: str
+
+
+@dataclass
+class EntityPortDiff:
+    """Represents the differences between entities for porting.
+
+    This class holds the lists of deleted, modified, and created entities as part
+    of the porting process.
+    """
+
+    deleted: list[Entity] = field(default_factory=list)
+    modified: list[Entity] = field(default_factory=list)
+    created: list[Entity] = field(default_factory=list)

port_ocean/core/ocean_types.py

@@ -1,11 +1,28 @@
-from typing import TypedDict, Any, AsyncIterator, Callable, Awaitable
-
+from typing import (
+    TypedDict,
+    Any,
+    AsyncIterator,
+    Callable,
+    Awaitable,
+    NamedTuple,
+)
+
+from dataclasses import field
 from port_ocean.core.models import Entity
 
+RAW_ITEM = dict[Any, Any]
+RAW_RESULT = list[RAW_ITEM]
+ASYNC_GENERATOR_RESYNC_TYPE = AsyncIterator[RAW_RESULT]
+RESYNC_RESULT = list[RAW_ITEM | ASYNC_GENERATOR_RESYNC_TYPE]
+
+LISTENER_RESULT = Awaitable[RAW_RESULT] | ASYNC_GENERATOR_RESYNC_TYPE
+RESYNC_EVENT_LISTENER = Callable[[str], LISTENER_RESULT]
+START_EVENT_LISTENER = Callable[[], Awaitable[None]]
+
 
 class RawEntityDiff(TypedDict):
-    before: list[dict[Any, Any]]
-    after: list[dict[Any, Any]]
+    before: list[RAW_ITEM]
+    after: list[RAW_ITEM]
 
 
 class EntityDiff(TypedDict):
@@ -13,14 +30,15 @@ class EntityDiff(TypedDict):
     after: list[Entity]
 
 
-RAW_ITEM = dict[Any, Any]
-RAW_RESULT = list[RAW_ITEM]
-ASYNC_GENERATOR_RESYNC_TYPE = AsyncIterator[RAW_RESULT]
-RESYNC_RESULT = list[RAW_ITEM | ASYNC_GENERATOR_RESYNC_TYPE]
+class EntitySelectorDiff(NamedTuple):
+    passed: list[Entity]
+    failed: list[Entity]
 
-LISTENER_RESULT = Awaitable[RAW_RESULT] | ASYNC_GENERATOR_RESYNC_TYPE
-RESYNC_EVENT_LISTENER = Callable[[str], LISTENER_RESULT]
-START_EVENT_LISTENER = Callable[[], Awaitable[None]]
+
+class CalculationResult(NamedTuple):
+    entity_selector_diff: EntitySelectorDiff
+    errors: list[Exception]
+    misonfigured_entity_keys: dict[str, str] = field(default_factory=dict)
 
 
 class IntegrationEventsCallbacks(TypedDict):

port_ocean/core/utils/entity_topological_sorter.py

@@ -0,0 +1,90 @@
+from typing import Any, Generator
+from port_ocean.context import event
+from port_ocean.core.models import Entity
+
+from loguru import logger
+
+from graphlib import TopologicalSorter, CycleError
+from typing import Set
+
+from port_ocean.exceptions.core import OceanAbortException
+
+Node = tuple[str, str]
+
+
+class EntityTopologicalSorter:
+    def __init__(self) -> None:
+        self.entities: list[Entity] = []
+
+    def register_entity(
+        self,
+        entity: Entity,
+    ) -> None:
+        logger.debug(
+            f"Will retry upserting entity - {entity.identifier} at the end of resync"
+        )
+        self.entities.append(entity)
+
+    def should_execute(self) -> int:
+        return not event.event.port_app_config.create_missing_related_entities
+
+    def get_entities_count(self) -> int:
+        return len(self.entities)
+
+    def get_entities(self, sorted: bool = True) -> Generator[Entity, Any, None]:
+        if not sorted:
+            for entity in self.entities:
+                yield entity
+            return
+
+        sorted_and_mapped = EntityTopologicalSorter.order_by_entities_dependencies(
+            self.entities
+        )
+        for entity in sorted_and_mapped:
+            yield entity
+
+    @staticmethod
+    def node(entity: Entity) -> Node:
+        return entity.identifier, entity.blueprint
+
+    @staticmethod
+    def order_by_entities_dependencies(entities: list[Entity]) -> list[Entity]:
+        nodes: dict[Node, Set[Node]] = {}
+        entities_map = {}
+        for entity in entities:
+            nodes[EntityTopologicalSorter.node(entity)] = set()
+            entities_map[EntityTopologicalSorter.node(entity)] = entity
+
+        for entity in entities:
+            relation_target_ids: list[str] = sum(
+                [
+                    identifiers if isinstance(identifiers, list) else [identifiers]
+                    for identifiers in entity.relations.values()
+                    if identifiers is not None
+                ],
+                [],
+            )
+            related_entities = [
+                related
+                for related in entities
+                if related.identifier in relation_target_ids
+            ]
+
+            for related_entity in related_entities:
+                if (
+                    entity.blueprint is not related_entity.blueprint
+                    or entity.identifier is not related_entity.identifier
+                ):
+                    nodes[EntityTopologicalSorter.node(entity)].add(
+                        EntityTopologicalSorter.node(related_entity)
+                    )
+
+        sort_op = TopologicalSorter(nodes)
+        try:
+            return [entities_map[item] for item in sort_op.static_order()]
+        except CycleError as ex:
+            raise OceanAbortException(
+                "Cannot order entities due to cyclic dependencies. \n"
+                "If you do want to have cyclic dependencies, please make sure to set the keys"
+                " 'createMissingRelatedEntities' and 'deleteDependentEntities' in the integration config in Port."
+            ) from ex