port-ocean 0.5.6__py3-none-any.whl → 0.17.8__py3-none-any.whl

port_ocean/core/event_listener/polling.py

@@ -10,6 +10,7 @@ from port_ocean.core.event_listener.base import (
     EventListenerSettings,
 )
 from port_ocean.utils.repeat import repeat_every
+from port_ocean.utils.signal import signal_handler
 
 
 class PollingEventListenerSettings(EventListenerSettings):
@@ -26,9 +27,6 @@ class PollingEventListenerSettings(EventListenerSettings):
     resync_on_start: bool = True
     interval: int = 60
 
-    def to_request(self) -> dict[str, Any]:
-        return {}
-
 
 class PollingEventListener(BaseEventListener):
     """
@@ -48,7 +46,16 @@ class PollingEventListener(BaseEventListener):
     ):
         super().__init__(events)
         self.event_listener_config = event_listener_config
-        self._last_updated_at = None
+
+    def should_resync(self, last_updated_at: str) -> bool:
+        _last_updated_at = (
+            ocean.app.resync_state_updater.last_integration_state_updated_at
+        )
+
+        if _last_updated_at is None:
+            return self.event_listener_config.resync_on_start
+
+        return _last_updated_at != last_updated_at
 
     async def _start(self) -> None:
         """
@@ -68,18 +75,13 @@ class PollingEventListener(BaseEventListener):
             integration = await ocean.app.port_client.get_current_integration()
             last_updated_at = integration["updatedAt"]
 
-            should_resync = (
-                self._last_updated_at is not None
-                or self.event_listener_config.resync_on_start
-            ) and self._last_updated_at != last_updated_at
-
-            if should_resync:
+            if self.should_resync(last_updated_at):
                 logger.info("Detected change in integration, resyncing")
-                self._last_updated_at = last_updated_at
-                running_task: Task[Any] = get_event_loop().create_task(
-                    self.events["on_resync"]({})  # type: ignore
+                ocean.app.resync_state_updater.last_integration_state_updated_at = (
+                    last_updated_at
                 )
-                self._tasks_to_close.append(running_task)
+                running_task: Task[Any] = get_event_loop().create_task(self._resync({}))
+                signal_handler.register(running_task.cancel)
 
                 await running_task
 

port_ocean/core/event_listener/webhooks_only.py

@@ -0,0 +1,41 @@
+from typing import Literal
+
+from loguru import logger
+
+from port_ocean.core.event_listener.base import (
+    BaseEventListener,
+    EventListenerEvents,
+    EventListenerSettings,
+)
+
+
+class WebhooksOnlyEventListenerSettings(EventListenerSettings):
+    """
+    This class inherits from `EventListenerSettings`, which provides a foundation for creating event listener settings.
+    """
+
+    type: Literal["WEBHOOKS_ONLY"]
+    should_resync: bool = False
+
+
+class WebhooksOnlyEventListener(BaseEventListener):
+    """
+    No resync event listener.
+
+    It is used to handle events exclusively through webhooks without supporting resync events.
+
+    Parameters:
+        events (EventListenerEvents): A dictionary containing event types and their corresponding event handlers.
+        event_listener_config (OnceEventListenerSettings): The event listener configuration settings.
+    """
+
+    def __init__(
+        self,
+        events: EventListenerEvents,
+        event_listener_config: WebhooksOnlyEventListenerSettings,
+    ):
+        super().__init__(events)
+        self.event_listener_config = event_listener_config
+
+    async def _start(self) -> None:
+        logger.info("Starting Webhooks-only event listener")

port_ocean/core/handlers/__init__.py

@@ -4,7 +4,7 @@ from .entities_state_applier.base import (
 from .entities_state_applier.port.applier import (
     HttpEntitiesStateApplier,
 )
-from .entity_processor.base import BaseEntityProcessor, EntityPortDiff
+from .entity_processor.base import BaseEntityProcessor
 from .entity_processor.jq_entity_processor import (
     JQEntityProcessor,
 )
@@ -12,7 +12,6 @@ from .port_app_config.api import APIPortAppConfig
 from .port_app_config.base import BasePortAppConfig
 
 __all__ = [
-    "EntityPortDiff",
     "BaseEntityProcessor",
     "JQEntityProcessor",
     "BasePortAppConfig",

port_ocean/core/handlers/entities_state_applier/base.py

@@ -47,12 +47,15 @@ class BaseEntitiesStateApplier(BaseHandler):
     @abstractmethod
     async def upsert(
         self, entities: list[Entity], user_agent_type: UserAgentType
-    ) -> None:
+    ) -> list[Entity]:
         """Upsert (insert or update) the given entities into the state.
 
         Args:
             entities (list[Entity]): The entities to be upserted.
             user_agent_type (UserAgentType): The user agent responsible for the upsert.
+
+        Returns:
+            list[Entity]: The upserted entities.
         """
         pass
 

port_ocean/core/handlers/entities_state_applier/port/applier.py

@@ -1,6 +1,3 @@
-import asyncio
-from itertools import chain
-
 from loguru import logger
 
 from port_ocean.clients.port.types import UserAgentType
@@ -11,17 +8,11 @@ from port_ocean.core.handlers.entities_state_applier.base import (
 from port_ocean.core.handlers.entities_state_applier.port.get_related_entities import (
     get_related_entities,
 )
-from port_ocean.core.handlers.entities_state_applier.port.order_by_entities_dependencies import (
-    order_by_entities_dependencies,
-)
-from port_ocean.core.handlers.entities_state_applier.port.validate_entity_relations import (
-    validate_entity_relations,
-)
-from port_ocean.core.handlers.entity_processor.base import EntityPortDiff
+
 from port_ocean.core.models import Entity
 from port_ocean.core.ocean_types import EntityDiff
-from port_ocean.core.utils import is_same_entity, get_unique, get_port_diff
-from port_ocean.exceptions.core import RelationValidationException
+from port_ocean.core.utils.entity_topological_sorter import EntityTopologicalSorter
+from port_ocean.core.utils.utils import is_same_entity, get_port_diff
 
 
 class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
@@ -32,63 +23,17 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
     through HTTP requests.
     """
 
-    async def _validate_delete_dependent_entities(self, entities: list[Entity]) -> None:
-        logger.info("Validated deleted entities")
-        if not event.port_app_config.delete_dependent_entities:
-            dependent_entities = await asyncio.gather(
-                *(
-                    self.context.port_client.search_dependent_entities(entity)
-                    for entity in entities
-                )
-            )
-            new_dependent_entities = get_unique(
-                [
-                    entity
-                    for entity in chain.from_iterable(dependent_entities)
-                    if not any(is_same_entity(item, entity) for item in entities)
-                ]
-            )
-
-            if new_dependent_entities:
-                raise RelationValidationException(
-                    f"Must enable delete_dependent_entities flag or delete all dependent entities: "
-                    f" {[(dep.blueprint, dep.identifier) for dep in new_dependent_entities]}"
-                )
-
-    async def _validate_entity_diff(self, diff: EntityPortDiff) -> None:
-        config = event.port_app_config
-        await self._validate_delete_dependent_entities(diff.deleted)
-        modified_or_created_entities = diff.modified + diff.created
-
-        if modified_or_created_entities and not config.create_missing_related_entities:
-            logger.info("Validating modified or created entities")
-
-            await asyncio.gather(
-                *(
-                    self.context.port_client.validate_entity_payload(
-                        entity,
-                        config.enable_merge_entity,
-                        create_missing_related_entities=config.create_missing_related_entities,
-                    )
-                    for entity in modified_or_created_entities
-                )
-            )
-
-        if not event.port_app_config.delete_dependent_entities:
-            logger.info("Validating no relation blocks the operation")
-            await validate_entity_relations(diff, self.context.port_client)
-
-    async def _delete_diff(
+    async def _safe_delete(
         self,
         entities_to_delete: list[Entity],
-        created_entities: list[Entity],
+        entities_to_protect: list[Entity],
         user_agent_type: UserAgentType,
     ) -> None:
         if not entities_to_delete:
             return
 
         related_entities = await get_related_entities(
-            created_entities, self.context.port_client
+            entities_to_protect, self.context.port_client
         )
 
         allowed_entities_to_delete = []
@@ -98,7 +43,8 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
                 is_same_entity(entity, entity_to_delete) for entity in related_entities
             )
             is_part_of_created = any(
-                is_same_entity(entity, entity_to_delete) for entity in created_entities
+                is_same_entity(entity, entity_to_delete)
+                for entity in entities_to_protect
             )
             if is_part_of_related:
                 if event.port_app_config.create_missing_related_entities:
@@ -119,21 +65,14 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
         user_agent_type: UserAgentType,
     ) -> None:
         diff = get_port_diff(entities["before"], entities["after"])
+        kept_entities = diff.created + diff.modified
 
         logger.info(
             f"Updating entity diff (created: {len(diff.created)}, deleted: {len(diff.deleted)}, modified: {len(diff.modified)})"
         )
-        await self._validate_entity_diff(diff)
+        modified_entities = await self.upsert(kept_entities, user_agent_type)
 
-        logger.info("Upserting new entities")
-        await self.upsert(diff.created, user_agent_type)
-        logger.info("Upserting modified entities")
-        await self.upsert(diff.modified, user_agent_type)
-
-        logger.info("Deleting diff entities")
-        await self._delete_diff(
-            diff.deleted, diff.created + diff.modified, user_agent_type
-        )
+        await self._safe_delete(diff.deleted, modified_entities, user_agent_type)
 
     async def delete_diff(
         self,
@@ -145,39 +84,40 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
         if not diff.deleted:
             return
 
+        kept_entities = diff.created + diff.modified
+
         logger.info(
-            f"Updating entity diff (created: {len(diff.created)}, deleted: {len(diff.deleted)}, modified: {len(diff.modified)})"
+            f"Determining entities to delete ({len(diff.deleted)}/{len(kept_entities)})"
         )
-        await self._validate_entity_diff(diff)
 
-        logger.info("Deleting diff entities")
-        await self._delete_diff(
-            diff.deleted, diff.created + diff.modified, user_agent_type
-        )
+        await self._safe_delete(diff.deleted, kept_entities, user_agent_type)
 
     async def upsert(
         self, entities: list[Entity], user_agent_type: UserAgentType
-    ) -> None:
+    ) -> list[Entity]:
         logger.info(f"Upserting {len(entities)} entities")
+        modified_entities: list[Entity] = []
         if event.port_app_config.create_missing_related_entities:
-            await self.context.port_client.batch_upsert_entities(
+            modified_entities = await self.context.port_client.batch_upsert_entities(
                 entities,
                 event.port_app_config.get_port_request_options(),
                 user_agent_type,
                 should_raise=False,
             )
         else:
-            ordered_created_entities = reversed(
-                order_by_entities_dependencies(entities)
-            )
-
-            for entity in ordered_created_entities:
-                await self.context.port_client.upsert_entity(
+            for entity in entities:
+                upsertedEntity = await self.context.port_client.upsert_entity(
                     entity,
                     event.port_app_config.get_port_request_options(),
                     user_agent_type,
                     should_raise=False,
                 )
+                if upsertedEntity:
+                    modified_entities.append(upsertedEntity)
+                # condition to false to differentiate from `result_entity.is_using_search_identifier`
+                if upsertedEntity is False:
+                    event.entity_topological_sorter.register_entity(entity)
+        return modified_entities
 
     async def delete(
         self, entities: list[Entity], user_agent_type: UserAgentType
@@ -191,7 +131,9 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
                 should_raise=False,
             )
         else:
-            ordered_deleted_entities = order_by_entities_dependencies(entities)
+            ordered_deleted_entities = (
+                EntityTopologicalSorter.order_by_entities_dependencies(entities)
+            )
 
             for entity in ordered_deleted_entities:
                 await self.context.port_client.delete_entity(

port_ocean/core/handlers/entities_state_applier/port/order_by_entities_dependencies.py

@@ -14,7 +14,6 @@ def node(entity: Entity) -> Node:
 def order_by_entities_dependencies(entities: list[Entity]) -> list[Entity]:
     nodes: dict[Node, Set[Node]] = {}
     entities_map = {}
-
     for entity in entities:
         nodes[node(entity)] = set()
         entities_map[node(entity)] = entity
@@ -33,7 +32,11 @@ def order_by_entities_dependencies(entities: list[Entity]) -> list[Entity]:
         ]
 
         for related_entity in related_entities:
-            nodes[node(entity)].add(node(related_entity))
+            if (
+                entity.blueprint is not related_entity.blueprint
+                or entity.identifier is not related_entity.identifier
+            ):
+                nodes[node(entity)].add(node(related_entity))
 
     sort_op = TopologicalSorter(nodes)
     try:

port_ocean/core/handlers/entity_processor/base.py

@@ -1,25 +1,14 @@
 from abc import abstractmethod
-from dataclasses import dataclass, field
 
 from loguru import logger
 
 from port_ocean.core.handlers.base import BaseHandler
 from port_ocean.core.handlers.port_app_config.models import ResourceConfig
-from port_ocean.core.models import Entity
-from port_ocean.core.ocean_types import RawEntityDiff, EntityDiff
-
-
-@dataclass
-class EntityPortDiff:
-    """Represents the differences between entities for porting.
-
-    This class holds the lists of deleted, modified, and created entities as part
-    of the porting process.
-    """
-
-    deleted: list[Entity] = field(default_factory=list)
-    modified: list[Entity] = field(default_factory=list)
-    created: list[Entity] = field(default_factory=list)
+from port_ocean.core.ocean_types import (
+    RAW_ITEM,
+    CalculationResult,
+    EntitySelectorDiff,
+)
 
 
 class BaseEntityProcessor(BaseHandler):
@@ -33,21 +22,36 @@ class BaseEntityProcessor(BaseHandler):
 
     @abstractmethod
     async def _parse_items(
-        self, mapping: ResourceConfig, raw_data: RawEntityDiff
-    ) -> EntityDiff:
+        self,
+        mapping: ResourceConfig,
+        raw_data: list[RAW_ITEM],
+        parse_all: bool = False,
+        send_raw_data_examples_amount: int = 0,
+    ) -> CalculationResult:
         pass
 
     async def parse_items(
-        self, mapping: ResourceConfig, raw_data: RawEntityDiff
-    ) -> EntityDiff:
+        self,
+        mapping: ResourceConfig,
+        raw_data: list[RAW_ITEM],
+        parse_all: bool = False,
+        send_raw_data_examples_amount: int = 0,
+    ) -> CalculationResult:
         """Public method to parse raw entity data and map it to an EntityDiff.
 
         Args:
             mapping (ResourceConfig): The configuration for entity mapping.
-            raw_data (RawEntityDiff): The raw data to be parsed.
+            raw_data (list[RawEntity]): The raw data to be parsed.
+            parse_all (bool): Whether to parse all data or just data that passed the selector.
+            send_raw_data_examples_amount (bool): Whether to send example data to the integration service.
 
         Returns:
             EntityDiff: The parsed entity differences.
         """
         with logger.contextualize(kind=mapping.kind):
-            return await self._parse_items(mapping, raw_data)
+            if not raw_data:
+                return CalculationResult(EntitySelectorDiff([], []), [])
+
+            return await self._parse_items(
+                mapping, raw_data, parse_all, send_raw_data_examples_amount
+            )