port-ocean 0.17.7__py3-none-any.whl → 0.18.0__py3-none-any.whl

port_ocean/clients/port/mixins/entities.py

@@ -98,10 +98,22 @@ class EntityClientMixin:
         if result_entity.is_using_search_identifier:
             return None
 
-        # In order to save memory we'll keep only the identifier, blueprint and relations of the
-        # upserted entity result for later calculations
+        return self._reduce_entity(result_entity)
+
+    @staticmethod
+    def _reduce_entity(entity: Entity) -> Entity:
+        """
+        Reduces an entity to only keep identifier, blueprint and processed relations.
+        This helps save memory by removing unnecessary data.
+
+        Args:
+            entity: The entity to reduce
+
+        Returns:
+            Entity: A new entity with only the essential data
+        """
         reduced_entity = Entity(
-            identifier=result_entity.identifier, blueprint=result_entity.blueprint
+            identifier=entity.identifier, blueprint=entity.blueprint
         )
 
         # Turning dict typed relations (raw search relations) is required
@@ -109,7 +121,7 @@ class EntityClientMixin:
         # and ignore the ones that don't as they weren't upserted
         reduced_entity.relations = {
             key: None if isinstance(relation, dict) else relation
-            for key, relation in result_entity.relations.items()
+            for key, relation in entity.relations.items()
         }
 
         return reduced_entity
@@ -202,7 +214,10 @@ class EntityClientMixin:
         )
 
     async def search_entities(
-        self, user_agent_type: UserAgentType, query: dict[Any, Any] | None = None
+        self,
+        user_agent_type: UserAgentType,
+        query: dict[Any, Any] | None = None,
+        parameters_to_include: list[str] | None = None,
     ) -> list[Entity]:
         default_query = {
             "combinator": "and",
@@ -232,7 +247,7 @@ class EntityClientMixin:
             headers=await self.auth.headers(user_agent_type),
             params={
                 "exclude_calculated_properties": "true",
-                "include": ["blueprint", "identifier"],
+                "include": parameters_to_include or ["blueprint", "identifier"],
             },
             extensions={"retryable": True},
         )

port_ocean/core/integrations/mixins/sync_raw.py

@@ -28,7 +28,7 @@ from port_ocean.core.ocean_types import (
     RAW_ITEM,
     CalculationResult,
 )
-from port_ocean.core.utils.utils import zip_and_sum, gather_and_split_errors_from_results
+from port_ocean.core.utils.utils import resolve_entities_diff, zip_and_sum, gather_and_split_errors_from_results
 from port_ocean.exceptions.core import OceanAbortException
 
 SEND_RAW_DATA_EXAMPLES_AMOUNT = 5
@@ -130,24 +130,128 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
             )
         )
 
+    def _construct_search_query_for_entities(self, entities: list[Entity]) -> dict:
+        """Create a query to search for entities by their identifiers.
+
+        Args:
+            entities (list[Entity]): List of entities to search for.
+
+        Returns:
+            dict: Query structure for searching entities by identifier.
+        """
+        return {
+            "combinator": "and",
+            "rules": [
+                {
+                    "combinator": "or",
+                    "rules": [
+                        {
+                            "property": "$identifier",
+                            "operator": "=",
+                            "value": entity.identifier,
+                        }
+                        for entity in entities
+                    ]
+                }
+            ]
+        }
+
+    async def _map_entities_compared_with_port(
+        self,
+        entities: list[Entity],
+        resource: ResourceConfig,
+        user_agent_type: UserAgentType,
+    ) -> list[Entity]:
+        if not entities:
+            return []
+
+        if entities[0].is_using_search_identifier or entities[0].is_using_search_relation:
+            return entities
+
+        BATCH_SIZE = 50
+        entities_at_port_with_properties = []
+
+        # Process entities in batches
+        for start_index in range(0, len(entities), BATCH_SIZE):
+            entities_batch = entities[start_index:start_index + BATCH_SIZE]
+            batch_results = await self._fetch_entities_batch_from_port(
+                entities_batch,
+                resource,
+                user_agent_type
+            )
+            entities_at_port_with_properties.extend(batch_results)
+
+        logger.info("Got entities from port with properties and relations", port_entities=len(entities_at_port_with_properties))
+
+        if len(entities_at_port_with_properties) > 0:
+            return resolve_entities_diff(entities, entities_at_port_with_properties)
+        return entities
+
+    async def _fetch_entities_batch_from_port(
+        self,
+        entities_batch: list[Entity],
+        resource: ResourceConfig,
+        user_agent_type: UserAgentType,
+    ) -> list[Entity]:
+        query = self._construct_search_query_for_entities(entities_batch)
+        return await ocean.port_client.search_entities(
+            user_agent_type,
+            parameters_to_include=["blueprint", "identifier"] + (
+                ["title"] if resource.port.entity.mappings.title != None else []
+            ) + (
+                ["team"] if resource.port.entity.mappings.team != None else []
+            ) + [
+                f"properties.{prop}" for prop in resource.port.entity.mappings.properties
+            ] + [
+                f"relations.{relation}" for relation in resource.port.entity.mappings.relations
+            ],
+            query=query
+        )
+
     async def _register_resource_raw(
         self,
         resource: ResourceConfig,
         results: list[dict[Any, Any]],
         user_agent_type: UserAgentType,
         parse_all: bool = False,
-        send_raw_data_examples_amount: int = 0,
+        send_raw_data_examples_amount: int = 0
     ) -> CalculationResult:
         objects_diff = await self._calculate_raw(
             [(resource, results)], parse_all, send_raw_data_examples_amount
         )
-        modified_objects = await self.entities_state_applier.upsert(
-            objects_diff[0].entity_selector_diff.passed, user_agent_type
+        modified_objects = []
+
+        if ocean.app.is_saas():
+            try:
+                changed_entities = await self._map_entities_compared_with_port(
+                    objects_diff[0].entity_selector_diff.passed,
+                    resource,
+                    user_agent_type
+                )
+
+                if changed_entities:
+                    logger.info("Upserting changed entities", changed_entities=len(changed_entities),
+                        total_entities=len(objects_diff[0].entity_selector_diff.passed))
+                    await self.entities_state_applier.upsert(
+                        changed_entities, user_agent_type
+                    )
+                else:
+                    logger.info("Entities in batch didn't changed since last sync, skipping", total_entities=len(objects_diff[0].entity_selector_diff.passed))
+
+                modified_objects = [ocean.port_client._reduce_entity(entity) for entity in objects_diff[0].entity_selector_diff.passed]
+            except Exception as e:
+                logger.warning(f"Failed to resolve batch entities with Port, falling back to upserting all entities: {str(e)}")
+                modified_objects = await self.entities_state_applier.upsert(
+                    objects_diff[0].entity_selector_diff.passed, user_agent_type
+                    )
+        else:
+            modified_objects = await self.entities_state_applier.upsert(
+                objects_diff[0].entity_selector_diff.passed, user_agent_type
         )
         return CalculationResult(
             objects_diff[0].entity_selector_diff._replace(passed=modified_objects),
             errors=objects_diff[0].errors,
-            misonfigured_entity_keys=objects_diff[0].misonfigured_entity_keys,
+            misonfigured_entity_keys=objects_diff[0].misonfigured_entity_keys
         )
 
     async def _unregister_resource_raw(
@@ -186,14 +290,17 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         send_raw_data_examples_amount = (
             SEND_RAW_DATA_EXAMPLES_AMOUNT if ocean.config.send_raw_data_examples else 0
         )
-        all_entities, register_errors,_ = await self._register_resource_raw(
-            resource_config,
-            raw_results,
-            user_agent_type,
-            send_raw_data_examples_amount=send_raw_data_examples_amount,
-        )
-        errors.extend(register_errors)
-        passed_entities = list(all_entities.passed)
+
+        passed_entities = []
+        if raw_results:
+            calculation_result = await self._register_resource_raw(
+                resource_config,
+                raw_results,
+                user_agent_type,
+                send_raw_data_examples_amount=send_raw_data_examples_amount
+            )
+            errors.extend(calculation_result.errors)
+            passed_entities = list(calculation_result.entity_selector_diff.passed)
 
         for generator in async_generators:
             try:
@@ -203,14 +310,14 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                             0, send_raw_data_examples_amount - len(passed_entities)
                         )
 
-                    entities, register_errors,_ = await self._register_resource_raw(
+                    calculation_result = await self._register_resource_raw(
                         resource_config,
                         items,
                         user_agent_type,
-                        send_raw_data_examples_amount=send_raw_data_examples_amount,
+                        send_raw_data_examples_amount=send_raw_data_examples_amount
                     )
-                    errors.extend(register_errors)
-                    passed_entities.extend(entities.passed)
+                    errors.extend(calculation_result.errors)
+                    passed_entities.extend(calculation_result.entity_selector_diff.passed)
             except* OceanAbortException as error:
                 errors.append(error)
 
@@ -446,9 +553,6 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
 
             try:
                 did_fetched_current_state = True
-                entities_at_port = await ocean.port_client.search_entities(
-                    user_agent_type
-                )
             except httpx.HTTPError as e:
                 logger.warning(
                     "Failed to fetch the current state of entities at Port. "
@@ -461,6 +565,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
 
             creation_results: list[tuple[list[Entity], list[Exception]]] = []
 
+
             try:
                 for resource in app_config.resources:
                     # create resource context per resource kind, so resync method could have access to the resource
@@ -471,7 +576,6 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                         )
 
                         event.on_abort(lambda: task.cancel())
-
                         creation_results.append(await task)
 
                 await self.sort_and_upsert_failed_entities(user_agent_type)
@@ -487,7 +591,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                     return
 
                 logger.info("Starting resync diff calculation")
-                flat_created_entities, errors = zip_and_sum(creation_results) or [
+                generated_entities, errors = zip_and_sum(creation_results) or [
                     [],
                     [],
                 ]
@@ -504,10 +608,14 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                     logger.error(message, exc_info=error_group)
                 else:
                     logger.info(
-                        f"Running resync diff calculation, number of entities at Port before resync: {len(entities_at_port)}, number of entities created during sync: {len(flat_created_entities)}"
+                        f"Running resync diff calculation, number of entities created during sync: {len(generated_entities)}"
+                    )
+                    entities_at_port = await ocean.port_client.search_entities(
+                        user_agent_type
                     )
                     await self.entities_state_applier.delete_diff(
-                        {"before": entities_at_port, "after": flat_created_entities},
+                        {"before": entities_at_port, "after": generated_entities},
                         user_agent_type,
                     )
+
                     logger.info("Resync finished successfully")

port_ocean/core/models.py

@@ -43,6 +43,10 @@ class Entity(BaseModel):
     def is_using_search_identifier(self) -> bool:
         return isinstance(self.identifier, dict)
 
+    @property
+    def is_using_search_relation(self) -> bool:
+        return any(isinstance(relation, dict) for relation in self.relations.values())
+
 
 class BlueprintRelation(BaseModel):
     many: bool

port_ocean/core/utils/utils.py

@@ -1,9 +1,12 @@
 import asyncio
+import hashlib
+import json
 from typing import Iterable, Any, TypeVar, Callable, Awaitable
 
 from loguru import logger
 from pydantic import parse_obj_as, ValidationError
 
+
 from port_ocean.clients.port.client import PortClient
 from port_ocean.core.models import Entity, Runtime
 from port_ocean.core.models import EntityPortDiff
@@ -76,10 +79,7 @@ async def gather_and_split_errors_from_results(
     return valid_items, errors
 
 
-def get_port_diff(
-    before: Iterable[Entity],
-    after: Iterable[Entity],
-) -> EntityPortDiff:
+def get_port_diff(before: Iterable[Entity], after: Iterable[Entity]) -> EntityPortDiff:
     before_dict = {}
     after_dict = {}
     created = []
@@ -107,3 +107,79 @@ def get_port_diff(
             deleted.append(obj)
 
     return EntityPortDiff(created=created, modified=modified, deleted=deleted)
+
+
+def are_teams_different(
+    first_team: str | None | list[Any], second_team: str | None | list[Any]
+) -> bool:
+    if isinstance(first_team, list) and isinstance(second_team, list):
+        return sorted(first_team) != sorted(second_team)
+    return first_team != second_team
+
+
+def are_entities_fields_equal(
+    first_entity_field: dict[str, Any], second_entity_field: dict[str, Any]
+) -> bool:
+    """
+    Compare two entity fields by serializing them to JSON and comparing their SHA-256 hashes.
+
+    Args:
+        first_entity_field: First entity field dictionary to compare
+        second_entity_field: Second entity field dictionary to compare
+
+    Returns:
+        bool: True if the entity fields have identical content
+    """
+    first_props = json.dumps(first_entity_field, sort_keys=True)
+    second_props = json.dumps(second_entity_field, sort_keys=True)
+    first_hash = hashlib.sha256(first_props.encode()).hexdigest()
+    second_hash = hashlib.sha256(second_props.encode()).hexdigest()
+    return first_hash == second_hash
+
+
+def are_entities_different(first_entity: Entity, second_entity: Entity) -> bool:
+    if first_entity.title != second_entity.title:
+        return True
+    if are_teams_different(first_entity.team, second_entity.team):
+        return True
+    if not are_entities_fields_equal(first_entity.properties, second_entity.properties):
+        return True
+    if not are_entities_fields_equal(first_entity.relations, second_entity.relations):
+        return True
+
+    return False
+
+
+def resolve_entities_diff(
+    source_entities: list[Entity], target_entities: list[Entity]
+) -> list[Entity]:
+    """
+    Maps the entities into filtered list of source entities, excluding matches found in target that needs to be upserted
+    Args:
+        source_entities: List of entities from third party source
+        target_entities: List of existing Port entities
+
+    Returns:
+        list[Entity]: Filtered list of source entities, excluding matches found in target
+    """
+    target_entities_dict = {}
+    source_entities_dict = {}
+    changed_entities = []
+
+    for entity in target_entities:
+        key = (entity.identifier, entity.blueprint)
+        target_entities_dict[key] = entity
+
+    for entity in source_entities:
+        if entity.is_using_search_identifier or entity.is_using_search_relation:
+            return source_entities
+        key = (entity.identifier, entity.blueprint)
+        source_entities_dict[key] = entity
+
+        entity_at_target = target_entities_dict.get(key, None)
+        if entity_at_target is None:
+            changed_entities.append(entity)
+        elif are_entities_different(entity, target_entities_dict[key]):
+            changed_entities.append(entity)
+
+    return changed_entities