port-ocean 0.15.3__tar.gz → 0.16.1__tar.gz

{port_ocean-0.15.3 → port_ocean-0.16.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.15.3
+Version: 0.16.1
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.15.3 → port_ocean-0.16.1}/port_ocean/clients/port/mixins/entities.py

@@ -1,5 +1,5 @@
 import asyncio
-from typing import Any
+from typing import Any, Literal
 from urllib.parse import quote_plus
 
 import httpx
@@ -11,7 +11,8 @@ from port_ocean.clients.port.utils import (
     handle_status_code,
     PORT_HTTP_MAX_CONNECTIONS_LIMIT,
 )
-from port_ocean.core.models import Entity
+from port_ocean.core.models import Entity, PortAPIErrorMessage
+from starlette import status
 
 
 class EntityClientMixin:
@@ -29,7 +30,27 @@ class EntityClientMixin:
         request_options: RequestOptions,
         user_agent_type: UserAgentType | None = None,
         should_raise: bool = True,
-    ) -> Entity | None:
+    ) -> Entity | None | Literal[False]:
+        """
+        This function upserts an entity into Port.
+
+        Usage:
+        ```python
+            upsertedEntity = await self.context.port_client.upsert_entity(
+                            entity,
+                            event.port_app_config.get_port_request_options(),
+                            user_agent_type,
+                            should_raise=False,
+                        )
+        ```
+        :param entity: An Entity to be upserted
+        :param request_options: A dictionary specifying how to upsert the entity
+        :param user_agent_type: a UserAgentType specifying who is preforming the action
+        :param should_raise: A boolean specifying whether the error should be raised or handled silently
+        :return: [Entity] if the upsert occured successfully
+        :return: [None] will be returned if entity is using search identifier
+        :return: [False] will be returned if upsert failed because of unmet dependency
+        """
         validation_only = request_options["validation_only"]
         async with self.semaphore:
             logger.debug(
@@ -50,13 +71,21 @@ class EntityClientMixin:
                 },
                 extensions={"retryable": True},
             )
-
         if response.is_error:
             logger.error(
                 f"Error {'Validating' if validation_only else 'Upserting'} "
                 f"entity: {entity.identifier} of "
                 f"blueprint: {entity.blueprint}"
             )
+            result = response.json()
+
+            if (
+                response.status_code == status.HTTP_404_NOT_FOUND
+                and not result.get("ok")
+                and result.get("error") == PortAPIErrorMessage.NOT_FOUND.value
+            ):
+                # Return false to differentiate from `result_entity.is_using_search_identifier`
+                return False
         handle_status_code(response, should_raise)
         result = response.json()
 

{port_ocean-0.15.3 → port_ocean-0.16.1}/port_ocean/context/event.py

@@ -14,6 +14,7 @@ from typing import (
 from uuid import uuid4
 
 from loguru import logger
+from port_ocean.core.utils.entity_topological_sorter import EntityTopologicalSorter
 from pydispatch import dispatcher  # type: ignore
 from werkzeug.local import LocalStack, LocalProxy
 
@@ -24,6 +25,7 @@ from port_ocean.exceptions.context import (
 )
 from port_ocean.utils.misc import get_time
 
+
 if TYPE_CHECKING:
     from port_ocean.core.handlers.port_app_config.models import (
         ResourceConfig,
@@ -50,6 +52,9 @@ class EventContext:
     _parent_event: Optional["EventContext"] = None
     _event_id: str = field(default_factory=lambda: str(uuid4()))
     _on_abort_callbacks: list[AbortCallbackFunction] = field(default_factory=list)
+    entity_topological_sorter: EntityTopologicalSorter = field(
+        default_factory=EntityTopologicalSorter
+    )
 
     def on_abort(self, func: AbortCallbackFunction) -> None:
         self._on_abort_callbacks.append(func)
@@ -129,6 +134,11 @@ async def event_context(
 ) -> AsyncIterator[EventContext]:
     parent = parent_override or _event_context_stack.top
     parent_attributes = parent.attributes if parent else {}
+    entity_topological_sorter = (
+        parent.entity_topological_sorter
+        if parent and parent.entity_topological_sorter
+        else EntityTopologicalSorter()
+    )
 
     attributes = {**parent_attributes, **(attributes or {})}
     new_event = EventContext(
@@ -138,6 +148,7 @@ async def event_context(
         _parent_event=parent,
         # inherit port app config from parent event, so it can be used in nested events
         _port_app_config=parent.port_app_config if parent else None,
+        entity_topological_sorter=entity_topological_sorter,
     )
     _event_context_stack.push(new_event)
 

{port_ocean-0.15.3 → port_ocean-0.16.1}/port_ocean/core/defaults/initialize.py

@@ -14,7 +14,7 @@ from port_ocean.core.defaults.common import (
 )
 from port_ocean.core.handlers.port_app_config.models import PortAppConfig
 from port_ocean.core.models import Blueprint
-from port_ocean.core.utils import gather_and_split_errors_from_results
+from port_ocean.core.utils.utils import gather_and_split_errors_from_results
 from port_ocean.exceptions.port_defaults import (
     AbortDefaultCreationError,
 )

{port_ocean-0.15.3 → port_ocean-0.16.1}/port_ocean/core/handlers/entities_state_applier/port/applier.py

@@ -8,12 +8,11 @@ from port_ocean.core.handlers.entities_state_applier.base import (
 from port_ocean.core.handlers.entities_state_applier.port.get_related_entities import (
     get_related_entities,
 )
-from port_ocean.core.handlers.entities_state_applier.port.order_by_entities_dependencies import (
-    order_by_entities_dependencies,
-)
+
 from port_ocean.core.models import Entity
 from port_ocean.core.ocean_types import EntityDiff
-from port_ocean.core.utils import is_same_entity, get_port_diff
+from port_ocean.core.utils.entity_topological_sorter import EntityTopologicalSorter
+from port_ocean.core.utils.utils import is_same_entity, get_port_diff
 
 
 class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
@@ -106,19 +105,7 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
                 should_raise=False,
             )
         else:
-            entities_with_search_identifier: list[Entity] = []
-            entities_without_search_identifier: list[Entity] = []
             for entity in entities:
-                if entity.is_using_search_identifier:
-                    entities_with_search_identifier.append(entity)
-                else:
-                    entities_without_search_identifier.append(entity)
-
-            ordered_created_entities = reversed(
-                entities_with_search_identifier
-                + order_by_entities_dependencies(entities_without_search_identifier)
-            )
-            for entity in ordered_created_entities:
                 upsertedEntity = await self.context.port_client.upsert_entity(
                     entity,
                     event.port_app_config.get_port_request_options(),
@@ -127,6 +114,9 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
                 )
                 if upsertedEntity:
                     modified_entities.append(upsertedEntity)
+                # condition to false to differentiate from `result_entity.is_using_search_identifier`
+                if upsertedEntity is False:
+                    event.entity_topological_sorter.register_entity(entity)
         return modified_entities
 
     async def delete(
@@ -141,7 +131,9 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
                 should_raise=False,
             )
         else:
-            ordered_deleted_entities = order_by_entities_dependencies(entities)
+            ordered_deleted_entities = (
+                EntityTopologicalSorter.order_by_entities_dependencies(entities)
+            )
 
             for entity in ordered_deleted_entities:
                 await self.context.port_client.delete_entity(

{port_ocean-0.15.3 → port_ocean-0.16.1}/port_ocean/core/handlers/entities_state_applier/port/order_by_entities_dependencies.py

@@ -14,7 +14,6 @@ def node(entity: Entity) -> Node:
 def order_by_entities_dependencies(entities: list[Entity]) -> list[Entity]:
     nodes: dict[Node, Set[Node]] = {}
     entities_map = {}
-
     for entity in entities:
         nodes[node(entity)] = set()
         entities_map[node(entity)] = entity
@@ -33,7 +32,11 @@ def order_by_entities_dependencies(entities: list[Entity]) -> list[Entity]:
         ]
 
         for related_entity in related_entities:
-            nodes[node(entity)].add(node(related_entity))
+            if (
+                entity.blueprint is not related_entity.blueprint
+                or entity.identifier is not related_entity.identifier
+            ):
+                nodes[node(entity)].add(node(related_entity))
 
     sort_op = TopologicalSorter(nodes)
     try:

{port_ocean-0.15.3 → port_ocean-0.16.1}/port_ocean/core/handlers/entity_processor/jq_entity_processor.py

@@ -16,7 +16,10 @@ from port_ocean.core.ocean_types import (
     EntitySelectorDiff,
     CalculationResult,
 )
-from port_ocean.core.utils import gather_and_split_errors_from_results, zip_and_sum
+from port_ocean.core.utils.utils import (
+    gather_and_split_errors_from_results,
+    zip_and_sum,
+)
 from port_ocean.exceptions.core import EntityProcessorException
 from port_ocean.utils.queue_utils import process_in_queue
 
@@ -31,6 +34,7 @@ class MappedEntity:
     entity: dict[str, Any] = field(default_factory=dict)
     did_entity_pass_selector: bool = False
     raw_data: Optional[dict[str, Any]] = None
+    misconfigurations: dict[str, str] = field(default_factory=dict)
 
 
 class JQEntityProcessor(BaseEntityProcessor):
@@ -92,21 +96,37 @@ class JQEntityProcessor(BaseEntityProcessor):
         )
 
     async def _search_as_object(
-        self, data: dict[str, Any], obj: dict[str, Any]
+        self,
+        data: dict[str, Any],
+        obj: dict[str, Any],
+        misconfigurations: dict[str, str] | None = None,
     ) -> dict[str, Any | None]:
+        """
+        Identify and extract the relevant value for the chosen key and populate it into the entity
+        :param data: the property itself that holds the key and the value, it is being passed to the task and we get back a task item,
+            if the data is a dict, we will recursively call this function again.
+        :param obj: the key that we want its value to be mapped into our entity.
+        :param misconfigurations: due to the recursive nature of this function,
+            we aim to have a dict that represents all of the misconfigured properties and when used recursively,
+            we pass this reference to misfoncigured object to add the relevant misconfigured keys.
+        :return: Mapped object with found value.
+        """
+
         search_tasks: dict[
             str, Task[dict[str, Any | None]] | list[Task[dict[str, Any | None]]]
         ] = {}
         for key, value in obj.items():
             if isinstance(value, list):
                 search_tasks[key] = [
-                    asyncio.create_task(self._search_as_object(data, obj))
+                    asyncio.create_task(
+                        self._search_as_object(data, obj, misconfigurations)
+                    )
                     for obj in value
                 ]
 
             elif isinstance(value, dict):
                 search_tasks[key] = asyncio.create_task(
-                    self._search_as_object(data, value)
+                    self._search_as_object(data, value, misconfigurations)
                 )
             else:
                 search_tasks[key] = asyncio.create_task(self._search(data, value))
@@ -115,12 +135,20 @@ class JQEntityProcessor(BaseEntityProcessor):
         for key, task in search_tasks.items():
             try:
                 if isinstance(task, list):
-                    result[key] = [await task for task in task]
+                    result_list = []
+                    for task in task:
+                        task_result = await task
+                        if task_result is None and misconfigurations is not None:
+                            misconfigurations[key] = obj[key]
+                        result_list.append(task_result)
+                    result[key] = result_list
                 else:
-                    result[key] = await task
+                    task_result = await task
+                    if task_result is None and misconfigurations is not None:
+                        misconfigurations[key] = obj[key]
+                    result[key] = task_result
             except Exception:
                 result[key] = None
-
         return result
 
     async def _get_mapped_entity(
@@ -132,11 +160,15 @@ class JQEntityProcessor(BaseEntityProcessor):
     ) -> MappedEntity:
         should_run = await self._search_as_bool(data, selector_query)
         if parse_all or should_run:
-            mapped_entity = await self._search_as_object(data, raw_entity_mappings)
+            misconfigurations: dict[str, str] = {}
+            mapped_entity = await self._search_as_object(
+                data, raw_entity_mappings, misconfigurations
+            )
             return MappedEntity(
                 mapped_entity,
                 did_entity_pass_selector=should_run,
                 raw_data=data if should_run else None,
+                misconfigurations=misconfigurations,
             )
 
         return MappedEntity()
@@ -218,7 +250,11 @@ class JQEntityProcessor(BaseEntityProcessor):
         passed_entities = []
         failed_entities = []
         examples_to_send: list[dict[str, Any]] = []
+        entity_misconfigurations: dict[str, str] = {}
+        missing_required_fields: bool = False
         for result in calculated_entities_results:
+            if len(result.misconfigurations) > 0:
+                entity_misconfigurations |= result.misconfigurations
             if result.entity.get("identifier") and result.entity.get("blueprint"):
                 parsed_entity = Entity.parse_obj(result.entity)
                 if result.did_entity_pass_selector:
@@ -230,6 +266,12 @@ class JQEntityProcessor(BaseEntityProcessor):
                         examples_to_send.append(result.raw_data)
                 else:
                     failed_entities.append(parsed_entity)
+            else:
+                missing_required_fields = True
+        if len(entity_misconfigurations) > 0:
+            logger.info(
+                f"The mapping resulted with invalid values for{" identifier, blueprint," if missing_required_fields else " "} properties. Mapping result: {entity_misconfigurations}"
+            )
         if (
             not calculated_entities_results
             and raw_results
@@ -245,4 +287,5 @@ class JQEntityProcessor(BaseEntityProcessor):
         return CalculationResult(
             EntitySelectorDiff(passed=passed_entities, failed=failed_entities),
             errors,
+            misonfigured_entity_keys=entity_misconfigurations,
         )

{port_ocean-0.15.3 → port_ocean-0.16.1}/port_ocean/core/integrations/mixins/sync_raw.py

@@ -1,4 +1,5 @@
 import asyncio
+from graphlib import CycleError
 import inspect
 import typing
 from typing import Callable, Awaitable, Any
@@ -27,7 +28,7 @@ from port_ocean.core.ocean_types import (
     RAW_ITEM,
     CalculationResult,
 )
-from port_ocean.core.utils import zip_and_sum, gather_and_split_errors_from_results
+from port_ocean.core.utils.utils import zip_and_sum, gather_and_split_errors_from_results
 from port_ocean.exceptions.core import OceanAbortException
 
 SEND_RAW_DATA_EXAMPLES_AMOUNT = 5
@@ -184,7 +185,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         send_raw_data_examples_amount = (
             SEND_RAW_DATA_EXAMPLES_AMOUNT if ocean.config.send_raw_data_examples else 0
         )
-        all_entities, register_errors = await self._register_resource_raw(
+        all_entities, register_errors,_ = await self._register_resource_raw(
             resource_config,
             raw_results,
             user_agent_type,
@@ -201,7 +202,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                             0, send_raw_data_examples_amount - len(passed_entities)
                         )
 
-                    entities, register_errors = await self._register_resource_raw(
+                    entities, register_errors,_ = await self._register_resource_raw(
                         resource_config,
                         items,
                         user_agent_type,
@@ -396,7 +397,20 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                 {"before": entities_before_flatten, "after": entities_after_flatten},
                 user_agent_type,
             )
-
+    async def sort_and_upsert_failed_entities(self,user_agent_type: UserAgentType)->None:
+        try:
+            if not event.entity_topological_sorter.should_execute():
+                return None
+            logger.info(f"Executings topological sort of {event.entity_topological_sorter.get_entities_count()} entities failed to upsert.",failed_toupsert_entities_count=event.entity_topological_sorter.get_entities_count())
+
+            for entity in event.entity_topological_sorter.get_entities():
+                await self.entities_state_applier.context.port_client.upsert_entity(entity,event.port_app_config.get_port_request_options(),user_agent_type,should_raise=False)
+
+        except OceanAbortException as ocean_abort:
+            logger.info(f"Failed topological sort of failed to upsert entites - trying to upsert unordered {event.entity_topological_sorter.get_entities_count()} entities.",failed_topological_sort_entities_count=event.entity_topological_sorter.get_entities_count() )
+            if isinstance(ocean_abort.__cause__,CycleError):
+                for entity in event.entity_topological_sorter.get_entities(False):
+                    await self.entities_state_applier.context.port_client.upsert_entity(entity,event.port_app_config.get_port_request_options(),user_agent_type,should_raise=False)
     async def sync_raw_all(
         self,
         _: dict[Any, Any] | None = None,
@@ -426,6 +440,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                 use_cache=False
             )
             logger.info(f"Resync will use the following mappings: {app_config.dict()}")
+
             try:
                 did_fetched_current_state = True
                 entities_at_port = await ocean.port_client.search_entities(
@@ -455,6 +470,8 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                         event.on_abort(lambda: task.cancel())
 
                         creation_results.append(await task)
+
+                await self.sort_and_upsert_failed_entities(user_agent_type)
             except asyncio.CancelledError as e:
                 logger.warning("Resync aborted successfully, skipping delete phase. This leads to an incomplete state")
                 raise

{port_ocean-0.15.3 → port_ocean-0.16.1}/port_ocean/core/integrations/mixins/utils.py

@@ -9,7 +9,7 @@ from port_ocean.core.ocean_types import (
     RESYNC_EVENT_LISTENER,
     RESYNC_RESULT,
 )
-from port_ocean.core.utils import validate_result
+from port_ocean.core.utils.utils import validate_result
 from port_ocean.exceptions.core import (
     RawObjectValidationException,
     OceanAbortException,

{port_ocean-0.15.3 → port_ocean-0.16.1}/port_ocean/core/models.py

@@ -27,6 +27,10 @@ class Runtime(Enum):
         ) or installation_type == self.value
 
 
+class PortAPIErrorMessage(Enum):
+    NOT_FOUND = "not_found"
+
+
 class Entity(BaseModel):
     identifier: Any
     blueprint: Any

{port_ocean-0.15.3 → port_ocean-0.16.1}/port_ocean/core/ocean_types.py

@@ -1,5 +1,13 @@
-from typing import TypedDict, Any, AsyncIterator, Callable, Awaitable, NamedTuple
-
+from typing import (
+    TypedDict,
+    Any,
+    AsyncIterator,
+    Callable,
+    Awaitable,
+    NamedTuple,
+)
+
+from dataclasses import field
 from port_ocean.core.models import Entity
 
 RAW_ITEM = dict[Any, Any]
@@ -30,6 +38,7 @@ class EntitySelectorDiff(NamedTuple):
 class CalculationResult(NamedTuple):
     entity_selector_diff: EntitySelectorDiff
     errors: list[Exception]
+    misonfigured_entity_keys: dict[str, str] = field(default_factory=dict)
 
 
 class IntegrationEventsCallbacks(TypedDict):

port_ocean-0.16.1/port_ocean/core/utils/entity_topological_sorter.py

@@ -0,0 +1,90 @@
+from typing import Any, Generator
+from port_ocean.context import event
+from port_ocean.core.models import Entity
+
+from loguru import logger
+
+from graphlib import TopologicalSorter, CycleError
+from typing import Set
+
+from port_ocean.exceptions.core import OceanAbortException
+
+Node = tuple[str, str]
+
+
+class EntityTopologicalSorter:
+    def __init__(self) -> None:
+        self.entities: list[Entity] = []
+
+    def register_entity(
+        self,
+        entity: Entity,
+    ) -> None:
+        logger.debug(
+            f"Will retry upserting entity - {entity.identifier} at the end of resync"
+        )
+        self.entities.append(entity)
+
+    def should_execute(self) -> int:
+        return not event.event.port_app_config.create_missing_related_entities
+
+    def get_entities_count(self) -> int:
+        return len(self.entities)
+
+    def get_entities(self, sorted: bool = True) -> Generator[Entity, Any, None]:
+        if not sorted:
+            for entity in self.entities:
+                yield entity
+            return
+
+        sorted_and_mapped = EntityTopologicalSorter.order_by_entities_dependencies(
+            self.entities
+        )
+        for entity in sorted_and_mapped:
+            yield entity
+
+    @staticmethod
+    def node(entity: Entity) -> Node:
+        return entity.identifier, entity.blueprint
+
+    @staticmethod
+    def order_by_entities_dependencies(entities: list[Entity]) -> list[Entity]:
+        nodes: dict[Node, Set[Node]] = {}
+        entities_map = {}
+        for entity in entities:
+            nodes[EntityTopologicalSorter.node(entity)] = set()
+            entities_map[EntityTopologicalSorter.node(entity)] = entity
+
+        for entity in entities:
+            relation_target_ids: list[str] = sum(
+                [
+                    identifiers if isinstance(identifiers, list) else [identifiers]
+                    for identifiers in entity.relations.values()
+                    if identifiers is not None
+                ],
+                [],
+            )
+            related_entities = [
+                related
+                for related in entities
+                if related.identifier in relation_target_ids
+            ]
+
+            for related_entity in related_entities:
+                if (
+                    entity.blueprint is not related_entity.blueprint
+                    or entity.identifier is not related_entity.identifier
+                ):
+                    nodes[EntityTopologicalSorter.node(entity)].add(
+                        EntityTopologicalSorter.node(related_entity)
+                    )
+
+        sort_op = TopologicalSorter(nodes)
+        try:
+            return [entities_map[item] for item in sort_op.static_order()]
+        except CycleError as ex:
+            raise OceanAbortException(
+                "Cannot order entities due to cyclic dependencies. \n"
+                "If you do want to have cyclic dependencies, please make sure to set the keys"
+                " 'createMissingRelatedEntities' and 'deleteDependentEntities' in the integration config in Port."
+            ) from ex

{port_ocean-0.15.3 → port_ocean-0.16.1}/port_ocean/run.py

@@ -9,7 +9,7 @@ from port_ocean.bootstrap import create_default_app
 from port_ocean.config.dynamic import default_config_factory
 from port_ocean.config.settings import ApplicationSettings, LogLevelType
 from port_ocean.core.defaults.initialize import initialize_defaults
-from port_ocean.core.utils import validate_integration_runtime
+from port_ocean.core.utils.utils import validate_integration_runtime
 from port_ocean.log.logger_setup import setup_logger
 from port_ocean.ocean import Ocean
 from port_ocean.utils.misc import get_spec_file, load_module

{port_ocean-0.15.3 → port_ocean-0.16.1}/port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py

@@ -269,3 +269,37 @@ class TestJQEntityProcessor:
             assert len(result.entity_selector_diff.passed) == 1
             assert result.entity_selector_diff.passed[0].properties.get("foo") == "bar"
             assert not result.errors
+
+    async def test_parse_items_wrong_mapping(
+        self, mocked_processor: JQEntityProcessor
+    ) -> None:
+        mapping = Mock()
+        mapping.port.entity.mappings.dict.return_value = {
+            "title": ".foo",
+            "identifier": ".ark",
+            "blueprint": ".baz",
+            "properties": {
+                "description": ".bazbar",
+                "url": ".foobar",
+                "defaultBranch": ".bar.baz",
+            },
+        }
+        mapping.port.items_to_parse = None
+        mapping.selector.query = "true"
+        raw_results = [
+            {
+                "foo": "bar",
+                "baz": "bazbar",
+                "bar": {"foobar": "barfoo", "baz": "barbaz"},
+            },
+            {"foo": "bar", "baz": "bazbar", "bar": {"foobar": "foobar"}},
+        ]
+        result = await mocked_processor._parse_items(mapping, raw_results)
+        assert len(result.misonfigured_entity_keys) > 0
+        assert len(result.misonfigured_entity_keys) == 4
+        assert result.misonfigured_entity_keys == {
+            "identifier": ".ark",
+            "description": ".bazbar",
+            "url": ".foobar",
+            "defaultBranch": ".bar.baz",
+        }