port-ocean 0.15.2__tar.gz → 0.16.0__tar.gz

{port_ocean-0.15.2 → port_ocean-0.16.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.15.2
+Version: 0.16.0
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.15.2 → port_ocean-0.16.0}/port_ocean/cli/commands/new.py

@@ -77,18 +77,15 @@ def new(path: str, is_private_integration: bool) -> None:
     )
     console.print("Here are your next steps:\n", style="bold")
     console.print(
-        "⚓️ Install necessary packages: Run [bold][blue]make install[/blue][/bold] to install all required packages for your project.\n"
-        f"▶️ [bold][blue]cd {path}/{name} && make install && . .venv/bin/activate[/blue][/bold]\n"
+        f"⚓️ Install necessary packages: Run [bold][blue]cd {path}/{name} && make install && . .venv/bin/activate[/blue][/bold] to install all required packages for your project."
     )
     console.print(
-        f"⚓️ Copy example env file: Run [bold][blue]cp {path}/{name}.env.example {path}/{name}/.env [/blue][/bold] and set your port credentials in the created file.\n"
+        "⚓️ Copy example env file: Run [bold][blue]cp .env.example .env [/blue][/bold] and update your integration's configuration in the .env file."
     )
     console.print(
-        "⚓️ Set sail with [blue]Ocean[/blue]: Run [bold][blue]ocean sail[/blue] <path_to_integration>[/bold] to run the project using Ocean.\n"
-        f"▶️ [bold][blue]ocean sail {path}/{name}[/blue][/bold] \n"
+        "⚓️ Set sail with [blue]Ocean[/blue]: Run [bold][blue]ocean sail[/blue][/bold] to run the project using Ocean."
     )
     if not final_private_integration:
         console.print(
-            "⚓️ Smooth sailing with [blue]Make[/blue]: Alternatively, you can run [bold][blue]make run[/blue][/bold] to launch your project using Make. \n"
-            f"▶️ [bold][blue]make run {path}/{name}[/blue][/bold]"
+            f"⚓️ Smooth sailing with [blue]Make[/blue]: Alternatively, you can run [bold][blue]make run {path}/{name}[/blue][/bold] to launch your project using Make."
         )

{port_ocean-0.15.2 → port_ocean-0.16.0}/port_ocean/cli/cookiecutter/cookiecutter.json

@@ -6,6 +6,9 @@
     "email": "Your address email <you@example.com>",
     "release_date": "{% now 'local' %}",
     "is_private_integration": true,
+    "port_client_id": "you can find it using: https://docs.getport.io/build-your-software-catalog/custom-integration/api/#find-your-port-credentials",
+    "port_client_secret": "you can find it using: https://docs.getport.io/build-your-software-catalog/custom-integration/api/#find-your-port-credentials",
+    "is_us_region": false,
     "_extensions": [
         "jinja2_time.TimeExtension",
         "extensions.VersionExtension"

port_ocean-0.16.0/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.env.example

@@ -0,0 +1,6 @@
+OCEAN__PORT__CLIENT_ID={{ cookiecutter.port_client_id }}
+OCEAN__PORT__CLIENT_SECRET={{ cookiecutter.port_client_secret }}
+OCEAN__INTEGRATION__IDENTIFIER={{ cookiecutter.integration_slug }}
+OCEAN__PORT__BASE_URL={% if cookiecutter.is_us_region %}https://api.us.getport.io{% else %}https://api.getport.io{% endif %}
+OCEAN__EVENT_LISTENER__TYPE=POLLING
+OCEAN__INITIALIZE_PORT_RESOURCES=true

{port_ocean-0.15.2 → port_ocean-0.16.0}/port_ocean/clients/port/mixins/entities.py

@@ -1,5 +1,5 @@
 import asyncio
-from typing import Any
+from typing import Any, Literal
 from urllib.parse import quote_plus
 
 import httpx
@@ -11,7 +11,8 @@ from port_ocean.clients.port.utils import (
     handle_status_code,
     PORT_HTTP_MAX_CONNECTIONS_LIMIT,
 )
-from port_ocean.core.models import Entity
+from port_ocean.core.models import Entity, PortAPIErrorMessage
+from starlette import status
 
 
 class EntityClientMixin:
@@ -29,7 +30,27 @@ class EntityClientMixin:
         request_options: RequestOptions,
         user_agent_type: UserAgentType | None = None,
         should_raise: bool = True,
-    ) -> Entity | None:
+    ) -> Entity | None | Literal[False]:
+        """
+        This function upserts an entity into Port.
+
+        Usage:
+        ```python
+            upsertedEntity = await self.context.port_client.upsert_entity(
+                            entity,
+                            event.port_app_config.get_port_request_options(),
+                            user_agent_type,
+                            should_raise=False,
+                        )
+        ```
+        :param entity: An Entity to be upserted
+        :param request_options: A dictionary specifying how to upsert the entity
+        :param user_agent_type: a UserAgentType specifying who is preforming the action
+        :param should_raise: A boolean specifying whether the error should be raised or handled silently
+        :return: [Entity] if the upsert occured successfully
+        :return: [None] will be returned if entity is using search identifier
+        :return: [False] will be returned if upsert failed because of unmet dependency
+        """
         validation_only = request_options["validation_only"]
         async with self.semaphore:
             logger.debug(
@@ -50,13 +71,21 @@ class EntityClientMixin:
                 },
                 extensions={"retryable": True},
             )
-
         if response.is_error:
             logger.error(
                 f"Error {'Validating' if validation_only else 'Upserting'} "
                 f"entity: {entity.identifier} of "
                 f"blueprint: {entity.blueprint}"
             )
+            result = response.json()
+
+            if (
+                response.status_code == status.HTTP_404_NOT_FOUND
+                and not result.get("ok")
+                and result.get("error") == PortAPIErrorMessage.NOT_FOUND.value
+            ):
+                # Return false to differentiate from `result_entity.is_using_search_identifier`
+                return False
         handle_status_code(response, should_raise)
         result = response.json()
 

{port_ocean-0.15.2 → port_ocean-0.16.0}/port_ocean/context/event.py

@@ -14,6 +14,7 @@ from typing import (
 from uuid import uuid4
 
 from loguru import logger
+from port_ocean.core.utils.entity_topological_sorter import EntityTopologicalSorter
 from pydispatch import dispatcher  # type: ignore
 from werkzeug.local import LocalStack, LocalProxy
 
@@ -24,6 +25,7 @@ from port_ocean.exceptions.context import (
 )
 from port_ocean.utils.misc import get_time
 
+
 if TYPE_CHECKING:
     from port_ocean.core.handlers.port_app_config.models import (
         ResourceConfig,
@@ -50,6 +52,9 @@ class EventContext:
     _parent_event: Optional["EventContext"] = None
     _event_id: str = field(default_factory=lambda: str(uuid4()))
     _on_abort_callbacks: list[AbortCallbackFunction] = field(default_factory=list)
+    entity_topological_sorter: EntityTopologicalSorter = field(
+        default_factory=EntityTopologicalSorter
+    )
 
     def on_abort(self, func: AbortCallbackFunction) -> None:
         self._on_abort_callbacks.append(func)
@@ -129,6 +134,11 @@ async def event_context(
 ) -> AsyncIterator[EventContext]:
     parent = parent_override or _event_context_stack.top
     parent_attributes = parent.attributes if parent else {}
+    entity_topological_sorter = (
+        parent.entity_topological_sorter
+        if parent and parent.entity_topological_sorter
+        else EntityTopologicalSorter()
+    )
 
     attributes = {**parent_attributes, **(attributes or {})}
     new_event = EventContext(
@@ -138,6 +148,7 @@ async def event_context(
         _parent_event=parent,
         # inherit port app config from parent event, so it can be used in nested events
         _port_app_config=parent.port_app_config if parent else None,
+        entity_topological_sorter=entity_topological_sorter,
     )
     _event_context_stack.push(new_event)
 

{port_ocean-0.15.2 → port_ocean-0.16.0}/port_ocean/core/defaults/initialize.py

@@ -14,7 +14,7 @@ from port_ocean.core.defaults.common import (
 )
 from port_ocean.core.handlers.port_app_config.models import PortAppConfig
 from port_ocean.core.models import Blueprint
-from port_ocean.core.utils import gather_and_split_errors_from_results
+from port_ocean.core.utils.utils import gather_and_split_errors_from_results
 from port_ocean.exceptions.port_defaults import (
     AbortDefaultCreationError,
 )

{port_ocean-0.15.2 → port_ocean-0.16.0}/port_ocean/core/handlers/entities_state_applier/port/applier.py

@@ -8,12 +8,11 @@ from port_ocean.core.handlers.entities_state_applier.base import (
 from port_ocean.core.handlers.entities_state_applier.port.get_related_entities import (
     get_related_entities,
 )
-from port_ocean.core.handlers.entities_state_applier.port.order_by_entities_dependencies import (
-    order_by_entities_dependencies,
-)
+
 from port_ocean.core.models import Entity
 from port_ocean.core.ocean_types import EntityDiff
-from port_ocean.core.utils import is_same_entity, get_port_diff
+from port_ocean.core.utils.entity_topological_sorter import EntityTopologicalSorter
+from port_ocean.core.utils.utils import is_same_entity, get_port_diff
 
 
 class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
@@ -106,19 +105,7 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
                 should_raise=False,
             )
         else:
-            entities_with_search_identifier: list[Entity] = []
-            entities_without_search_identifier: list[Entity] = []
             for entity in entities:
-                if entity.is_using_search_identifier:
-                    entities_with_search_identifier.append(entity)
-                else:
-                    entities_without_search_identifier.append(entity)
-
-            ordered_created_entities = reversed(
-                entities_with_search_identifier
-                + order_by_entities_dependencies(entities_without_search_identifier)
-            )
-            for entity in ordered_created_entities:
                 upsertedEntity = await self.context.port_client.upsert_entity(
                     entity,
                     event.port_app_config.get_port_request_options(),
@@ -127,6 +114,9 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
                 )
                 if upsertedEntity:
                     modified_entities.append(upsertedEntity)
+                # condition to false to differentiate from `result_entity.is_using_search_identifier`
+                if upsertedEntity is False:
+                    event.entity_topological_sorter.register_entity(entity)
         return modified_entities
 
     async def delete(
@@ -141,7 +131,9 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
                 should_raise=False,
             )
         else:
-            ordered_deleted_entities = order_by_entities_dependencies(entities)
+            ordered_deleted_entities = (
+                EntityTopologicalSorter.order_by_entities_dependencies(entities)
+            )
 
             for entity in ordered_deleted_entities:
                 await self.context.port_client.delete_entity(

{port_ocean-0.15.2 → port_ocean-0.16.0}/port_ocean/core/handlers/entities_state_applier/port/order_by_entities_dependencies.py

@@ -14,7 +14,6 @@ def node(entity: Entity) -> Node:
 def order_by_entities_dependencies(entities: list[Entity]) -> list[Entity]:
     nodes: dict[Node, Set[Node]] = {}
     entities_map = {}
-
     for entity in entities:
         nodes[node(entity)] = set()
         entities_map[node(entity)] = entity
@@ -33,7 +32,11 @@ def order_by_entities_dependencies(entities: list[Entity]) -> list[Entity]:
         ]
 
         for related_entity in related_entities:
-            nodes[node(entity)].add(node(related_entity))
+            if (
+                entity.blueprint is not related_entity.blueprint
+                or entity.identifier is not related_entity.identifier
+            ):
+                nodes[node(entity)].add(node(related_entity))
 
     sort_op = TopologicalSorter(nodes)
     try:

{port_ocean-0.15.2 → port_ocean-0.16.0}/port_ocean/core/handlers/entity_processor/jq_entity_processor.py

@@ -16,7 +16,10 @@ from port_ocean.core.ocean_types import (
     EntitySelectorDiff,
     CalculationResult,
 )
-from port_ocean.core.utils import gather_and_split_errors_from_results, zip_and_sum
+from port_ocean.core.utils.utils import (
+    gather_and_split_errors_from_results,
+    zip_and_sum,
+)
 from port_ocean.exceptions.core import EntityProcessorException
 from port_ocean.utils.queue_utils import process_in_queue
 

{port_ocean-0.15.2 → port_ocean-0.16.0}/port_ocean/core/integrations/mixins/sync_raw.py

@@ -1,4 +1,5 @@
 import asyncio
+from graphlib import CycleError
 import inspect
 import typing
 from typing import Callable, Awaitable, Any
@@ -27,7 +28,7 @@ from port_ocean.core.ocean_types import (
     RAW_ITEM,
     CalculationResult,
 )
-from port_ocean.core.utils import zip_and_sum, gather_and_split_errors_from_results
+from port_ocean.core.utils.utils import zip_and_sum, gather_and_split_errors_from_results
 from port_ocean.exceptions.core import OceanAbortException
 
 SEND_RAW_DATA_EXAMPLES_AMOUNT = 5
@@ -396,7 +397,20 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                 {"before": entities_before_flatten, "after": entities_after_flatten},
                 user_agent_type,
             )
-
+    async def sort_and_upsert_failed_entities(self,user_agent_type: UserAgentType)->None:
+        try:
+            if not event.entity_topological_sorter.should_execute():
+                return None
+            logger.info(f"Executings topological sort of {event.entity_topological_sorter.get_entities_count()} entities failed to upsert.",failed_toupsert_entities_count=event.entity_topological_sorter.get_entities_count())
+
+            for entity in event.entity_topological_sorter.get_entities():
+                await self.entities_state_applier.context.port_client.upsert_entity(entity,event.port_app_config.get_port_request_options(),user_agent_type,should_raise=False)
+
+        except OceanAbortException as ocean_abort:
+            logger.info(f"Failed topological sort of failed to upsert entites - trying to upsert unordered {event.entity_topological_sorter.get_entities_count()} entities.",failed_topological_sort_entities_count=event.entity_topological_sorter.get_entities_count() )
+            if isinstance(ocean_abort.__cause__,CycleError):
+                for entity in event.entity_topological_sorter.get_entities(False):
+                    await self.entities_state_applier.context.port_client.upsert_entity(entity,event.port_app_config.get_port_request_options(),user_agent_type,should_raise=False)
     async def sync_raw_all(
         self,
         _: dict[Any, Any] | None = None,
@@ -426,6 +440,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                 use_cache=False
             )
             logger.info(f"Resync will use the following mappings: {app_config.dict()}")
+
             try:
                 did_fetched_current_state = True
                 entities_at_port = await ocean.port_client.search_entities(
@@ -455,6 +470,8 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                         event.on_abort(lambda: task.cancel())
 
                         creation_results.append(await task)
+
+                await self.sort_and_upsert_failed_entities(user_agent_type)
             except asyncio.CancelledError as e:
                 logger.warning("Resync aborted successfully, skipping delete phase. This leads to an incomplete state")
                 raise

{port_ocean-0.15.2 → port_ocean-0.16.0}/port_ocean/core/integrations/mixins/utils.py

@@ -9,7 +9,7 @@ from port_ocean.core.ocean_types import (
     RESYNC_EVENT_LISTENER,
     RESYNC_RESULT,
 )
-from port_ocean.core.utils import validate_result
+from port_ocean.core.utils.utils import validate_result
 from port_ocean.exceptions.core import (
     RawObjectValidationException,
     OceanAbortException,

{port_ocean-0.15.2 → port_ocean-0.16.0}/port_ocean/core/models.py

@@ -27,6 +27,10 @@ class Runtime(Enum):
         ) or installation_type == self.value
 
 
+class PortAPIErrorMessage(Enum):
+    NOT_FOUND = "not_found"
+
+
 class Entity(BaseModel):
     identifier: Any
     blueprint: Any

port_ocean-0.16.0/port_ocean/core/utils/entity_topological_sorter.py

@@ -0,0 +1,90 @@
+from typing import Any, Generator
+from port_ocean.context import event
+from port_ocean.core.models import Entity
+
+from loguru import logger
+
+from graphlib import TopologicalSorter, CycleError
+from typing import Set
+
+from port_ocean.exceptions.core import OceanAbortException
+
+Node = tuple[str, str]
+
+
+class EntityTopologicalSorter:
+    def __init__(self) -> None:
+        self.entities: list[Entity] = []
+
+    def register_entity(
+        self,
+        entity: Entity,
+    ) -> None:
+        logger.debug(
+            f"Will retry upserting entity - {entity.identifier} at the end of resync"
+        )
+        self.entities.append(entity)
+
+    def should_execute(self) -> int:
+        return not event.event.port_app_config.create_missing_related_entities
+
+    def get_entities_count(self) -> int:
+        return len(self.entities)
+
+    def get_entities(self, sorted: bool = True) -> Generator[Entity, Any, None]:
+        if not sorted:
+            for entity in self.entities:
+                yield entity
+            return
+
+        sorted_and_mapped = EntityTopologicalSorter.order_by_entities_dependencies(
+            self.entities
+        )
+        for entity in sorted_and_mapped:
+            yield entity
+
+    @staticmethod
+    def node(entity: Entity) -> Node:
+        return entity.identifier, entity.blueprint
+
+    @staticmethod
+    def order_by_entities_dependencies(entities: list[Entity]) -> list[Entity]:
+        nodes: dict[Node, Set[Node]] = {}
+        entities_map = {}
+        for entity in entities:
+            nodes[EntityTopologicalSorter.node(entity)] = set()
+            entities_map[EntityTopologicalSorter.node(entity)] = entity
+
+        for entity in entities:
+            relation_target_ids: list[str] = sum(
+                [
+                    identifiers if isinstance(identifiers, list) else [identifiers]
+                    for identifiers in entity.relations.values()
+                    if identifiers is not None
+                ],
+                [],
+            )
+            related_entities = [
+                related
+                for related in entities
+                if related.identifier in relation_target_ids
+            ]
+
+            for related_entity in related_entities:
+                if (
+                    entity.blueprint is not related_entity.blueprint
+                    or entity.identifier is not related_entity.identifier
+                ):
+                    nodes[EntityTopologicalSorter.node(entity)].add(
+                        EntityTopologicalSorter.node(related_entity)
+                    )
+
+        sort_op = TopologicalSorter(nodes)
+        try:
+            return [entities_map[item] for item in sort_op.static_order()]
+        except CycleError as ex:
+            raise OceanAbortException(
+                "Cannot order entities due to cyclic dependencies. \n"
+                "If you do want to have cyclic dependencies, please make sure to set the keys"
+                " 'createMissingRelatedEntities' and 'deleteDependentEntities' in the integration config in Port."
+            ) from ex

port_ocean-0.16.0/port_ocean/debug_cli.py

@@ -0,0 +1,5 @@
+from port_ocean.cli.commands.main import cli_start
+
+
+if __name__ == "__main__":
+    cli_start()

{port_ocean-0.15.2 → port_ocean-0.16.0}/port_ocean/run.py

@@ -9,7 +9,7 @@ from port_ocean.bootstrap import create_default_app
 from port_ocean.config.dynamic import default_config_factory
 from port_ocean.config.settings import ApplicationSettings, LogLevelType
 from port_ocean.core.defaults.initialize import initialize_defaults
-from port_ocean.core.utils import validate_integration_runtime
+from port_ocean.core.utils.utils import validate_integration_runtime
 from port_ocean.log.logger_setup import setup_logger
 from port_ocean.ocean import Ocean
 from port_ocean.utils.misc import get_spec_file, load_module