port-ocean 0.5.22__tar.gz → 0.5.24.dev1__tar.gz

{port_ocean-0.5.22 → port_ocean-0.5.24.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.5.22
+Version: 0.5.24.dev1
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.5.22 → port_ocean-0.5.24.dev1}/port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11-slim-buster
+FROM python:3.11-slim-bookworm
 
 ENV LIBRDKAFKA_VERSION 1.9.2
 

{port_ocean-0.5.22 → port_ocean-0.5.24.dev1}/port_ocean/clients/port/mixins/entities.py

@@ -32,7 +32,7 @@ class EntityClientMixin:
     ) -> None:
         validation_only = request_options["validation_only"]
         async with self.semaphore:
-            logger.info(
+            logger.debug(
                 f"{'Validating' if validation_only else 'Upserting'} entity: {entity.identifier} of blueprint: {entity.blueprint}"
             )
             headers = await self.auth.headers(user_agent_type)

{port_ocean-0.5.22 → port_ocean-0.5.24.dev1}/port_ocean/core/defaults/initialize.py

@@ -2,8 +2,8 @@ import asyncio
 from typing import Type, Any
 
 import httpx
-from starlette import status
 from loguru import logger
+from starlette import status
 
 from port_ocean.clients.port.client import PortClient
 from port_ocean.clients.port.types import UserAgentType
@@ -12,6 +12,7 @@ from port_ocean.context.ocean import ocean
 from port_ocean.core.defaults.common import Defaults, get_port_integration_defaults
 from port_ocean.core.handlers.port_app_config.models import PortAppConfig
 from port_ocean.core.models import Blueprint
+from port_ocean.core.utils import gather_and_split_errors_from_results
 from port_ocean.exceptions.port_defaults import (
     AbortDefaultCreationError,
 )
@@ -65,43 +66,30 @@ async def _create_resources(
         defaults.blueprints
     )
 
-    blueprints_results = await asyncio.gather(
-        *(
+    blueprints_results, _ = await gather_and_split_errors_from_results(
+        [
             port_client.get_blueprint(blueprint["identifier"], should_log=False)
             for blueprint in creation_stage
-        ),
-        return_exceptions=True,
+        ],
+        lambda item: isinstance(item, Blueprint),
     )
 
-    existing_blueprints = [
-        result.identifier
-        for result in blueprints_results
-        if not isinstance(result, httpx.HTTPStatusError)
-        and isinstance(result, Blueprint)
-    ]
-
-    if existing_blueprints:
+    if blueprints_results:
         logger.info(
-            f"Blueprints already exist: {existing_blueprints}. Skipping integration default creation..."
+            f"Blueprints already exist: {[result.identifier for result in blueprints_results]}. Skipping integration default creation..."
         )
         return
 
-    create_results = await asyncio.gather(
-        *(
+    created_blueprints, errors = await gather_and_split_errors_from_results(
+        (
             port_client.create_blueprint(
                 blueprint, user_agent_type=UserAgentType.exporter
             )
             for blueprint in creation_stage
-        ),
-        return_exceptions=True,
+        )
     )
 
-    errors = [result for result in create_results if isinstance(result, Exception)]
-    created_blueprints = [
-        result["identifier"]
-        for result in create_results
-        if not isinstance(result, BaseException)
-    ]
+    created_blueprints_identifiers = [bp["identifier"] for bp in created_blueprints]
 
     if errors:
         for error in errors:
@@ -110,8 +98,8 @@ async def _create_resources(
                     f"Failed to create resources: {error.response.text}. Rolling back changes..."
                 )
 
-        raise AbortDefaultCreationError(created_blueprints, errors)
-    created_pages = []
+        raise AbortDefaultCreationError(created_blueprints_identifiers, errors)
+    created_pages_identifiers = []
     try:
         for patch_stage in blueprint_patches:
             await asyncio.gather(
@@ -141,19 +129,11 @@ async def _create_resources(
             )
         )
 
-        create_pages_result = await asyncio.gather(
-            *(port_client.create_page(page) for page in defaults.pages),
-            return_exceptions=True,
+        created_pages, pages_errors = await gather_and_split_errors_from_results(
+            (port_client.create_page(page) for page in defaults.pages)
         )
-
-        created_pages = [
-            result.get("identifier", "")
-            for result in create_pages_result
-            if not isinstance(result, BaseException)
-        ]
-
-        pages_errors = [
-            result for result in create_pages_result if isinstance(result, Exception)
+        created_pages_identifiers = [
+            page.get("identifier", "") for page in created_pages
         ]
 
         if pages_errors:
@@ -164,7 +144,9 @@ async def _create_resources(
                     )
 
             raise AbortDefaultCreationError(
-                created_blueprints, pages_errors, created_pages
+                created_blueprints_identifiers,
+                pages_errors,
+                created_pages_identifiers,
             )
 
         await port_client.create_integration(
@@ -176,7 +158,9 @@ async def _create_resources(
         logger.error(
             f"Failed to create resources: {err.response.text}. Rolling back changes..."
         )
-        raise AbortDefaultCreationError(created_blueprints, [err], created_pages)
+        raise AbortDefaultCreationError(
+            created_blueprints_identifiers, [err], created_pages
+        )
 
 
 async def _initialize_defaults(

{port_ocean-0.5.22 → port_ocean-0.5.24.dev1}/port_ocean/core/handlers/__init__.py

@@ -4,7 +4,7 @@ from .entities_state_applier.base import (
 from .entities_state_applier.port.applier import (
     HttpEntitiesStateApplier,
 )
-from .entity_processor.base import BaseEntityProcessor, EntityPortDiff
+from .entity_processor.base import BaseEntityProcessor
 from .entity_processor.jq_entity_processor import (
     JQEntityProcessor,
 )
@@ -12,7 +12,6 @@ from .port_app_config.api import APIPortAppConfig
 from .port_app_config.base import BasePortAppConfig
 
 __all__ = [
-    "EntityPortDiff",
     "BaseEntityProcessor",
     "JQEntityProcessor",
     "BasePortAppConfig",

{port_ocean-0.5.22 → port_ocean-0.5.24.dev1}/port_ocean/core/handlers/entity_processor/base.py

@@ -1,30 +1,16 @@
 from abc import abstractmethod
-from dataclasses import dataclass, field
 
 from loguru import logger
 
 from port_ocean.core.handlers.base import BaseHandler
 from port_ocean.core.handlers.port_app_config.models import ResourceConfig
-from port_ocean.core.models import Entity
 from port_ocean.core.ocean_types import (
     RAW_ITEM,
+    CalculationResult,
     EntitySelectorDiff,
 )
 
 
-@dataclass
-class EntityPortDiff:
-    """Represents the differences between entities for porting.
-
-    This class holds the lists of deleted, modified, and created entities as part
-    of the porting process.
-    """
-
-    deleted: list[Entity] = field(default_factory=list)
-    modified: list[Entity] = field(default_factory=list)
-    created: list[Entity] = field(default_factory=list)
-
-
 class BaseEntityProcessor(BaseHandler):
     """Abstract base class for processing and parsing entities.
 
@@ -41,7 +27,7 @@ class BaseEntityProcessor(BaseHandler):
         raw_data: list[RAW_ITEM],
         parse_all: bool = False,
         send_raw_data_examples_amount: int = 0,
-    ) -> EntitySelectorDiff:
+    ) -> CalculationResult:
         pass
 
     async def parse_items(
@@ -50,7 +36,7 @@ class BaseEntityProcessor(BaseHandler):
         raw_data: list[RAW_ITEM],
         parse_all: bool = False,
         send_raw_data_examples_amount: int = 0,
-    ) -> EntitySelectorDiff:
+    ) -> CalculationResult:
         """Public method to parse raw entity data and map it to an EntityDiff.
 
         Args:
@@ -63,6 +49,9 @@ class BaseEntityProcessor(BaseHandler):
             EntityDiff: The parsed entity differences.
         """
         with logger.contextualize(kind=mapping.kind):
+            if not raw_data:
+                return CalculationResult(EntitySelectorDiff([], []), [])
+
             return await self._parse_items(
                 mapping, raw_data, parse_all, send_raw_data_examples_amount
             )

{port_ocean-0.5.22 → port_ocean-0.5.24.dev1}/port_ocean/core/handlers/entity_processor/jq_entity_processor.py

@@ -14,7 +14,9 @@ from port_ocean.core.models import Entity
 from port_ocean.core.ocean_types import (
     RAW_ITEM,
     EntitySelectorDiff,
+    CalculationResult,
 )
+from port_ocean.core.utils import gather_and_split_errors_from_results, zip_and_sum
 from port_ocean.exceptions.core import EntityProcessorException
 from port_ocean.utils.queue_utils import process_in_queue
 
@@ -111,32 +113,34 @@ class JQEntityProcessor(BaseEntityProcessor):
         items_to_parse: str | None,
         selector_query: str,
         parse_all: bool = False,
-    ) -> list[MappedEntity]:
+    ) -> tuple[list[MappedEntity], list[Exception]]:
+        raw_data = [data.copy()]
         if items_to_parse:
             items = await self._search(data, items_to_parse)
-            if isinstance(items, list):
-                return await asyncio.gather(
-                    *[
-                        self._get_mapped_entity(
-                            {"item": item, **data},
-                            raw_entity_mappings,
-                            selector_query,
-                            parse_all,
-                        )
-                        for item in items
-                    ]
+            if not isinstance(items, list):
+                logger.warning(
+                    f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items)}."
+                    f" Skipping..."
                 )
-            logger.warning(
-                f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items)}."
-                f" Skipping..."
-            )
-        else:
-            return [
-                await self._get_mapped_entity(
-                    data, raw_entity_mappings, selector_query, parse_all
+                return [], []
+            raw_data = [{"item": item, **data} for item in items]
+
+        entities, errors = await gather_and_split_errors_from_results(
+            [
+                self._get_mapped_entity(
+                    raw,
+                    raw_entity_mappings,
+                    selector_query,
+                    parse_all,
                 )
+                for raw in raw_data
             ]
-        return [MappedEntity()]
+        )
+        if errors:
+            logger.error(
+                f"Failed to calculate entities with {len(errors)} errors. errors: {errors}"
+            )
+        return entities, errors
 
     @staticmethod
     async def _send_examples(data: list[dict[str, Any]], kind: str) -> None:
@@ -157,37 +161,44 @@ class JQEntityProcessor(BaseEntityProcessor):
         raw_results: list[RAW_ITEM],
         parse_all: bool = False,
         send_raw_data_examples_amount: int = 0,
-    ) -> EntitySelectorDiff:
+    ) -> CalculationResult:
         raw_entity_mappings: dict[str, Any] = mapping.port.entity.mappings.dict(
             exclude_unset=True
         )
-
-        calculated_entities_results = await process_in_queue(
-            raw_results,
-            self._calculate_entity,
-            raw_entity_mappings,
-            mapping.port.items_to_parse,
-            mapping.selector.query,
-            parse_all,
+        logger.info(f"Parsing {len(raw_results)} raw results into entities")
+        calculated_entities_results, errors = zip_and_sum(
+            await process_in_queue(
+                raw_results,
+                self._calculate_entity,
+                raw_entity_mappings,
+                mapping.port.items_to_parse,
+                mapping.selector.query,
+                parse_all,
+            )
+        )
+        logger.debug(
+            f"Finished parsing raw results into entities with {len(errors)} errors. errors: {errors}"
         )
 
         passed_entities = []
         failed_entities = []
         examples_to_send: list[dict[str, Any]] = []
-        for entities_results in calculated_entities_results:
-            for result in entities_results:
-                if result.entity.get("identifier") and result.entity.get("blueprint"):
-                    parsed_entity = Entity.parse_obj(result.entity)
-                    if result.did_entity_pass_selector:
-                        passed_entities.append(parsed_entity)
-                        if (
-                            len(examples_to_send) < send_raw_data_examples_amount
-                            and result.raw_data is not None
-                        ):
-                            examples_to_send.append(result.raw_data)
-                    else:
-                        failed_entities.append(parsed_entity)
+        for result in calculated_entities_results:
+            if result.entity.get("identifier") and result.entity.get("blueprint"):
+                parsed_entity = Entity.parse_obj(result.entity)
+                if result.did_entity_pass_selector:
+                    passed_entities.append(parsed_entity)
+                    if (
+                        len(examples_to_send) < send_raw_data_examples_amount
+                        and result.raw_data is not None
+                    ):
+                        examples_to_send.append(result.raw_data)
+                else:
+                    failed_entities.append(parsed_entity)
 
         await self._send_examples(examples_to_send, mapping.kind)
 
-        return EntitySelectorDiff(passed=passed_entities, failed=failed_entities)
+        return CalculationResult(
+            EntitySelectorDiff(passed=passed_entities, failed=failed_entities),
+            errors,
+        )

{port_ocean-0.5.22 → port_ocean-0.5.24.dev1}/port_ocean/core/integrations/mixins/sync_raw.py

@@ -3,6 +3,7 @@ import inspect
 import typing
 from typing import Callable, Awaitable, Any
 
+import httpx
 from loguru import logger
 
 from port_ocean.clients.port.types import UserAgentType
@@ -24,12 +25,11 @@ from port_ocean.core.ocean_types import (
     RawEntityDiff,
     ASYNC_GENERATOR_RESYNC_TYPE,
     RAW_ITEM,
-    EntitySelectorDiff,
+    CalculationResult,
 )
-from port_ocean.core.utils import zip_and_sum
+from port_ocean.core.utils import zip_and_sum, gather_and_split_errors_from_results
 from port_ocean.exceptions.core import OceanAbortException
 
-
 SEND_RAW_DATA_EXAMPLES_AMOUNT = 5
 
 
@@ -101,21 +101,13 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         logger.info(
             f"Found {len(tasks) + len(results)} resync tasks for {resource_config.kind}"
         )
-
-        results_with_error = await asyncio.gather(*tasks, return_exceptions=True)
+        successful_results, errors = await gather_and_split_errors_from_results(tasks)
         results.extend(
             sum(
-                [
-                    result
-                    for result in results_with_error
-                    if not isinstance(result, Exception)
-                ],
+                successful_results,
                 [],
             )
         )
-        errors = [
-            result for result in results_with_error if isinstance(result, Exception)
-        ]
 
         logger.info(
             f"Triggered {len(tasks)} tasks for {resource_config.kind}, failed: {len(errors)}"
@@ -128,7 +120,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         raw_diff: list[tuple[ResourceConfig, list[RAW_ITEM]]],
         parse_all: bool = False,
         send_raw_data_examples_amount: int = 0,
-    ) -> list[EntitySelectorDiff]:
+    ) -> list[CalculationResult]:
         return await asyncio.gather(
             *(
                 self.entity_processor.parse_items(
@@ -145,12 +137,12 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         user_agent_type: UserAgentType,
         parse_all: bool = False,
         send_raw_data_examples_amount: int = 0,
-    ) -> EntitySelectorDiff:
+    ) -> CalculationResult:
         objects_diff = await self._calculate_raw(
             [(resource, results)], parse_all, send_raw_data_examples_amount
         )
         await self.entities_state_applier.upsert(
-            objects_diff[0].passed, user_agent_type
+            objects_diff[0].entity_selector_diff.passed, user_agent_type
         )
 
         return objects_diff[0]
@@ -160,13 +152,15 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         resource: ResourceConfig,
         results: list[RAW_ITEM],
         user_agent_type: UserAgentType,
-    ) -> list[Entity]:
+    ) -> tuple[list[Entity], list[Exception]]:
         objects_diff = await self._calculate_raw([(resource, results)])
+        entities_selector_diff, errors = objects_diff[0]
 
-        entities_after: list[Entity] = objects_diff[0].passed
-        await self.entities_state_applier.delete(entities_after, user_agent_type)
+        await self.entities_state_applier.delete(
+            entities_selector_diff.passed, user_agent_type
+        )
         logger.info("Finished unregistering change")
-        return entities_after
+        return entities_selector_diff.passed, errors
 
     async def _register_in_batches(
         self, resource_config: ResourceConfig, user_agent_type: UserAgentType
@@ -183,39 +177,38 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         send_raw_data_examples_amount = (
             SEND_RAW_DATA_EXAMPLES_AMOUNT if ocean.config.send_raw_data_examples else 0
         )
-        entities = (
-            await self._register_resource_raw(
-                resource_config,
-                raw_results,
-                user_agent_type,
-                send_raw_data_examples_amount=send_raw_data_examples_amount,
-            )
-        ).passed
+        all_entities, register_errors = await self._register_resource_raw(
+            resource_config,
+            raw_results,
+            user_agent_type,
+            send_raw_data_examples_amount=send_raw_data_examples_amount,
+        )
+        errors.extend(register_errors)
+        passed_entities = list(all_entities.passed)
 
         for generator in async_generators:
             try:
                 async for items in generator:
                     if send_raw_data_examples_amount > 0:
                         send_raw_data_examples_amount = max(
-                            0, send_raw_data_examples_amount - len(entities)
+                            0, send_raw_data_examples_amount - len(passed_entities)
                         )
-                    entities.extend(
-                        (
-                            await self._register_resource_raw(
-                                resource_config,
-                                items,
-                                user_agent_type,
-                                send_raw_data_examples_amount=send_raw_data_examples_amount,
-                            )
-                        ).passed
+
+                    entities, register_errors = await self._register_resource_raw(
+                        resource_config,
+                        items,
+                        user_agent_type,
+                        send_raw_data_examples_amount=send_raw_data_examples_amount,
                     )
+                    errors.extend(register_errors)
+                    passed_entities.extend(entities.passed)
             except* OceanAbortException as error:
                 errors.append(error)
 
         logger.info(
-            f"Finished registering change for {len(results)} raw results for kind: {resource_config.kind}. {len(entities)} entities were affected"
+            f"Finished registering change for {len(results)} raw results for kind: {resource_config.kind}. {len(passed_entities)} entities were affected"
         )
-        return entities, errors
+        return passed_entities, errors
 
     async def register_raw(
         self,
@@ -244,16 +237,26 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         if not resource_mappings:
             return []
 
-        diffs: list[EntitySelectorDiff] = await asyncio.gather(
-            *(
-                self._register_resource_raw(resource, results, user_agent_type, True)
-                for resource in resource_mappings
+        diffs, errors = zip(
+            await asyncio.gather(
+                *(
+                    self._register_resource_raw(
+                        resource, results, user_agent_type, True
+                    )
+                    for resource in resource_mappings
+                )
             )
         )
 
-        registered_entities, entities_to_delete = zip_and_sum(
-            (entities_diff.passed, entities_diff.failed) for entities_diff in diffs
-        )
+        if errors:
+            message = f"Failed to register {len(errors)} entities. Skipping delete phase due to incomplete state"
+            logger.error(message, exc_info=errors)
+            raise ExceptionGroup(
+                message,
+                errors,
+            )
+
+        registered_entities, entities_to_delete = zip_and_sum(diffs)
 
         registered_entities_attributes = {
             (entity.identifier, entity.blueprint) for entity in registered_entities
@@ -306,13 +309,23 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
             resource for resource in config.resources if resource.kind == kind
         ]
 
-        return await asyncio.gather(
+        entities, errors = await asyncio.gather(
             *(
                 self._unregister_resource_raw(resource, results, user_agent_type)
                 for resource in resource_mappings
             )
         )
 
+        if errors:
+            message = f"Failed to unregister all entities with {len(errors)} errors"
+            logger.error(message, exc_info=errors)
+            raise ExceptionGroup(
+                message,
+                errors,
+            )
+
+        return entities
+
     async def update_raw_diff(
         self,
         kind: str,
@@ -337,14 +350,16 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         with logger.contextualize(kind=kind):
             logger.info(f"Found {len(resource_mappings)} resources for {kind}")
 
-            entities_before = await self._calculate_raw(
-                [
-                    (mapping, raw_desired_state["before"])
-                    for mapping in resource_mappings
-                ]
+            entities_before, _ = zip(
+                await self._calculate_raw(
+                    [
+                        (mapping, raw_desired_state["before"])
+                        for mapping in resource_mappings
+                    ]
+                )
             )
 
-            entities_after = await self._calculate_raw(
+            entities_after, after_errors = await self._calculate_raw(
                 [(mapping, raw_desired_state["after"]) for mapping in resource_mappings]
             )
 
@@ -356,6 +371,11 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                 (entities_diff.passed for entities_diff in entities_after), []
             )
 
+            if after_errors:
+                message = f"Failed to calculate diff for entities with {len(after_errors)} errors. Skipping delete phase due to incomplete state"
+                logger.error(message, exc_info=after_errors)
+                entities_before_flatten = []
+
             await self.entities_state_applier.apply_diff(
                 {"before": entities_before_flatten, "after": entities_after_flatten},
                 user_agent_type,
@@ -390,6 +410,16 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                 use_cache=False
             )
 
+            try:
+                entities_at_port = await ocean.port_client.search_entities(
+                    user_agent_type
+                )
+            except httpx.HTTPError as e:
+                logger.warning(
+                    "Failed to fetch the current state of entities at Port. Skipping delete phase due to unknown initial state"
+                )
+                entities_at_port = []
+
             creation_results: list[tuple[list[Entity], list[Exception]]] = []
 
             try:
@@ -407,6 +437,13 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
             except asyncio.CancelledError as e:
                 logger.warning("Resync aborted successfully")
             else:
+                if not entities_at_port:
+                    logger.warning(
+                        "Due to an error before the resync, the previous state of entities at Port is unknown."
+                        " Skipping delete phase due to unknown initial state."
+                    )
+                    return
+
                 logger.info("Starting resync diff calculation")
                 flat_created_entities, errors = zip_and_sum(creation_results) or [
                     [],
@@ -424,11 +461,8 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
 
                     logger.error(message, exc_info=error_group)
                 else:
-                    entities_at_port = await ocean.port_client.search_entities(
-                        user_agent_type
-                    )
                     logger.info(
-                        f"Running resync diff calculation, number of entities found at Port: {len(entities_at_port)}, number of entities found during sync: {len(flat_created_entities)}"
+                        f"Running resync diff calculation, number of entities at Port before resync: {len(entities_at_port)}, number of entities created during sync: {len(flat_created_entities)}"
                     )
                     await self.entities_state_applier.delete_diff(
                         {"before": entities_at_port, "after": flat_created_entities},

{port_ocean-0.5.22 → port_ocean-0.5.24.dev1}/port_ocean/core/models.py

@@ -1,3 +1,4 @@
+from dataclasses import dataclass, field
 from typing import Any
 
 from pydantic import BaseModel
@@ -34,3 +35,16 @@ class Migration(BaseModel):
     sourceBlueprint: str
     mapping: dict[str, Any]
     status: str
+
+
+@dataclass
+class EntityPortDiff:
+    """Represents the differences between entities for porting.
+
+    This class holds the lists of deleted, modified, and created entities as part
+    of the porting process.
+    """
+
+    deleted: list[Entity] = field(default_factory=list)
+    modified: list[Entity] = field(default_factory=list)
+    created: list[Entity] = field(default_factory=list)