port-ocean 0.5.21__py3-none-any.whl → 0.5.24__py3-none-any.whl

port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11-slim-buster
+FROM python:3.11-slim-bookworm
 
 ENV LIBRDKAFKA_VERSION 1.9.2
 

port_ocean/clients/port/mixins/entities.py

@@ -32,7 +32,7 @@ class EntityClientMixin:
     ) -> None:
         validation_only = request_options["validation_only"]
         async with self.semaphore:
-            logger.info(
+            logger.debug(
                 f"{'Validating' if validation_only else 'Upserting'} entity: {entity.identifier} of blueprint: {entity.blueprint}"
             )
             headers = await self.auth.headers(user_agent_type)

port_ocean/clients/port/mixins/integrations.py

@@ -1,4 +1,5 @@
 from typing import Any, TYPE_CHECKING, Optional, TypedDict
+from urllib.parse import quote_plus
 
 import httpx
 from loguru import logger
@@ -145,7 +146,7 @@ class IntegrationClientMixin:
         logger.debug(f"Ingesting examples for kind: {kind}")
         headers = await self.auth.headers()
         response = await self.client.post(
-            f"{self.auth.api_url}/integration/{self.integration_identifier}/kinds/{kind}/examples",
+            f"{self.auth.api_url}/integration/{quote_plus(self.integration_identifier)}/kinds/{quote_plus(kind)}/examples",
             headers=headers,
             json={
                 "examples": sensitive_log_filter.mask_object(data, full_hide=True),

port_ocean/core/defaults/initialize.py

@@ -2,8 +2,8 @@ import asyncio
 from typing import Type, Any
 
 import httpx
-from starlette import status
 from loguru import logger
+from starlette import status
 
 from port_ocean.clients.port.client import PortClient
 from port_ocean.clients.port.types import UserAgentType
@@ -12,6 +12,7 @@ from port_ocean.context.ocean import ocean
 from port_ocean.core.defaults.common import Defaults, get_port_integration_defaults
 from port_ocean.core.handlers.port_app_config.models import PortAppConfig
 from port_ocean.core.models import Blueprint
+from port_ocean.core.utils import gather_and_split_errors_from_results
 from port_ocean.exceptions.port_defaults import (
     AbortDefaultCreationError,
 )
@@ -65,43 +66,30 @@ async def _create_resources(
         defaults.blueprints
     )
 
-    blueprints_results = await asyncio.gather(
-        *(
+    blueprints_results, _ = await gather_and_split_errors_from_results(
+        [
             port_client.get_blueprint(blueprint["identifier"], should_log=False)
             for blueprint in creation_stage
-        ),
-        return_exceptions=True,
+        ],
+        lambda item: isinstance(item, Blueprint),
     )
 
-    existing_blueprints = [
-        result.identifier
-        for result in blueprints_results
-        if not isinstance(result, httpx.HTTPStatusError)
-        and isinstance(result, Blueprint)
-    ]
-
-    if existing_blueprints:
+    if blueprints_results:
         logger.info(
-            f"Blueprints already exist: {existing_blueprints}. Skipping integration default creation..."
+            f"Blueprints already exist: {[result.identifier for result in blueprints_results]}. Skipping integration default creation..."
         )
         return
 
-    create_results = await asyncio.gather(
-        *(
+    created_blueprints, errors = await gather_and_split_errors_from_results(
+        (
             port_client.create_blueprint(
                 blueprint, user_agent_type=UserAgentType.exporter
             )
             for blueprint in creation_stage
-        ),
-        return_exceptions=True,
+        )
     )
 
-    errors = [result for result in create_results if isinstance(result, Exception)]
-    created_blueprints = [
-        result["identifier"]
-        for result in create_results
-        if not isinstance(result, BaseException)
-    ]
+    created_blueprints_identifiers = [bp["identifier"] for bp in created_blueprints]
 
     if errors:
         for error in errors:
@@ -110,8 +98,8 @@ async def _create_resources(
                     f"Failed to create resources: {error.response.text}. Rolling back changes..."
                 )
 
-        raise AbortDefaultCreationError(created_blueprints, errors)
-    created_pages = []
+        raise AbortDefaultCreationError(created_blueprints_identifiers, errors)
+    created_pages_identifiers = []
     try:
         for patch_stage in blueprint_patches:
             await asyncio.gather(
@@ -141,19 +129,11 @@ async def _create_resources(
             )
         )
 
-        create_pages_result = await asyncio.gather(
-            *(port_client.create_page(page) for page in defaults.pages),
-            return_exceptions=True,
+        created_pages, pages_errors = await gather_and_split_errors_from_results(
+            (port_client.create_page(page) for page in defaults.pages)
         )
-
-        created_pages = [
-            result.get("identifier", "")
-            for result in create_pages_result
-            if not isinstance(result, BaseException)
-        ]
-
-        pages_errors = [
-            result for result in create_pages_result if isinstance(result, Exception)
+        created_pages_identifiers = [
+            page.get("identifier", "") for page in created_pages
         ]
 
         if pages_errors:
@@ -164,7 +144,9 @@ async def _create_resources(
                     )
 
             raise AbortDefaultCreationError(
-                created_blueprints, pages_errors, created_pages
+                created_blueprints_identifiers,
+                pages_errors,
+                created_pages_identifiers,
             )
 
         await port_client.create_integration(
@@ -176,7 +158,9 @@ async def _create_resources(
         logger.error(
             f"Failed to create resources: {err.response.text}. Rolling back changes..."
         )
-        raise AbortDefaultCreationError(created_blueprints, [err], created_pages)
+        raise AbortDefaultCreationError(
+            created_blueprints_identifiers, [err], created_pages_identifiers
+        )
 
 
 async def _initialize_defaults(

port_ocean/core/handlers/__init__.py

@@ -4,7 +4,7 @@ from .entities_state_applier.base import (
 from .entities_state_applier.port.applier import (
     HttpEntitiesStateApplier,
 )
-from .entity_processor.base import BaseEntityProcessor, EntityPortDiff
+from .entity_processor.base import BaseEntityProcessor
 from .entity_processor.jq_entity_processor import (
     JQEntityProcessor,
 )
@@ -12,7 +12,6 @@ from .port_app_config.api import APIPortAppConfig
 from .port_app_config.base import BasePortAppConfig
 
 __all__ = [
-    "EntityPortDiff",
     "BaseEntityProcessor",
     "JQEntityProcessor",
     "BasePortAppConfig",

port_ocean/core/handlers/entity_processor/base.py

@@ -1,30 +1,16 @@
 from abc import abstractmethod
-from dataclasses import dataclass, field
 
 from loguru import logger
 
 from port_ocean.core.handlers.base import BaseHandler
 from port_ocean.core.handlers.port_app_config.models import ResourceConfig
-from port_ocean.core.models import Entity
 from port_ocean.core.ocean_types import (
     RAW_ITEM,
+    CalculationResult,
     EntitySelectorDiff,
 )
 
 
-@dataclass
-class EntityPortDiff:
-    """Represents the differences between entities for porting.
-
-    This class holds the lists of deleted, modified, and created entities as part
-    of the porting process.
-    """
-
-    deleted: list[Entity] = field(default_factory=list)
-    modified: list[Entity] = field(default_factory=list)
-    created: list[Entity] = field(default_factory=list)
-
-
 class BaseEntityProcessor(BaseHandler):
     """Abstract base class for processing and parsing entities.
 
@@ -41,7 +27,7 @@ class BaseEntityProcessor(BaseHandler):
         raw_data: list[RAW_ITEM],
         parse_all: bool = False,
         send_raw_data_examples_amount: int = 0,
-    ) -> EntitySelectorDiff:
+    ) -> CalculationResult:
         pass
 
     async def parse_items(
@@ -50,7 +36,7 @@ class BaseEntityProcessor(BaseHandler):
         raw_data: list[RAW_ITEM],
         parse_all: bool = False,
         send_raw_data_examples_amount: int = 0,
-    ) -> EntitySelectorDiff:
+    ) -> CalculationResult:
         """Public method to parse raw entity data and map it to an EntityDiff.
 
         Args:
@@ -63,6 +49,9 @@ class BaseEntityProcessor(BaseHandler):
             EntityDiff: The parsed entity differences.
         """
         with logger.contextualize(kind=mapping.kind):
+            if not raw_data:
+                return CalculationResult(EntitySelectorDiff([], []), [])
+
             return await self._parse_items(
                 mapping, raw_data, parse_all, send_raw_data_examples_amount
             )

port_ocean/core/handlers/entity_processor/jq_entity_processor.py

@@ -14,7 +14,9 @@ from port_ocean.core.models import Entity
 from port_ocean.core.ocean_types import (
     RAW_ITEM,
     EntitySelectorDiff,
+    CalculationResult,
 )
+from port_ocean.core.utils import gather_and_split_errors_from_results, zip_and_sum
 from port_ocean.exceptions.core import EntityProcessorException
 from port_ocean.utils.queue_utils import process_in_queue
 
@@ -111,32 +113,34 @@ class JQEntityProcessor(BaseEntityProcessor):
         items_to_parse: str | None,
         selector_query: str,
         parse_all: bool = False,
-    ) -> list[MappedEntity]:
+    ) -> tuple[list[MappedEntity], list[Exception]]:
+        raw_data = [data.copy()]
         if items_to_parse:
             items = await self._search(data, items_to_parse)
-            if isinstance(items, list):
-                return await asyncio.gather(
-                    *[
-                        self._get_mapped_entity(
-                            {"item": item, **data},
-                            raw_entity_mappings,
-                            selector_query,
-                            parse_all,
-                        )
-                        for item in items
-                    ]
+            if not isinstance(items, list):
+                logger.warning(
+                    f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items)}."
+                    f" Skipping..."
                 )
-            logger.warning(
-                f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items)}."
-                f" Skipping..."
-            )
-        else:
-            return [
-                await self._get_mapped_entity(
-                    data, raw_entity_mappings, selector_query, parse_all
+                return [], []
+            raw_data = [{"item": item, **data} for item in items]
+
+        entities, errors = await gather_and_split_errors_from_results(
+            [
+                self._get_mapped_entity(
+                    raw,
+                    raw_entity_mappings,
+                    selector_query,
+                    parse_all,
                 )
+                for raw in raw_data
             ]
-        return [MappedEntity()]
+        )
+        if errors:
+            logger.error(
+                f"Failed to calculate entities with {len(errors)} errors. errors: {errors}"
+            )
+        return entities, errors
 
     @staticmethod
     async def _send_examples(data: list[dict[str, Any]], kind: str) -> None:
@@ -157,37 +161,44 @@ class JQEntityProcessor(BaseEntityProcessor):
         raw_results: list[RAW_ITEM],
         parse_all: bool = False,
         send_raw_data_examples_amount: int = 0,
-    ) -> EntitySelectorDiff:
+    ) -> CalculationResult:
         raw_entity_mappings: dict[str, Any] = mapping.port.entity.mappings.dict(
             exclude_unset=True
         )
-
-        calculated_entities_results = await process_in_queue(
-            raw_results,
-            self._calculate_entity,
-            raw_entity_mappings,
-            mapping.port.items_to_parse,
-            mapping.selector.query,
-            parse_all,
+        logger.info(f"Parsing {len(raw_results)} raw results into entities")
+        calculated_entities_results, errors = zip_and_sum(
+            await process_in_queue(
+                raw_results,
+                self._calculate_entity,
+                raw_entity_mappings,
+                mapping.port.items_to_parse,
+                mapping.selector.query,
+                parse_all,
+            )
+        )
+        logger.debug(
+            f"Finished parsing raw results into entities with {len(errors)} errors. errors: {errors}"
         )
 
         passed_entities = []
         failed_entities = []
         examples_to_send: list[dict[str, Any]] = []
-        for entities_results in calculated_entities_results:
-            for result in entities_results:
-                if result.entity.get("identifier") and result.entity.get("blueprint"):
-                    parsed_entity = Entity.parse_obj(result.entity)
-                    if result.did_entity_pass_selector:
-                        passed_entities.append(parsed_entity)
-                        if (
-                            len(examples_to_send) < send_raw_data_examples_amount
-                            and result.raw_data is not None
-                        ):
-                            examples_to_send.append(result.raw_data)
-                    else:
-                        failed_entities.append(parsed_entity)
+        for result in calculated_entities_results:
+            if result.entity.get("identifier") and result.entity.get("blueprint"):
+                parsed_entity = Entity.parse_obj(result.entity)
+                if result.did_entity_pass_selector:
+                    passed_entities.append(parsed_entity)
+                    if (
+                        len(examples_to_send) < send_raw_data_examples_amount
+                        and result.raw_data is not None
+                    ):
+                        examples_to_send.append(result.raw_data)
+                else:
+                    failed_entities.append(parsed_entity)
 
         await self._send_examples(examples_to_send, mapping.kind)
 
-        return EntitySelectorDiff(passed=passed_entities, failed=failed_entities)
+        return CalculationResult(
+            EntitySelectorDiff(passed=passed_entities, failed=failed_entities),
+            errors,
+        )

port_ocean/core/integrations/mixins/sync_raw.py

@@ -3,6 +3,7 @@ import inspect
 import typing
 from typing import Callable, Awaitable, Any
 
+import httpx
 from loguru import logger
 
 from port_ocean.clients.port.types import UserAgentType
@@ -24,12 +25,11 @@ from port_ocean.core.ocean_types import (
     RawEntityDiff,
     ASYNC_GENERATOR_RESYNC_TYPE,
     RAW_ITEM,
-    EntitySelectorDiff,
+    CalculationResult,
 )
-from port_ocean.core.utils import zip_and_sum
+from port_ocean.core.utils import zip_and_sum, gather_and_split_errors_from_results
 from port_ocean.exceptions.core import OceanAbortException
 
-
 SEND_RAW_DATA_EXAMPLES_AMOUNT = 5
 
 
@@ -101,21 +101,13 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         logger.info(
             f"Found {len(tasks) + len(results)} resync tasks for {resource_config.kind}"
         )
-
-        results_with_error = await asyncio.gather(*tasks, return_exceptions=True)
+        successful_results, errors = await gather_and_split_errors_from_results(tasks)
         results.extend(
             sum(
-                [
-                    result
-                    for result in results_with_error
-                    if not isinstance(result, Exception)
-                ],
+                successful_results,
                 [],
             )
         )
-        errors = [
-            result for result in results_with_error if isinstance(result, Exception)
-        ]
 
         logger.info(
             f"Triggered {len(tasks)} tasks for {resource_config.kind}, failed: {len(errors)}"
@@ -128,7 +120,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         raw_diff: list[tuple[ResourceConfig, list[RAW_ITEM]]],
         parse_all: bool = False,
         send_raw_data_examples_amount: int = 0,
-    ) -> list[EntitySelectorDiff]:
+    ) -> list[CalculationResult]:
         return await asyncio.gather(
             *(
                 self.entity_processor.parse_items(
@@ -145,12 +137,12 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         user_agent_type: UserAgentType,
         parse_all: bool = False,
         send_raw_data_examples_amount: int = 0,
-    ) -> EntitySelectorDiff:
+    ) -> CalculationResult:
         objects_diff = await self._calculate_raw(
             [(resource, results)], parse_all, send_raw_data_examples_amount
         )
         await self.entities_state_applier.upsert(
-            objects_diff[0].passed, user_agent_type
+            objects_diff[0].entity_selector_diff.passed, user_agent_type
         )
 
         return objects_diff[0]
@@ -160,13 +152,15 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         resource: ResourceConfig,
         results: list[RAW_ITEM],
         user_agent_type: UserAgentType,
-    ) -> list[Entity]:
+    ) -> tuple[list[Entity], list[Exception]]:
         objects_diff = await self._calculate_raw([(resource, results)])
+        entities_selector_diff, errors = objects_diff[0]
 
-        entities_after: list[Entity] = objects_diff[0].passed
-        await self.entities_state_applier.delete(entities_after, user_agent_type)
+        await self.entities_state_applier.delete(
+            entities_selector_diff.passed, user_agent_type
+        )
         logger.info("Finished unregistering change")
-        return entities_after
+        return entities_selector_diff.passed, errors
 
     async def _register_in_batches(
         self, resource_config: ResourceConfig, user_agent_type: UserAgentType
@@ -183,39 +177,38 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         send_raw_data_examples_amount = (
             SEND_RAW_DATA_EXAMPLES_AMOUNT if ocean.config.send_raw_data_examples else 0
         )
-        entities = (
-            await self._register_resource_raw(
-                resource_config,
-                raw_results,
-                user_agent_type,
-                send_raw_data_examples_amount=send_raw_data_examples_amount,
-            )
-        ).passed
+        all_entities, register_errors = await self._register_resource_raw(
+            resource_config,
+            raw_results,
+            user_agent_type,
+            send_raw_data_examples_amount=send_raw_data_examples_amount,
+        )
+        errors.extend(register_errors)
+        passed_entities = list(all_entities.passed)
 
         for generator in async_generators:
             try:
                 async for items in generator:
                     if send_raw_data_examples_amount > 0:
                         send_raw_data_examples_amount = max(
-                            0, send_raw_data_examples_amount - len(entities)
+                            0, send_raw_data_examples_amount - len(passed_entities)
                         )
-                    entities.extend(
-                        (
-                            await self._register_resource_raw(
-                                resource_config,
-                                items,
-                                user_agent_type,
-                                send_raw_data_examples_amount=send_raw_data_examples_amount,
-                            )
-                        ).passed
+
+                    entities, register_errors = await self._register_resource_raw(
+                        resource_config,
+                        items,
+                        user_agent_type,
+                        send_raw_data_examples_amount=send_raw_data_examples_amount,
                     )
+                    errors.extend(register_errors)
+                    passed_entities.extend(entities.passed)
             except* OceanAbortException as error:
                 errors.append(error)
 
         logger.info(
-            f"Finished registering change for {len(results)} raw results for kind: {resource_config.kind}. {len(entities)} entities were affected"
+            f"Finished registering change for {len(results)} raw results for kind: {resource_config.kind}. {len(passed_entities)} entities were affected"
         )
-        return entities, errors
+        return passed_entities, errors
 
     async def register_raw(
         self,
@@ -244,16 +237,26 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         if not resource_mappings:
             return []
 
-        diffs: list[EntitySelectorDiff] = await asyncio.gather(
-            *(
-                self._register_resource_raw(resource, results, user_agent_type, True)
-                for resource in resource_mappings
+        diffs, errors = zip(
+            await asyncio.gather(
+                *(
+                    self._register_resource_raw(
+                        resource, results, user_agent_type, True
+                    )
+                    for resource in resource_mappings
+                )
             )
         )
 
-        registered_entities, entities_to_delete = zip_and_sum(
-            (entities_diff.passed, entities_diff.failed) for entities_diff in diffs
-        )
+        if errors:
+            message = f"Failed to register {len(errors)} entities. Skipping delete phase due to incomplete state"
+            logger.error(message, exc_info=errors)
+            raise ExceptionGroup(
+                message,
+                errors,
+            )
+
+        registered_entities, entities_to_delete = zip_and_sum(diffs)
 
         registered_entities_attributes = {
             (entity.identifier, entity.blueprint) for entity in registered_entities
@@ -306,13 +309,23 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
             resource for resource in config.resources if resource.kind == kind
         ]
 
-        return await asyncio.gather(
+        entities, errors = await asyncio.gather(
             *(
                 self._unregister_resource_raw(resource, results, user_agent_type)
                 for resource in resource_mappings
             )
         )
 
+        if errors:
+            message = f"Failed to unregister all entities with {len(errors)} errors"
+            logger.error(message, exc_info=errors)
+            raise ExceptionGroup(
+                message,
+                errors,
+            )
+
+        return entities
+
     async def update_raw_diff(
         self,
         kind: str,
@@ -337,14 +350,16 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         with logger.contextualize(kind=kind):
             logger.info(f"Found {len(resource_mappings)} resources for {kind}")
 
-            entities_before = await self._calculate_raw(
-                [
-                    (mapping, raw_desired_state["before"])
-                    for mapping in resource_mappings
-                ]
+            entities_before, _ = zip(
+                await self._calculate_raw(
+                    [
+                        (mapping, raw_desired_state["before"])
+                        for mapping in resource_mappings
+                    ]
+                )
             )
 
-            entities_after = await self._calculate_raw(
+            entities_after, after_errors = await self._calculate_raw(
                 [(mapping, raw_desired_state["after"]) for mapping in resource_mappings]
             )
 
@@ -356,6 +371,14 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                 (entities_diff.passed for entities_diff in entities_after), []
             )
 
+            if after_errors:
+                message = (
+                    f"Failed to calculate diff for entities with {len(after_errors)} errors. "
+                    f"Skipping delete phase due to incomplete state"
+                )
+                logger.error(message, exc_info=after_errors)
+                entities_before_flatten = []
+
             await self.entities_state_applier.apply_diff(
                 {"before": entities_before_flatten, "after": entities_after_flatten},
                 user_agent_type,
@@ -390,6 +413,20 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                 use_cache=False
             )
 
+            try:
+                entities_at_port = await ocean.port_client.search_entities(
+                    user_agent_type
+                )
+            except httpx.HTTPError as e:
+                logger.warning(
+                    "Failed to fetch the current state of entities at Port. "
+                    "Skipping delete phase due to unknown initial state. "
+                    f"Error: {e}\n"
+                    f"Response status code: {e.response.status_code if isinstance(e, httpx.HTTPStatusError) else None}\n"
+                    f"Response content: {e.response.text if isinstance(e, httpx.HTTPStatusError) else None}\n"
+                )
+                entities_at_port = []
+
             creation_results: list[tuple[list[Entity], list[Exception]]] = []
 
             try:
@@ -407,6 +444,13 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
             except asyncio.CancelledError as e:
                 logger.warning("Resync aborted successfully")
             else:
+                if not entities_at_port:
+                    logger.warning(
+                        "Due to an error before the resync, the previous state of entities at Port is unknown."
+                        " Skipping delete phase due to unknown initial state."
+                    )
+                    return
+
                 logger.info("Starting resync diff calculation")
                 flat_created_entities, errors = zip_and_sum(creation_results) or [
                     [],
@@ -424,11 +468,8 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
 
                     logger.error(message, exc_info=error_group)
                 else:
-                    entities_at_port = await ocean.port_client.search_entities(
-                        user_agent_type
-                    )
                     logger.info(
-                        f"Running resync diff calculation, number of entities found at Port: {len(entities_at_port)}, number of entities found during sync: {len(flat_created_entities)}"
+                        f"Running resync diff calculation, number of entities at Port before resync: {len(entities_at_port)}, number of entities created during sync: {len(flat_created_entities)}"
                     )
                     await self.entities_state_applier.delete_diff(
                         {"before": entities_at_port, "after": flat_created_entities},

port_ocean/core/models.py

@@ -1,3 +1,4 @@
+from dataclasses import dataclass, field
 from typing import Any
 
 from pydantic import BaseModel
@@ -34,3 +35,16 @@ class Migration(BaseModel):
     sourceBlueprint: str
     mapping: dict[str, Any]
     status: str
+
+
+@dataclass
+class EntityPortDiff:
+    """Represents the differences between entities for porting.
+
+    This class holds the lists of deleted, modified, and created entities as part
+    of the porting process.
+    """
+
+    deleted: list[Entity] = field(default_factory=list)
+    modified: list[Entity] = field(default_factory=list)
+    created: list[Entity] = field(default_factory=list)

port_ocean/core/ocean_types.py

@@ -2,7 +2,6 @@ from typing import TypedDict, Any, AsyncIterator, Callable, Awaitable, NamedTupl
 
 from port_ocean.core.models import Entity
 
-
 RAW_ITEM = dict[Any, Any]
 RAW_RESULT = list[RAW_ITEM]
 ASYNC_GENERATOR_RESYNC_TYPE = AsyncIterator[RAW_RESULT]
@@ -28,6 +27,11 @@ class EntitySelectorDiff(NamedTuple):
     failed: list[Entity]
 
 
+class CalculationResult(NamedTuple):
+    entity_selector_diff: EntitySelectorDiff
+    errors: list[Exception]
+
+
 class IntegrationEventsCallbacks(TypedDict):
     start: list[START_EVENT_LISTENER]
     resync: dict[str | None, list[RESYNC_EVENT_LISTENER]]

port_ocean/core/utils.py

@@ -1,9 +1,10 @@
-from typing import Iterable, Any, TypeVar
+import asyncio
+from typing import Iterable, Any, TypeVar, Callable, Awaitable
 
 from pydantic import parse_obj_as, ValidationError
 
-from port_ocean.core.handlers.entity_processor.base import EntityPortDiff
 from port_ocean.core.models import Entity
+from port_ocean.core.models import EntityPortDiff
 from port_ocean.core.ocean_types import RAW_RESULT
 from port_ocean.exceptions.core import RawObjectValidationException
 
@@ -28,6 +29,31 @@ def is_same_entity(first_entity: Entity, second_entity: Entity) -> bool:
     )
 
 
+Q = TypeVar("Q")
+
+
+async def gather_and_split_errors_from_results(
+    task: Iterable[Awaitable[Q]],
+    result_threshold_validation: Callable[[Q | Exception], bool] | None = None,
+) -> tuple[list[Q], list[Exception]]:
+    valid_items: list[Q] = []
+    errors: list[Exception] = []
+    results = await asyncio.gather(*task, return_exceptions=True)
+    for item in results:
+        # return_exceptions will also catch Python BaseException which also includes KeyboardInterrupt, SystemExit, GeneratorExit
+        # https://docs.python.org/3/library/asyncio-task.html#asyncio.gather
+        # These exceptions should be raised and not caught for the application to exit properly.
+        # https://stackoverflow.com/a/17802352
+        if isinstance(item, BaseException):
+            raise item
+        elif isinstance(item, Exception):
+            errors.append(item)
+        elif not result_threshold_validation or result_threshold_validation(item):
+            valid_items.append(item)
+
+    return valid_items, errors
+
+
 def get_port_diff(
     before: Iterable[Entity],
     after: Iterable[Entity],

port_ocean/utils/queue_utils.py

@@ -11,14 +11,18 @@ async def _start_processor_worker(
     queue: Queue[Any | None],
     func: Callable[..., Coroutine[Any, Any, T]],
     results: list[T],
+    errors: list[Exception],
 ) -> None:
     while True:
-        raw_params = await queue.get()
         try:
+            raw_params = await queue.get()
             if raw_params is None:
                 return
-            logger.debug(f"Processing {raw_params[0]}")
+            logger.debug("Processing async task")
             results.append(await func(*raw_params))
+        except Exception as e:
+            logger.error(f"Error processing task: {e}")
+            errors.append(e)
         finally:
             queue.task_done()
 
@@ -59,11 +63,12 @@ async def process_in_queue(
     queue: Queue[Any | None] = Queue(maxsize=concurrency * 2)
     tasks: list[Task[Any]] = []
     processing_results: list[T] = []
+    errors: list[Exception] = []
 
     for i in range(concurrency):
         tasks.append(
             asyncio.create_task(
-                _start_processor_worker(queue, func, processing_results)
+                _start_processor_worker(queue, func, processing_results, errors)
             )
         )
 
@@ -77,5 +82,7 @@ async def process_in_queue(
 
     await queue.join()
     await asyncio.gather(*tasks)
+    if errors:
+        raise ExceptionGroup("Error processing tasks", errors)
 
     return processing_results

{port_ocean-0.5.21.dist-info → port_ocean-0.5.24.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.5.21
+Version: 0.5.24
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.5.21.dist-info → port_ocean-0.5.24.dist-info}/RECORD

@@ -22,7 +22,7 @@ port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.gitignore,sha256=
 port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/resources/.gitignore,sha256=kCpRPdl3S_jqYYZaOrc0-xa6-l3KqVjNRXc6jCkd_-Q,12
 port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/.port/spec.yaml,sha256=jTYvu-Iayl-bxc917Y7ejcC9KyvH-LSq4-bdYZCYsuM,457
 port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/CHANGELOG.md,sha256=nzAmB0Bjnd2eZo79OjrlyVOdpTBHTmTxvO7c2C8Q-VQ,292
-port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/Dockerfile,sha256=Hh1dBnL959V2n28pmqFpXSrNvSMQjX6fDCUos8ITiu0,326
+port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/Dockerfile,sha256=LsH3vZqqEJkzeQG44cE7JkvPAuh_WPSqYam4YoMvG3M,328
 port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/Makefile,sha256=kTa72qEp8pi-joOH6zl8oeIgjEHSCF628p2074yHHNA,1779
 port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/README.md,sha256=5VZmgDRW9gO4d8UuzkujslOIDfIDBiAGL2Hd74HK770,468
 port_ocean/cli/cookiecutter/{{cookiecutter.integration_slug}}/changelog/.gitignore,sha256=kCpRPdl3S_jqYYZaOrc0-xa6-l3KqVjNRXc6jCkd_-Q,12
@@ -39,8 +39,8 @@ port_ocean/clients/port/authentication.py,sha256=t3z6h4vld-Tzkpth15sstaMJg0rccX-
 port_ocean/clients/port/client.py,sha256=3GYCM0ZkX3pB6sNoOb-7_6dm0Jr5_vqhflD9iltf_As,2640
 port_ocean/clients/port/mixins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/clients/port/mixins/blueprints.py,sha256=BiqkhvDFdkySWgL1NHI-LAQ9ieZWazZAojPo9E8d7U4,4575
-port_ocean/clients/port/mixins/entities.py,sha256=Lg5Sa6jQuhDTQKLURVavqXlBQt4-XPUUigB9JqQ1X0k,7364
-port_ocean/clients/port/mixins/integrations.py,sha256=EDjWnDS55rbGyZ0lctxdxHm-KaKVsGqZG_dEX9He5gg,5883
+port_ocean/clients/port/mixins/entities.py,sha256=pMERHqy1keb5w2k2xQsNuyZaKSAZ5ijVA4pzxEbAatY,7365
+port_ocean/clients/port/mixins/integrations.py,sha256=ypRYi7_VlnP2YEPgE8zS-_6WgPzUmsRi95K_pwVIpKE,5943
 port_ocean/clients/port/mixins/migrations.py,sha256=A6896oJF6WbFL2WroyTkMzr12yhVyWqGoq9dtLNSKBY,1457
 port_ocean/clients/port/retry_transport.py,sha256=PtIZOAZ6V-ncpVysRUsPOgt8Sf01QLnTKB5YeKBxkJk,1861
 port_ocean/clients/port/types.py,sha256=nvlgiAq4WH5_F7wQbz_GAWl-faob84LVgIjZ2Ww5mTk,451
@@ -59,7 +59,7 @@ port_ocean/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/core/defaults/__init__.py,sha256=8qCZg8n06WAdMu9s_FiRtDYLGPGHbOuS60vapeUoAks,142
 port_ocean/core/defaults/clean.py,sha256=S3UAfca-oU89WJKIB4OgGjGjPr0vxBQ2aRZsLTZhQ04,2185
 port_ocean/core/defaults/common.py,sha256=QnLFTkT3yIWIRtLQb7fUvvfe5AfInYJy0q5LjlzHkOw,3553
-port_ocean/core/defaults/initialize.py,sha256=z7IdE_WN1_gF5_60OwSlid9-di7pYOJhzX8VdFjkXdc,7702
+port_ocean/core/defaults/initialize.py,sha256=Cm1QFO7CLoPDdbJVYycbkWlfJfq7hlGbdRj49clBNUM,7432
 port_ocean/core/event_listener/__init__.py,sha256=mzJ33wRq0kh60fpVdOHVmvMTUQIvz3vxmifyBgwDn0E,889
 port_ocean/core/event_listener/base.py,sha256=4RujgPz4VfDFlviu4qLGJFnJougSCL-Ewf0rfTQfwgc,1133
 port_ocean/core/event_listener/factory.py,sha256=AYYfSHPAF7P5H-uQECXT0JVJjKDHrYkWJJBSL4mGkg8,3697
@@ -67,7 +67,7 @@ port_ocean/core/event_listener/http.py,sha256=UjONC_OODHjpGjvsBPAvO6zGzosdmv5Hx9
 port_ocean/core/event_listener/kafka.py,sha256=0U_TwmlmtS8N2lprkCmnyOmdqOAvghWxHhSfyu46kEs,6875
 port_ocean/core/event_listener/once.py,sha256=KdvUqjIcyp8XqhY1GfR_KYJfParFIRrjCtcMf3JMegk,2150
 port_ocean/core/event_listener/polling.py,sha256=3UxjgQJly5y-hA8R798oFWb7bFsYMxSc6GRozA3biiM,3539
-port_ocean/core/handlers/__init__.py,sha256=R2HIRD8JTzupY4mXuXveMBQbuiPiiB3Oivmpc5C2jjI,610
+port_ocean/core/handlers/__init__.py,sha256=d7ShmS90gLRzGKJA6oNy2Zs_dF2yjkmYZInRhBnO9Rw,572
 port_ocean/core/handlers/base.py,sha256=cTarblazu8yh8xz2FpB-dzDKuXxtoi143XJgPbV_DcM,157
 port_ocean/core/handlers/entities_state_applier/__init__.py,sha256=kgLZDCeCEzi4r-0nzW9k78haOZNf6PX7mJOUr34A4c8,173
 port_ocean/core/handlers/entities_state_applier/base.py,sha256=FMsrBOVgaO4o7B1klLDY8fobTUDvyrerCKCICyYtkXs,2193
@@ -76,8 +76,8 @@ port_ocean/core/handlers/entities_state_applier/port/applier.py,sha256=G6RyPgBAy
 port_ocean/core/handlers/entities_state_applier/port/get_related_entities.py,sha256=1zncwCbE-Gej0xaWKlzZgoXxOBe9bgs_YxlZ8QW3NdI,1751
 port_ocean/core/handlers/entities_state_applier/port/order_by_entities_dependencies.py,sha256=82BvU8t5w9uhsxX8hbnwuRPuWhW3cMeuT_5sVIkip1I,1550
 port_ocean/core/handlers/entity_processor/__init__.py,sha256=FvFCunFg44wNQoqlybem9MthOs7p1Wawac87uSXz9U8,156
-port_ocean/core/handlers/entity_processor/base.py,sha256=3e5QHlSjE7CCAv0WFSyQ7AhFEOS6-9eHj-ZYXGuvqFQ,2216
-port_ocean/core/handlers/entity_processor/jq_entity_processor.py,sha256=mF3ZyO1aSVxfefoeXSlTaIAFdoqITdsEuxlo2d1dUxM,6913
+port_ocean/core/handlers/entity_processor/base.py,sha256=udR0w5TstTOS5xOfTjAZIEdldn4xr6Oyb3DylatYX3Q,1869
+port_ocean/core/handlers/entity_processor/jq_entity_processor.py,sha256=CLg3FxHDs-GIXmRFC0r_BlJd9VGIi9ySjbWkYfB73i8,7330
 port_ocean/core/handlers/port_app_config/__init__.py,sha256=8AAT5OthiVM7KCcM34iEgEeXtn2pRMrT4Dze5r1Ixbk,134
 port_ocean/core/handlers/port_app_config/api.py,sha256=6VbKPwFzsWG0IYsVD81hxSmfqtHUFqrfUuj1DBX5g4w,853
 port_ocean/core/handlers/port_app_config/base.py,sha256=oufdNLzUmEgJY5PgIz75zDlowWrjA-y8WR4UnM58E58,2897
@@ -88,11 +88,11 @@ port_ocean/core/integrations/mixins/__init__.py,sha256=FA1FEKMM6P-L2_m7Q4L20mFa4
 port_ocean/core/integrations/mixins/events.py,sha256=Ddfx2L4FpghV38waF8OfVeOV0bHBxNIgjU-q5ffillI,2341
 port_ocean/core/integrations/mixins/handler.py,sha256=mZ7-0UlG3LcrwJttFbMe-R4xcOU2H_g33tZar7PwTv8,3771
 port_ocean/core/integrations/mixins/sync.py,sha256=TKqRytxXONVhuCo3CB3rDvWNbITnZz33TYTDs3SWWVk,3880
-port_ocean/core/integrations/mixins/sync_raw.py,sha256=DQ5pNqUedYXXykuMqecMhqi0GwMAvMWUZsTcjmkycxY,16137
+port_ocean/core/integrations/mixins/sync_raw.py,sha256=GmH5YPVnYO6mKEhoRer9F-vYy8mxPspaSWw2y9JJhgc,17954
 port_ocean/core/integrations/mixins/utils.py,sha256=7y1rGETZIjOQadyIjFJXIHKkQFKx_SwiP-TrAIsyyLY,2303
-port_ocean/core/models.py,sha256=bDO_I4Yd33TEZIh2QSV8UwXQIuwE7IgrINkYDHI0dkc,714
-port_ocean/core/ocean_types.py,sha256=ltnn22eRuDMFW02kIgmIAu6S06-i9jJV2NJ-MZcwwj0,879
-port_ocean/core/utils.py,sha256=6rYtsb1bjW8owxWngGiV3awMLZkP3tXZdxXClmRD1SU,1824
+port_ocean/core/models.py,sha256=YPIU7V3GKeDXIVkNzZn0w17YN2Akl-D8nFs-l4bEKcU,1143
+port_ocean/core/ocean_types.py,sha256=3_d8-n626f1kWLQ_Jxw194LEyrOVupz05qs_Y1pvB-A,990
+port_ocean/core/utils.py,sha256=rP8Fo2TSdSye4w0S8W0Vi4lpMJUOIbZ9kmtCkbGKxPY,2855
 port_ocean/exceptions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 port_ocean/exceptions/api.py,sha256=TLmTMqn4uHGaHgZK8PMIJ0TVJlPB4iP7xl9rx7GtCyY,426
 port_ocean/exceptions/base.py,sha256=uY4DX7fIITDFfemCJDWpaZi3bD51lcANc5swpoNvMJA,46
@@ -118,12 +118,12 @@ port_ocean/utils/async_http.py,sha256=arnH458TExn2Dju_Sy6pHas_vF5RMWnOp-jBz5WAAc
 port_ocean/utils/async_iterators.py,sha256=buFBiPdsqkNMCk91h6ZG8hJa181j7RjgHajbfgeB8A8,1608
 port_ocean/utils/cache.py,sha256=3KItZDE2yVrbVDr-hoM8lNna8s2dlpxhP4ICdLjH4LQ,2231
 port_ocean/utils/misc.py,sha256=WZjrEDRfyeqbesVt_Nkp2yjazbKF-BOnxJMFAI721yQ,1965
-port_ocean/utils/queue_utils.py,sha256=Pzb6e8PcjylZpXcb9EEIK-QcTty_E2k1egMiJF5J_8Q,2500
+port_ocean/utils/queue_utils.py,sha256=KWWl8YVnG-glcfIHhM6nefY-2sou_C6DVP1VynQwzB4,2762
 port_ocean/utils/repeat.py,sha256=0EFWM9d8lLXAhZmAyczY20LAnijw6UbIECf5lpGbOas,3231
 port_ocean/utils/signal.py,sha256=K-6kKFQTltcmKDhtyZAcn0IMa3sUpOHGOAUdWKgx0_E,1369
 port_ocean/version.py,sha256=UsuJdvdQlazzKGD3Hd5-U7N69STh8Dq9ggJzQFnu9fU,177
-port_ocean-0.5.21.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-port_ocean-0.5.21.dist-info/METADATA,sha256=w90EuOVc4M2mYy6KoaeLE03v0AoH_0SOgwf8RKy_e30,6554
-port_ocean-0.5.21.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-port_ocean-0.5.21.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
-port_ocean-0.5.21.dist-info/RECORD,,
+port_ocean-0.5.24.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+port_ocean-0.5.24.dist-info/METADATA,sha256=5EZHZqQ-zCeTf4LVLveW1-L-35tCSQ745mYQJ_FmTCM,6554
+port_ocean-0.5.24.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+port_ocean-0.5.24.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
+port_ocean-0.5.24.dist-info/RECORD,,