port-ocean 0.5.6__py3-none-any.whl → 0.17.8__py3-none-any.whl

port_ocean/core/handlers/entity_processor/jq_entity_processor.py

@@ -1,15 +1,39 @@
 import asyncio
-import functools
+from asyncio import Task
+from dataclasses import dataclass, field
 from functools import lru_cache
-from typing import Any
-
-import pyjq as jq  # type: ignore
+from typing import Any, Optional
+import jq  # type: ignore
+from loguru import logger
 
+from port_ocean.context.ocean import ocean
 from port_ocean.core.handlers.entity_processor.base import BaseEntityProcessor
 from port_ocean.core.handlers.port_app_config.models import ResourceConfig
 from port_ocean.core.models import Entity
-from port_ocean.core.ocean_types import RawEntityDiff, EntityDiff
+from port_ocean.core.ocean_types import (
+    RAW_ITEM,
+    EntitySelectorDiff,
+    CalculationResult,
+)
+from port_ocean.core.utils.utils import (
+    gather_and_split_errors_from_results,
+    zip_and_sum,
+)
 from port_ocean.exceptions.core import EntityProcessorException
+from port_ocean.utils.queue_utils import process_in_queue
+
+
+@dataclass
+class MappedEntity:
+    """Represents the entity after applying the mapping
+
+    This class holds the mapping entity along with the selector boolean value and optionally the raw data.
+    """
+
+    entity: dict[str, Any] = field(default_factory=dict)
+    did_entity_pass_selector: bool = False
+    raw_data: Optional[dict[str, Any]] = None
+    misconfigurations: dict[str, str] = field(default_factory=dict)
 
 
 class JQEntityProcessor(BaseEntityProcessor):
@@ -22,38 +46,102 @@ class JQEntityProcessor(BaseEntityProcessor):
 
     @lru_cache
     def _compile(self, pattern: str) -> Any:
+        if not ocean.config.allow_environment_variables_jq_access:
+            pattern = "def env: {}; {} as $ENV | " + pattern
         return jq.compile(pattern)
 
+    @staticmethod
+    def _stop_iterator_handler(func: Any) -> Any:
+        """
+        Wrap the function to handle StopIteration exceptions.
+        Prevents StopIteration from stopping the thread and skipping further queue processing.
+        """
+
+        def inner() -> Any:
+            try:
+                return func()
+            except StopIteration:
+                return None
+
+        return inner
+
+    @staticmethod
+    def _notify_mapping_issues(
+        entity_misconfigurations: dict[str, str],
+        missing_required_fields: bool,
+        entity_mapping_fault_counter: int,
+    ) -> None:
+
+        if len(entity_misconfigurations) > 0:
+            logger.info(
+                f"Unable to find valid data for: {entity_misconfigurations} (null, missing, or misconfigured)"
+            )
+        if missing_required_fields:
+            logger.info(
+                f"{entity_mapping_fault_counter} transformations of batch failed due to empty, null or missing values"
+            )
+
     async def _search(self, data: dict[str, Any], pattern: str) -> Any:
         try:
             loop = asyncio.get_event_loop()
             compiled_pattern = self._compile(pattern)
-            first_value_callable = functools.partial(compiled_pattern.first, data)
-            return await loop.run_in_executor(None, first_value_callable)
-        except Exception:
+            func = compiled_pattern.input_value(data)
+            return await loop.run_in_executor(
+                None, self._stop_iterator_handler(func.first)
+            )
+        except Exception as exc:
+            logger.debug(
+                f"Search failed for pattern '{pattern}' in data: {data}, Error: {exc}"
+            )
             return None
 
     async def _search_as_bool(self, data: dict[str, Any], pattern: str) -> bool:
         loop = asyncio.get_event_loop()
+
         compiled_pattern = self._compile(pattern)
-        first_value_callable = functools.partial(compiled_pattern.first, data)
-        value = await loop.run_in_executor(None, first_value_callable)
+        func = compiled_pattern.input_value(data)
 
+        value = await loop.run_in_executor(
+            None, self._stop_iterator_handler(func.first)
+        )
         if isinstance(value, bool):
             return value
-
         raise EntityProcessorException(
-            f"Expected boolean value, got {type(value)} instead"
+            f"Expected boolean value, got value:{value} of type: {type(value)} instead"
         )
 
     async def _search_as_object(
-        self, data: dict[str, Any], obj: dict[str, Any]
+        self,
+        data: dict[str, Any],
+        obj: dict[str, Any],
+        misconfigurations: dict[str, str] | None = None,
     ) -> dict[str, Any | None]:
-        search_tasks = {}
+        """
+        Identify and extract the relevant value for the chosen key and populate it into the entity
+        :param data: the property itself that holds the key and the value, it is being passed to the task and we get back a task item,
+            if the data is a dict, we will recursively call this function again.
+        :param obj: the key that we want its value to be mapped into our entity.
+        :param misconfigurations: due to the recursive nature of this function,
+            we aim to have a dict that represents all of the misconfigured properties and when used recursively,
+            we pass this reference to misfoncigured object to add the relevant misconfigured keys.
+        :return: Mapped object with found value.
+        """
+
+        search_tasks: dict[
+            str, Task[dict[str, Any | None]] | list[Task[dict[str, Any | None]]]
+        ] = {}
         for key, value in obj.items():
-            if isinstance(value, dict):
+            if isinstance(value, list):
+                search_tasks[key] = [
+                    asyncio.create_task(
+                        self._search_as_object(data, obj, misconfigurations)
+                    )
+                    for obj in value
+                ]
+
+            elif isinstance(value, dict):
                 search_tasks[key] = asyncio.create_task(
-                    self._search_as_object(data, value)
+                    self._search_as_object(data, value, misconfigurations)
                 )
             else:
                 search_tasks[key] = asyncio.create_task(self._search(data, value))
@@ -61,45 +149,165 @@ class JQEntityProcessor(BaseEntityProcessor):
         result: dict[str, Any | None] = {}
         for key, task in search_tasks.items():
             try:
-                result[key] = await task
+                if isinstance(task, list):
+                    result_list = []
+                    for task in task:
+                        task_result = await task
+                        if task_result is None and misconfigurations is not None:
+                            misconfigurations[key] = obj[key]
+                        result_list.append(task_result)
+                    result[key] = result_list
+                else:
+                    task_result = await task
+                    if task_result is None and misconfigurations is not None:
+                        misconfigurations[key] = obj[key]
+                    result[key] = task_result
             except Exception:
                 result[key] = None
-
         return result
 
-    async def _calculate_entities(
-        self, mapping: ResourceConfig, raw_data: list[dict[str, Any]]
-    ) -> list[Entity]:
-        async def calculate_raw(data: dict[str, Any]) -> dict[str, Any]:
-            should_run = await self._search_as_bool(data, mapping.selector.query)
-            if should_run and mapping.port.entity:
-                return await self._search_as_object(
-                    data, mapping.port.entity.mappings.dict(exclude_unset=True)
+    async def _get_mapped_entity(
+        self,
+        data: dict[str, Any],
+        raw_entity_mappings: dict[str, Any],
+        selector_query: str,
+        parse_all: bool = False,
+    ) -> MappedEntity:
+        should_run = await self._search_as_bool(data, selector_query)
+        if parse_all or should_run:
+            misconfigurations: dict[str, str] = {}
+            mapped_entity = await self._search_as_object(
+                data, raw_entity_mappings, misconfigurations
+            )
+            return MappedEntity(
+                mapped_entity,
+                did_entity_pass_selector=should_run,
+                raw_data=data if should_run else None,
+                misconfigurations=misconfigurations,
+            )
+
+        return MappedEntity()
+
+    async def _calculate_entity(
+        self,
+        data: dict[str, Any],
+        raw_entity_mappings: dict[str, Any],
+        items_to_parse: str | None,
+        selector_query: str,
+        parse_all: bool = False,
+    ) -> tuple[list[MappedEntity], list[Exception]]:
+        raw_data = [data.copy()]
+        if items_to_parse:
+            items = await self._search(data, items_to_parse)
+            if not isinstance(items, list):
+                logger.warning(
+                    f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items)}."
+                    f" Skipping..."
                 )
-            return {}
+                return [], []
+            raw_data = [{"item": item, **data} for item in items]
 
-        entities_tasks = [asyncio.create_task(calculate_raw(data)) for data in raw_data]
-        entities = await asyncio.gather(*entities_tasks)
+        entities, errors = await gather_and_split_errors_from_results(
+            [
+                self._get_mapped_entity(
+                    raw,
+                    raw_entity_mappings,
+                    selector_query,
+                    parse_all,
+                )
+                for raw in raw_data
+            ]
+        )
+        if errors:
+            logger.error(
+                f"Failed to calculate entities with {len(errors)} errors. errors: {errors}"
+            )
+        return entities, errors
 
-        return [
-            Entity.parse_obj(entity_data)
-            for entity_data in filter(
-                lambda entity: entity.get("identifier") and entity.get("blueprint"),
-                entities,
+    @staticmethod
+    async def _send_examples(data: list[dict[str, Any]], kind: str) -> None:
+        try:
+            if data:
+                await ocean.port_client.ingest_integration_kind_examples(
+                    kind, data, should_log=False
+                )
+        except Exception as ex:
+            logger.warning(
+                f"Failed to send raw data example {ex}",
+                exc_info=True,
             )
-        ]
 
     async def _parse_items(
-        self, mapping: ResourceConfig, raw_results: RawEntityDiff
-    ) -> EntityDiff:
-        entities_before: list[Entity] = await self._calculate_entities(
-            mapping, raw_results["before"]
+        self,
+        mapping: ResourceConfig,
+        raw_results: list[RAW_ITEM],
+        parse_all: bool = False,
+        send_raw_data_examples_amount: int = 0,
+    ) -> CalculationResult:
+        raw_entity_mappings: dict[str, Any] = mapping.port.entity.mappings.dict(
+            exclude_unset=True
+        )
+        logger.info(f"Parsing {len(raw_results)} raw results into entities")
+        calculated_entities_results, errors = zip_and_sum(
+            await process_in_queue(
+                raw_results,
+                self._calculate_entity,
+                raw_entity_mappings,
+                mapping.port.items_to_parse,
+                mapping.selector.query,
+                parse_all,
+            )
         )
-        entities_after: list[Entity] = await self._calculate_entities(
-            mapping, raw_results["after"]
+        logger.debug(
+            f"Finished parsing raw results into entities with {len(errors)} errors. errors: {errors}"
         )
 
-        return {
-            "before": entities_before,
-            "after": entities_after,
-        }
+        passed_entities = []
+        failed_entities = []
+        examples_to_send: list[dict[str, Any]] = []
+        entity_misconfigurations: dict[str, str] = {}
+        missing_required_fields: bool = False
+        entity_mapping_fault_counter: int = 0
+
+        for result in calculated_entities_results:
+            if len(result.misconfigurations) > 0:
+                entity_misconfigurations |= result.misconfigurations
+
+            if result.entity.get("identifier") and result.entity.get("blueprint"):
+                parsed_entity = Entity.parse_obj(result.entity)
+                if result.did_entity_pass_selector:
+                    passed_entities.append(parsed_entity)
+                    if (
+                        len(examples_to_send) < send_raw_data_examples_amount
+                        and result.raw_data is not None
+                    ):
+                        examples_to_send.append(result.raw_data)
+                else:
+                    failed_entities.append(parsed_entity)
+            else:
+                missing_required_fields = True
+                entity_mapping_fault_counter += 1
+
+        self._notify_mapping_issues(
+            entity_misconfigurations,
+            missing_required_fields,
+            entity_mapping_fault_counter,
+        )
+
+        if (
+            not calculated_entities_results
+            and raw_results
+            and send_raw_data_examples_amount > 0
+        ):
+            logger.warning(
+                f"No entities were parsed from {len(raw_results)} raw results, sending raw data examples"
+            )
+            examples_to_send = raw_results[:send_raw_data_examples_amount]
+
+        await self._send_examples(examples_to_send, mapping.kind)
+
+        return CalculationResult(
+            EntitySelectorDiff(passed=passed_entities, failed=failed_entities),
+            errors,
+            misonfigured_entity_keys=entity_misconfigurations,
+        )

port_ocean/core/handlers/port_app_config/base.py

@@ -5,8 +5,37 @@ from loguru import logger
 from pydantic import ValidationError
 
 from port_ocean.context.event import event
+from port_ocean.context.ocean import PortOceanContext
 from port_ocean.core.handlers.base import BaseHandler
 from port_ocean.core.handlers.port_app_config.models import PortAppConfig
+from port_ocean.utils.misc import get_time
+
+
+class PortAppConfigCache:
+    _port_app_config: PortAppConfig | None
+    _retrieval_time: float
+
+    def __init__(self, cache_ttl: int):
+        self._port_app_config = None
+        self._cache_ttl = cache_ttl
+
+    @property
+    def port_app_config(self) -> PortAppConfig:
+        if self._port_app_config is None:
+            raise ValueError("Port app config is not set")
+        return self._port_app_config
+
+    @port_app_config.setter
+    def port_app_config(self, value: PortAppConfig) -> None:
+        self._retrieval_time = get_time()
+        self._port_app_config = value
+
+    @property
+    def is_cache_invalid(self) -> bool:
+        return (
+            not self._port_app_config
+            or self._retrieval_time + self._cache_ttl < get_time()
+        )
 
 
 class BasePortAppConfig(BaseHandler):
@@ -21,24 +50,35 @@ class BasePortAppConfig(BaseHandler):
 
     CONFIG_CLASS: Type[PortAppConfig] = PortAppConfig
 
+    def __init__(self, context: PortOceanContext):
+        super().__init__(context)
+        self._app_config_cache = PortAppConfigCache(
+            self.context.config.port.port_app_config_cache_ttl
+        )
+
     @abstractmethod
     async def _get_port_app_config(self) -> dict[str, Any]:
         pass
 
-    async def get_port_app_config(self) -> PortAppConfig:
-        """Retrieve and parse the port application configuration.
+    async def get_port_app_config(self, use_cache: bool = True) -> PortAppConfig:
+        """
+        Retrieve and parse the port application configuration.
 
-        Returns:
-            PortAppConfig: The parsed port application configuration.
+        :param use_cache: Determines whether to use the cached port-app-config if it exists, or to fetch it regardless
+        :return: The parsed port application configuration.
         """
-        raw_config = await self._get_port_app_config()
-        try:
-            config = self.CONFIG_CLASS.parse_obj(raw_config)
-        except ValidationError:
-            logger.error(
-                "Invalid port app config found. Please check that the integration has been configured correctly."
-            )
-            raise
-
-        event.port_app_config = config
-        return config
+        if not use_cache or self._app_config_cache.is_cache_invalid:
+            raw_config = await self._get_port_app_config()
+            try:
+                self._app_config_cache.port_app_config = self.CONFIG_CLASS.parse_obj(
+                    raw_config
+                )
+            except ValidationError:
+                logger.error(
+                    "Invalid port app config found. Please check that the integration has been configured correctly."
+                )
+                logger.warning(f"Invalid port app config: {raw_config}")
+                raise
+
+        event.port_app_config = self._app_config_cache.port_app_config
+        return self._app_config_cache.port_app_config

port_ocean/core/handlers/port_app_config/models.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from typing import Any
 
 from pydantic import BaseModel, Field
@@ -5,20 +7,37 @@ from pydantic import BaseModel, Field
 from port_ocean.clients.port.types import RequestOptions
 
 
+class Rule(BaseModel):
+    property: str
+    operator: str
+    value: str
+
+
+class IngestSearchQuery(BaseModel):
+    combinator: str
+    rules: list[Rule | IngestSearchQuery]
+
+
 class EntityMapping(BaseModel):
-    identifier: str
+    identifier: str | IngestSearchQuery
     title: str | None
     blueprint: str
     team: str | None
     properties: dict[str, str] = Field(default_factory=dict)
-    relations: dict[str, str] = Field(default_factory=dict)
+    relations: dict[str, str | IngestSearchQuery] = Field(default_factory=dict)
 
+    @property
+    def is_using_search_identifier(self) -> bool:
+        return isinstance(self.identifier, dict)
 
-class PortResourceConfig(BaseModel):
-    class MappingsConfig(BaseModel):
-        mappings: EntityMapping
 
+class MappingsConfig(BaseModel):
+    mappings: EntityMapping
+
+
+class PortResourceConfig(BaseModel):
     entity: MappingsConfig
+    items_to_parse: str | None = Field(alias="itemsToParse")
 
 
 class Selector(BaseModel):

port_ocean/core/handlers/resync_state_updater/__init__.py

@@ -0,0 +1,5 @@
+from .updater import ResyncStateUpdater
+
+__all__ = [
+    "ResyncStateUpdater",
+]

port_ocean/core/handlers/resync_state_updater/updater.py

@@ -0,0 +1,84 @@
+import datetime
+from typing import Any, Literal
+from port_ocean.clients.port.client import PortClient
+from port_ocean.utils.misc import IntegrationStateStatus
+from port_ocean.utils.time import get_next_occurrence
+
+
+class ResyncStateUpdater:
+    def __init__(self, port_client: PortClient, scheduled_resync_interval: int | None):
+        self.port_client = port_client
+        self.initiated_at = datetime.datetime.now(tz=datetime.timezone.utc)
+        self.scheduled_resync_interval = scheduled_resync_interval
+
+        # This is used to differ between integration changes that require a full resync and state changes
+        # So that the polling event-listener can decide whether to perform a full resync or not
+        # TODO: remove this once we separate the state from the integration
+        self.last_integration_state_updated_at: str = ""
+
+    def _calculate_next_scheduled_resync(
+        self,
+        interval: int | None = None,
+        custom_start_time: datetime.datetime | None = None,
+    ) -> str | None:
+        if interval is None:
+            return None
+        return get_next_occurrence(
+            interval * 60, custom_start_time or self.initiated_at
+        ).isoformat()
+
+    async def update_before_resync(
+        self,
+        interval: int | None = None,
+        custom_start_time: datetime.datetime | None = None,
+    ) -> None:
+        _interval = interval or self.scheduled_resync_interval
+        nest_resync = self._calculate_next_scheduled_resync(
+            _interval, custom_start_time
+        )
+        state: dict[str, Any] = {
+            "status": IntegrationStateStatus.Running.value,
+            "lastResyncEnd": None,
+            "lastResyncStart": datetime.datetime.now(
+                tz=datetime.timezone.utc
+            ).isoformat(),
+            "nextResync": nest_resync,
+            "intervalInMinuets": _interval,
+        }
+
+        integration = await self.port_client.update_integration_state(
+            state, should_raise=False
+        )
+        if integration:
+            self.last_integration_state_updated_at = integration["resyncState"][
+                "updatedAt"
+            ]
+
+    async def update_after_resync(
+        self,
+        status: Literal[
+            IntegrationStateStatus.Completed, IntegrationStateStatus.Failed
+        ] = IntegrationStateStatus.Completed,
+        interval: int | None = None,
+        custom_start_time: datetime.datetime | None = None,
+    ) -> None:
+        _interval = interval or self.scheduled_resync_interval
+        nest_resync = self._calculate_next_scheduled_resync(
+            _interval, custom_start_time
+        )
+        state: dict[str, Any] = {
+            "status": status.value,
+            "lastResyncEnd": datetime.datetime.now(
+                tz=datetime.timezone.utc
+            ).isoformat(),
+            "nextResync": nest_resync,
+            "intervalInMinuets": _interval,
+        }
+
+        integration = await self.port_client.update_integration_state(
+            state, should_raise=False
+        )
+        if integration:
+            self.last_integration_state_updated_at = integration["resyncState"][
+                "updatedAt"
+            ]

port_ocean/core/integrations/base.py

@@ -54,24 +54,22 @@ class BaseIntegration(SyncRawMixin, SyncMixin):
         """
         Initializes handlers, establishes integration at the specified port, and starts the event listener.
         """
-        logger.info("Starting integration")
+        logger.info(
+            "Starting integration",
+            integration_type=self.context.config.integration.type,
+        )
         if self.started:
             raise IntegrationAlreadyStartedException("Integration already started")
 
         if (
             not self.event_strategy["resync"]
             and self.__class__._on_resync == BaseIntegration._on_resync
+            and self.context.config.event_listener.should_resync
         ):
             raise NotImplementedError("on_resync is not implemented")
 
         await self.initialize_handlers()
 
-        logger.info("Initializing integration at port")
-        await self.context.port_client.initialize_integration(
-            self.context.config.integration.type,
-            self.context.config.event_listener.to_request(),
-        )
-
         self.started = True
 
         async with event_context(

port_ocean/core/integrations/mixins/events.py

@@ -46,7 +46,7 @@ class EventsMixin:
         return func
 
     def on_resync(
-        self, func: RESYNC_EVENT_LISTENER, kind: str | None = None
+        self, func: RESYNC_EVENT_LISTENER| None, kind: str | None = None
     ) -> RESYNC_EVENT_LISTENER:
         """Register a function as a listener for a "resync" event.
 
@@ -57,6 +57,8 @@ class EventsMixin:
         Returns:
             RESYNC_EVENT_LISTENER: The input function, unchanged.
         """
+        if func is None:
+            return None
         if kind is None:
             logger.debug(f"Registering resync event listener any kind")
         else:

port_ocean/core/integrations/mixins/sync.py

@@ -97,9 +97,11 @@ class SyncMixin(HandlerMixin):
         """
         entities_at_port = await ocean.port_client.search_entities(user_agent_type)
 
-        await self.entities_state_applier.upsert(entities, user_agent_type)
+        modified_entities = await self.entities_state_applier.upsert(
+            entities, user_agent_type
+        )
         await self.entities_state_applier.delete_diff(
-            {"before": entities_at_port, "after": entities}, user_agent_type
+            {"before": entities_at_port, "after": modified_entities}, user_agent_type
         )
 
         logger.info("Finished syncing change")