port-ocean 0.28.8__tar.gz → 0.28.11__tar.gz

{port_ocean-0.28.8 → port_ocean-0.28.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.28.8
+Version: 0.28.11
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.28.8 → port_ocean-0.28.11}/integrations/_infra/Dockerfile.Deb

@@ -48,7 +48,6 @@ RUN apt-get update \
         curl \
         acl \
         sudo \
-        jq \
     && apt-get clean
 
 LABEL INTEGRATION_VERSION=${INTEGRATION_VERSION}

{port_ocean-0.28.8 → port_ocean-0.28.11}/integrations/_infra/Dockerfile.local

@@ -26,7 +26,6 @@ RUN apt-get update \
         python3-pip \
         python3-poetry \
         build-essential\
-        jq \
         git \
         python3-venv \
         acl \

{port_ocean-0.28.8 → port_ocean-0.28.11}/port_ocean/clients/port/mixins/integrations.py

@@ -301,7 +301,7 @@ class IntegrationClientMixin:
             headers=headers,
             json={
                 "items": raw_data,
-                "extractionTimestamp": datetime.now().isoformat(),
+                "extractionTimestamp": int(datetime.now().timestamp()),
             },
         )
         handle_port_status_code(response, should_log=False)

port_ocean-0.28.11/port_ocean/core/handlers/entity_processor/jq_entity_processor.py

@@ -0,0 +1,357 @@
+import asyncio
+from asyncio import Task
+from dataclasses import dataclass, field
+
+from functools import lru_cache
+from typing import Any, Optional
+import jq  # type: ignore
+from loguru import logger
+
+from port_ocean.context.ocean import ocean
+from port_ocean.core.handlers.entity_processor.base import BaseEntityProcessor
+from port_ocean.core.handlers.port_app_config.models import ResourceConfig
+from port_ocean.core.models import Entity
+from port_ocean.core.ocean_types import (
+    RAW_ITEM,
+    EntitySelectorDiff,
+    CalculationResult,
+)
+from port_ocean.core.utils.utils import (
+    gather_and_split_errors_from_results,
+    zip_and_sum,
+)
+from port_ocean.exceptions.core import EntityProcessorException
+from port_ocean.utils.queue_utils import process_in_queue
+
+
+class ExampleStates:
+    __succeed: list[dict[str, Any]]
+    __errors: list[dict[str, Any]]
+    __max_size: int
+
+    def __init__(self, max_size: int = 0) -> None:
+        """
+        Store two sequences:
+          - succeed: items that succeeded
+          - errors:  items that failed
+        """
+        self.__succeed = []
+        self.__errors = []
+        self.__max_size = max_size
+
+    def add_example(self, succeed: bool, item: dict[str, Any]) -> None:
+        if succeed:
+            self.__succeed.append(item)
+        else:
+            self.__errors.append(item)
+
+    def __len__(self) -> int:
+        """
+        Total number of items (successes + errors).
+        """
+        return len(self.__succeed) + len(self.__errors)
+
+    def get_examples(self, number: int = 0) -> list[dict[str, Any]]:
+        """
+        Return a list of up to number items, taking successes first,
+        """
+        if number <= 0:
+            number = self.__max_size
+        # how many from succeed?
+        s_count = min(number, len(self.__succeed))
+        result = list(self.__succeed[:s_count])
+        # how many more from errors?
+        e_count = number - s_count
+        if e_count > 0:
+            result.extend(self.__errors[:e_count])
+        return result
+
+
+@dataclass
+class MappedEntity:
+    """Represents the entity after applying the mapping
+
+    This class holds the mapping entity along with the selector boolean value and optionally the raw data.
+    """
+
+    entity: dict[str, Any] = field(default_factory=dict)
+    did_entity_pass_selector: bool = False
+    raw_data: Optional[dict[str, Any]] = None
+    misconfigurations: dict[str, str] = field(default_factory=dict)
+
+
+class JQEntityProcessor(BaseEntityProcessor):
+    """Processes and parses entities using JQ expressions.
+
+    This class extends the BaseEntityProcessor and provides methods for processing and
+    parsing entities based on PyJQ queries. It supports compiling and executing PyJQ patterns,
+    searching for data in dictionaries, and transforming data based on object mappings.
+    """
+
+    @lru_cache
+    def _compile(self, pattern: str) -> Any:
+        if not ocean.config.allow_environment_variables_jq_access:
+            pattern = "def env: {}; {} as $ENV | " + pattern
+        return jq.compile(pattern)
+
+    @staticmethod
+    def _stop_iterator_handler(func: Any) -> Any:
+        """
+        Wrap the function to handle StopIteration exceptions.
+        Prevents StopIteration from stopping the thread and skipping further queue processing.
+        """
+
+        def inner() -> Any:
+            try:
+                return func()
+            except StopIteration:
+                return None
+
+        return inner
+
+    @staticmethod
+    def _notify_mapping_issues(
+        entity_misconfigurations: dict[str, str],
+        missing_required_fields: bool,
+        entity_mapping_fault_counter: int,
+    ) -> None:
+
+        if len(entity_misconfigurations) > 0:
+            logger.info(
+                f"Unable to find valid data for: {entity_misconfigurations} (null, missing, or misconfigured)"
+            )
+        if missing_required_fields:
+            logger.info(
+                f"{entity_mapping_fault_counter} transformations of batch failed due to empty, null or missing values"
+            )
+
+    async def _search(self, data: dict[str, Any], pattern: str) -> Any:
+        try:
+            loop = asyncio.get_event_loop()
+            compiled_pattern = self._compile(pattern)
+            func = compiled_pattern.input_value(data)
+            return await loop.run_in_executor(
+                None, self._stop_iterator_handler(func.first)
+            )
+        except Exception as exc:
+            logger.debug(
+                f"Search failed for pattern '{pattern}' in data: {data}, Error: {exc}"
+            )
+            return None
+
+    async def _search_as_bool(self, data: dict[str, Any], pattern: str) -> bool:
+        loop = asyncio.get_event_loop()
+
+        compiled_pattern = self._compile(pattern)
+        func = compiled_pattern.input_value(data)
+
+        value = await loop.run_in_executor(
+            None, self._stop_iterator_handler(func.first)
+        )
+        if isinstance(value, bool):
+            return value
+        raise EntityProcessorException(
+            f"Expected boolean value, got value:{value} of type: {type(value)} instead"
+        )
+
+    async def _search_as_object(
+        self,
+        data: dict[str, Any],
+        obj: dict[str, Any],
+        misconfigurations: dict[str, str] | None = None,
+    ) -> dict[str, Any | None]:
+        """
+        Identify and extract the relevant value for the chosen key and populate it into the entity
+        :param data: the property itself that holds the key and the value, it is being passed to the task and we get back a task item,
+            if the data is a dict, we will recursively call this function again.
+        :param obj: the key that we want its value to be mapped into our entity.
+        :param misconfigurations: due to the recursive nature of this function,
+            we aim to have a dict that represents all of the misconfigured properties and when used recursively,
+            we pass this reference to misfoncigured object to add the relevant misconfigured keys.
+        :return: Mapped object with found value.
+        """
+
+        search_tasks: dict[
+            str, Task[dict[str, Any | None]] | list[Task[dict[str, Any | None]]]
+        ] = {}
+        for key, value in obj.items():
+            if isinstance(value, list):
+                search_tasks[key] = [
+                    asyncio.create_task(
+                        self._search_as_object(data, obj, misconfigurations)
+                    )
+                    for obj in value
+                ]
+
+            elif isinstance(value, dict):
+                search_tasks[key] = asyncio.create_task(
+                    self._search_as_object(data, value, misconfigurations)
+                )
+            else:
+                search_tasks[key] = asyncio.create_task(self._search(data, value))
+
+        result: dict[str, Any | None] = {}
+        for key, task in search_tasks.items():
+            try:
+                if isinstance(task, list):
+                    result_list = []
+                    for task in task:
+                        task_result = await task
+                        if task_result is None and misconfigurations is not None:
+                            misconfigurations[key] = obj[key]
+                        result_list.append(task_result)
+                    result[key] = result_list
+                else:
+                    task_result = await task
+                    if task_result is None and misconfigurations is not None:
+                        misconfigurations[key] = obj[key]
+                    result[key] = task_result
+            except Exception:
+                result[key] = None
+        return result
+
+    async def _get_mapped_entity(
+        self,
+        data: dict[str, Any],
+        raw_entity_mappings: dict[str, Any],
+        selector_query: str,
+        parse_all: bool = False,
+    ) -> MappedEntity:
+        should_run = await self._search_as_bool(data, selector_query)
+        if parse_all or should_run:
+            misconfigurations: dict[str, str] = {}
+            mapped_entity = await self._search_as_object(
+                data, raw_entity_mappings, misconfigurations
+            )
+            return MappedEntity(
+                mapped_entity,
+                did_entity_pass_selector=should_run,
+                raw_data=data,
+                misconfigurations=misconfigurations,
+            )
+
+        return MappedEntity(
+            {},
+            did_entity_pass_selector=False,
+            raw_data=data,
+            misconfigurations={},
+        )
+
+    async def _calculate_entity(
+        self,
+        data: dict[str, Any],
+        raw_entity_mappings: dict[str, Any],
+        items_to_parse: str | None,
+        items_to_parse_name: str,
+        selector_query: str,
+        parse_all: bool = False,
+    ) -> tuple[list[MappedEntity], list[Exception]]:
+        raw_data = [data.copy()]
+        if not ocean.config.yield_items_to_parse:
+            if items_to_parse:
+                items = await self._search(data, items_to_parse)
+                if not isinstance(items, list):
+                    logger.warning(
+                        f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items)}."
+                        f" Skipping..."
+                    )
+                    return [], []
+                raw_data = [{items_to_parse_name: item, **data} for item in items]
+
+        entities, errors = await gather_and_split_errors_from_results(
+            [
+                self._get_mapped_entity(
+                    raw,
+                    raw_entity_mappings,
+                    selector_query,
+                    parse_all,
+                )
+                for raw in raw_data
+            ]
+        )
+        if errors:
+            logger.error(
+                f"Failed to calculate entities with {len(errors)} errors. errors: {errors}"
+            )
+        return entities, errors
+
+    @staticmethod
+    async def _send_examples(data: list[dict[str, Any]], kind: str) -> None:
+        try:
+            if data:
+                await ocean.port_client.ingest_integration_kind_examples(
+                    kind, data, should_log=False
+                )
+        except Exception as ex:
+            logger.warning(
+                f"Failed to send raw data example {ex}",
+                exc_info=True,
+            )
+
+    async def _parse_items(
+        self,
+        mapping: ResourceConfig,
+        raw_results: list[RAW_ITEM],
+        parse_all: bool = False,
+        send_raw_data_examples_amount: int = 0,
+    ) -> CalculationResult:
+        raw_entity_mappings: dict[str, Any] = mapping.port.entity.mappings.dict(
+            exclude_unset=True
+        )
+        logger.info(f"Parsing {len(raw_results)} raw results into entities")
+        calculated_entities_results, errors = zip_and_sum(
+            await process_in_queue(
+                raw_results,
+                self._calculate_entity,
+                raw_entity_mappings,
+                mapping.port.items_to_parse,
+                mapping.port.items_to_parse_name,
+                mapping.selector.query,
+                parse_all,
+            )
+        )
+        logger.debug(
+            f"Finished parsing raw results into entities with {len(errors)} errors. errors: {errors}"
+        )
+
+        passed_entities = []
+        failed_entities = []
+        examples_to_send = ExampleStates(send_raw_data_examples_amount)
+        entity_misconfigurations: dict[str, str] = {}
+        missing_required_fields: bool = False
+        entity_mapping_fault_counter: int = 0
+        for result in calculated_entities_results:
+            if len(result.misconfigurations) > 0:
+                entity_misconfigurations |= result.misconfigurations
+
+            if (
+                len(examples_to_send) < send_raw_data_examples_amount
+                and result.raw_data is not None
+            ):
+                examples_to_send.add_example(
+                    result.did_entity_pass_selector, result.raw_data
+                )
+
+            if result.entity.get("identifier") and result.entity.get("blueprint"):
+                parsed_entity = Entity.parse_obj(result.entity)
+                if result.did_entity_pass_selector:
+                    passed_entities.append(parsed_entity)
+                else:
+                    failed_entities.append(parsed_entity)
+            else:
+                missing_required_fields = True
+                entity_mapping_fault_counter += 1
+
+        self._notify_mapping_issues(
+            entity_misconfigurations,
+            missing_required_fields,
+            entity_mapping_fault_counter,
+        )
+
+        await self._send_examples(examples_to_send.get_examples(), mapping.kind)
+
+        return CalculationResult(
+            EntitySelectorDiff(passed=passed_entities, failed=failed_entities),
+            errors,
+            misconfigured_entity_keys=entity_misconfigurations,
+        )

{port_ocean-0.28.8 → port_ocean-0.28.11}/port_ocean/core/handlers/port_app_config/models.py

@@ -29,7 +29,9 @@ class EntityMapping(BaseModel):
 
     @property
     def is_using_search_identifier(self) -> bool:
-        return isinstance(self.identifier, dict)
+        return isinstance(self.identifier, dict) or isinstance(
+            self.identifier, IngestSearchQuery
+        )
 
 
 class MappingsConfig(BaseModel):
@@ -39,7 +41,7 @@ class MappingsConfig(BaseModel):
 class PortResourceConfig(BaseModel):
     entity: MappingsConfig
     items_to_parse: str | None = Field(alias="itemsToParse")
-    items_to_parse_name: str = Field(alias="itemsToParseName", default="item")
+    items_to_parse_name: str | None = Field(alias="itemsToParseName", default="item")
 
 
 class Selector(BaseModel):

{port_ocean-0.28.8 → port_ocean-0.28.11}/port_ocean/core/integrations/mixins/sync_raw.py

@@ -117,7 +117,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                 logger.info(
                     f"Found async generator function for {resource_config.kind} name: {task.__qualname__}"
                 )
-                results.append(resync_generator_wrapper(task, resource_config.kind, resource_config.port.items_to_parse_name, resource_config.port.items_to_parse))
+                results.append(resync_generator_wrapper(task, resource_config.kind,resource_config.port.items_to_parse))
             else:
                 logger.info(
                     f"Found sync function for {resource_config.kind} name: {task.__qualname__}"

port_ocean-0.28.11/port_ocean/core/integrations/mixins/utils.py

@@ -0,0 +1,136 @@
+from contextlib import contextmanager
+from typing import Awaitable, Generator, Callable, cast
+
+from loguru import logger
+
+import asyncio
+import multiprocessing
+
+from port_ocean.core.handlers.entity_processor.jq_entity_processor import JQEntityProcessor
+from port_ocean.core.handlers.port_app_config.models import ResourceConfig
+from port_ocean.core.ocean_types import (
+    ASYNC_GENERATOR_RESYNC_TYPE,
+    RAW_RESULT,
+    RESYNC_EVENT_LISTENER,
+    RESYNC_RESULT,
+)
+from port_ocean.core.utils.utils import validate_result
+from port_ocean.exceptions.core import (
+    RawObjectValidationException,
+    OceanAbortException,
+    KindNotImplementedException,
+)
+
+from port_ocean.utils.async_http import _http_client
+from port_ocean.clients.port.utils import _http_client as _port_http_client
+from port_ocean.helpers.metric.metric import MetricType, MetricPhase
+from port_ocean.context.ocean import ocean
+
+@contextmanager
+def resync_error_handling() -> Generator[None, None, None]:
+    try:
+        yield
+    except RawObjectValidationException as error:
+        err_msg = f"Failed to validate raw data for returned data from resync function, error: {error}"
+        logger.exception(err_msg)
+        raise OceanAbortException(err_msg) from error
+    except StopAsyncIteration:
+        raise
+    except Exception as error:
+        err_msg = f"Failed to execute resync function, error: {error}"
+        logger.exception(err_msg)
+        raise OceanAbortException(err_msg) from error
+
+
+async def resync_function_wrapper(
+    fn: Callable[[str], Awaitable[RAW_RESULT]], kind: str
+) -> RAW_RESULT:
+    with resync_error_handling():
+        results = await fn(kind)
+        return validate_result(results)
+
+
+async def resync_generator_wrapper(
+    fn: Callable[[str], ASYNC_GENERATOR_RESYNC_TYPE], kind: str, items_to_parse: str | None = None
+) -> ASYNC_GENERATOR_RESYNC_TYPE:
+    generator = fn(kind)
+    errors = []
+    try:
+        while True:
+            try:
+                with resync_error_handling():
+                    result = await anext(generator)
+                    if not ocean.config.yield_items_to_parse:
+                        yield validate_result(result)
+                    else:
+                        batch_size = ocean.config.yield_items_to_parse_batch_size
+                        if items_to_parse:
+                            for data in result:
+                                items = await cast(JQEntityProcessor, ocean.app.integration.entity_processor)._search(data, items_to_parse)
+                                if not isinstance(items, list):
+                                    logger.warning(
+                                        f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items)}."
+                                        f" Skipping..."
+                                    )
+                                    yield []
+                                raw_data = [{"item": item, **data} for item in items]
+                                while True:
+                                    raw_data_batch = raw_data[:batch_size]
+                                    yield raw_data_batch
+                                    raw_data = raw_data[batch_size:]
+                                    if len(raw_data) == 0:
+                                        break
+                        else:
+                            yield validate_result(result)
+            except OceanAbortException as error:
+                errors.append(error)
+                ocean.metrics.inc_metric(
+                    name=MetricType.OBJECT_COUNT_NAME,
+                    labels=[ocean.metrics.current_resource_kind(), MetricPhase.EXTRACT , MetricPhase.ExtractResult.FAILED],
+                    value=1
+                )
+    except StopAsyncIteration:
+        if errors:
+            raise ExceptionGroup(
+                "At least one of the resync generator iterations failed", errors
+            )
+
+
+def is_resource_supported(
+    kind: str, resync_event_mapping: dict[str | None, list[RESYNC_EVENT_LISTENER]]
+) -> bool:
+    return bool(resync_event_mapping[kind] or resync_event_mapping[None])
+
+
+def unsupported_kind_response(
+    kind: str, available_resync_kinds: list[str]
+) -> tuple[RESYNC_RESULT, list[Exception]]:
+    logger.error(f"Kind {kind} is not supported in this integration")
+    return [], [KindNotImplementedException(kind, available_resync_kinds)]
+
+
+class ProcessWrapper(multiprocessing.Process):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    async def join_async(self) -> None:
+        while self.exitcode is None:
+            await asyncio.sleep(2)
+        if self.exitcode != 0:
+            logger.error(f"Process {self.pid} failed with exit code {self.exitcode}")
+        else:
+            logger.info(f"Process {self.pid} finished with exit code {self.exitcode}")
+        return super().join()
+
+def clear_http_client_context() -> None:
+    try:
+        while _http_client.top is not None:
+            _http_client.pop()
+    except (RuntimeError, AttributeError):
+        pass
+
+    try:
+        while _port_http_client.top is not None:
+            _port_http_client.pop()
+    except (RuntimeError, AttributeError):
+        pass

{port_ocean-0.28.8 → port_ocean-0.28.11}/port_ocean/core/utils/utils.py

@@ -4,7 +4,7 @@ import json
 from typing import Iterable, Any, TypeVar, Callable, Awaitable
 
 from loguru import logger
-from pydantic import parse_obj_as, ValidationError
+from pydantic import BaseModel, parse_obj_as, ValidationError
 
 
 from port_ocean.clients.port.client import PortClient
@@ -79,6 +79,19 @@ async def gather_and_split_errors_from_results(
     return valid_items, errors
 
 
+def _get_entity_key(entity: Entity) -> tuple[str, str]:
+    identifier = entity.identifier
+    if isinstance(identifier, BaseModel):
+        identifier = identifier.dict()
+
+    key_part = (
+        json.dumps(identifier, sort_keys=True)
+        if isinstance(identifier, dict)
+        else str(identifier)
+    )
+    return key_part, entity.blueprint
+
+
 def get_port_diff(before: Iterable[Entity], after: Iterable[Entity]) -> EntityPortDiff:
     before_dict = {}
     after_dict = {}
@@ -88,12 +101,10 @@ def get_port_diff(before: Iterable[Entity], after: Iterable[Entity]) -> EntityPo
 
     # Create dictionaries for before and after lists
     for entity in before:
-        key = (entity.identifier, entity.blueprint)
-        before_dict[key] = entity
+        before_dict[_get_entity_key(entity)] = entity
 
     for entity in after:
-        key = (entity.identifier, entity.blueprint)
-        after_dict[key] = entity
+        after_dict[_get_entity_key(entity)] = entity
 
     # Find created, modified, and deleted objects
     for key, obj in after_dict.items():

{port_ocean-0.28.8 → port_ocean-0.28.11}/port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py

@@ -50,7 +50,7 @@ class TestJQEntityProcessor:
         raw_entity_mappings = {"foo": ".foo"}
         selector_query = '.foo == "bar"'
         result = await mocked_processor._get_mapped_entity(
-            data, raw_entity_mappings, None, selector_query
+            data, raw_entity_mappings, selector_query
         )
         assert result.entity == {"foo": "bar"}
         assert result.did_entity_pass_selector is True