port-ocean 0.30.6__py3-none-any.whl → 0.31.0__py3-none-any.whl

integrations/_infra/Dockerfile.Deb

@@ -48,7 +48,7 @@ RUN apt-get update \
         curl \
         acl \
         sudo \
-        jq \
+        libyaml-dev \
     && apt-get clean
 
 LABEL INTEGRATION_VERSION=${INTEGRATION_VERSION}

integrations/_infra/Dockerfile.local

@@ -26,10 +26,10 @@ RUN apt-get update \
         python3-pip \
         python3-poetry \
         build-essential\
-        jq \
         git \
         python3-venv \
         acl \
+        libyaml-dev \
     && apt-get clean
 
 ARG BUILD_CONTEXT
@@ -51,7 +51,6 @@ RUN rm -rf .venv-docker ${BUILD_CONTEXT}/.venv-docker
 RUN python3 -m venv .venv-docker
 RUN python3 -m venv ${BUILD_CONTEXT}/.venv-docker
 
-
 WORKDIR /app/${BUILD_CONTEXT}
 
 WORKDIR /app
@@ -60,6 +59,7 @@ RUN chown -R ocean:appgroup /app/${BUILD_CONTEXT} && chmod -R 755 /app/${BUILD_C
 RUN chown -R ocean:appgroup /tmp/ocean && chmod -R 755 /tmp/ocean
 # Add ocean user to ssl certs group
 RUN setfacl -m u:ocean:rwX /etc/ssl/certs
+
 USER ocean
 
 ENTRYPOINT ["./integrations/_infra/entry_local.sh"]

integrations/_infra/entry_local.sh

@@ -23,4 +23,5 @@ source .venv-docker/bin/activate
 python -m pip install -e ../../
 
 python -m pip install debugpy
-python -m debugpy --listen 0.0.0.0:5678 --wait-for-client debug.py
+# python -m debugpy --listen 0.0.0.0:5678 --wait-for-client debug.py
+make run

integrations/_infra/hosts

@@ -0,0 +1,4 @@
+192.168.65.254 localhost
+192.168.65.254 api.localhost
+192.168.65.254 gw.localhost
+192.168.65.254 ingest.localhost

port_ocean/config/settings.py

@@ -132,8 +132,7 @@ class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
     upsert_entities_batch_max_length: int = 20
     upsert_entities_batch_max_size_in_bytes: int = 1024 * 1024
     lakehouse_enabled: bool = False
-    yield_items_to_parse: bool = True
-    yield_items_to_parse_batch_size: int = 10
+    yield_items_to_parse_batch_size: int = 500
 
     streaming: StreamingSettings = Field(default_factory=lambda: StreamingSettings())
     actions_processor: ActionsProcessorSettings = Field(

port_ocean/core/handlers/entity_processor/jq_entity_processor.py

@@ -11,11 +11,6 @@ from loguru import logger
 
 from port_ocean.context.ocean import ocean
 from port_ocean.core.handlers.entity_processor.base import BaseEntityProcessor
-from port_ocean.core.handlers.entity_processor.jq_input_evaluator import (
-    InputClassifyingResult,
-    can_expression_run_with_no_input,
-    classify_input,
-)
 from port_ocean.core.handlers.port_app_config.models import ResourceConfig
 from port_ocean.core.models import Entity
 from port_ocean.core.ocean_types import (
@@ -144,55 +139,32 @@ class JQEntityProcessor(BaseEntityProcessor):
         entity_mapping_fault_counter: int,
     ) -> None:
         if len(entity_misconfigurations) > 0:
-            logger.info(
+            logger.error(
                 f"Unable to find valid data for: {entity_misconfigurations} (null, missing, or misconfigured)"
             )
         if missing_required_fields:
-            logger.info(
+            logger.error(
                 f"{entity_mapping_fault_counter} transformations of batch failed due to empty, null or missing values"
             )
 
     async def _search(self, data: dict[str, Any], pattern: str) -> Any:
         try:
-            loop = asyncio.get_event_loop()
             compiled_pattern = self._compile(pattern)
             func = compiled_pattern.input_value(data)
-            return await loop.run_in_executor(
-                None, self._stop_iterator_handler(func.first)
-            )
+            return func.first()
         except Exception as exc:
             logger.error(
                 f"Search failed for pattern '{pattern}' in data: {data}, Error: {exc}"
             )
             return None
 
-    @lru_cache
-    async def _search_stringified(self, data: str, pattern: str) -> Any:
-        try:
-            loop = asyncio.get_event_loop()
-            compiled_pattern = self._compile(pattern)
-            func = compiled_pattern.input_text(data)
-            return await loop.run_in_executor(
-                None, self._stop_iterator_handler(func.first)
-            )
-        except Exception as exc:
-            logger.debug(
-                f"Search failed for pattern '{pattern}' in data: {data}, Error: {exc}"
-            )
-            return None
-
     async def _search_as_bool(self, data: dict[str, Any] | str, pattern: str) -> bool:
-        loop = asyncio.get_event_loop()
 
         compiled_pattern = self._compile(pattern)
-        if isinstance(data, str):
-            func = compiled_pattern.input_text(data)
-        else:
-            func = compiled_pattern.input_value(data)
 
-        value = await loop.run_in_executor(
-            None, self._stop_iterator_handler(func.first)
-        )
+        func = compiled_pattern.input_value(data)
+
+        value = func.first()
         if isinstance(value, bool):
             return value
         raise EntityProcessorException(
@@ -201,7 +173,7 @@ class JQEntityProcessor(BaseEntityProcessor):
 
     async def _search_as_object(
         self,
-        data: dict[str, Any] | str,
+        data: dict[str, Any],
         obj: dict[str, Any],
         misconfigurations: dict[str, str] | None = None,
     ) -> dict[str, Any | None]:
@@ -233,12 +205,7 @@ class JQEntityProcessor(BaseEntityProcessor):
                     self._search_as_object(data, value, misconfigurations)
                 )
             else:
-                if isinstance(data, str):
-                    search_tasks[key] = asyncio.create_task(
-                        self._search_stringified(data, value)
-                    )
-                else:
-                    search_tasks[key] = asyncio.create_task(self._search(data, value))
+                search_tasks[key] = asyncio.create_task(self._search(data, value))
 
         result: dict[str, Any | None] = {}
         for key, task in search_tasks.items():
@@ -262,153 +229,40 @@ class JQEntityProcessor(BaseEntityProcessor):
 
     async def _get_mapped_entity(
         self,
-        data: dict[str, Any] | tuple[dict[str, Any], str],
+        data: dict[str, Any],
         raw_entity_mappings: dict[str, Any],
-        items_to_parse_key: str | None,
         selector_query: str,
         parse_all: bool = False,
     ) -> MappedEntity:
-        should_run = await self._should_map_entity(
-            data, selector_query, items_to_parse_key
-        )
+        should_run = await self._search_as_bool(data, selector_query)
         if parse_all or should_run:
-            misconfigurations, mapped_entity = await self._map_entity(
-                data, raw_entity_mappings, items_to_parse_key
+            misconfigurations: dict[str, str] = {}
+            mapped_entity = await self._search_as_object(
+                data, raw_entity_mappings, misconfigurations
             )
             return MappedEntity(
                 mapped_entity,
                 did_entity_pass_selector=should_run,
-                raw_data=data,
+                raw_data=data if should_run else None,
                 misconfigurations=misconfigurations,
             )
 
-        return MappedEntity(
-            {},
-            did_entity_pass_selector=False,
-            raw_data=data,
-            misconfigurations={},
-        )
-
-    async def _map_entity(
-        self,
-        data: dict[str, Any] | tuple[dict[str, Any], str],
-        raw_entity_mappings: dict[str, Any],
-        items_to_parse_key: str | None,
-    ) -> tuple[dict[str, str], dict[str, Any]]:
-        if not items_to_parse_key:
-            # No items to parse, map the entity and return the misconfigurations and the mapped entity
-            misconfigurations: dict[str, str] = {}
-            data_to_search = data if isinstance(data, dict) else data[0]
-            mapped_entity = await self._search_as_object(
-                data_to_search, raw_entity_mappings, misconfigurations
-            )
-            return misconfigurations, mapped_entity
-
-        modified_data: tuple[dict[str, Any], str | dict[str, Any]] = (
-            data
-            if isinstance(data, tuple)
-            else (
-                {items_to_parse_key: data[items_to_parse_key]},
-                data,
-            )
-        )
-
-        misconfigurations_item: dict[str, str] = {}
-        misconfigurations_all: dict[str, str] = {}
-        # Map the entity with jq expressions that classified as single item expressions with the single item as input
-        mapped_entity_item = await self._search_as_object(
-            modified_data[0], raw_entity_mappings["item"], misconfigurations_item
-        )
-        # To Prevent misclassification data loss, we merge the expressions that classified as single item expressions and resulted as misconfigured
-        # into the expressions that classified as all expressions
-        if misconfigurations_item:
-            # The misconfigurations dict not contains the mapping expressions themselves, so we need to filter the original mapping by the misconfigured keys
-            filtered_item_mappings = self._filter_mappings_by_keys(
-                raw_entity_mappings["item"], list(misconfigurations_item.keys())
-            )
-            raw_entity_mappings["all"] = self._deep_merge(
-                raw_entity_mappings["all"], filtered_item_mappings
-            )
-        # Map the entity with jq expressions that classified as all expressions with the whole data as input
-        mapped_entity_all = await self._search_as_object(
-            modified_data[1], raw_entity_mappings["all"], misconfigurations_all
-        )
-        # Map the entity with jq expressions that classified as no input required expressions with empty object as input
-        mapped_entity_empty = await self._search_as_object(
-            {}, raw_entity_mappings["empty"], misconfigurations_all
-        )
-        # Merge the mapped entities
-        mapped_entity = self._deep_merge(mapped_entity_item, mapped_entity_all)
-        mapped_entity = self._deep_merge(mapped_entity, mapped_entity_empty)
-        return misconfigurations_all, mapped_entity
-
-    async def _should_map_entity(
-        self,
-        data: dict[str, Any] | tuple[dict[str, Any], str],
-        selector_query: str,
-        items_to_parse_key: str | None,
-    ) -> bool:
-        if can_expression_run_with_no_input(selector_query):
-            return await self._search_as_bool({}, selector_query)
-        if isinstance(data, tuple):
-            return await self._search_as_bool(
-                data[0], selector_query
-            ) or await self._search_as_bool(data[1], selector_query)
-        if items_to_parse_key:
-            return await self._search_as_bool(
-                data[items_to_parse_key], selector_query
-            ) or await self._search_as_bool(data, selector_query)
-        return await self._search_as_bool(data, selector_query)
+        return MappedEntity()
 
     async def _calculate_entity(
         self,
         data: dict[str, Any],
         raw_entity_mappings: dict[str, Any],
-        items_to_parse: str | None,
-        items_to_parse_name: str,
         selector_query: str,
         parse_all: bool = False,
     ) -> tuple[list[MappedEntity], list[Exception]]:
-        raw_data: list[dict[str, Any]] | list[tuple[dict[str, Any], str]] = [
-            data.copy()
-        ]
-        items_to_parse_key = None
-        if items_to_parse:
-            items_to_parse_key = items_to_parse_name
-            if not ocean.config.yield_items_to_parse:
-                if isinstance(data, dict) and data.get("__type") == "path":
-                    file_path = data.get("file", {}).get("content", {}).get("path")
-                    with open(file_path, "r") as f:
-                        data["file"]["content"] = json.loads(f.read())
-                items = await self._search(data, items_to_parse)
-                if not isinstance(items, list):
-                    logger.warning(
-                        f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items)}."
-                        f" Skipping..."
-                    )
-                    return [], []
-                raw_all_payload_stringified = json.dumps(data)
-                raw_data = [
-                    ({items_to_parse_name: item}, raw_all_payload_stringified)
-                    for item in items
-                ]
-            single_item_mappings, all_items_mappings, empty_items_mappings = (
-                self._build_raw_entity_mappings(
-                    raw_entity_mappings, items_to_parse_name
-                )
-            )
-            raw_entity_mappings = {
-                "item": single_item_mappings,
-                "all": all_items_mappings,
-                "empty": empty_items_mappings,
-            }
+        raw_data = [data.copy()]
 
         entities, errors = await gather_and_split_errors_from_results(
             [
                 self._get_mapped_entity(
                     raw,
                     raw_entity_mappings,
-                    items_to_parse_key,
                     selector_query,
                     parse_all,
                 )
@@ -421,324 +275,6 @@ class JQEntityProcessor(BaseEntityProcessor):
             )
         return entities, errors
 
-    def _build_raw_entity_mappings(
-        self, raw_entity_mappings: dict[str, Any], items_to_parse_name: str
-    ) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]:
-        """
-        Build the raw entity mappings for the items to parse.
-        The mappings are grouped by the input classifying result.
-        There are 3 input classifying results:
-            - NONE: The expression can be executed with no input
-            - SINGLE: The expression can be executed on a single item
-            - ALL: The expression can be executed on all the data
-        """
-        mappings: dict[InputClassifyingResult, dict[str, Any]] = {
-            InputClassifyingResult.NONE: {},
-            InputClassifyingResult.SINGLE: {},
-            InputClassifyingResult.ALL: {},
-        }
-        for key, value in raw_entity_mappings.items():
-            if isinstance(value, str):
-                # Direct string values (identifier, title, icon, blueprint, team)
-                self.group_string_mapping_value(
-                    items_to_parse_name,
-                    mappings,
-                    key,
-                    value,
-                )
-            elif isinstance(value, dict):
-                # Complex objects (IngestSearchQuery for identifier/team, properties, relations)
-                self.group_complex_mapping_value(
-                    items_to_parse_name,
-                    mappings,
-                    key,
-                    value,
-                )
-        return (
-            mappings[InputClassifyingResult.SINGLE],
-            mappings[InputClassifyingResult.ALL],
-            mappings[InputClassifyingResult.NONE],
-        )
-
-    def group_complex_mapping_value(
-        self,
-        pattern: str,
-        mappings: dict[InputClassifyingResult, dict[str, Any]],
-        key: str,
-        value: dict[str, Any],
-    ) -> None:
-        if key in ["properties", "relations"]:
-            mapping_dicts: dict[InputClassifyingResult, dict[str, Any]] = {
-                InputClassifyingResult.SINGLE: {},
-                InputClassifyingResult.ALL: {},
-                InputClassifyingResult.NONE: {},
-            }
-            # For properties and relations, filter the dictionary values
-            for dict_key, dict_value in value.items():
-                if isinstance(dict_value, str):
-                    self.group_string_mapping_value(
-                        pattern,
-                        mapping_dicts,
-                        dict_key,
-                        dict_value,
-                    )
-                elif isinstance(dict_value, dict):
-                    # Handle IngestSearchQuery objects
-                    self.group_search_query_mapping_value(
-                        pattern,
-                        mapping_dicts[InputClassifyingResult.SINGLE],
-                        mapping_dicts[InputClassifyingResult.ALL],
-                        dict_key,
-                        dict_value,
-                    )
-            for input_classifying_result, mapping_dict in mapping_dicts.items():
-                if mapping_dict:
-                    mappings[input_classifying_result][key] = mapping_dict
-        else:
-            # For identifier/team IngestSearchQuery objects
-            self.group_search_query_mapping_value(
-                pattern,
-                mappings[InputClassifyingResult.SINGLE],
-                mappings[InputClassifyingResult.ALL],
-                key,
-                value,
-            )
-
-    def group_search_query_mapping_value(
-        self,
-        pattern: str,
-        single_item_dict: dict[str, Any],
-        all_item_dict: dict[str, Any],
-        dict_key: str,
-        dict_value: dict[str, Any],
-    ) -> None:
-        if self._classify_search_query(dict_value, pattern):
-            single_item_dict[dict_key] = dict_value
-        else:
-            all_item_dict[dict_key] = dict_value
-
-    def group_string_mapping_value(
-        self,
-        pattern: str,
-        mappings: dict[InputClassifyingResult, dict[str, Any]],
-        key: str,
-        value: str,
-    ) -> None:
-        input_evaluation_result = classify_input(value, pattern)
-        mappings[input_evaluation_result][key] = value
-
-    def _classify_search_query(self, query_dict: dict[str, Any], pattern: str) -> bool:
-        """
-        Classify the input required to run jq expressions of an IngestSearchQuery
-        If at least one rule contains a value that includes the pattern, return True
-        Otherwise, return False
-        Example:
-        The pattern is item
-        relations:
-            someRelation:
-              combinator: "and"
-              rules:
-                - operator: "="
-                  property: "someProperty"
-                  value: .field
-                - combinator: "or"
-                  rules:
-                    - operator: "="
-                      property: "anotherProperty"
-                      value: .item.something
-                    - operator: "="
-                      property: "yetAnotherProperty"
-                      value: .yetAnotherValue
-        One value is .item.something, which is a single item based expression, so it will be classified as SINGLE
-        """
-        if "rules" not in query_dict:
-            return False
-
-        rules = query_dict["rules"]
-        if not isinstance(rules, list):
-            return False
-
-        # Check if any rule contains a value that includes the pattern
-        for rule in rules:
-            if isinstance(rule, dict) and self._is_rule_or_query_contains_pattern(
-                rule, pattern
-            ):
-                return True
-        return False
-
-    def _is_rule_or_query_contains_pattern(
-        self, rule: dict[str, Any], pattern: str
-    ) -> bool:
-        """
-        Check if a rule or query contains a value that includes the pattern
-        If the value is a single item based expression, return True
-        Otherwise, return False
-        Example:
-        The pattern is item
-        The rule is:
-                - combinator: "or"
-                  rules:
-                    - operator: "="
-                      property: "anotherProperty"
-                      value: .item.something
-                    - operator: "="
-                      property: "yetAnotherProperty"
-                      value: .yetAnotherValue
-        This rule is not a single rule (not contains a value property) but a search query rule (contains a rules property)
-        so we need to recursively check the rules property to check if at least one rule contains a value that includes the pattern
-        In this case, one value is .item.something, which is a single item based expression, so it will be classified as SINGLE ==> true
-        """
-        if "value" in rule and isinstance(rule["value"], str):
-            # Use evaluate_input to check if the pattern is relevant for this value
-            input_evaluation_result = classify_input(rule["value"], pattern)
-            if input_evaluation_result == InputClassifyingResult.SINGLE:
-                return True
-        # Recursively check nested IngestSearchQuery objects
-        elif "rules" in rule:
-            if self._classify_search_query(rule, pattern):
-                return True
-        return False
-
-    def _filter_mappings_by_keys(
-        self, mappings: dict[str, Any], target_keys: list[str]
-    ) -> dict[str, Any]:
-        """
-        Filter mappings to preserve structure with only the specified keys present.
-        Recursively handles nested dictionaries and lists, searching for keys at any level.
-
-        Args:
-            mappings: The dictionary containing mapping configurations to filter
-            target_keys: List of keys to preserve in the filtered result
-
-        Returns:
-            A filtered dictionary containing only the specified keys and their nested structures
-
-        Examples:
-            # Basic filtering with direct keys
-            mappings = {
-                "name": "John",
-                "age": 30,
-                "city": "New York",
-                "country": "USA"
-            }
-            target_keys = ["name", "age"]
-            result = self._filter_mappings_by_keys(mappings, target_keys)
-            # Returns: {"name": "John", "age": 30}
-
-            # Nested dictionary filtering
-            mappings = {
-                "user": {
-                    "profile": {"name": "John", "email": "john@example.com"},
-                    "settings": {"theme": "dark", "notifications": True}
-                },
-                "metadata": {"created_at": "2023-01-01", "version": "1.0"}
-            }
-            target_keys = ["name", "email", "created_at"]
-            result = self._filter_mappings_by_keys(mappings, target_keys)
-            # Returns: {
-            #     "user": {
-            #         "profile": {"name": "John", "email": "john@example.com"}
-            #     },
-            #     "metadata": {"created_at": "2023-01-01"}
-            # }
-
-            # Empty target keys returns empty dict
-            result = self._filter_mappings_by_keys(mappings, [])
-            # Returns: {}
-        """
-        if not target_keys:
-            return {}
-
-        filtered_mappings: dict[str, Any] = {}
-
-        for key, value in mappings.items():
-            filtered_value = self._process_mapping_value(key, value, target_keys)
-
-            # Include if it's a direct match or contains nested target keys
-            if key in target_keys or filtered_value:
-                filtered_mappings[key] = filtered_value
-
-        return filtered_mappings
-
-    def _process_mapping_value(
-        self, key: str, value: Any, target_keys: list[str]
-    ) -> Any:
-        """
-        Process a single mapping value, handling different types recursively.
-
-        This helper method is used by _filter_mappings_by_keys to process individual
-        key-value pairs. It handles both simple values and nested dictionaries,
-        applying the filtering logic recursively to maintain the hierarchical structure.
-
-        Args:
-            key: The key of the current mapping item being processed
-            value: The value associated with the key (can be any type)
-            target_keys: List of keys to preserve in the filtered result
-
-        Returns:
-            The processed value if it should be included, None otherwise.
-            For dictionaries, returns the filtered dictionary or None if empty.
-            For simple values, returns the value if the key is in target_keys, None otherwise.
-
-        Examples:
-            # Simple value processing - key in target_keys
-            result = self._process_mapping_value("name", "John", ["name", "age"])
-            # Returns: "John"
-
-            # Simple value processing - key not in target_keys
-            result = self._process_mapping_value("city", "New York", ["name", "age"])
-            # Returns: None
-
-            # Dictionary processing with nested target keys
-            nested_dict = {
-                "profile": {"name": "John", "email": "john@example.com"},
-                "settings": {"theme": "dark"}
-            }
-            result = self._process_mapping_value("user", nested_dict, ["name", "email"])
-            # Returns: {"profile": {"name": "John", "email": "john@example.com"}}
-
-            # Dictionary processing with no matching target keys
-            result = self._process_mapping_value("user", nested_dict, ["version"])
-            # Returns: None
-        """
-        if isinstance(value, dict):
-            # Recursively filter nested dictionary
-            filtered_dict = self._filter_mappings_by_keys(value, target_keys)
-            return filtered_dict if filtered_dict else None
-        else:
-            # Return simple values as-is
-            return value if key in target_keys else None
-
-    def _deep_merge(
-        self, dict1: dict[str, Any], dict2: dict[str, Any]
-    ) -> dict[str, Any]:
-        """
-        Deep merge two dictionaries, preserving nested structures.
-        Values from dict2 override values from dict1 for the same keys.
-        """
-        result = dict1.copy()
-
-        for key, value in dict2.items():
-            if (
-                key in result
-                and isinstance(result[key], dict)
-                and isinstance(value, dict)
-            ):
-                # Recursively merge nested dictionaries
-                result[key] = self._deep_merge(result[key], value)
-            elif (
-                key in result
-                and isinstance(result[key], list)
-                and isinstance(value, list)
-            ):
-                # Merge lists by extending
-                result[key].extend(value)
-            else:
-                # Override with value from dict2
-                result[key] = value
-
-        return result
-
     @staticmethod
     async def _send_examples(data: list[dict[str, Any]], kind: str) -> None:
         try:
@@ -768,8 +304,6 @@ class JQEntityProcessor(BaseEntityProcessor):
                 raw_results,
                 self._calculate_entity,
                 raw_entity_mappings,
-                mapping.port.items_to_parse,
-                mapping.port.items_to_parse_name,
                 mapping.selector.query,
                 parse_all,
             )