port-ocean 0.28.4__py3-none-any.whl → 0.28.7__py3-none-any.whl

integrations/_infra/Dockerfile.Deb

@@ -48,6 +48,7 @@ RUN apt-get update \
         curl \
         acl \
         sudo \
+        jq \
     && apt-get clean
 
 LABEL INTEGRATION_VERSION=${INTEGRATION_VERSION}
@@ -69,18 +70,22 @@ COPY --from=base /app/.venv /app/.venv
 COPY ./integrations/_infra/init.sh /app/init.sh
 
 USER root
+
 # Ensure that ocean is available for all in path
 RUN chmod a+x /app/.venv/bin/ocean
 
 RUN chmod a+x /app/init.sh
 RUN ln -s /app/.venv/bin/ocean /usr/bin/ocean
+
 # Add ocean user to ssl certs group
-RUN setfacl -m u:ocean:rwX /etc/ssl/certs
+RUN setfacl -R -m u:ocean:rwX /etc/ssl/certs \
+    && setfacl -d -m u:ocean:rwX /etc/ssl/certs
 
 # Allow ocean user to run update-ca-certificates without password (secure, limited sudo)
 RUN echo "ocean ALL=(root) NOPASSWD: /usr/sbin/update-ca-certificates" >> /etc/sudoers.d/ocean-certs \
     && chmod 440 /etc/sudoers.d/ocean-certs
 
 USER ocean
+
 # Run the application
 CMD ["bash", "/app/init.sh"]

integrations/_infra/Dockerfile.local

@@ -26,6 +26,7 @@ RUN apt-get update \
         python3-pip \
         python3-poetry \
         build-essential\
+        jq \
         git \
         python3-venv \
         acl \

port_ocean/core/handlers/entity_processor/jq_entity_processor.py

@@ -1,12 +1,11 @@
 import asyncio
 from asyncio import Task
 from dataclasses import dataclass, field
-
 from functools import lru_cache
+import json
 from typing import Any, Optional
 import jq  # type: ignore
 from loguru import logger
-
 from port_ocean.context.ocean import ocean
 from port_ocean.core.handlers.entity_processor.base import BaseEntityProcessor
 from port_ocean.core.handlers.port_app_config.models import ResourceConfig
@@ -22,6 +21,11 @@ from port_ocean.core.utils.utils import (
 )
 from port_ocean.exceptions.core import EntityProcessorException
 from port_ocean.utils.queue_utils import process_in_queue
+from port_ocean.core.handlers.entity_processor.jq_input_evaluator import (
+    InputEvaluationResult,
+    evaluate_input,
+    should_shortcut_no_input,
+)
 
 
 class ExampleStates:
@@ -76,7 +80,7 @@ class MappedEntity:
 
     entity: dict[str, Any] = field(default_factory=dict)
     did_entity_pass_selector: bool = False
-    raw_data: Optional[dict[str, Any]] = None
+    raw_data: Optional[dict[str, Any] | tuple[dict[str, Any], str]] = None
     misconfigurations: dict[str, str] = field(default_factory=dict)
 
 
@@ -133,17 +137,35 @@ class JQEntityProcessor(BaseEntityProcessor):
             return await loop.run_in_executor(
                 None, self._stop_iterator_handler(func.first)
             )
+        except Exception as exc:
+            logger.error(
+                f"Search failed for pattern '{pattern}' in data: {data}, Error: {exc}"
+            )
+            return None
+
+    @lru_cache
+    async def _search_stringified(self, data: str, pattern: str) -> Any:
+        try:
+            loop = asyncio.get_event_loop()
+            compiled_pattern = self._compile(pattern)
+            func = compiled_pattern.input_text(data)
+            return await loop.run_in_executor(
+                None, self._stop_iterator_handler(func.first)
+            )
         except Exception as exc:
             logger.debug(
                 f"Search failed for pattern '{pattern}' in data: {data}, Error: {exc}"
             )
             return None
 
-    async def _search_as_bool(self, data: dict[str, Any], pattern: str) -> bool:
+    async def _search_as_bool(self, data: dict[str, Any] | str, pattern: str) -> bool:
         loop = asyncio.get_event_loop()
 
         compiled_pattern = self._compile(pattern)
-        func = compiled_pattern.input_value(data)
+        if isinstance(data, str):
+            func = compiled_pattern.input_text(data)
+        else:
+            func = compiled_pattern.input_value(data)
 
         value = await loop.run_in_executor(
             None, self._stop_iterator_handler(func.first)
@@ -156,7 +178,7 @@ class JQEntityProcessor(BaseEntityProcessor):
 
     async def _search_as_object(
         self,
-        data: dict[str, Any],
+        data: dict[str, Any] | str,
         obj: dict[str, Any],
         misconfigurations: dict[str, str] | None = None,
     ) -> dict[str, Any | None]:
@@ -188,7 +210,12 @@ class JQEntityProcessor(BaseEntityProcessor):
                     self._search_as_object(data, value, misconfigurations)
                 )
             else:
-                search_tasks[key] = asyncio.create_task(self._search(data, value))
+                if isinstance(data, str):
+                    search_tasks[key] = asyncio.create_task(
+                        self._search_stringified(data, value)
+                    )
+                else:
+                    search_tasks[key] = asyncio.create_task(self._search(data, value))
 
         result: dict[str, Any | None] = {}
         for key, task in search_tasks.items():
@@ -212,16 +239,18 @@ class JQEntityProcessor(BaseEntityProcessor):
 
     async def _get_mapped_entity(
         self,
-        data: dict[str, Any],
+        data: dict[str, Any] | tuple[dict[str, Any], str],
         raw_entity_mappings: dict[str, Any],
+        items_to_parse_key: str | None,
         selector_query: str,
         parse_all: bool = False,
     ) -> MappedEntity:
-        should_run = await self._search_as_bool(data, selector_query)
+        should_run = await self._should_map_entity(
+            data, selector_query, items_to_parse_key
+        )
         if parse_all or should_run:
-            misconfigurations: dict[str, str] = {}
-            mapped_entity = await self._search_as_object(
-                data, raw_entity_mappings, misconfigurations
+            misconfigurations, mapped_entity = await self._map_entity(
+                data, raw_entity_mappings, items_to_parse_key
             )
             return MappedEntity(
                 mapped_entity,
@@ -237,6 +266,69 @@ class JQEntityProcessor(BaseEntityProcessor):
             misconfigurations={},
         )
 
+    async def _map_entity(
+        self,
+        data: dict[str, Any] | tuple[dict[str, Any], str],
+        raw_entity_mappings: dict[str, Any],
+        items_to_parse_key: str | None,
+    ) -> tuple[dict[str, str], dict[str, Any]]:
+        if not items_to_parse_key:
+            misconfigurations: dict[str, str] = {}
+            data_to_search = data if isinstance(data, dict) else data[0]
+            mapped_entity = await self._search_as_object(
+                data_to_search, raw_entity_mappings, misconfigurations
+            )
+            return misconfigurations, mapped_entity
+
+        modified_data: tuple[dict[str, Any], str | dict[str, Any]] = (
+            data
+            if isinstance(data, tuple)
+            else (
+                {items_to_parse_key: data[items_to_parse_key]},
+                data,
+            )
+        )
+
+        misconfigurations_item: dict[str, str] = {}
+        misconfigurations_all: dict[str, str] = {}
+        mapped_entity_item = await self._search_as_object(
+            modified_data[0], raw_entity_mappings["item"], misconfigurations_item
+        )
+        if misconfigurations_item:
+            filtered_item_mappings = self._filter_mappings_by_keys(
+                raw_entity_mappings["item"], list(misconfigurations_item.keys())
+            )
+            raw_entity_mappings["all"] = self._deep_merge(
+                raw_entity_mappings["all"], filtered_item_mappings
+            )
+        mapped_entity_all = await self._search_as_object(
+            modified_data[1], raw_entity_mappings["all"], misconfigurations_all
+        )
+        mapped_entity_empty = await self._search_as_object(
+            {}, raw_entity_mappings["empty"], misconfigurations_all
+        )
+        mapped_entity = self._deep_merge(mapped_entity_item, mapped_entity_all)
+        mapped_entity = self._deep_merge(mapped_entity, mapped_entity_empty)
+        return misconfigurations_all, mapped_entity
+
+    async def _should_map_entity(
+        self,
+        data: dict[str, Any] | tuple[dict[str, Any], str],
+        selector_query: str,
+        items_to_parse_key: str | None,
+    ) -> bool:
+        if should_shortcut_no_input(selector_query):
+            return await self._search_as_bool({}, selector_query)
+        if isinstance(data, tuple):
+            return await self._search_as_bool(
+                data[0], selector_query
+            ) or await self._search_as_bool(data[1], selector_query)
+        if items_to_parse_key:
+            return await self._search_as_bool(
+                data[items_to_parse_key], selector_query
+            ) or await self._search_as_bool(data, selector_query)
+        return await self._search_as_bool(data, selector_query)
+
     async def _calculate_entity(
         self,
         data: dict[str, Any],
@@ -246,9 +338,16 @@ class JQEntityProcessor(BaseEntityProcessor):
         selector_query: str,
         parse_all: bool = False,
     ) -> tuple[list[MappedEntity], list[Exception]]:
-        raw_data = [data.copy()]
-        if not ocean.config.yield_items_to_parse:
-            if items_to_parse:
+        raw_data: list[dict[str, Any]] | list[tuple[dict[str, Any], str]] = [
+            data.copy()
+        ]
+        items_to_parse_key = None
+        if items_to_parse:
+            items_to_parse_key = items_to_parse_name
+            if not ocean.config.yield_items_to_parse:
+                if data.get("file", {}).get("content", {}).get("path", None):
+                    with open(data["file"]["content"]["path"], "r") as f:
+                        data["file"]["content"] = json.loads(f.read())
                 items = await self._search(data, items_to_parse)
                 if not isinstance(items, list):
                     logger.warning(
@@ -256,13 +355,28 @@ class JQEntityProcessor(BaseEntityProcessor):
                         f" Skipping..."
                     )
                     return [], []
-                raw_data = [{items_to_parse_name: item, **data} for item in items]
+                raw_all_payload_stringified = json.dumps(data)
+                raw_data = [
+                    ({items_to_parse_name: item}, raw_all_payload_stringified)
+                    for item in items
+                ]
+            single_item_mappings, all_items_mappings, empty_items_mappings = (
+                self._build_raw_entity_mappings(
+                    raw_entity_mappings, items_to_parse_name
+                )
+            )
+            raw_entity_mappings = {
+                "item": single_item_mappings,
+                "all": all_items_mappings,
+                "empty": empty_items_mappings,
+            }
 
         entities, errors = await gather_and_split_errors_from_results(
             [
                 self._get_mapped_entity(
                     raw,
                     raw_entity_mappings,
+                    items_to_parse_key,
                     selector_query,
                     parse_all,
                 )
@@ -275,6 +389,201 @@ class JQEntityProcessor(BaseEntityProcessor):
             )
         return entities, errors
 
+    def _build_raw_entity_mappings(
+        self, raw_entity_mappings: dict[str, Any], items_to_parse_name: str
+    ) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]:
+        """Filter entity mappings to only include values that start with f'.{items_to_parse_name}'"""
+        mappings: dict[InputEvaluationResult, dict[str, Any]] = {
+            InputEvaluationResult.NONE: {},
+            InputEvaluationResult.SINGLE: {},
+            InputEvaluationResult.ALL: {},
+        }
+        pattern = f".{items_to_parse_name}"
+        for key, value in raw_entity_mappings.items():
+            if isinstance(value, str):
+                # Direct string values (identifier, title, icon, blueprint, team)
+                self.group_string_mapping_value(
+                    pattern,
+                    mappings,
+                    key,
+                    value,
+                )
+            elif isinstance(value, dict):
+                # Complex objects (IngestSearchQuery for identifier/team, properties, relations)
+                self.group_complex_mapping_value(
+                    pattern,
+                    mappings,
+                    key,
+                    value,
+                )
+        return (
+            mappings[InputEvaluationResult.SINGLE],
+            mappings[InputEvaluationResult.ALL],
+            mappings[InputEvaluationResult.NONE],
+        )
+
+    def group_complex_mapping_value(
+        self,
+        pattern: str,
+        mappings: dict[InputEvaluationResult, dict[str, Any]],
+        key: str,
+        value: dict[str, Any],
+    ) -> None:
+        mapping_dicts: dict[InputEvaluationResult, dict[str, Any]] = {
+            InputEvaluationResult.SINGLE: {},
+            InputEvaluationResult.ALL: {},
+            InputEvaluationResult.NONE: {},
+        }
+        if key in ["properties", "relations"]:
+            # For properties and relations, filter the dictionary values
+            for dict_key, dict_value in value.items():
+                if isinstance(dict_value, str):
+                    self.group_string_mapping_value(
+                        pattern,
+                        mapping_dicts,
+                        dict_key,
+                        dict_value,
+                    )
+                elif isinstance(dict_value, dict):
+                    # Handle IngestSearchQuery objects
+                    self.group_search_query_mapping_value(
+                        pattern,
+                        mapping_dicts[InputEvaluationResult.SINGLE],
+                        mapping_dicts[InputEvaluationResult.ALL],
+                        dict_key,
+                        dict_value,
+                    )
+        else:
+            # For identifier/team IngestSearchQuery objects
+            self.group_search_query_mapping_value(
+                pattern,
+                mapping_dicts[InputEvaluationResult.SINGLE],
+                mapping_dicts[InputEvaluationResult.ALL],
+                key,
+                value,
+            )
+        if mapping_dicts[InputEvaluationResult.SINGLE]:
+            mappings[InputEvaluationResult.SINGLE][key] = mapping_dicts[
+                InputEvaluationResult.SINGLE
+            ][key]
+        if mapping_dicts[InputEvaluationResult.ALL]:
+            mappings[InputEvaluationResult.ALL][key] = mapping_dicts[
+                InputEvaluationResult.ALL
+            ][key]
+        if mapping_dicts[InputEvaluationResult.NONE]:
+            mappings[InputEvaluationResult.NONE][key] = mapping_dicts[
+                InputEvaluationResult.NONE
+            ][key]
+
+    def group_search_query_mapping_value(
+        self,
+        pattern: str,
+        single_item_dict: dict[str, Any],
+        all_item_dict: dict[str, Any],
+        dict_key: str,
+        dict_value: dict[str, Any],
+    ) -> None:
+        if self._should_keep_ingest_search_query(dict_value, pattern):
+            single_item_dict[dict_key] = dict_value
+        else:
+            all_item_dict[dict_key] = dict_value
+
+    def group_string_mapping_value(
+        self,
+        pattern: str,
+        mappings: dict[InputEvaluationResult, dict[str, Any]],
+        key: str,
+        value: str,
+    ) -> None:
+        input_evaluation_result = evaluate_input(value, pattern)
+        mappings[input_evaluation_result][key] = value
+
+    def _should_keep_ingest_search_query(
+        self, query_dict: dict[str, Any], pattern: str
+    ) -> bool:
+        """Check if an IngestSearchQuery should be kept based on its rules"""
+        if "rules" not in query_dict:
+            return False
+
+        rules = query_dict["rules"]
+        if not isinstance(rules, list):
+            return False
+
+        # Check if any rule contains a value starting with the pattern
+        for rule in rules:
+            if isinstance(rule, dict):
+                if "value" in rule and isinstance(rule["value"], str):
+                    if pattern in rule["value"]:
+                        return True
+                # Recursively check nested IngestSearchQuery objects
+                elif "rules" in rule:
+                    if self._should_keep_ingest_search_query(rule, pattern):
+                        return True
+        return False
+
+    def _filter_mappings_by_keys(
+        self, mappings: dict[str, Any], target_keys: list[str]
+    ) -> dict[str, Any]:
+        """
+        Filter mappings to preserve structure with only the specified keys present.
+        Recursively handles nested dictionaries and lists, searching for keys at any level.
+        """
+        if not target_keys:
+            return {}
+
+        filtered_mappings: dict[str, Any] = {}
+
+        for key, value in mappings.items():
+            filtered_value = self._process_mapping_value(key, value, target_keys)
+
+            # Include if it's a direct match or contains nested target keys
+            if key in target_keys or filtered_value:
+                filtered_mappings[key] = filtered_value
+
+        return filtered_mappings
+
+    def _process_mapping_value(
+        self, key: str, value: Any, target_keys: list[str]
+    ) -> Any:
+        """Process a single mapping value, handling different types recursively."""
+        if isinstance(value, dict):
+            # Recursively filter nested dictionary
+            filtered_dict = self._filter_mappings_by_keys(value, target_keys)
+            return filtered_dict if filtered_dict else None
+        else:
+            # Return simple values as-is
+            return value if key in target_keys else None
+
+    def _deep_merge(
+        self, dict1: dict[str, Any], dict2: dict[str, Any]
+    ) -> dict[str, Any]:
+        """
+        Deep merge two dictionaries, preserving nested structures.
+        Values from dict2 override values from dict1 for the same keys.
+        """
+        result = dict1.copy()
+
+        for key, value in dict2.items():
+            if (
+                key in result
+                and isinstance(result[key], dict)
+                and isinstance(value, dict)
+            ):
+                # Recursively merge nested dictionaries
+                result[key] = self._deep_merge(result[key], value)
+            elif (
+                key in result
+                and isinstance(result[key], list)
+                and isinstance(value, list)
+            ):
+                # Merge lists by extending
+                result[key].extend(value)
+            else:
+                # Override with value from dict2
+                result[key] = value
+
+        return result
+
     @staticmethod
     async def _send_examples(data: list[dict[str, Any]], kind: str) -> None:
         try:
@@ -329,7 +638,10 @@ class JQEntityProcessor(BaseEntityProcessor):
                 and result.raw_data is not None
             ):
                 examples_to_send.add_example(
-                    result.did_entity_pass_selector, result.raw_data
+                    result.did_entity_pass_selector,
+                    self._get_raw_data_for_example(
+                        result.raw_data, mapping.port.items_to_parse_name
+                    ),
                 )
 
             if result.entity.get("identifier") and result.entity.get("blueprint"):
@@ -355,3 +667,13 @@ class JQEntityProcessor(BaseEntityProcessor):
             errors,
             misconfigured_entity_keys=entity_misconfigurations,
         )
+
+    def _get_raw_data_for_example(
+        self,
+        data: dict[str, Any] | tuple[dict[str, Any], str],
+        items_to_parse_name: str,
+    ) -> dict[str, Any]:
+        if isinstance(data, tuple):
+            raw_data = json.loads(data[1])
+            return {items_to_parse_name: data[0], **raw_data}
+        return data

port_ocean/core/handlers/entity_processor/jq_input_evaluator.py

@@ -0,0 +1,69 @@
+import re
+from enum import Enum
+
+
+class InputEvaluationResult(Enum):
+    NONE = 1
+    SINGLE = 2
+    ALL = 3
+
+
+# Conservative allowlist: truly nullary jq expressions
+_ALLOWLIST_PATTERNS = [
+    r"^\s*null\s*$",  # null
+    r"^\s*true\s*$",  # true
+    r"^\s*false\s*$",  # false
+    r"^\s*-?\d+(\.\d+)?\s*$",  # number literal
+    r'^\s*".*"\s*$',  # string literal (simple heuristic)
+    r"^\s*\[.*\]\s*$",  # array literal (includes [])
+    r"^\s*\{.*\}\s*$",  # object literal (includes {})
+    r"^\s*range\s*\(.*\)\s*$",  # range(...)
+    r"^\s*empty\s*$",  # empty
+]
+
+# Functions/filters that (even without ".") still require/assume input
+_INPUT_DEPENDENT_FUNCS = r"""
+\b(
+    map|select|reverse|sort|sort_by|unique|unique_by|group_by|flatten|transpose|
+    split|explode|join|add|length|has|in|index|indices|contains|
+    paths|leaf_paths|keys|keys_unsorted|values|to_entries|with_entries|from_entries|
+    del|delpaths|walk|reduce|foreach|input|inputs|limit|first|last|nth|
+    while|until|recurse|recurse_down|bsearch|combinations|permutations
+)\b
+"""
+
+_INPUT_DEPENDENT_RE = re.compile(_INPUT_DEPENDENT_FUNCS, re.VERBOSE)
+
+
+def should_shortcut_no_input(selector_query: str) -> bool:
+    """
+    Returns True if the jq expression can be executed without providing any JSON input.
+    Conservative: requires NO '.' and must match a known nullary-safe pattern.
+    """
+    if "." in selector_query:
+        return False  # explicit JSON reference -> needs input
+
+    # If it contains any known input-dependent functions, don't shortcut
+    if _INPUT_DEPENDENT_RE.search(selector_query):
+        return False
+
+    # Allow only if it matches one of the nullary-safe patterns
+    for pat in _ALLOWLIST_PATTERNS:
+        if re.match(pat, selector_query):
+            return True
+
+    return False
+
+
+def evaluate_input(
+    selector_query: str, single_item_key: str | None = None
+) -> InputEvaluationResult:
+    """
+    Returns the input evaluation result for the jq expression.
+    Conservative: requires NO '.' and must match a known nullary-safe pattern.
+    """
+    if should_shortcut_no_input(selector_query):
+        return InputEvaluationResult.NONE
+    if single_item_key and single_item_key in selector_query:
+        return InputEvaluationResult.SINGLE
+    return InputEvaluationResult.ALL

port_ocean/core/handlers/port_app_config/models.py

@@ -39,7 +39,7 @@ class MappingsConfig(BaseModel):
 class PortResourceConfig(BaseModel):
     entity: MappingsConfig
     items_to_parse: str | None = Field(alias="itemsToParse")
-    items_to_parse_name: str | None = Field(alias="itemsToParseName", default="item")
+    items_to_parse_name: str = Field(alias="itemsToParseName", default="item")
 
 
 class Selector(BaseModel):

port_ocean/core/integrations/mixins/sync_raw.py

@@ -117,7 +117,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                 logger.info(
                     f"Found async generator function for {resource_config.kind} name: {task.__qualname__}"
                 )
-                results.append(resync_generator_wrapper(task, resource_config.kind,resource_config.port.items_to_parse))
+                results.append(resync_generator_wrapper(task, resource_config.kind, resource_config.port.items_to_parse_name, resource_config.port.items_to_parse))
             else:
                 logger.info(
                     f"Found sync function for {resource_config.kind} name: {task.__qualname__}"

port_ocean/core/integrations/mixins/utils.py

@@ -2,12 +2,11 @@ from contextlib import contextmanager
 from typing import Awaitable, Generator, Callable, cast
 
 from loguru import logger
-
 import asyncio
 import multiprocessing
-
+import re
+import json
 from port_ocean.core.handlers.entity_processor.jq_entity_processor import JQEntityProcessor
-from port_ocean.core.handlers.port_app_config.models import ResourceConfig
 from port_ocean.core.ocean_types import (
     ASYNC_GENERATOR_RESYNC_TYPE,
     RAW_RESULT,
@@ -20,11 +19,60 @@ from port_ocean.exceptions.core import (
     OceanAbortException,
     KindNotImplementedException,
 )
-
+import os
 from port_ocean.utils.async_http import _http_client
 from port_ocean.clients.port.utils import _http_client as _port_http_client
 from port_ocean.helpers.metric.metric import MetricType, MetricPhase
 from port_ocean.context.ocean import ocean
+import subprocess
+import tempfile
+import stat
+import ijson
+from typing import Any, AsyncGenerator
+
+def _process_path_type_items(
+    result: RAW_RESULT, items_to_parse: str | None = None
+) -> RAW_RESULT:
+    """
+    Process items in the result array to check for "__type": "path" fields.
+    If found, read the file contents and load them into a "content" field.
+    Skip processing if we're on the items_to_parse branch.
+    """
+    if not isinstance(result, list):
+        return result
+
+    # Skip processing if we're on the items_to_parse branch
+    if items_to_parse:
+        return result
+
+    processed_result = []
+    for item in result:
+        if isinstance(item, dict) and item.get("__type") == "path":
+            try:
+                # Read the file content and parse as JSON
+                file_path = item.get("file", {}).get("content", {}).get("path")
+                if file_path and os.path.exists(file_path):
+                    with open(file_path, "r") as f:
+                        content = json.loads(f.read())
+                    # Create a copy of the item with the content field
+                    processed_item = item.copy()
+                    processed_item["content"] = content
+                    processed_result.append(processed_item)
+                else:
+                    # If file doesn't exist, keep the original item
+                    processed_result.append(item)
+            except (json.JSONDecodeError, IOError, OSError) as e:
+                logger.warning(
+                    f"Failed to read or parse file content for path "
+                    f"{item.get('file', {}).get('content', {}).get('path')}: {e}"
+                )
+                # Keep the original item if there's an error
+                processed_result.append(item)
+        else:
+            # Keep non-path type items as is
+            processed_result.append(item)
+
+    return processed_result
 
 @contextmanager
 def resync_error_handling() -> Generator[None, None, None]:
@@ -47,11 +95,12 @@ async def resync_function_wrapper(
 ) -> RAW_RESULT:
     with resync_error_handling():
         results = await fn(kind)
-        return validate_result(results)
+        validated_results = validate_result(results)
+        return _process_path_type_items(validated_results)
 
 
 async def resync_generator_wrapper(
-    fn: Callable[[str], ASYNC_GENERATOR_RESYNC_TYPE], kind: str, items_to_parse: str | None = None
+    fn: Callable[[str], ASYNC_GENERATOR_RESYNC_TYPE], kind: str, items_to_parse_name: str, items_to_parse: str | None = None
 ) -> ASYNC_GENERATOR_RESYNC_TYPE:
     generator = fn(kind)
     errors = []
@@ -61,27 +110,23 @@ async def resync_generator_wrapper(
                 with resync_error_handling():
                     result = await anext(generator)
                     if not ocean.config.yield_items_to_parse:
-                        yield validate_result(result)
+                        validated_result = validate_result(result)
+                        processed_result = _process_path_type_items(validated_result)
+                        yield processed_result
                     else:
-                        batch_size = ocean.config.yield_items_to_parse_batch_size
                         if items_to_parse:
                             for data in result:
-                                items = await cast(JQEntityProcessor, ocean.app.integration.entity_processor)._search(data, items_to_parse)
-                                if not isinstance(items, list):
-                                    logger.warning(
-                                        f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items)}."
-                                        f" Skipping..."
-                                    )
-                                    yield []
-                                raw_data = [{"item": item, **data} for item in items]
-                                while True:
-                                    raw_data_batch = raw_data[:batch_size]
-                                    yield raw_data_batch
-                                    raw_data = raw_data[batch_size:]
-                                    if len(raw_data) == 0:
-                                        break
+                                data_path: str | None = None
+                                if isinstance(data, dict) and data.get("file") is not None:
+                                    content = data["file"].get("content") if isinstance(data["file"].get("content"), dict) else {}
+                                    data_path = content.get("path", None)
+                                bulks = get_items_to_parse_bulks(data, data_path, items_to_parse, items_to_parse_name, data.get("__base_jq", ".file.content"))
+                                async for bulk in bulks:
+                                    yield bulk
                         else:
-                            yield validate_result(result)
+                            validated_result = validate_result(result)
+                            processed_result = _process_path_type_items(validated_result, items_to_parse)
+                            yield processed_result
             except OceanAbortException as error:
                 errors.append(error)
                 ocean.metrics.inc_metric(
@@ -101,6 +146,104 @@ def is_resource_supported(
 ) -> bool:
     return bool(resync_event_mapping[kind] or resync_event_mapping[None])
 
+def _validate_jq_expression(expression: str) -> None:
+    """Validate jq expression to prevent command injection."""
+    try:
+        _ = cast(JQEntityProcessor, ocean.app.integration.entity_processor)._compile(expression)
+    except Exception as e:
+        raise ValueError(f"Invalid jq expression: {e}") from e
+    # Basic validation - reject expressions that could be dangerous
+    # Check for dangerous patterns (include, import, module)
+    dangerous_patterns = ['include', 'import', 'module', 'env']
+    for pattern in dangerous_patterns:
+        # Use word boundary regex to match only complete words, not substrings
+        if re.search(rf'\b{re.escape(pattern)}\b', expression):
+            raise ValueError(f"Potentially dangerous pattern '{pattern}' found in jq expression")
+
+    # Special handling for 'env' - block environment variable access
+    if re.search(r'(?<!\w)\$ENV(?:\.)?', expression):
+        raise ValueError("Environment variable access '$ENV.' found in jq expression")
+    if re.search(r'\benv\.', expression):
+        raise ValueError("Environment variable access 'env.' found in jq expression")
+
+def _create_secure_temp_file(suffix: str = ".json") -> str:
+    """Create a secure temporary file with restricted permissions."""
+    # Create temp directory if it doesn't exist
+    temp_dir = "/tmp/ocean"
+    os.makedirs(temp_dir, exist_ok=True)
+
+    # Create temporary file with secure permissions
+    fd, temp_path = tempfile.mkstemp(suffix=suffix, dir=temp_dir)
+    try:
+        # Set restrictive permissions (owner read/write only)
+        os.chmod(temp_path, stat.S_IRUSR | stat.S_IWUSR)
+        return temp_path
+    finally:
+        os.close(fd)
+
+async def get_items_to_parse_bulks(raw_data: dict[Any, Any], data_path: str, items_to_parse: str, items_to_parse_name: str, base_jq: str) -> AsyncGenerator[list[dict[str, Any]], None]:
+    # Validate inputs to prevent command injection
+    _validate_jq_expression(items_to_parse)
+    items_to_parse = items_to_parse.replace(base_jq, ".") if data_path else items_to_parse
+
+    temp_data_path = None
+    temp_output_path = None
+
+    try:
+        # Create secure temporary files
+        if not data_path:
+            raw_data_serialized = json.dumps(raw_data)
+            temp_data_path = _create_secure_temp_file("_input.json")
+            with open(temp_data_path, "w") as f:
+                f.write(raw_data_serialized)
+            data_path = temp_data_path
+
+        temp_output_path = _create_secure_temp_file("_parsed.json")
+
+        delete_target = items_to_parse.split('|', 1)[0].strip() if not items_to_parse.startswith('map(') else '.'
+        base_jq_object_string = await _build_base_jq_object_string(raw_data, base_jq, delete_target)
+
+        # Build jq expression safely
+        jq_expression = f""". as $all
+      | ($all | {items_to_parse}) as $items
+      | $items
+      | map({{{items_to_parse_name}: ., {base_jq_object_string}}})"""
+
+        # Use subprocess with list arguments instead of shell=True
+        jq_args = ["/bin/jq", jq_expression, data_path]
+
+        with open(temp_output_path, "w") as output_file:
+            result = subprocess.run(
+                jq_args,
+                stdout=output_file,
+                stderr=subprocess.PIPE,
+                text=True,
+                check=False  # Don't raise exception, handle errors manually
+            )
+
+        if result.returncode != 0:
+            logger.error(f"Failed to parse items for JQ expression {items_to_parse}, error: {result.stderr}")
+            yield []
+        else:
+            with open(temp_output_path, "r") as f:
+                events_stream = get_events_as_a_stream(f, 'item', ocean.config.yield_items_to_parse_batch_size)
+                for items_bulk in events_stream:
+                    yield items_bulk
+
+    except ValueError as e:
+        logger.error(f"Invalid jq expression: {e}")
+        yield []
+    except Exception as e:
+        logger.error(f"Failed to parse items for JQ expression {items_to_parse}, error: {e}")
+        yield []
+    finally:
+        # Cleanup temporary files
+        for temp_path in [temp_data_path, temp_output_path]:
+            if temp_path and os.path.exists(temp_path):
+                try:
+                    os.remove(temp_path)
+                except OSError as e:
+                    logger.warning(f"Failed to cleanup temporary file {temp_path}: {e}")
 
 def unsupported_kind_response(
     kind: str, available_resync_kinds: list[str]
@@ -108,6 +251,44 @@ def unsupported_kind_response(
     logger.error(f"Kind {kind} is not supported in this integration")
     return [], [KindNotImplementedException(kind, available_resync_kinds)]
 
+async def _build_base_jq_object_string(raw_data: dict[Any, Any], base_jq: str, delete_target: str) -> str:
+    base_jq_object_before_parsing = await cast(JQEntityProcessor, ocean.app.integration.entity_processor)._search(raw_data, f"{base_jq} = {json.dumps("__all")}")
+    base_jq_object_before_parsing_serialized = json.dumps(base_jq_object_before_parsing)
+    base_jq_object_before_parsing_serialized = base_jq_object_before_parsing_serialized[1:-1] if len(base_jq_object_before_parsing_serialized) >= 2 else base_jq_object_before_parsing_serialized
+    base_jq_object_before_parsing_serialized = base_jq_object_before_parsing_serialized.replace("\"__all\"", f"(($all | del({delete_target})) // {{}})")
+    return base_jq_object_before_parsing_serialized
+
+
+def get_events_as_a_stream(
+        stream: Any,
+        target_items: str = "item",
+        max_buffer_size_mb: int = 1
+    ) -> Generator[list[dict[str, Any]], None, None]:
+        events = ijson.sendable_list()
+        coro = ijson.items_coro(events, target_items)
+
+        # Convert MB to bytes for the buffer size
+        buffer_size = max_buffer_size_mb * 1024 * 1024
+
+        # Read chunks from the stream until exhausted
+        while True:
+            chunk = stream.read(buffer_size)
+            if not chunk:  # End of stream
+                break
+
+            # Convert string to bytes if necessary (for text mode files)
+            if isinstance(chunk, str):
+                chunk = chunk.encode('utf-8')
+
+            coro.send(chunk)
+            yield events
+            del events[:]
+        try:
+            coro.close()
+        finally:
+            if events:
+                yield events
+                events[:] = []
 
 class ProcessWrapper(multiprocessing.Process):
     def __init__(self, *args, **kwargs):
@@ -134,3 +315,34 @@ def clear_http_client_context() -> None:
             _port_http_client.pop()
     except (RuntimeError, AttributeError):
         pass
+
+class _AiterReader:
+    """
+    Wraps an iterable of byte chunks (e.g., response.iter_bytes())
+    and exposes a .read(n) method that ijson expects.
+    """
+    def __init__(self, iterable):
+        self._iter = iter(iterable)
+        self._buf = bytearray()
+        self._eof = False
+
+    def read(self, n=-1):
+        # If n < 0, return everything until EOF
+        if n is None or n < 0:
+            chunks = [bytes(self._buf)]
+            self._buf.clear()
+            chunks.extend(self._iter)  # drain the iterator
+            return b"".join(chunks)
+
+        # Fill buffer until we have n bytes or hit EOF
+        while len(self._buf) < n and not self._eof:
+            try:
+                self._buf.extend(next(self._iter))
+            except StopIteration:
+                self._eof = True
+                break
+
+        # Serve up to n bytes
+        out = bytes(self._buf[:n])
+        del self._buf[:n]
+        return out

port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py

@@ -50,7 +50,7 @@ class TestJQEntityProcessor:
         raw_entity_mappings = {"foo": ".foo"}
         selector_query = '.foo == "bar"'
         result = await mocked_processor._get_mapped_entity(
-            data, raw_entity_mappings, selector_query
+            data, raw_entity_mappings, None, selector_query
         )
         assert result.entity == {"foo": "bar"}
         assert result.did_entity_pass_selector is True

{port_ocean-0.28.4.dist-info → port_ocean-0.28.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.28.4
+Version: 0.28.7
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.28.4.dist-info → port_ocean-0.28.7.dist-info}/RECORD

@@ -1,9 +1,9 @@
-integrations/_infra/Dockerfile.Deb,sha256=weK3VrskDEsuvYaxerak-XIqDgTx_wPorAFQQjYH_FU,2493
+integrations/_infra/Dockerfile.Deb,sha256=ZqAg-p3GbLaneWS0sIcUDHp1FLwLoxHLvsKT5H8sCLc,2562
 integrations/_infra/Dockerfile.alpine,sha256=7E4Sb-8supsCcseerHwTkuzjHZoYcaHIyxiBZ-wewo0,3482
 integrations/_infra/Dockerfile.base.builder,sha256=ESe1PKC6itp_AuXawbLI75k1Kruny6NTANaTinxOgVs,743
 integrations/_infra/Dockerfile.base.runner,sha256=uAcs2IsxrAAUHGXt_qULA5INr-HFguf5a5fCKiqEzbY,384
 integrations/_infra/Dockerfile.dockerignore,sha256=CM1Fxt3I2AvSvObuUZRmy5BNLSGC7ylnbpWzFgD4cso,1163
-integrations/_infra/Dockerfile.local,sha256=FFX9RvFqlaHvhUrRnnzUl0zQp2oKDFVRGkXJQPMQ7cI,1650
+integrations/_infra/Dockerfile.local,sha256=yLkNs8AB1QMsSXyb2OOo0F8cPXeNF9bb2pzAt2d9fZ8,1663
 integrations/_infra/Makefile,sha256=YgLKvuF_Dw4IA7X98Nus6zIW_3cJ60M1QFGs3imj5c4,2430
 integrations/_infra/README.md,sha256=ZtJFSMCTU5zTeM8ddRuW1ZL1ga8z7Ic2F3mxmgOSjgo,1195
 integrations/_infra/entry_local.sh,sha256=Sn2TexTEpruH2ixIAGsk-fZV6Y7pT3jd2Pi9TxBeFuw,633
@@ -101,11 +101,12 @@ port_ocean/core/handlers/entities_state_applier/port/get_related_entities.py,sha
 port_ocean/core/handlers/entities_state_applier/port/order_by_entities_dependencies.py,sha256=lyv6xKzhYfd6TioUgR3AVRSJqj7JpAaj1LxxU2xAqeo,1720
 port_ocean/core/handlers/entity_processor/__init__.py,sha256=FvFCunFg44wNQoqlybem9MthOs7p1Wawac87uSXz9U8,156
 port_ocean/core/handlers/entity_processor/base.py,sha256=PsnpNRqjHth9xwOvDRe7gKu8cjnVV0XGmTIHGvOelX0,1867
-port_ocean/core/handlers/entity_processor/jq_entity_processor.py,sha256=qvPMbIH1XRvaZ-TvW7lw9k4W27ZPCHcXGSdqnZ0wblw,12970
+port_ocean/core/handlers/entity_processor/jq_entity_processor.py,sha256=yMymDTiEvIvqA01yB08A6A1_cPH0Pnp1zs0O1v38RJQ,25730
+port_ocean/core/handlers/entity_processor/jq_input_evaluator.py,sha256=fcCt35pPi-Myv27ZI_HNhCoqvntU2sVWM2aGJ7yrkHQ,2300
 port_ocean/core/handlers/port_app_config/__init__.py,sha256=8AAT5OthiVM7KCcM34iEgEeXtn2pRMrT4Dze5r1Ixbk,134
 port_ocean/core/handlers/port_app_config/api.py,sha256=r_Th66NEw38IpRdnXZcRvI8ACfvxW_A6V62WLwjWXlQ,1044
 port_ocean/core/handlers/port_app_config/base.py,sha256=Sup4-X_a7JGa27rMy_OgqGIjFHMlKBpKevicaK3AeHU,2919
-port_ocean/core/handlers/port_app_config/models.py,sha256=M9NJRRacvpZQYCFgHGlRwrdgumbIIJ82WpCyvtOUf5w,3019
+port_ocean/core/handlers/port_app_config/models.py,sha256=SMeId2M5Po3_NbJi1mGMduz3VaV9DY20O0tIqBAxEZw,3012
 port_ocean/core/handlers/queue/__init__.py,sha256=yzgicE_jAR1wtljFKxgyG6j-HbLcG_Zze5qw1kkALUI,171
 port_ocean/core/handlers/queue/abstract_queue.py,sha256=SaivrYbqg8qsX6wtQlJZyxgcbdMD5B9NZG3byN9AvrI,782
 port_ocean/core/handlers/queue/group_queue.py,sha256=JvvJOwz9z_aI4CjPr7yQX-0rOgqLI5wMdxWk2x5x-34,4989
@@ -123,8 +124,8 @@ port_ocean/core/integrations/mixins/events.py,sha256=2L7P3Jhp8XBqddh2_o9Cn4N261n
 port_ocean/core/integrations/mixins/handler.py,sha256=mZ7-0UlG3LcrwJttFbMe-R4xcOU2H_g33tZar7PwTv8,3771
 port_ocean/core/integrations/mixins/live_events.py,sha256=zM24dhNc7uHx9XYZ6toVhDADPA90EnpOmZxgDegFZbA,4196
 port_ocean/core/integrations/mixins/sync.py,sha256=Vm_898pLKBwfVewtwouDWsXoxcOLicnAy6pzyqqk6U8,4053
-port_ocean/core/integrations/mixins/sync_raw.py,sha256=49P9b4Fc5L3NUYmv0W2fzwJ5hariuDqQ0frURw-9o54,40929
-port_ocean/core/integrations/mixins/utils.py,sha256=ytnFX7Lyv6N3CgBnOXxYaI1cRDq5Z4NDrVFiwE6bn-M,5250
+port_ocean/core/integrations/mixins/sync_raw.py,sha256=Zga3fSxALuXmAMKmIS0hZYWRe22lSGhiSVFWUCI4f1U,40972
+port_ocean/core/integrations/mixins/utils.py,sha256=aIIMCGb2_ezc5d8NEbKwh9cJ_E0C1VIkyqO8jVENH-Q,14120
 port_ocean/core/models.py,sha256=DNbKpStMINI2lIekKprTqBevqkw_wFuFayN19w1aDfQ,2893
 port_ocean/core/ocean_types.py,sha256=bkLlTd8XfJK6_JDl0eXUHfE_NygqgiInSMwJ4YJH01Q,1399
 port_ocean/core/utils/entity_topological_sorter.py,sha256=MDUjM6OuDy4Xj68o-7InNN0w1jqjxeDfeY8U02vySNI,3081
@@ -170,7 +171,7 @@ port_ocean/tests/core/conftest.py,sha256=0Oql7R1iTbjPyNdUoO6M21IKknLwnCIgDRz2JQ7
 port_ocean/tests/core/defaults/test_common.py,sha256=sR7RqB3ZYV6Xn6NIg-c8k5K6JcGsYZ2SCe_PYX5vLYM,5560
 port_ocean/tests/core/event_listener/test_kafka.py,sha256=PH90qk2fvdrQOSZD2QrvkGy8w_WoYb_KHGnqJ6PLHAo,2681
 port_ocean/tests/core/handlers/entities_state_applier/test_applier.py,sha256=7XWgwUB9uVYRov4VbIz1A-7n2YLbHTTYT-4rKJxjB0A,10711
-port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py,sha256=TjSj8ssIqH23VJlO5PGovbudCqDbuE2-54iNQsD9K-I,14099
+port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py,sha256=JcDyC7bI4KqIcPpS2S3Hju0mNRfKIoPHdgKVCRBey_E,14105
 port_ocean/tests/core/handlers/mixins/test_live_events.py,sha256=Sbv9IZAGQoZDhf27xDjMMVYxUSie9mHltDtxLSqckmM,12548
 port_ocean/tests/core/handlers/mixins/test_sync_raw.py,sha256=-Jd2rUG63fZM8LuyKtCp1tt4WEqO2m5woESjs1c91sU,44428
 port_ocean/tests/core/handlers/port_app_config/test_api.py,sha256=eJZ6SuFBLz71y4ca3DNqKag6d6HUjNJS0aqQPwiLMTI,1999
@@ -208,8 +209,8 @@ port_ocean/utils/repeat.py,sha256=U2OeCkHPWXmRTVoPV-VcJRlQhcYqPWI5NfmPlb1JIbc,32
 port_ocean/utils/signal.py,sha256=J1sI-e_32VHP_VUa5bskLMFoJjJOAk5isrnewKDikUI,2125
 port_ocean/utils/time.py,sha256=pufAOH5ZQI7gXvOvJoQXZXZJV-Dqktoj9Qp9eiRwmJ4,1939
 port_ocean/version.py,sha256=UsuJdvdQlazzKGD3Hd5-U7N69STh8Dq9ggJzQFnu9fU,177
-port_ocean-0.28.4.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-port_ocean-0.28.4.dist-info/METADATA,sha256=oUkOlsaK4ol1Aq5d0wDkwNf6inmpdjLOf1lBB9ImJD8,7015
-port_ocean-0.28.4.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-port_ocean-0.28.4.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
-port_ocean-0.28.4.dist-info/RECORD,,
+port_ocean-0.28.7.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+port_ocean-0.28.7.dist-info/METADATA,sha256=gvbQSwADuQQrcqf9it-ShnXH-82xm4KUTYvqpn_rf0A,7015
+port_ocean-0.28.7.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+port_ocean-0.28.7.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
+port_ocean-0.28.7.dist-info/RECORD,,