port-ocean 0.28.8__py3-none-any.whl → 0.28.9__py3-none-any.whl

integrations/_infra/Dockerfile.Deb

@@ -48,7 +48,6 @@ RUN apt-get update \
         curl \
         acl \
         sudo \
-        jq \
     && apt-get clean
 
 LABEL INTEGRATION_VERSION=${INTEGRATION_VERSION}

integrations/_infra/Dockerfile.local

@@ -26,7 +26,6 @@ RUN apt-get update \
         python3-pip \
         python3-poetry \
         build-essential\
-        jq \
         git \
         python3-venv \
         acl \

port_ocean/core/handlers/entity_processor/jq_entity_processor.py

@@ -1,11 +1,12 @@
 import asyncio
 from asyncio import Task
 from dataclasses import dataclass, field
+
 from functools import lru_cache
-import json
 from typing import Any, Optional
 import jq  # type: ignore
 from loguru import logger
+
 from port_ocean.context.ocean import ocean
 from port_ocean.core.handlers.entity_processor.base import BaseEntityProcessor
 from port_ocean.core.handlers.port_app_config.models import ResourceConfig
@@ -21,11 +22,6 @@ from port_ocean.core.utils.utils import (
 )
 from port_ocean.exceptions.core import EntityProcessorException
 from port_ocean.utils.queue_utils import process_in_queue
-from port_ocean.core.handlers.entity_processor.jq_input_evaluator import (
-    InputEvaluationResult,
-    evaluate_input,
-    should_shortcut_no_input,
-)
 
 
 class ExampleStates:
@@ -80,7 +76,7 @@ class MappedEntity:
 
     entity: dict[str, Any] = field(default_factory=dict)
     did_entity_pass_selector: bool = False
-    raw_data: Optional[dict[str, Any] | tuple[dict[str, Any], str]] = None
+    raw_data: Optional[dict[str, Any]] = None
     misconfigurations: dict[str, str] = field(default_factory=dict)
 
 
@@ -137,35 +133,17 @@ class JQEntityProcessor(BaseEntityProcessor):
             return await loop.run_in_executor(
                 None, self._stop_iterator_handler(func.first)
             )
-        except Exception as exc:
-            logger.error(
-                f"Search failed for pattern '{pattern}' in data: {data}, Error: {exc}"
-            )
-            return None
-
-    @lru_cache
-    async def _search_stringified(self, data: str, pattern: str) -> Any:
-        try:
-            loop = asyncio.get_event_loop()
-            compiled_pattern = self._compile(pattern)
-            func = compiled_pattern.input_text(data)
-            return await loop.run_in_executor(
-                None, self._stop_iterator_handler(func.first)
-            )
         except Exception as exc:
             logger.debug(
                 f"Search failed for pattern '{pattern}' in data: {data}, Error: {exc}"
             )
             return None
 
-    async def _search_as_bool(self, data: dict[str, Any] | str, pattern: str) -> bool:
+    async def _search_as_bool(self, data: dict[str, Any], pattern: str) -> bool:
         loop = asyncio.get_event_loop()
 
         compiled_pattern = self._compile(pattern)
-        if isinstance(data, str):
-            func = compiled_pattern.input_text(data)
-        else:
-            func = compiled_pattern.input_value(data)
+        func = compiled_pattern.input_value(data)
 
         value = await loop.run_in_executor(
             None, self._stop_iterator_handler(func.first)
@@ -178,7 +156,7 @@ class JQEntityProcessor(BaseEntityProcessor):
 
     async def _search_as_object(
         self,
-        data: dict[str, Any] | str,
+        data: dict[str, Any],
         obj: dict[str, Any],
         misconfigurations: dict[str, str] | None = None,
     ) -> dict[str, Any | None]:
@@ -210,12 +188,7 @@ class JQEntityProcessor(BaseEntityProcessor):
                     self._search_as_object(data, value, misconfigurations)
                 )
             else:
-                if isinstance(data, str):
-                    search_tasks[key] = asyncio.create_task(
-                        self._search_stringified(data, value)
-                    )
-                else:
-                    search_tasks[key] = asyncio.create_task(self._search(data, value))
+                search_tasks[key] = asyncio.create_task(self._search(data, value))
 
         result: dict[str, Any | None] = {}
         for key, task in search_tasks.items():
@@ -239,18 +212,16 @@ class JQEntityProcessor(BaseEntityProcessor):
 
     async def _get_mapped_entity(
         self,
-        data: dict[str, Any] | tuple[dict[str, Any], str],
+        data: dict[str, Any],
         raw_entity_mappings: dict[str, Any],
-        items_to_parse_key: str | None,
         selector_query: str,
         parse_all: bool = False,
     ) -> MappedEntity:
-        should_run = await self._should_map_entity(
-            data, selector_query, items_to_parse_key
-        )
+        should_run = await self._search_as_bool(data, selector_query)
         if parse_all or should_run:
-            misconfigurations, mapped_entity = await self._map_entity(
-                data, raw_entity_mappings, items_to_parse_key
+            misconfigurations: dict[str, str] = {}
+            mapped_entity = await self._search_as_object(
+                data, raw_entity_mappings, misconfigurations
             )
             return MappedEntity(
                 mapped_entity,
@@ -266,69 +237,6 @@ class JQEntityProcessor(BaseEntityProcessor):
             misconfigurations={},
         )
 
-    async def _map_entity(
-        self,
-        data: dict[str, Any] | tuple[dict[str, Any], str],
-        raw_entity_mappings: dict[str, Any],
-        items_to_parse_key: str | None,
-    ) -> tuple[dict[str, str], dict[str, Any]]:
-        if not items_to_parse_key:
-            misconfigurations: dict[str, str] = {}
-            data_to_search = data if isinstance(data, dict) else data[0]
-            mapped_entity = await self._search_as_object(
-                data_to_search, raw_entity_mappings, misconfigurations
-            )
-            return misconfigurations, mapped_entity
-
-        modified_data: tuple[dict[str, Any], str | dict[str, Any]] = (
-            data
-            if isinstance(data, tuple)
-            else (
-                {items_to_parse_key: data[items_to_parse_key]},
-                data,
-            )
-        )
-
-        misconfigurations_item: dict[str, str] = {}
-        misconfigurations_all: dict[str, str] = {}
-        mapped_entity_item = await self._search_as_object(
-            modified_data[0], raw_entity_mappings["item"], misconfigurations_item
-        )
-        if misconfigurations_item:
-            filtered_item_mappings = self._filter_mappings_by_keys(
-                raw_entity_mappings["item"], list(misconfigurations_item.keys())
-            )
-            raw_entity_mappings["all"] = self._deep_merge(
-                raw_entity_mappings["all"], filtered_item_mappings
-            )
-        mapped_entity_all = await self._search_as_object(
-            modified_data[1], raw_entity_mappings["all"], misconfigurations_all
-        )
-        mapped_entity_empty = await self._search_as_object(
-            {}, raw_entity_mappings["empty"], misconfigurations_all
-        )
-        mapped_entity = self._deep_merge(mapped_entity_item, mapped_entity_all)
-        mapped_entity = self._deep_merge(mapped_entity, mapped_entity_empty)
-        return misconfigurations_all, mapped_entity
-
-    async def _should_map_entity(
-        self,
-        data: dict[str, Any] | tuple[dict[str, Any], str],
-        selector_query: str,
-        items_to_parse_key: str | None,
-    ) -> bool:
-        if should_shortcut_no_input(selector_query):
-            return await self._search_as_bool({}, selector_query)
-        if isinstance(data, tuple):
-            return await self._search_as_bool(
-                data[0], selector_query
-            ) or await self._search_as_bool(data[1], selector_query)
-        if items_to_parse_key:
-            return await self._search_as_bool(
-                data[items_to_parse_key], selector_query
-            ) or await self._search_as_bool(data, selector_query)
-        return await self._search_as_bool(data, selector_query)
-
     async def _calculate_entity(
         self,
         data: dict[str, Any],
@@ -338,16 +246,9 @@ class JQEntityProcessor(BaseEntityProcessor):
         selector_query: str,
         parse_all: bool = False,
     ) -> tuple[list[MappedEntity], list[Exception]]:
-        raw_data: list[dict[str, Any]] | list[tuple[dict[str, Any], str]] = [
-            data.copy()
-        ]
-        items_to_parse_key = None
-        if items_to_parse:
-            items_to_parse_key = items_to_parse_name
-            if not ocean.config.yield_items_to_parse:
-                if data.get("file", {}).get("content", {}).get("path", None):
-                    with open(data["file"]["content"]["path"], "r") as f:
-                        data["file"]["content"] = json.loads(f.read())
+        raw_data = [data.copy()]
+        if not ocean.config.yield_items_to_parse:
+            if items_to_parse:
                 items = await self._search(data, items_to_parse)
                 if not isinstance(items, list):
                     logger.warning(
@@ -355,28 +256,13 @@ class JQEntityProcessor(BaseEntityProcessor):
                         f" Skipping..."
                     )
                     return [], []
-                raw_all_payload_stringified = json.dumps(data)
-                raw_data = [
-                    ({items_to_parse_name: item}, raw_all_payload_stringified)
-                    for item in items
-                ]
-            single_item_mappings, all_items_mappings, empty_items_mappings = (
-                self._build_raw_entity_mappings(
-                    raw_entity_mappings, items_to_parse_name
-                )
-            )
-            raw_entity_mappings = {
-                "item": single_item_mappings,
-                "all": all_items_mappings,
-                "empty": empty_items_mappings,
-            }
+                raw_data = [{items_to_parse_name: item, **data} for item in items]
 
         entities, errors = await gather_and_split_errors_from_results(
             [
                 self._get_mapped_entity(
                     raw,
                     raw_entity_mappings,
-                    items_to_parse_key,
                     selector_query,
                     parse_all,
                 )
@@ -389,201 +275,6 @@ class JQEntityProcessor(BaseEntityProcessor):
             )
         return entities, errors
 
-    def _build_raw_entity_mappings(
-        self, raw_entity_mappings: dict[str, Any], items_to_parse_name: str
-    ) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]:
-        """Filter entity mappings to only include values that start with f'.{items_to_parse_name}'"""
-        mappings: dict[InputEvaluationResult, dict[str, Any]] = {
-            InputEvaluationResult.NONE: {},
-            InputEvaluationResult.SINGLE: {},
-            InputEvaluationResult.ALL: {},
-        }
-        pattern = f".{items_to_parse_name}"
-        for key, value in raw_entity_mappings.items():
-            if isinstance(value, str):
-                # Direct string values (identifier, title, icon, blueprint, team)
-                self.group_string_mapping_value(
-                    pattern,
-                    mappings,
-                    key,
-                    value,
-                )
-            elif isinstance(value, dict):
-                # Complex objects (IngestSearchQuery for identifier/team, properties, relations)
-                self.group_complex_mapping_value(
-                    pattern,
-                    mappings,
-                    key,
-                    value,
-                )
-        return (
-            mappings[InputEvaluationResult.SINGLE],
-            mappings[InputEvaluationResult.ALL],
-            mappings[InputEvaluationResult.NONE],
-        )
-
-    def group_complex_mapping_value(
-        self,
-        pattern: str,
-        mappings: dict[InputEvaluationResult, dict[str, Any]],
-        key: str,
-        value: dict[str, Any],
-    ) -> None:
-        mapping_dicts: dict[InputEvaluationResult, dict[str, Any]] = {
-            InputEvaluationResult.SINGLE: {},
-            InputEvaluationResult.ALL: {},
-            InputEvaluationResult.NONE: {},
-        }
-        if key in ["properties", "relations"]:
-            # For properties and relations, filter the dictionary values
-            for dict_key, dict_value in value.items():
-                if isinstance(dict_value, str):
-                    self.group_string_mapping_value(
-                        pattern,
-                        mapping_dicts,
-                        dict_key,
-                        dict_value,
-                    )
-                elif isinstance(dict_value, dict):
-                    # Handle IngestSearchQuery objects
-                    self.group_search_query_mapping_value(
-                        pattern,
-                        mapping_dicts[InputEvaluationResult.SINGLE],
-                        mapping_dicts[InputEvaluationResult.ALL],
-                        dict_key,
-                        dict_value,
-                    )
-        else:
-            # For identifier/team IngestSearchQuery objects
-            self.group_search_query_mapping_value(
-                pattern,
-                mapping_dicts[InputEvaluationResult.SINGLE],
-                mapping_dicts[InputEvaluationResult.ALL],
-                key,
-                value,
-            )
-        if mapping_dicts[InputEvaluationResult.SINGLE]:
-            mappings[InputEvaluationResult.SINGLE][key] = mapping_dicts[
-                InputEvaluationResult.SINGLE
-            ][key]
-        if mapping_dicts[InputEvaluationResult.ALL]:
-            mappings[InputEvaluationResult.ALL][key] = mapping_dicts[
-                InputEvaluationResult.ALL
-            ][key]
-        if mapping_dicts[InputEvaluationResult.NONE]:
-            mappings[InputEvaluationResult.NONE][key] = mapping_dicts[
-                InputEvaluationResult.NONE
-            ][key]
-
-    def group_search_query_mapping_value(
-        self,
-        pattern: str,
-        single_item_dict: dict[str, Any],
-        all_item_dict: dict[str, Any],
-        dict_key: str,
-        dict_value: dict[str, Any],
-    ) -> None:
-        if self._should_keep_ingest_search_query(dict_value, pattern):
-            single_item_dict[dict_key] = dict_value
-        else:
-            all_item_dict[dict_key] = dict_value
-
-    def group_string_mapping_value(
-        self,
-        pattern: str,
-        mappings: dict[InputEvaluationResult, dict[str, Any]],
-        key: str,
-        value: str,
-    ) -> None:
-        input_evaluation_result = evaluate_input(value, pattern)
-        mappings[input_evaluation_result][key] = value
-
-    def _should_keep_ingest_search_query(
-        self, query_dict: dict[str, Any], pattern: str
-    ) -> bool:
-        """Check if an IngestSearchQuery should be kept based on its rules"""
-        if "rules" not in query_dict:
-            return False
-
-        rules = query_dict["rules"]
-        if not isinstance(rules, list):
-            return False
-
-        # Check if any rule contains a value starting with the pattern
-        for rule in rules:
-            if isinstance(rule, dict):
-                if "value" in rule and isinstance(rule["value"], str):
-                    if pattern in rule["value"]:
-                        return True
-                # Recursively check nested IngestSearchQuery objects
-                elif "rules" in rule:
-                    if self._should_keep_ingest_search_query(rule, pattern):
-                        return True
-        return False
-
-    def _filter_mappings_by_keys(
-        self, mappings: dict[str, Any], target_keys: list[str]
-    ) -> dict[str, Any]:
-        """
-        Filter mappings to preserve structure with only the specified keys present.
-        Recursively handles nested dictionaries and lists, searching for keys at any level.
-        """
-        if not target_keys:
-            return {}
-
-        filtered_mappings: dict[str, Any] = {}
-
-        for key, value in mappings.items():
-            filtered_value = self._process_mapping_value(key, value, target_keys)
-
-            # Include if it's a direct match or contains nested target keys
-            if key in target_keys or filtered_value:
-                filtered_mappings[key] = filtered_value
-
-        return filtered_mappings
-
-    def _process_mapping_value(
-        self, key: str, value: Any, target_keys: list[str]
-    ) -> Any:
-        """Process a single mapping value, handling different types recursively."""
-        if isinstance(value, dict):
-            # Recursively filter nested dictionary
-            filtered_dict = self._filter_mappings_by_keys(value, target_keys)
-            return filtered_dict if filtered_dict else None
-        else:
-            # Return simple values as-is
-            return value if key in target_keys else None
-
-    def _deep_merge(
-        self, dict1: dict[str, Any], dict2: dict[str, Any]
-    ) -> dict[str, Any]:
-        """
-        Deep merge two dictionaries, preserving nested structures.
-        Values from dict2 override values from dict1 for the same keys.
-        """
-        result = dict1.copy()
-
-        for key, value in dict2.items():
-            if (
-                key in result
-                and isinstance(result[key], dict)
-                and isinstance(value, dict)
-            ):
-                # Recursively merge nested dictionaries
-                result[key] = self._deep_merge(result[key], value)
-            elif (
-                key in result
-                and isinstance(result[key], list)
-                and isinstance(value, list)
-            ):
-                # Merge lists by extending
-                result[key].extend(value)
-            else:
-                # Override with value from dict2
-                result[key] = value
-
-        return result
-
     @staticmethod
     async def _send_examples(data: list[dict[str, Any]], kind: str) -> None:
         try:
@@ -638,10 +329,7 @@ class JQEntityProcessor(BaseEntityProcessor):
                 and result.raw_data is not None
             ):
                 examples_to_send.add_example(
-                    result.did_entity_pass_selector,
-                    self._get_raw_data_for_example(
-                        result.raw_data, mapping.port.items_to_parse_name
-                    ),
+                    result.did_entity_pass_selector, result.raw_data
                 )
 
             if result.entity.get("identifier") and result.entity.get("blueprint"):
@@ -667,13 +355,3 @@ class JQEntityProcessor(BaseEntityProcessor):
             errors,
             misconfigured_entity_keys=entity_misconfigurations,
         )
-
-    def _get_raw_data_for_example(
-        self,
-        data: dict[str, Any] | tuple[dict[str, Any], str],
-        items_to_parse_name: str,
-    ) -> dict[str, Any]:
-        if isinstance(data, tuple):
-            raw_data = json.loads(data[1])
-            return {items_to_parse_name: data[0], **raw_data}
-        return data

port_ocean/core/handlers/port_app_config/models.py

@@ -39,7 +39,7 @@ class MappingsConfig(BaseModel):
 class PortResourceConfig(BaseModel):
     entity: MappingsConfig
     items_to_parse: str | None = Field(alias="itemsToParse")
-    items_to_parse_name: str = Field(alias="itemsToParseName", default="item")
+    items_to_parse_name: str | None = Field(alias="itemsToParseName", default="item")
 
 
 class Selector(BaseModel):

port_ocean/core/integrations/mixins/sync_raw.py

@@ -117,7 +117,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                 logger.info(
                     f"Found async generator function for {resource_config.kind} name: {task.__qualname__}"
                 )
-                results.append(resync_generator_wrapper(task, resource_config.kind, resource_config.port.items_to_parse_name, resource_config.port.items_to_parse))
+                results.append(resync_generator_wrapper(task, resource_config.kind,resource_config.port.items_to_parse))
             else:
                 logger.info(
                     f"Found sync function for {resource_config.kind} name: {task.__qualname__}"

port_ocean/core/integrations/mixins/utils.py

@@ -2,11 +2,12 @@ from contextlib import contextmanager
 from typing import Awaitable, Generator, Callable, cast
 
 from loguru import logger
+
 import asyncio
 import multiprocessing
-import re
-import json
+
 from port_ocean.core.handlers.entity_processor.jq_entity_processor import JQEntityProcessor
+from port_ocean.core.handlers.port_app_config.models import ResourceConfig
 from port_ocean.core.ocean_types import (
     ASYNC_GENERATOR_RESYNC_TYPE,
     RAW_RESULT,
@@ -19,60 +20,11 @@ from port_ocean.exceptions.core import (
     OceanAbortException,
     KindNotImplementedException,
 )
-import os
+
 from port_ocean.utils.async_http import _http_client
 from port_ocean.clients.port.utils import _http_client as _port_http_client
 from port_ocean.helpers.metric.metric import MetricType, MetricPhase
 from port_ocean.context.ocean import ocean
-import subprocess
-import tempfile
-import stat
-import ijson
-from typing import Any, AsyncGenerator
-
-def _process_path_type_items(
-    result: RAW_RESULT, items_to_parse: str | None = None
-) -> RAW_RESULT:
-    """
-    Process items in the result array to check for "__type": "path" fields.
-    If found, read the file contents and load them into a "content" field.
-    Skip processing if we're on the items_to_parse branch.
-    """
-    if not isinstance(result, list):
-        return result
-
-    # Skip processing if we're on the items_to_parse branch
-    if items_to_parse:
-        return result
-
-    processed_result = []
-    for item in result:
-        if isinstance(item, dict) and item.get("__type") == "path":
-            try:
-                # Read the file content and parse as JSON
-                file_path = item.get("file", {}).get("content", {}).get("path")
-                if file_path and os.path.exists(file_path):
-                    with open(file_path, "r") as f:
-                        content = json.loads(f.read())
-                    # Create a copy of the item with the content field
-                    processed_item = item.copy()
-                    processed_item["content"] = content
-                    processed_result.append(processed_item)
-                else:
-                    # If file doesn't exist, keep the original item
-                    processed_result.append(item)
-            except (json.JSONDecodeError, IOError, OSError) as e:
-                logger.warning(
-                    f"Failed to read or parse file content for path "
-                    f"{item.get('file', {}).get('content', {}).get('path')}: {e}"
-                )
-                # Keep the original item if there's an error
-                processed_result.append(item)
-        else:
-            # Keep non-path type items as is
-            processed_result.append(item)
-
-    return processed_result
 
 @contextmanager
 def resync_error_handling() -> Generator[None, None, None]:
@@ -95,12 +47,11 @@ async def resync_function_wrapper(
 ) -> RAW_RESULT:
     with resync_error_handling():
         results = await fn(kind)
-        validated_results = validate_result(results)
-        return _process_path_type_items(validated_results)
+        return validate_result(results)
 
 
 async def resync_generator_wrapper(
-    fn: Callable[[str], ASYNC_GENERATOR_RESYNC_TYPE], kind: str, items_to_parse_name: str, items_to_parse: str | None = None
+    fn: Callable[[str], ASYNC_GENERATOR_RESYNC_TYPE], kind: str, items_to_parse: str | None = None
 ) -> ASYNC_GENERATOR_RESYNC_TYPE:
     generator = fn(kind)
     errors = []
@@ -110,23 +61,27 @@ async def resync_generator_wrapper(
                 with resync_error_handling():
                     result = await anext(generator)
                     if not ocean.config.yield_items_to_parse:
-                        validated_result = validate_result(result)
-                        processed_result = _process_path_type_items(validated_result)
-                        yield processed_result
+                        yield validate_result(result)
                     else:
+                        batch_size = ocean.config.yield_items_to_parse_batch_size
                         if items_to_parse:
                             for data in result:
-                                data_path: str | None = None
-                                if isinstance(data, dict) and data.get("file") is not None:
-                                    content = data["file"].get("content") if isinstance(data["file"].get("content"), dict) else {}
-                                    data_path = content.get("path", None)
-                                bulks = get_items_to_parse_bulks(data, data_path, items_to_parse, items_to_parse_name, data.get("__base_jq", ".file.content"))
-                                async for bulk in bulks:
-                                    yield bulk
+                                items = await cast(JQEntityProcessor, ocean.app.integration.entity_processor)._search(data, items_to_parse)
+                                if not isinstance(items, list):
+                                    logger.warning(
+                                        f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items)}."
+                                        f" Skipping..."
+                                    )
+                                    yield []
+                                raw_data = [{"item": item, **data} for item in items]
+                                while True:
+                                    raw_data_batch = raw_data[:batch_size]
+                                    yield raw_data_batch
+                                    raw_data = raw_data[batch_size:]
+                                    if len(raw_data) == 0:
+                                        break
                         else:
-                            validated_result = validate_result(result)
-                            processed_result = _process_path_type_items(validated_result, items_to_parse)
-                            yield processed_result
+                            yield validate_result(result)
             except OceanAbortException as error:
                 errors.append(error)
                 ocean.metrics.inc_metric(
@@ -146,104 +101,6 @@ def is_resource_supported(
 ) -> bool:
     return bool(resync_event_mapping[kind] or resync_event_mapping[None])
 
-def _validate_jq_expression(expression: str) -> None:
-    """Validate jq expression to prevent command injection."""
-    try:
-        _ = cast(JQEntityProcessor, ocean.app.integration.entity_processor)._compile(expression)
-    except Exception as e:
-        raise ValueError(f"Invalid jq expression: {e}") from e
-    # Basic validation - reject expressions that could be dangerous
-    # Check for dangerous patterns (include, import, module)
-    dangerous_patterns = ['include', 'import', 'module', 'env']
-    for pattern in dangerous_patterns:
-        # Use word boundary regex to match only complete words, not substrings
-        if re.search(rf'\b{re.escape(pattern)}\b', expression):
-            raise ValueError(f"Potentially dangerous pattern '{pattern}' found in jq expression")
-
-    # Special handling for 'env' - block environment variable access
-    if re.search(r'(?<!\w)\$ENV(?:\.)?', expression):
-        raise ValueError("Environment variable access '$ENV.' found in jq expression")
-    if re.search(r'\benv\.', expression):
-        raise ValueError("Environment variable access 'env.' found in jq expression")
-
-def _create_secure_temp_file(suffix: str = ".json") -> str:
-    """Create a secure temporary file with restricted permissions."""
-    # Create temp directory if it doesn't exist
-    temp_dir = "/tmp/ocean"
-    os.makedirs(temp_dir, exist_ok=True)
-
-    # Create temporary file with secure permissions
-    fd, temp_path = tempfile.mkstemp(suffix=suffix, dir=temp_dir)
-    try:
-        # Set restrictive permissions (owner read/write only)
-        os.chmod(temp_path, stat.S_IRUSR | stat.S_IWUSR)
-        return temp_path
-    finally:
-        os.close(fd)
-
-async def get_items_to_parse_bulks(raw_data: dict[Any, Any], data_path: str, items_to_parse: str, items_to_parse_name: str, base_jq: str) -> AsyncGenerator[list[dict[str, Any]], None]:
-    # Validate inputs to prevent command injection
-    _validate_jq_expression(items_to_parse)
-    items_to_parse = items_to_parse.replace(base_jq, ".") if data_path else items_to_parse
-
-    temp_data_path = None
-    temp_output_path = None
-
-    try:
-        # Create secure temporary files
-        if not data_path:
-            raw_data_serialized = json.dumps(raw_data)
-            temp_data_path = _create_secure_temp_file("_input.json")
-            with open(temp_data_path, "w") as f:
-                f.write(raw_data_serialized)
-            data_path = temp_data_path
-
-        temp_output_path = _create_secure_temp_file("_parsed.json")
-
-        delete_target = items_to_parse.split('|', 1)[0].strip() if not items_to_parse.startswith('map(') else '.'
-        base_jq_object_string = await _build_base_jq_object_string(raw_data, base_jq, delete_target)
-
-        # Build jq expression safely
-        jq_expression = f""". as $all
-      | ($all | {items_to_parse}) as $items
-      | $items
-      | map({{{items_to_parse_name}: ., {base_jq_object_string}}})"""
-
-        # Use subprocess with list arguments instead of shell=True
-        jq_args = ["/bin/jq", jq_expression, data_path]
-
-        with open(temp_output_path, "w") as output_file:
-            result = subprocess.run(
-                jq_args,
-                stdout=output_file,
-                stderr=subprocess.PIPE,
-                text=True,
-                check=False  # Don't raise exception, handle errors manually
-            )
-
-        if result.returncode != 0:
-            logger.error(f"Failed to parse items for JQ expression {items_to_parse}, error: {result.stderr}")
-            yield []
-        else:
-            with open(temp_output_path, "r") as f:
-                events_stream = get_events_as_a_stream(f, 'item', ocean.config.yield_items_to_parse_batch_size)
-                for items_bulk in events_stream:
-                    yield items_bulk
-
-    except ValueError as e:
-        logger.error(f"Invalid jq expression: {e}")
-        yield []
-    except Exception as e:
-        logger.error(f"Failed to parse items for JQ expression {items_to_parse}, error: {e}")
-        yield []
-    finally:
-        # Cleanup temporary files
-        for temp_path in [temp_data_path, temp_output_path]:
-            if temp_path and os.path.exists(temp_path):
-                try:
-                    os.remove(temp_path)
-                except OSError as e:
-                    logger.warning(f"Failed to cleanup temporary file {temp_path}: {e}")
 
 def unsupported_kind_response(
     kind: str, available_resync_kinds: list[str]
@@ -251,44 +108,6 @@ def unsupported_kind_response(
     logger.error(f"Kind {kind} is not supported in this integration")
     return [], [KindNotImplementedException(kind, available_resync_kinds)]
 
-async def _build_base_jq_object_string(raw_data: dict[Any, Any], base_jq: str, delete_target: str) -> str:
-    base_jq_object_before_parsing = await cast(JQEntityProcessor, ocean.app.integration.entity_processor)._search(raw_data, f"{base_jq} = {json.dumps("__all")}")
-    base_jq_object_before_parsing_serialized = json.dumps(base_jq_object_before_parsing)
-    base_jq_object_before_parsing_serialized = base_jq_object_before_parsing_serialized[1:-1] if len(base_jq_object_before_parsing_serialized) >= 2 else base_jq_object_before_parsing_serialized
-    base_jq_object_before_parsing_serialized = base_jq_object_before_parsing_serialized.replace("\"__all\"", f"(($all | del({delete_target})) // {{}})")
-    return base_jq_object_before_parsing_serialized
-
-
-def get_events_as_a_stream(
-        stream: Any,
-        target_items: str = "item",
-        max_buffer_size_mb: int = 1
-    ) -> Generator[list[dict[str, Any]], None, None]:
-        events = ijson.sendable_list()
-        coro = ijson.items_coro(events, target_items)
-
-        # Convert MB to bytes for the buffer size
-        buffer_size = max_buffer_size_mb * 1024 * 1024
-
-        # Read chunks from the stream until exhausted
-        while True:
-            chunk = stream.read(buffer_size)
-            if not chunk:  # End of stream
-                break
-
-            # Convert string to bytes if necessary (for text mode files)
-            if isinstance(chunk, str):
-                chunk = chunk.encode('utf-8')
-
-            coro.send(chunk)
-            yield events
-            del events[:]
-        try:
-            coro.close()
-        finally:
-            if events:
-                yield events
-                events[:] = []
 
 class ProcessWrapper(multiprocessing.Process):
     def __init__(self, *args, **kwargs):
@@ -315,34 +134,3 @@ def clear_http_client_context() -> None:
             _port_http_client.pop()
     except (RuntimeError, AttributeError):
         pass
-
-class _AiterReader:
-    """
-    Wraps an iterable of byte chunks (e.g., response.iter_bytes())
-    and exposes a .read(n) method that ijson expects.
-    """
-    def __init__(self, iterable):
-        self._iter = iter(iterable)
-        self._buf = bytearray()
-        self._eof = False
-
-    def read(self, n=-1):
-        # If n < 0, return everything until EOF
-        if n is None or n < 0:
-            chunks = [bytes(self._buf)]
-            self._buf.clear()
-            chunks.extend(self._iter)  # drain the iterator
-            return b"".join(chunks)
-
-        # Fill buffer until we have n bytes or hit EOF
-        while len(self._buf) < n and not self._eof:
-            try:
-                self._buf.extend(next(self._iter))
-            except StopIteration:
-                self._eof = True
-                break
-
-        # Serve up to n bytes
-        out = bytes(self._buf[:n])
-        del self._buf[:n]
-        return out

port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py

@@ -50,7 +50,7 @@ class TestJQEntityProcessor:
         raw_entity_mappings = {"foo": ".foo"}
         selector_query = '.foo == "bar"'
         result = await mocked_processor._get_mapped_entity(
-            data, raw_entity_mappings, None, selector_query
+            data, raw_entity_mappings, selector_query
         )
         assert result.entity == {"foo": "bar"}
         assert result.did_entity_pass_selector is True

{port_ocean-0.28.8.dist-info → port_ocean-0.28.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.28.8
+Version: 0.28.9
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.28.8.dist-info → port_ocean-0.28.9.dist-info}/RECORD

@@ -1,9 +1,9 @@
-integrations/_infra/Dockerfile.Deb,sha256=ZqAg-p3GbLaneWS0sIcUDHp1FLwLoxHLvsKT5H8sCLc,2562
+integrations/_infra/Dockerfile.Deb,sha256=QNyStzc0Zov1e3sejWna84yhrdOPO8Ogc-r_he3fYT4,2549
 integrations/_infra/Dockerfile.alpine,sha256=7E4Sb-8supsCcseerHwTkuzjHZoYcaHIyxiBZ-wewo0,3482
 integrations/_infra/Dockerfile.base.builder,sha256=ESe1PKC6itp_AuXawbLI75k1Kruny6NTANaTinxOgVs,743
 integrations/_infra/Dockerfile.base.runner,sha256=uAcs2IsxrAAUHGXt_qULA5INr-HFguf5a5fCKiqEzbY,384
 integrations/_infra/Dockerfile.dockerignore,sha256=CM1Fxt3I2AvSvObuUZRmy5BNLSGC7ylnbpWzFgD4cso,1163
-integrations/_infra/Dockerfile.local,sha256=yLkNs8AB1QMsSXyb2OOo0F8cPXeNF9bb2pzAt2d9fZ8,1663
+integrations/_infra/Dockerfile.local,sha256=FFX9RvFqlaHvhUrRnnzUl0zQp2oKDFVRGkXJQPMQ7cI,1650
 integrations/_infra/Makefile,sha256=YgLKvuF_Dw4IA7X98Nus6zIW_3cJ60M1QFGs3imj5c4,2430
 integrations/_infra/README.md,sha256=ZtJFSMCTU5zTeM8ddRuW1ZL1ga8z7Ic2F3mxmgOSjgo,1195
 integrations/_infra/entry_local.sh,sha256=Sn2TexTEpruH2ixIAGsk-fZV6Y7pT3jd2Pi9TxBeFuw,633
@@ -101,12 +101,11 @@ port_ocean/core/handlers/entities_state_applier/port/get_related_entities.py,sha
 port_ocean/core/handlers/entities_state_applier/port/order_by_entities_dependencies.py,sha256=lyv6xKzhYfd6TioUgR3AVRSJqj7JpAaj1LxxU2xAqeo,1720
 port_ocean/core/handlers/entity_processor/__init__.py,sha256=FvFCunFg44wNQoqlybem9MthOs7p1Wawac87uSXz9U8,156
 port_ocean/core/handlers/entity_processor/base.py,sha256=PsnpNRqjHth9xwOvDRe7gKu8cjnVV0XGmTIHGvOelX0,1867
-port_ocean/core/handlers/entity_processor/jq_entity_processor.py,sha256=yMymDTiEvIvqA01yB08A6A1_cPH0Pnp1zs0O1v38RJQ,25730
-port_ocean/core/handlers/entity_processor/jq_input_evaluator.py,sha256=fcCt35pPi-Myv27ZI_HNhCoqvntU2sVWM2aGJ7yrkHQ,2300
+port_ocean/core/handlers/entity_processor/jq_entity_processor.py,sha256=qvPMbIH1XRvaZ-TvW7lw9k4W27ZPCHcXGSdqnZ0wblw,12970
 port_ocean/core/handlers/port_app_config/__init__.py,sha256=8AAT5OthiVM7KCcM34iEgEeXtn2pRMrT4Dze5r1Ixbk,134
 port_ocean/core/handlers/port_app_config/api.py,sha256=r_Th66NEw38IpRdnXZcRvI8ACfvxW_A6V62WLwjWXlQ,1044
 port_ocean/core/handlers/port_app_config/base.py,sha256=Sup4-X_a7JGa27rMy_OgqGIjFHMlKBpKevicaK3AeHU,2919
-port_ocean/core/handlers/port_app_config/models.py,sha256=SMeId2M5Po3_NbJi1mGMduz3VaV9DY20O0tIqBAxEZw,3012
+port_ocean/core/handlers/port_app_config/models.py,sha256=M9NJRRacvpZQYCFgHGlRwrdgumbIIJ82WpCyvtOUf5w,3019
 port_ocean/core/handlers/queue/__init__.py,sha256=yzgicE_jAR1wtljFKxgyG6j-HbLcG_Zze5qw1kkALUI,171
 port_ocean/core/handlers/queue/abstract_queue.py,sha256=SaivrYbqg8qsX6wtQlJZyxgcbdMD5B9NZG3byN9AvrI,782
 port_ocean/core/handlers/queue/group_queue.py,sha256=JvvJOwz9z_aI4CjPr7yQX-0rOgqLI5wMdxWk2x5x-34,4989
@@ -124,8 +123,8 @@ port_ocean/core/integrations/mixins/events.py,sha256=2L7P3Jhp8XBqddh2_o9Cn4N261n
 port_ocean/core/integrations/mixins/handler.py,sha256=mZ7-0UlG3LcrwJttFbMe-R4xcOU2H_g33tZar7PwTv8,3771
 port_ocean/core/integrations/mixins/live_events.py,sha256=zM24dhNc7uHx9XYZ6toVhDADPA90EnpOmZxgDegFZbA,4196
 port_ocean/core/integrations/mixins/sync.py,sha256=Vm_898pLKBwfVewtwouDWsXoxcOLicnAy6pzyqqk6U8,4053
-port_ocean/core/integrations/mixins/sync_raw.py,sha256=Zga3fSxALuXmAMKmIS0hZYWRe22lSGhiSVFWUCI4f1U,40972
-port_ocean/core/integrations/mixins/utils.py,sha256=aIIMCGb2_ezc5d8NEbKwh9cJ_E0C1VIkyqO8jVENH-Q,14120
+port_ocean/core/integrations/mixins/sync_raw.py,sha256=49P9b4Fc5L3NUYmv0W2fzwJ5hariuDqQ0frURw-9o54,40929
+port_ocean/core/integrations/mixins/utils.py,sha256=ytnFX7Lyv6N3CgBnOXxYaI1cRDq5Z4NDrVFiwE6bn-M,5250
 port_ocean/core/models.py,sha256=DNbKpStMINI2lIekKprTqBevqkw_wFuFayN19w1aDfQ,2893
 port_ocean/core/ocean_types.py,sha256=bkLlTd8XfJK6_JDl0eXUHfE_NygqgiInSMwJ4YJH01Q,1399
 port_ocean/core/utils/entity_topological_sorter.py,sha256=MDUjM6OuDy4Xj68o-7InNN0w1jqjxeDfeY8U02vySNI,3081
@@ -171,7 +170,7 @@ port_ocean/tests/core/conftest.py,sha256=0Oql7R1iTbjPyNdUoO6M21IKknLwnCIgDRz2JQ7
 port_ocean/tests/core/defaults/test_common.py,sha256=sR7RqB3ZYV6Xn6NIg-c8k5K6JcGsYZ2SCe_PYX5vLYM,5560
 port_ocean/tests/core/event_listener/test_kafka.py,sha256=PH90qk2fvdrQOSZD2QrvkGy8w_WoYb_KHGnqJ6PLHAo,2681
 port_ocean/tests/core/handlers/entities_state_applier/test_applier.py,sha256=7XWgwUB9uVYRov4VbIz1A-7n2YLbHTTYT-4rKJxjB0A,10711
-port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py,sha256=JcDyC7bI4KqIcPpS2S3Hju0mNRfKIoPHdgKVCRBey_E,14105
+port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py,sha256=TjSj8ssIqH23VJlO5PGovbudCqDbuE2-54iNQsD9K-I,14099
 port_ocean/tests/core/handlers/mixins/test_live_events.py,sha256=Sbv9IZAGQoZDhf27xDjMMVYxUSie9mHltDtxLSqckmM,12548
 port_ocean/tests/core/handlers/mixins/test_sync_raw.py,sha256=-Jd2rUG63fZM8LuyKtCp1tt4WEqO2m5woESjs1c91sU,44428
 port_ocean/tests/core/handlers/port_app_config/test_api.py,sha256=eJZ6SuFBLz71y4ca3DNqKag6d6HUjNJS0aqQPwiLMTI,1999
@@ -209,8 +208,8 @@ port_ocean/utils/repeat.py,sha256=U2OeCkHPWXmRTVoPV-VcJRlQhcYqPWI5NfmPlb1JIbc,32
 port_ocean/utils/signal.py,sha256=J1sI-e_32VHP_VUa5bskLMFoJjJOAk5isrnewKDikUI,2125
 port_ocean/utils/time.py,sha256=pufAOH5ZQI7gXvOvJoQXZXZJV-Dqktoj9Qp9eiRwmJ4,1939
 port_ocean/version.py,sha256=UsuJdvdQlazzKGD3Hd5-U7N69STh8Dq9ggJzQFnu9fU,177
-port_ocean-0.28.8.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-port_ocean-0.28.8.dist-info/METADATA,sha256=a0LUIWsaY3WN8PFHnIMqjp5XkyLlqo7_SH03tIdLug4,7015
-port_ocean-0.28.8.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-port_ocean-0.28.8.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
-port_ocean-0.28.8.dist-info/RECORD,,
+port_ocean-0.28.9.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+port_ocean-0.28.9.dist-info/METADATA,sha256=gd8ZvFlVK5p1ypLtnTiCIJwvdoMtXm4Tli18ebscTjs,7015
+port_ocean-0.28.9.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+port_ocean-0.28.9.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
+port_ocean-0.28.9.dist-info/RECORD,,

port_ocean/core/handlers/entity_processor/jq_input_evaluator.py

@@ -1,69 +0,0 @@
-import re
-from enum import Enum
-
-
-class InputEvaluationResult(Enum):
-    NONE = 1
-    SINGLE = 2
-    ALL = 3
-
-
-# Conservative allowlist: truly nullary jq expressions
-_ALLOWLIST_PATTERNS = [
-    r"^\s*null\s*$",  # null
-    r"^\s*true\s*$",  # true
-    r"^\s*false\s*$",  # false
-    r"^\s*-?\d+(\.\d+)?\s*$",  # number literal
-    r'^\s*".*"\s*$',  # string literal (simple heuristic)
-    r"^\s*\[.*\]\s*$",  # array literal (includes [])
-    r"^\s*\{.*\}\s*$",  # object literal (includes {})
-    r"^\s*range\s*\(.*\)\s*$",  # range(...)
-    r"^\s*empty\s*$",  # empty
-]
-
-# Functions/filters that (even without ".") still require/assume input
-_INPUT_DEPENDENT_FUNCS = r"""
-\b(
-    map|select|reverse|sort|sort_by|unique|unique_by|group_by|flatten|transpose|
-    split|explode|join|add|length|has|in|index|indices|contains|
-    paths|leaf_paths|keys|keys_unsorted|values|to_entries|with_entries|from_entries|
-    del|delpaths|walk|reduce|foreach|input|inputs|limit|first|last|nth|
-    while|until|recurse|recurse_down|bsearch|combinations|permutations
-)\b
-"""
-
-_INPUT_DEPENDENT_RE = re.compile(_INPUT_DEPENDENT_FUNCS, re.VERBOSE)
-
-
-def should_shortcut_no_input(selector_query: str) -> bool:
-    """
-    Returns True if the jq expression can be executed without providing any JSON input.
-    Conservative: requires NO '.' and must match a known nullary-safe pattern.
-    """
-    if "." in selector_query:
-        return False  # explicit JSON reference -> needs input
-
-    # If it contains any known input-dependent functions, don't shortcut
-    if _INPUT_DEPENDENT_RE.search(selector_query):
-        return False
-
-    # Allow only if it matches one of the nullary-safe patterns
-    for pat in _ALLOWLIST_PATTERNS:
-        if re.match(pat, selector_query):
-            return True
-
-    return False
-
-
-def evaluate_input(
-    selector_query: str, single_item_key: str | None = None
-) -> InputEvaluationResult:
-    """
-    Returns the input evaluation result for the jq expression.
-    Conservative: requires NO '.' and must match a known nullary-safe pattern.
-    """
-    if should_shortcut_no_input(selector_query):
-        return InputEvaluationResult.NONE
-    if single_item_key and single_item_key in selector_query:
-        return InputEvaluationResult.SINGLE
-    return InputEvaluationResult.ALL