port-ocean 0.28.11__py3-none-any.whl → 0.28.14__py3-none-any.whl

port_ocean/core/handlers/entity_processor/jq_input_evaluator.py

@@ -0,0 +1,137 @@
+import re
+from enum import Enum
+
+# This file is used to classify the input that a jq expression to run on.
+# It is used to determine if the jq expression can be executed without providing any JSON input (const expressions)
+# or on a single item (in items to parse situation)
+# or on all the data
+
+
+class InputClassifyingResult(Enum):
+    NONE = 1
+    SINGLE = 2
+    ALL = 3
+
+
+# Functions/filters that (even without ".") still require/assume input
+_INPUT_DEPENDENT_FUNCS = r"""
+\b(
+    map|select|reverse|sort|sort_by|unique|unique_by|group_by|flatten|transpose|
+    split|explode|join|add|length|has|in|index|indices|contains|
+    paths|leaf_paths|keys|keys_unsorted|values|to_entries|with_entries|from_entries|
+    del|delpaths|walk|reduce|foreach|input|inputs|limit|first|last|nth|
+    while|until|recurse|recurse_down|bsearch|combinations|permutations
+)\b
+"""
+
+_INPUT_DEPENDENT_RE = re.compile(_INPUT_DEPENDENT_FUNCS, re.VERBOSE)
+
+
+# String literal handling (jq uses double quotes for strings)
+_STRING_LITERAL_RE = re.compile(r'"(?:\\.|[^"\\])*"')
+_STRING_ONLY_RE = re.compile(r'^\s*"(?:\\.|[^"\\])*"\s*$')
+_NUMBER_ONLY_RE = re.compile(r"^\s*-?\d+(\.\d+)?\s*$")
+
+
+def _mask_strings(expr: str) -> str:
+    """
+    Replace string literals with 'S' strings so '.' inside quotes don't count.
+    Example:
+        - '"this is a string"' ---> 'S'
+        - '"sting" + .field'. ---> 'S + .field'
+    """
+    return _STRING_LITERAL_RE.sub("S", expr)
+
+
+def _mask_numbers(expr: str) -> str:
+    """
+    Replace number literals with 'N' so decimal points in numbers don't count as input references.
+    Example:
+        - '3.14' ---> 'N'
+        - '42 + 3.14' ---> 'N + N'
+    """
+    # Pattern to match numbers (integers and decimals, with optional sign)
+    number_pattern = re.compile(r"[-+]?\d+(?:\.\d+)?")
+    return number_pattern.sub("N", expr)
+
+
+def can_expression_run_with_no_input(selector_query: str) -> bool:
+    """
+    Returns True if the jq expression can be executed without providing any JSON input.
+    Rules:
+      - Whitespace-only => No Input Required
+      - A pure string literal => No Input Required (even if it contains '.')
+      - After masking strings, if it contains '.' => Input Required
+      - Disallow known input-dependent functions (functions that require input)
+      - After masking strings, if it contains only operators and numbers and 'S' => No Input Required
+      - Allow null/true/false/number/range/empty, and array/object literals that
+        don't reference input (no '.' after masking strings) => No Input Required
+    Example:
+        - blueprint: '"newRelicService"' in mapping, selector_query param would be '"newRelicService"' => No Input Required
+    """
+    s = selector_query.strip()
+    if s == "":
+        return True  # whitespace-only
+
+    # Pure string literal is nullary
+    if _STRING_ONLY_RE.match(s):
+        return True
+
+    # First mask strings, then mask numbers to prevent decimal points in numbers from being treated as input references
+    masked = _mask_strings(s).strip()
+    masked = _mask_numbers(masked).strip()
+
+    # If it contains any known input-dependent functions, don't shortcut
+    if _INPUT_DEPENDENT_RE.search(masked):
+        return False
+
+    # If it contains only operators and 'S'/'N', it can be executed with no input
+    # Example:
+    # - '"abc" + "def"' ---> 'S + S' => No Input Required
+    # - '3.14 + 2.5' ---> 'N + N' => No Input Required
+    # if re.fullmatch(
+    #     r"(?:S|N)(?:\s*[+\-*/]\s*(?:S|N))*",
+    #     masked,
+    # ):
+    #     return True
+
+    if "." not in masked:
+        return True
+
+    return False
+
+
+def _can_expression_run_on_single_item(expr: str, key: str) -> bool:
+    """
+    Detect `.key` outside of quotes, as a standalone path segment beginning
+    after a non-word boundary (start, space, |, (, [, {, , or :) and not part
+    of `.something.key`.
+    assuming key = 'item'
+    Examples:
+        - .item.yaeli => true
+        - map(.item.yaeli) => true
+        - .body.item => false
+    """
+    if not key:
+        return False
+
+    masked = _mask_strings(expr)
+    masked = _mask_numbers(masked)
+    pattern = re.compile(rf"(?<![A-Za-z0-9_])\.{re.escape(key)}(?![A-Za-z0-9_])")
+    return bool(pattern.search(masked))
+
+
+def classify_input(
+    selector_query: str, single_item_key: str | None = None
+) -> InputClassifyingResult:
+    """
+    Returns the input evaluation result for the jq expression.
+    Conservative: requires NO '.' and must match a known nullary-safe pattern.
+    """
+    if can_expression_run_with_no_input(selector_query):
+        return InputClassifyingResult.NONE
+    if single_item_key and _can_expression_run_on_single_item(
+        selector_query, single_item_key
+    ):
+        return InputClassifyingResult.SINGLE
+    return InputClassifyingResult.ALL

port_ocean/core/handlers/port_app_config/models.py

@@ -41,7 +41,7 @@ class MappingsConfig(BaseModel):
 class PortResourceConfig(BaseModel):
     entity: MappingsConfig
     items_to_parse: str | None = Field(alias="itemsToParse")
-    items_to_parse_name: str | None = Field(alias="itemsToParseName", default="item")
+    items_to_parse_name: str = Field(alias="itemsToParseName", default="item")
 
 
 class Selector(BaseModel):

port_ocean/core/integrations/mixins/sync_raw.py

@@ -117,7 +117,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                 logger.info(
                     f"Found async generator function for {resource_config.kind} name: {task.__qualname__}"
                 )
-                results.append(resync_generator_wrapper(task, resource_config.kind,resource_config.port.items_to_parse))
+                results.append(resync_generator_wrapper(task, resource_config.kind, resource_config.port.items_to_parse_name, resource_config.port.items_to_parse))
             else:
                 logger.info(
                     f"Found sync function for {resource_config.kind} name: {task.__qualname__}"

port_ocean/core/integrations/mixins/utils.py

@@ -2,12 +2,11 @@ from contextlib import contextmanager
 from typing import Awaitable, Generator, Callable, cast
 
 from loguru import logger
-
 import asyncio
 import multiprocessing
-
+import re
+import json
 from port_ocean.core.handlers.entity_processor.jq_entity_processor import JQEntityProcessor
-from port_ocean.core.handlers.port_app_config.models import ResourceConfig
 from port_ocean.core.ocean_types import (
     ASYNC_GENERATOR_RESYNC_TYPE,
     RAW_RESULT,
@@ -20,11 +19,66 @@ from port_ocean.exceptions.core import (
     OceanAbortException,
     KindNotImplementedException,
 )
-
+import os
 from port_ocean.utils.async_http import _http_client
 from port_ocean.clients.port.utils import _http_client as _port_http_client
 from port_ocean.helpers.metric.metric import MetricType, MetricPhase
 from port_ocean.context.ocean import ocean
+import subprocess
+import tempfile
+import stat
+import ijson
+from typing import Any, AsyncGenerator
+
+def _process_path_type_items(
+    result: RAW_RESULT, items_to_parse: str | None = None
+) -> RAW_RESULT:
+    """
+    Process items in the result array to check for "__type": "path" fields.
+    If found, read the file contents and load them into a "content" field.
+    Skip processing if we're on the items_to_parse branch.
+    """
+    if not isinstance(result, list):
+        return result
+
+    # Skip processing if we're on the items_to_parse branch
+    if items_to_parse:
+        return result
+
+    processed_result = []
+    for item in result:
+        if isinstance(item, dict) and item.get("__type") == "path":
+            try:
+                # Read the file content and parse as JSON
+                file_path = item.get("file", {}).get("content", {}).get("path")
+                if file_path and os.path.exists(file_path):
+                    with open(file_path, "r") as f:
+                        content = json.loads(f.read())
+                    # Create a copy of the item with the content field
+                    processed_item = item.copy()
+                    processed_item["content"] = content
+                    processed_result.append(processed_item)
+                else:
+                    # If file doesn't exist, keep the original item
+                    processed_result.append(item)
+            except (json.JSONDecodeError, IOError, OSError) as e:
+                if isinstance(item, dict) and item.get("file") is not None:
+                    content = item["file"].get("content") if isinstance(item["file"].get("content"), dict) else {}
+                    data_path = content.get("path", None)
+                    logger.warning(
+                        f"Failed to read or parse file content for path {data_path}: {e}"
+                    )
+                else:
+                    logger.warning(
+                        f"Failed to read or parse file content for unknown path: {e}. item: {json.dumps(item)}"
+                    )
+                # Keep the original item if there's an error
+                processed_result.append(item)
+        else:
+            # Keep non-path type items as is
+            processed_result.append(item)
+
+    return processed_result
 
 @contextmanager
 def resync_error_handling() -> Generator[None, None, None]:
@@ -47,11 +101,12 @@ async def resync_function_wrapper(
 ) -> RAW_RESULT:
     with resync_error_handling():
         results = await fn(kind)
-        return validate_result(results)
+        validated_results = validate_result(results)
+        return _process_path_type_items(validated_results)
 
 
 async def resync_generator_wrapper(
-    fn: Callable[[str], ASYNC_GENERATOR_RESYNC_TYPE], kind: str, items_to_parse: str | None = None
+    fn: Callable[[str], ASYNC_GENERATOR_RESYNC_TYPE], kind: str, items_to_parse_name: str, items_to_parse: str | None = None
 ) -> ASYNC_GENERATOR_RESYNC_TYPE:
     generator = fn(kind)
     errors = []
@@ -61,27 +116,23 @@ async def resync_generator_wrapper(
                 with resync_error_handling():
                     result = await anext(generator)
                     if not ocean.config.yield_items_to_parse:
-                        yield validate_result(result)
+                        validated_result = validate_result(result)
+                        processed_result = _process_path_type_items(validated_result)
+                        yield processed_result
                     else:
-                        batch_size = ocean.config.yield_items_to_parse_batch_size
                         if items_to_parse:
                             for data in result:
-                                items = await cast(JQEntityProcessor, ocean.app.integration.entity_processor)._search(data, items_to_parse)
-                                if not isinstance(items, list):
-                                    logger.warning(
-                                        f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items)}."
-                                        f" Skipping..."
-                                    )
-                                    yield []
-                                raw_data = [{"item": item, **data} for item in items]
-                                while True:
-                                    raw_data_batch = raw_data[:batch_size]
-                                    yield raw_data_batch
-                                    raw_data = raw_data[batch_size:]
-                                    if len(raw_data) == 0:
-                                        break
+                                data_path: str | None = None
+                                if isinstance(data, dict) and data.get("file") is not None:
+                                    content = data["file"].get("content") if isinstance(data["file"].get("content"), dict) else {}
+                                    data_path = content.get("path", None)
+                                bulks = get_items_to_parse_bulks(data, data_path, items_to_parse, items_to_parse_name, data.get("__base_jq", ".file.content"))
+                                async for bulk in bulks:
+                                    yield bulk
                         else:
-                            yield validate_result(result)
+                            validated_result = validate_result(result)
+                            processed_result = _process_path_type_items(validated_result, items_to_parse)
+                            yield processed_result
             except OceanAbortException as error:
                 errors.append(error)
                 ocean.metrics.inc_metric(
@@ -101,6 +152,104 @@ def is_resource_supported(
 ) -> bool:
     return bool(resync_event_mapping[kind] or resync_event_mapping[None])
 
+def _validate_jq_expression(expression: str) -> None:
+    """Validate jq expression to prevent command injection."""
+    try:
+        _ = cast(JQEntityProcessor, ocean.app.integration.entity_processor)._compile(expression)
+    except Exception as e:
+        raise ValueError(f"Invalid jq expression: {e}") from e
+    # Basic validation - reject expressions that could be dangerous
+    # Check for dangerous patterns (include, import, module)
+    dangerous_patterns = ['include', 'import', 'module', 'env', 'debug']
+    for pattern in dangerous_patterns:
+        # Use word boundary regex to match only complete words, not substrings
+        if re.search(rf'\b{re.escape(pattern)}\b', expression):
+            raise ValueError(f"Potentially dangerous pattern '{pattern}' found in jq expression")
+
+    # Special handling for 'env' - block environment variable access
+    if re.search(r'(?<!\w)\$ENV(?:\.)?', expression):
+        raise ValueError("Environment variable access '$ENV.' found in jq expression")
+    if re.search(r'\benv\.', expression):
+        raise ValueError("Environment variable access 'env.' found in jq expression")
+
+def _create_secure_temp_file(suffix: str = ".json") -> str:
+    """Create a secure temporary file with restricted permissions."""
+    # Create temp directory if it doesn't exist
+    temp_dir = "/tmp/ocean"
+    os.makedirs(temp_dir, exist_ok=True)
+
+    # Create temporary file with secure permissions
+    fd, temp_path = tempfile.mkstemp(suffix=suffix, dir=temp_dir)
+    try:
+        # Set restrictive permissions (owner read/write only)
+        os.chmod(temp_path, stat.S_IRUSR | stat.S_IWUSR)
+        return temp_path
+    finally:
+        os.close(fd)
+
+async def get_items_to_parse_bulks(raw_data: dict[Any, Any], data_path: str, items_to_parse: str, items_to_parse_name: str, base_jq: str) -> AsyncGenerator[list[dict[str, Any]], None]:
+    # Validate inputs to prevent command injection
+    _validate_jq_expression(items_to_parse)
+    items_to_parse = items_to_parse.replace(base_jq, ".") if data_path else items_to_parse
+
+    temp_data_path = None
+    temp_output_path = None
+
+    try:
+        # Create secure temporary files
+        if not data_path:
+            raw_data_serialized = json.dumps(raw_data)
+            temp_data_path = _create_secure_temp_file("_input.json")
+            with open(temp_data_path, "w") as f:
+                f.write(raw_data_serialized)
+            data_path = temp_data_path
+
+        temp_output_path = _create_secure_temp_file("_parsed.json")
+
+        delete_target = items_to_parse.split('|', 1)[0].strip() if not items_to_parse.startswith('map(') else '.'
+        base_jq_object_string = await _build_base_jq_object_string(raw_data, base_jq, delete_target)
+
+        # Build jq expression safely
+        jq_expression = f""". as $all
+      | ($all | {items_to_parse}) as $items
+      | $items
+      | map({{{items_to_parse_name}: ., {base_jq_object_string}}})"""
+
+        # Use subprocess with list arguments instead of shell=True
+        jq_args = ["/bin/jq", jq_expression, data_path]
+
+        with open(temp_output_path, "w") as output_file:
+            result = subprocess.run(
+                jq_args,
+                stdout=output_file,
+                stderr=subprocess.PIPE,
+                text=True,
+                check=False  # Don't raise exception, handle errors manually
+            )
+
+        if result.returncode != 0:
+            logger.error(f"Failed to parse items for JQ expression {items_to_parse}, error: {result.stderr}")
+            yield []
+        else:
+            with open(temp_output_path, "r") as f:
+                events_stream = get_events_as_a_stream(f, 'item', ocean.config.yield_items_to_parse_batch_size)
+                for items_bulk in events_stream:
+                    yield items_bulk
+
+    except ValueError as e:
+        logger.error(f"Invalid jq expression: {e}")
+        yield []
+    except Exception as e:
+        logger.error(f"Failed to parse items for JQ expression {items_to_parse}, error: {e}")
+        yield []
+    finally:
+        # Cleanup temporary files
+        for temp_path in [temp_data_path, temp_output_path]:
+            if temp_path and os.path.exists(temp_path):
+                try:
+                    os.remove(temp_path)
+                except OSError as e:
+                    logger.warning(f"Failed to cleanup temporary file {temp_path}: {e}")
 
 def unsupported_kind_response(
     kind: str, available_resync_kinds: list[str]
@@ -108,6 +257,44 @@ def unsupported_kind_response(
     logger.error(f"Kind {kind} is not supported in this integration")
     return [], [KindNotImplementedException(kind, available_resync_kinds)]
 
+async def _build_base_jq_object_string(raw_data: dict[Any, Any], base_jq: str, delete_target: str) -> str:
+    base_jq_object_before_parsing = await cast(JQEntityProcessor, ocean.app.integration.entity_processor)._search(raw_data, f"{base_jq} = {json.dumps("__all")}")
+    base_jq_object_before_parsing_serialized = json.dumps(base_jq_object_before_parsing)
+    base_jq_object_before_parsing_serialized = base_jq_object_before_parsing_serialized[1:-1] if len(base_jq_object_before_parsing_serialized) >= 2 else base_jq_object_before_parsing_serialized
+    base_jq_object_before_parsing_serialized = base_jq_object_before_parsing_serialized.replace("\"__all\"", f"(($all | del({delete_target})) // {{}})")
+    return base_jq_object_before_parsing_serialized
+
+
+def get_events_as_a_stream(
+        stream: Any,
+        target_items: str = "item",
+        max_buffer_size_mb: int = 1
+    ) -> Generator[list[dict[str, Any]], None, None]:
+        events = ijson.sendable_list()
+        coro = ijson.items_coro(events, target_items)
+
+        # Convert MB to bytes for the buffer size
+        buffer_size = max_buffer_size_mb * 1024 * 1024
+
+        # Read chunks from the stream until exhausted
+        while True:
+            chunk = stream.read(buffer_size)
+            if not chunk:  # End of stream
+                break
+
+            # Convert string to bytes if necessary (for text mode files)
+            if isinstance(chunk, str):
+                chunk = chunk.encode('utf-8')
+
+            coro.send(chunk)
+            yield events
+            del events[:]
+        try:
+            coro.close()
+        finally:
+            if events:
+                yield events
+                events[:] = []
 
 class ProcessWrapper(multiprocessing.Process):
     def __init__(self, *args, **kwargs):
@@ -134,3 +321,34 @@ def clear_http_client_context() -> None:
             _port_http_client.pop()
     except (RuntimeError, AttributeError):
         pass
+
+class _AiterReader:
+    """
+    Wraps an iterable of byte chunks (e.g., response.iter_bytes())
+    and exposes a .read(n) method that ijson expects.
+    """
+    def __init__(self, iterable):
+        self._iter = iter(iterable)
+        self._buf = bytearray()
+        self._eof = False
+
+    def read(self, n=-1):
+        # If n < 0, return everything until EOF
+        if n is None or n < 0:
+            chunks = [bytes(self._buf)]
+            self._buf.clear()
+            chunks.extend(self._iter)  # drain the iterator
+            return b"".join(chunks)
+
+        # Fill buffer until we have n bytes or hit EOF
+        while len(self._buf) < n and not self._eof:
+            try:
+                self._buf.extend(next(self._iter))
+            except StopIteration:
+                self._eof = True
+                break
+
+        # Serve up to n bytes
+        out = bytes(self._buf[:n])
+        del self._buf[:n]
+        return out