port-ocean 0.30.7__py3-none-any.whl → 0.31.0__py3-none-any.whl

port_ocean/core/integrations/mixins/utils.py

@@ -1,22 +1,14 @@
 import asyncio
-import json
 import multiprocessing
-import os
 import re
-import shutil
-import stat
-import subprocess
-import tempfile
 from contextlib import contextmanager
-from typing import Any, AsyncGenerator, Awaitable, Callable, Generator, cast
-import copy
+from typing import Any, AsyncGenerator, Awaitable, Callable, Generator
 
 import ijson
 from loguru import logger
 
 from port_ocean.clients.port.utils import _http_client as _port_http_client
 from port_ocean.context.ocean import ocean
-from port_ocean.core.handlers.entity_processor.jq_entity_processor import JQEntityProcessor
 from port_ocean.core.ocean_types import (
     ASYNC_GENERATOR_RESYNC_TYPE,
     RAW_RESULT,
@@ -32,122 +24,6 @@ from port_ocean.exceptions.core import (
 from port_ocean.helpers.metric.metric import MetricType, MetricPhase
 from port_ocean.utils.async_http import _http_client
 
-def _process_path_type_items(
-    result: RAW_RESULT, items_to_parse: str | None = None
-) -> RAW_RESULT:
-    """
-    Process items in the result array to check for "__type": "path" fields.
-    If found, read the file contents and load them into a "content" field.
-    Skip processing if we're on the items_to_parse branch.
-    """
-    if not isinstance(result, list):
-        return result
-
-    # Skip processing if we're on the items_to_parse branch
-    if items_to_parse:
-        return result
-
-    processed_result = []
-    for item in result:
-        if isinstance(item, dict) and item.get("__type") == "path":
-            try:
-                # Read the file content and parse as JSON
-                file_path = item.get("file", {}).get("content", {}).get("path")
-                if file_path and os.path.exists(file_path):
-                    with open(file_path, "r") as f:
-                        content = json.loads(f.read())
-                    # Create a copy of the item with the content field
-                    processed_item = item.copy()
-                    processed_item["file"]["content"] = content
-                    processed_result.append(processed_item)
-                else:
-                    # If file doesn't exist, keep the original item
-                    processed_result.append(item)
-            except (json.JSONDecodeError, IOError, OSError) as e:
-                if isinstance(item, dict) and item.get("file") is not None:
-                    content = item["file"].get("content") if isinstance(item["file"].get("content"), dict) else {}
-                    data_path = content.get("path", None)
-                    logger.warning(
-                        f"Failed to read or parse file content for path {data_path}: {e}"
-                    )
-                else:
-                    logger.warning(
-                        f"Failed to read or parse file content for unknown path: {e}. item: {json.dumps(item)}"
-                    )
-                # Keep the original item if there's an error
-                processed_result.append(item)
-        else:
-            # Keep non-path type items as is
-            processed_result.append(item)
-
-    return processed_result
-
-@contextmanager
-def resync_error_handling() -> Generator[None, None, None]:
-    try:
-        yield
-    except RawObjectValidationException as error:
-        err_msg = f"Failed to validate raw data for returned data from resync function, error: {error}"
-        logger.exception(err_msg)
-        raise OceanAbortException(err_msg) from error
-    except StopAsyncIteration:
-        raise
-    except Exception as error:
-        err_msg = f"Failed to execute resync function, error: {error}"
-        logger.exception(err_msg)
-        raise OceanAbortException(err_msg) from error
-
-
-async def resync_function_wrapper(
-    fn: Callable[[str], Awaitable[RAW_RESULT]], kind: str, items_to_parse: str | None = None
-) -> RAW_RESULT:
-    with resync_error_handling():
-        results = await fn(kind)
-        validated_results = validate_result(results)
-        return _process_path_type_items(validated_results, items_to_parse)
-
-
-async def resync_generator_wrapper(
-    fn: Callable[[str], ASYNC_GENERATOR_RESYNC_TYPE], kind: str, items_to_parse_name: str, items_to_parse: str | None = None
-) -> ASYNC_GENERATOR_RESYNC_TYPE:
-    generator = fn(kind)
-    errors = []
-    try:
-        while True:
-            try:
-                with resync_error_handling():
-                    result = await anext(generator)
-                    if not ocean.config.yield_items_to_parse:
-                        validated_result = validate_result(result)
-                        processed_result = _process_path_type_items(validated_result,items_to_parse)
-                        yield processed_result
-                    else:
-                        if items_to_parse:
-                            for data in result:
-                                data_path: str | None = None
-                                if isinstance(data, dict) and data.get("__type") == "path":
-                                    content = data.get("file", {}).get("content") if isinstance(data["file"].get("content"), dict) else {}
-                                    data_path = content.get("path", None)
-                                bulks = get_items_to_parse_bulks(data, data_path, items_to_parse, items_to_parse_name, data.get("__base_jq", ".file.content"))
-                                async for bulk in bulks:
-                                    yield bulk
-                        else:
-                            validated_result = validate_result(result)
-                            processed_result = _process_path_type_items(validated_result, items_to_parse)
-                            yield processed_result
-            except OceanAbortException as error:
-                errors.append(error)
-                ocean.metrics.inc_metric(
-                    name=MetricType.OBJECT_COUNT_NAME,
-                    labels=[ocean.metrics.current_resource_kind(), MetricPhase.EXTRACT , MetricPhase.ExtractResult.FAILED],
-                    value=1
-                )
-    except StopAsyncIteration:
-        if errors:
-            raise ExceptionGroup(
-                "At least one of the resync generator iterations failed", errors
-            )
-
 def extract_jq_deletion_path_revised(jq_expression: str) -> str | None:
     """
     Revised function to extract a simple path suitable for del() by analyzing pipe segments.
@@ -204,113 +80,90 @@ def extract_jq_deletion_path_revised(jq_expression: str) -> str | None:
     # Default case: No suitable path found after checking all segments
     return None
 
-
-def is_resource_supported(
-    kind: str, resync_event_mapping: dict[str | None, list[RESYNC_EVENT_LISTENER]]
-) -> bool:
-    return bool(resync_event_mapping[kind] or resync_event_mapping[None])
-
-def _validate_jq_expression(expression: str) -> None:
-    """Validate jq expression to prevent command injection."""
+@contextmanager
+def resync_error_handling() -> Generator[None, None, None]:
     try:
-        _ = cast(JQEntityProcessor, ocean.app.integration.entity_processor)._compile(expression)
-    except Exception as e:
-        raise ValueError(f"Invalid jq expression: {e}") from e
-    # Basic validation - reject expressions that could be dangerous
-    # Check for dangerous patterns (include, import, module)
-    dangerous_patterns = ['include', 'import', 'module', 'env', 'debug']
-    for pattern in dangerous_patterns:
-        # Use word boundary regex to match only complete words, not substrings
-        if re.search(rf'\b{re.escape(pattern)}\b', expression):
-            raise ValueError(f"Potentially dangerous pattern '{pattern}' found in jq expression")
-
-    # Special handling for 'env' - block environment variable access
-    if re.search(r'(?<!\w)\$ENV(?:\.)?', expression):
-        raise ValueError("Environment variable access '$ENV.' found in jq expression")
-    if re.search(r'\benv\.', expression):
-        raise ValueError("Environment variable access 'env.' found in jq expression")
+        yield
+    except RawObjectValidationException as error:
+        err_msg = f"Failed to validate raw data for returned data from resync function, error: {error}"
+        logger.exception(err_msg)
+        raise OceanAbortException(err_msg) from error
+    except StopAsyncIteration:
+        raise
+    except Exception as error:
+        err_msg = f"Failed to execute resync function, error: {error}"
+        logger.exception(err_msg)
+        raise OceanAbortException(err_msg) from error
 
-def _create_secure_temp_file(suffix: str = ".json") -> str:
-    """Create a secure temporary file with restricted permissions."""
-    # Create temp directory if it doesn't exist
-    temp_dir = "/tmp/ocean"
-    os.makedirs(temp_dir, exist_ok=True)
 
-    # Create temporary file with secure permissions
-    fd, temp_path = tempfile.mkstemp(suffix=suffix, dir=temp_dir)
-    try:
-        # Set restrictive permissions (owner read/write only)
-        os.chmod(temp_path, stat.S_IRUSR | stat.S_IWUSR)
-        return temp_path
-    finally:
-        os.close(fd)
+async def resync_function_wrapper(
+    fn: Callable[[str], Awaitable[RAW_RESULT]], kind: str, items_to_parse: str | None = None
+) -> RAW_RESULT:
+    with resync_error_handling():
+        results = await fn(kind)
+        return validate_result(results)
 
-async def get_items_to_parse_bulks(raw_data: dict[Any, Any], data_path: str, items_to_parse: str, items_to_parse_name: str, base_jq: str) -> AsyncGenerator[list[dict[str, Any]], None]:
-    # Validate inputs to prevent command injection
-    _validate_jq_expression(items_to_parse)
-    items_to_parse = items_to_parse.replace(base_jq, ".") if data_path else items_to_parse
+async def handle_items_to_parse(result: RAW_RESULT, items_to_parse_name: str, items_to_parse: str | None = None) -> AsyncGenerator[list[dict[str, Any]], None]:
+    delete_target = extract_jq_deletion_path_revised(items_to_parse) or '.'
+    jq_expression = f". | del({delete_target})"
+    batch_size = ocean.config.yield_items_to_parse_batch_size
 
-    temp_data_path = None
-    temp_output_path = None
+    for item in result:
+        lean_item = await ocean.app.integration.entity_processor._search(item, jq_expression)
+        items_to_parse_data = await ocean.app.integration.entity_processor._search(item, items_to_parse)
+        if not isinstance(items_to_parse_data, list):
+            logger.warning(
+                f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items_to_parse_data)}."
+                f" Skipping..."
+            )
+            continue
+        batch = []
+        while len(items_to_parse_data) > 0:
+            if (len(batch) >= batch_size):
+                yield batch
+                batch = []
+            merged_item = {**lean_item}
+            merged_item[items_to_parse_name] = items_to_parse_data.pop(0)
+            batch.append(merged_item)
+        if len(batch) > 0:
+            yield batch
 
+async def resync_generator_wrapper(
+    fn: Callable[[str], ASYNC_GENERATOR_RESYNC_TYPE], kind: str, items_to_parse_name: str, items_to_parse: str | None = None
+) -> ASYNC_GENERATOR_RESYNC_TYPE:
+    generator = fn(kind)
+    errors = []
     try:
-        is_path_type = True
-        # Create secure temporary files
-        if not data_path:
-            raw_data_serialized = json.dumps(raw_data)
-            temp_data_path = _create_secure_temp_file("_input.json")
-            with open(temp_data_path, "w") as f:
-                f.write(raw_data_serialized)
-            data_path = temp_data_path
-            is_path_type = False
-
-        temp_output_path = _create_secure_temp_file("_parsed.json")
-
-        delete_target = extract_jq_deletion_path_revised(items_to_parse) or '.'
-
-        # Build jq expression safely
-        jq_expression = f""". as $all
-      | ($all | {items_to_parse}) as $items
-      | $items
-      | {_build_mapping_jq_expression(items_to_parse_name, base_jq, delete_target, is_path_type)}"""
+        while True:
+            try:
+                with resync_error_handling():
+                    result = validate_result(await anext(generator))
 
-        # Use subprocess with list arguments instead of shell=True
+                    if items_to_parse:
+                        async for batch in handle_items_to_parse(result, items_to_parse_name, items_to_parse):
+                            yield batch
+                    else:
+                        yield result
 
-        jq_path = shutil.which("jq") or "/bin/jq"
-        jq_args = [jq_path, jq_expression, data_path]
 
-        with open(temp_output_path, "w") as output_file:
-            result = subprocess.run(
-                jq_args,
-                stdout=output_file,
-                stderr=subprocess.PIPE,
-                text=True,
-                check=False  # Don't raise exception, handle errors manually
+            except OceanAbortException as error:
+                errors.append(error)
+                ocean.metrics.inc_metric(
+                    name=MetricType.OBJECT_COUNT_NAME,
+                    labels=[ocean.metrics.current_resource_kind(), MetricPhase.EXTRACT , MetricPhase.ExtractResult.FAILED],
+                    value=1
+                )
+    except StopAsyncIteration:
+        if errors:
+            raise ExceptionGroup(
+                "At least one of the resync generator iterations failed", errors
             )
 
-        if result.returncode != 0:
-            logger.error(f"Failed to parse items for JQ expression {items_to_parse}, error: {result.stderr}")
-            yield []
-        else:
-            with open(temp_output_path, "r") as f:
-                events_stream = get_events_as_a_stream(f, 'item', ocean.config.yield_items_to_parse_batch_size)
-                for items_bulk in events_stream:
-                    yield items_bulk if not is_path_type else [merge_raw_data_to_item(item, raw_data) for item in items_bulk]
 
-    except ValueError as e:
-        logger.error(f"Invalid jq expression: {e}")
-        yield []
-    except Exception as e:
-        logger.error(f"Failed to parse items for JQ expression {items_to_parse}, error: {e}")
-        yield []
-    finally:
-        # Cleanup temporary files
-        for temp_path in [temp_data_path, temp_output_path]:
-            if temp_path and os.path.exists(temp_path):
-                try:
-                    os.remove(temp_path)
-                except OSError as e:
-                    logger.warning(f"Failed to cleanup temporary file {temp_path}: {e}")
+def is_resource_supported(
+    kind: str, resync_event_mapping: dict[str | None, list[RESYNC_EVENT_LISTENER]]
+) -> bool:
+    return bool(resync_event_mapping[kind] or resync_event_mapping[None])
 
 def unsupported_kind_response(
     kind: str, available_resync_kinds: list[str]
@@ -318,89 +171,6 @@ def unsupported_kind_response(
     logger.error(f"Kind {kind} is not supported in this integration")
     return [], [KindNotImplementedException(kind, available_resync_kinds)]
 
-def _build_mapping_jq_expression(items_to_parse_name: str, base_jq: str, delete_target: str, is_path_type: bool = False) -> str:
-    if is_path_type:
-        return f"map({{{items_to_parse_name}: . }} | {base_jq} = (($all | del({delete_target})) // {{}}))"
-    return f"map(($all | del({delete_target})) + {{{items_to_parse_name}: . }})"
-
-def merge_raw_data_to_item(item: dict[str, Any], raw_data: dict[str, Any]) -> dict[str, Any]:
-    return recursive_dict_merge(raw_data, item)
-
-def recursive_dict_merge(d1: dict[Any, Any], d2: dict[Any, Any]) -> dict[Any, Any]:
-    """
-    Recursively merges dict d2 into dict d1.
-
-    If a key exists in both dictionaries:
-    1. If the value in d2 is an empty dictionary (e.g., {}), it overwrites the value in d1.
-    2. If both values are non-empty dictionaries, they are merged recursively.
-    3. Otherwise, the value from d2 overwrites the value from d1.
-
-    The original dictionaries are not modified (d1 is copied).
-
-    Args:
-        d1: The base dictionary (will be copied and modified).
-        d2: The dictionary to merge into d1.
-
-    Returns:
-        The merged dictionary.
-    """
-    # Start with a copy of d1 to ensure d1 is not mutated
-    merged_dict = copy.deepcopy(d1)
-
-    for key, value in d2.items():
-        # Condition to trigger recursive deep merge:
-        # 1. Key exists in merged_dict
-        # 2. Both values are dictionaries
-        # 3. The value from d2 is NOT an empty dictionary ({}).
-        #    If d2's value is {}, we treat it as an explicit instruction to overwrite/clear.
-        is_deep_merge = (
-            key in merged_dict and
-            isinstance(merged_dict[key], dict) and
-            isinstance(value, dict) and
-            value != {}
-        )
-
-        if is_deep_merge:
-            # If both are dictionaries and d2 is not empty, recurse
-            merged_dict[key] = recursive_dict_merge(merged_dict[key], value)
-        else:
-            # Otherwise (new key, non-dict value, or explicit {} overwrite),
-            # overwrite the value from d1 with the value from d2
-            merged_dict[key] = value
-
-    return merged_dict
-
-def get_events_as_a_stream(
-        stream: Any,
-        target_items: str = "item",
-        max_buffer_size_mb: int = 1
-    ) -> Generator[list[dict[str, Any]], None, None]:
-        events = ijson.sendable_list()
-        coro = ijson.items_coro(events, target_items, use_float=True)
-
-        # Convert MB to bytes for the buffer size
-        buffer_size = max_buffer_size_mb * 1024 * 1024
-
-        # Read chunks from the stream until exhausted
-        while True:
-            chunk = stream.read(buffer_size)
-            if not chunk:  # End of stream
-                break
-
-            # Convert string to bytes if necessary (for text mode files)
-            if isinstance(chunk, str):
-                chunk = chunk.encode('utf-8')
-
-            coro.send(chunk)
-            yield events
-            del events[:]
-        try:
-            coro.close()
-        finally:
-            if events:
-                yield events
-                events[:] = []
-
 class ProcessWrapper(multiprocessing.Process):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)