port-ocean 0.28.5__py3-none-any.whl → 0.29.0__py3-none-any.whl

port_ocean/core/handlers/entity_processor/jq_input_evaluator.py

@@ -0,0 +1,137 @@
+import re
+from enum import Enum
+
+# This file is used to classify the input that a jq expression to run on.
+# It is used to determine if the jq expression can be executed without providing any JSON input (const expressions)
+# or on a single item (in items to parse situation)
+# or on all the data
+
+
+class InputClassifyingResult(Enum):
+    NONE = 1
+    SINGLE = 2
+    ALL = 3
+
+
+# Functions/filters that (even without ".") still require/assume input
+_INPUT_DEPENDENT_FUNCS = r"""
+\b(
+    map|select|reverse|sort|sort_by|unique|unique_by|group_by|flatten|transpose|
+    split|explode|join|add|length|has|in|index|indices|contains|
+    paths|leaf_paths|keys|keys_unsorted|values|to_entries|with_entries|from_entries|
+    del|delpaths|walk|reduce|foreach|input|inputs|limit|first|last|nth|
+    while|until|recurse|recurse_down|bsearch|combinations|permutations
+)\b
+"""
+
+_INPUT_DEPENDENT_RE = re.compile(_INPUT_DEPENDENT_FUNCS, re.VERBOSE)
+
+
+# String literal handling (jq uses double quotes for strings)
+_STRING_LITERAL_RE = re.compile(r'"(?:\\.|[^"\\])*"')
+_STRING_ONLY_RE = re.compile(r'^\s*"(?:\\.|[^"\\])*"\s*$')
+_NUMBER_ONLY_RE = re.compile(r"^\s*-?\d+(\.\d+)?\s*$")
+
+
+def _mask_strings(expr: str) -> str:
+    """
+    Replace string literals with 'S' strings so '.' inside quotes don't count.
+    Example:
+        - '"this is a string"' ---> 'S'
+        - '"sting" + .field'. ---> 'S + .field'
+    """
+    return _STRING_LITERAL_RE.sub("S", expr)
+
+
+def _mask_numbers(expr: str) -> str:
+    """
+    Replace number literals with 'N' so decimal points in numbers don't count as input references.
+    Example:
+        - '3.14' ---> 'N'
+        - '42 + 3.14' ---> 'N + N'
+    """
+    # Pattern to match numbers (integers and decimals, with optional sign)
+    number_pattern = re.compile(r"[-+]?\d+(?:\.\d+)?")
+    return number_pattern.sub("N", expr)
+
+
+def can_expression_run_with_no_input(selector_query: str) -> bool:
+    """
+    Returns True if the jq expression can be executed without providing any JSON input.
+    Rules:
+      - Whitespace-only => No Input Required
+      - A pure string literal => No Input Required (even if it contains '.')
+      - After masking strings, if it contains '.' => Input Required
+      - Disallow known input-dependent functions (functions that require input)
+      - After masking strings, if it contains only operators and numbers and 'S' => No Input Required
+      - Allow null/true/false/number/range/empty, and array/object literals that
+        don't reference input (no '.' after masking strings) => No Input Required
+    Example:
+        - blueprint: '"newRelicService"' in mapping, selector_query param would be '"newRelicService"' => No Input Required
+    """
+    s = selector_query.strip()
+    if s == "":
+        return True  # whitespace-only
+
+    # Pure string literal is nullary
+    if _STRING_ONLY_RE.match(s):
+        return True
+
+    # First mask strings, then mask numbers to prevent decimal points in numbers from being treated as input references
+    masked = _mask_strings(s).strip()
+    masked = _mask_numbers(masked).strip()
+
+    # If it contains any known input-dependent functions, don't shortcut
+    if _INPUT_DEPENDENT_RE.search(masked):
+        return False
+
+    # If it contains only operators and 'S'/'N', it can be executed with no input
+    # Example:
+    # - '"abc" + "def"' ---> 'S + S' => No Input Required
+    # - '3.14 + 2.5' ---> 'N + N' => No Input Required
+    # if re.fullmatch(
+    #     r"(?:S|N)(?:\s*[+\-*/]\s*(?:S|N))*",
+    #     masked,
+    # ):
+    #     return True
+
+    if "." not in masked:
+        return True
+
+    return False
+
+
+def _can_expression_run_on_single_item(expr: str, key: str) -> bool:
+    """
+    Detect `.key` outside of quotes, as a standalone path segment beginning
+    after a non-word boundary (start, space, |, (, [, {, , or :) and not part
+    of `.something.key`.
+    assuming key = 'item'
+    Examples:
+        - .item.yaeli => true
+        - map(.item.yaeli) => true
+        - .body.item => false
+    """
+    if not key:
+        return False
+
+    masked = _mask_strings(expr)
+    masked = _mask_numbers(masked)
+    pattern = re.compile(rf"(?<![A-Za-z0-9_])\.{re.escape(key)}(?![A-Za-z0-9_])")
+    return bool(pattern.search(masked))
+
+
+def classify_input(
+    selector_query: str, single_item_key: str | None = None
+) -> InputClassifyingResult:
+    """
+    Returns the input evaluation result for the jq expression.
+    Conservative: requires NO '.' and must match a known nullary-safe pattern.
+    """
+    if can_expression_run_with_no_input(selector_query):
+        return InputClassifyingResult.NONE
+    if single_item_key and _can_expression_run_on_single_item(
+        selector_query, single_item_key
+    ):
+        return InputClassifyingResult.SINGLE
+    return InputClassifyingResult.ALL

port_ocean/core/handlers/port_app_config/models.py

@@ -29,7 +29,9 @@ class EntityMapping(BaseModel):
 
     @property
     def is_using_search_identifier(self) -> bool:
-        return isinstance(self.identifier, dict)
+        return isinstance(self.identifier, dict) or isinstance(
+            self.identifier, IngestSearchQuery
+        )
 
 
 class MappingsConfig(BaseModel):
@@ -39,7 +41,7 @@ class MappingsConfig(BaseModel):
 class PortResourceConfig(BaseModel):
     entity: MappingsConfig
     items_to_parse: str | None = Field(alias="itemsToParse")
-    items_to_parse_name: str | None = Field(alias="itemsToParseName", default="item")
+    items_to_parse_name: str = Field(alias="itemsToParseName", default="item")
 
 
 class Selector(BaseModel):

port_ocean/core/handlers/webhook/abstract_webhook_processor.py

@@ -1,4 +1,5 @@
 from abc import ABC, abstractmethod
+from enum import StrEnum
 from loguru import logger
 
 from port_ocean.core.handlers.port_app_config.models import ResourceConfig
@@ -12,6 +13,17 @@ from .webhook_event import (
 )
 
 
+class WebhookProcessorType(StrEnum):
+    """Type of webhook processor"""
+
+    # For action-related webhooks
+    # (e.g. update finished action using the workflow runs webhook)
+    ACTION = "action"
+    # For regular webhooks
+    # (e.g. repository events that should be reflected as Entities in Port)
+    WEBHOOK = "webhook"
+
+
 class AbstractWebhookProcessor(ABC):
     """
     Abstract base class for webhook processors
@@ -47,6 +59,10 @@ class AbstractWebhookProcessor(ABC):
         self.event = event
         self.retry_count = 0
 
+    @classmethod
+    def get_processor_type(cls) -> WebhookProcessorType:
+        return WebhookProcessorType.WEBHOOK
+
     async def on_error(self, error: Exception) -> None:
         """Hook to handle errors during processing. Override if needed"""
         delay = self.calculate_retry_delay()

port_ocean/core/handlers/webhook/processor_manager.py

@@ -11,11 +11,17 @@ from port_ocean.core.handlers.queue.abstract_queue import AbstractQueue
 from port_ocean.core.integrations.mixins.events import EventsMixin
 from port_ocean.core.integrations.mixins.live_events import LiveEventsMixin
 from port_ocean.exceptions.webhook_processor import WebhookEventNotSupportedError
-from .webhook_event import WebhookEvent, WebhookEventRawResults, LiveEventTimestamp
+from port_ocean.core.handlers.webhook.webhook_event import (
+    WebhookEvent,
+    WebhookEventRawResults,
+    LiveEventTimestamp,
+)
 from port_ocean.context.event import event
 
-
-from .abstract_webhook_processor import AbstractWebhookProcessor
+from port_ocean.core.handlers.webhook.abstract_webhook_processor import (
+    AbstractWebhookProcessor,
+    WebhookProcessorType,
+)
 from port_ocean.utils.signal import SignalHandler
 from port_ocean.core.handlers.queue import LocalQueue
 
@@ -56,7 +62,7 @@ class LiveEventsProcessorManager(LiveEventsMixin, EventsMixin):
         while True:
             event = None
             matching_processors: List[
-                Tuple[ResourceConfig, AbstractWebhookProcessor]
+                Tuple[ResourceConfig | None, AbstractWebhookProcessor]
             ] = []
             try:
                 event = await queue.get()
@@ -133,16 +139,22 @@ class LiveEventsProcessorManager(LiveEventsMixin, EventsMixin):
 
     async def _extract_matching_processors(
         self, webhook_event: WebhookEvent, path: str
-    ) -> list[tuple[ResourceConfig, AbstractWebhookProcessor]]:
+    ) -> list[tuple[ResourceConfig | None, AbstractWebhookProcessor]]:
         """Find and extract the matching processor for an event"""
 
-        created_processors: list[tuple[ResourceConfig, AbstractWebhookProcessor]] = []
+        created_processors: list[
+            tuple[ResourceConfig | None, AbstractWebhookProcessor]
+        ] = []
         event_processor_names = []
 
         for processor_class in self._processors_classes[path]:
             processor = processor_class(webhook_event.clone())
             if await processor.should_process_event(webhook_event):
                 event_processor_names.append(processor.__class__.__name__)
+                if processor.get_processor_type() == WebhookProcessorType.ACTION:
+                    created_processors.append((None, processor))
+                    continue
+
                 kinds = await processor.get_matching_kinds(webhook_event)
                 for kind in kinds:
                     for resource in event.port_app_config.resources:
@@ -179,7 +191,10 @@ class LiveEventsProcessorManager(LiveEventsMixin, EventsMixin):
         event.set_timestamp(LiveEventTimestamp.FinishedProcessingWithError)
 
     async def _process_single_event(
-        self, processor: AbstractWebhookProcessor, path: str, resource: ResourceConfig
+        self,
+        processor: AbstractWebhookProcessor,
+        path: str,
+        resource: ResourceConfig | None,
     ) -> WebhookEventRawResults:
         """Process a single event with a specific processor"""
         try:
@@ -199,7 +214,7 @@ class LiveEventsProcessorManager(LiveEventsMixin, EventsMixin):
             raise
 
     async def _execute_processor(
-        self, processor: AbstractWebhookProcessor, resource: ResourceConfig
+        self, processor: AbstractWebhookProcessor, resource: ResourceConfig | None
     ) -> WebhookEventRawResults:
         """Execute a single processor within a max processing time"""
         try:
@@ -213,7 +228,7 @@ class LiveEventsProcessorManager(LiveEventsMixin, EventsMixin):
             )
 
     async def _process_webhook_request(
-        self, processor: AbstractWebhookProcessor, resource: ResourceConfig
+        self, processor: AbstractWebhookProcessor, resource: ResourceConfig | None
     ) -> WebhookEventRawResults:
         """Process a webhook request with retry logic
 
@@ -235,9 +250,10 @@ class LiveEventsProcessorManager(LiveEventsMixin, EventsMixin):
         while True:
             try:
                 webhook_event_raw_results = await processor.handle_event(
-                    payload, resource
+                    payload, resource  # type: ignore[arg-type]
                 )
-                webhook_event_raw_results.resource = resource
+                if resource is not None:
+                    webhook_event_raw_results.resource = resource
                 break
 
             except Exception as e:
@@ -258,7 +274,9 @@ class LiveEventsProcessorManager(LiveEventsMixin, EventsMixin):
         return webhook_event_raw_results
 
     def register_processor(
-        self, path: str, processor: Type[AbstractWebhookProcessor]
+        self,
+        path: str,
+        processor: Type[AbstractWebhookProcessor],
     ) -> None:
         """Register a webhook processor for a specific path with optional filter
 

port_ocean/core/integrations/mixins/sync_raw.py

@@ -24,7 +24,7 @@ from port_ocean.core.integrations.mixins.utils import (
     resync_generator_wrapper,
     resync_function_wrapper,
 )
-from port_ocean.core.models import Entity, ProcessExecutionMode
+from port_ocean.core.models import Entity, IntegrationFeatureFlag, ProcessExecutionMode
 from port_ocean.core.ocean_types import (
     RAW_RESULT,
     RESYNC_RESULT,
@@ -117,13 +117,13 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                 logger.info(
                     f"Found async generator function for {resource_config.kind} name: {task.__qualname__}"
                 )
-                results.append(resync_generator_wrapper(task, resource_config.kind,resource_config.port.items_to_parse))
+                results.append(resync_generator_wrapper(task, resource_config.kind, resource_config.port.items_to_parse_name, resource_config.port.items_to_parse))
             else:
                 logger.info(
                     f"Found sync function for {resource_config.kind} name: {task.__qualname__}"
                 )
                 task = typing.cast(Callable[[str], Awaitable[RAW_RESULT]], task)
-                tasks.append(resync_function_wrapper(task, resource_config.kind))
+                tasks.append(resync_function_wrapper(task, resource_config.kind, resource_config.port.items_to_parse))
 
         logger.info(
             f"Found {len(tasks) + len(results)} resync tasks for {resource_config.kind}"
@@ -478,7 +478,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
             bool: True if lakehouse data is enabled, False otherwise
         """
         flags = await ocean.port_client.get_organization_feature_flags()
-        if "LAKEHOUSE_ELIGIBLE" in flags and ocean.config.lakehouse_enabled:
+        if IntegrationFeatureFlag.LAKEHOUSE_ELIGIBLE in flags and ocean.config.lakehouse_enabled:
             return True
         return False
 

port_ocean/core/integrations/mixins/utils.py

@@ -1,13 +1,21 @@
+import asyncio
+import json
+import multiprocessing
+import os
+import re
+import shutil
+import stat
+import subprocess
+import tempfile
 from contextlib import contextmanager
-from typing import Awaitable, Generator, Callable, cast
+from typing import Any, AsyncGenerator, Awaitable, Callable, Generator, cast
 
+import ijson
 from loguru import logger
 
-import asyncio
-import multiprocessing
-
+from port_ocean.clients.port.utils import _http_client as _port_http_client
+from port_ocean.context.ocean import ocean
 from port_ocean.core.handlers.entity_processor.jq_entity_processor import JQEntityProcessor
-from port_ocean.core.handlers.port_app_config.models import ResourceConfig
 from port_ocean.core.ocean_types import (
     ASYNC_GENERATOR_RESYNC_TYPE,
     RAW_RESULT,
@@ -20,11 +28,58 @@ from port_ocean.exceptions.core import (
     OceanAbortException,
     KindNotImplementedException,
 )
-
-from port_ocean.utils.async_http import _http_client
-from port_ocean.clients.port.utils import _http_client as _port_http_client
 from port_ocean.helpers.metric.metric import MetricType, MetricPhase
-from port_ocean.context.ocean import ocean
+from port_ocean.utils.async_http import _http_client
+
+def _process_path_type_items(
+    result: RAW_RESULT, items_to_parse: str | None = None
+) -> RAW_RESULT:
+    """
+    Process items in the result array to check for "__type": "path" fields.
+    If found, read the file contents and load them into a "content" field.
+    Skip processing if we're on the items_to_parse branch.
+    """
+    if not isinstance(result, list):
+        return result
+
+    # Skip processing if we're on the items_to_parse branch
+    if items_to_parse:
+        return result
+
+    processed_result = []
+    for item in result:
+        if isinstance(item, dict) and item.get("__type") == "path":
+            try:
+                # Read the file content and parse as JSON
+                file_path = item.get("file", {}).get("content", {}).get("path")
+                if file_path and os.path.exists(file_path):
+                    with open(file_path, "r") as f:
+                        content = json.loads(f.read())
+                    # Create a copy of the item with the content field
+                    processed_item = item.copy()
+                    processed_item["file"]["content"] = content
+                    processed_result.append(processed_item)
+                else:
+                    # If file doesn't exist, keep the original item
+                    processed_result.append(item)
+            except (json.JSONDecodeError, IOError, OSError) as e:
+                if isinstance(item, dict) and item.get("file") is not None:
+                    content = item["file"].get("content") if isinstance(item["file"].get("content"), dict) else {}
+                    data_path = content.get("path", None)
+                    logger.warning(
+                        f"Failed to read or parse file content for path {data_path}: {e}"
+                    )
+                else:
+                    logger.warning(
+                        f"Failed to read or parse file content for unknown path: {e}. item: {json.dumps(item)}"
+                    )
+                # Keep the original item if there's an error
+                processed_result.append(item)
+        else:
+            # Keep non-path type items as is
+            processed_result.append(item)
+
+    return processed_result
 
 @contextmanager
 def resync_error_handling() -> Generator[None, None, None]:
@@ -43,15 +98,16 @@ def resync_error_handling() -> Generator[None, None, None]:
 
 
 async def resync_function_wrapper(
-    fn: Callable[[str], Awaitable[RAW_RESULT]], kind: str
+    fn: Callable[[str], Awaitable[RAW_RESULT]], kind: str, items_to_parse: str | None = None
 ) -> RAW_RESULT:
     with resync_error_handling():
         results = await fn(kind)
-        return validate_result(results)
+        validated_results = validate_result(results)
+        return _process_path_type_items(validated_results, items_to_parse)
 
 
 async def resync_generator_wrapper(
-    fn: Callable[[str], ASYNC_GENERATOR_RESYNC_TYPE], kind: str, items_to_parse: str | None = None
+    fn: Callable[[str], ASYNC_GENERATOR_RESYNC_TYPE], kind: str, items_to_parse_name: str, items_to_parse: str | None = None
 ) -> ASYNC_GENERATOR_RESYNC_TYPE:
     generator = fn(kind)
     errors = []
@@ -61,27 +117,23 @@ async def resync_generator_wrapper(
                 with resync_error_handling():
                     result = await anext(generator)
                     if not ocean.config.yield_items_to_parse:
-                        yield validate_result(result)
+                        validated_result = validate_result(result)
+                        processed_result = _process_path_type_items(validated_result,items_to_parse)
+                        yield processed_result
                     else:
-                        batch_size = ocean.config.yield_items_to_parse_batch_size
                         if items_to_parse:
                             for data in result:
-                                items = await cast(JQEntityProcessor, ocean.app.integration.entity_processor)._search(data, items_to_parse)
-                                if not isinstance(items, list):
-                                    logger.warning(
-                                        f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items)}."
-                                        f" Skipping..."
-                                    )
-                                    yield []
-                                raw_data = [{"item": item, **data} for item in items]
-                                while True:
-                                    raw_data_batch = raw_data[:batch_size]
-                                    yield raw_data_batch
-                                    raw_data = raw_data[batch_size:]
-                                    if len(raw_data) == 0:
-                                        break
+                                data_path: str | None = None
+                                if isinstance(data, dict) and data.get("file") is not None:
+                                    content = data["file"].get("content") if isinstance(data["file"].get("content"), dict) else {}
+                                    data_path = content.get("path", None)
+                                bulks = get_items_to_parse_bulks(data, data_path, items_to_parse, items_to_parse_name, data.get("__base_jq", ".file.content"))
+                                async for bulk in bulks:
+                                    yield bulk
                         else:
-                            yield validate_result(result)
+                            validated_result = validate_result(result)
+                            processed_result = _process_path_type_items(validated_result, items_to_parse)
+                            yield processed_result
             except OceanAbortException as error:
                 errors.append(error)
                 ocean.metrics.inc_metric(
@@ -101,6 +153,106 @@ def is_resource_supported(
 ) -> bool:
     return bool(resync_event_mapping[kind] or resync_event_mapping[None])
 
+def _validate_jq_expression(expression: str) -> None:
+    """Validate jq expression to prevent command injection."""
+    try:
+        _ = cast(JQEntityProcessor, ocean.app.integration.entity_processor)._compile(expression)
+    except Exception as e:
+        raise ValueError(f"Invalid jq expression: {e}") from e
+    # Basic validation - reject expressions that could be dangerous
+    # Check for dangerous patterns (include, import, module)
+    dangerous_patterns = ['include', 'import', 'module', 'env', 'debug']
+    for pattern in dangerous_patterns:
+        # Use word boundary regex to match only complete words, not substrings
+        if re.search(rf'\b{re.escape(pattern)}\b', expression):
+            raise ValueError(f"Potentially dangerous pattern '{pattern}' found in jq expression")
+
+    # Special handling for 'env' - block environment variable access
+    if re.search(r'(?<!\w)\$ENV(?:\.)?', expression):
+        raise ValueError("Environment variable access '$ENV.' found in jq expression")
+    if re.search(r'\benv\.', expression):
+        raise ValueError("Environment variable access 'env.' found in jq expression")
+
+def _create_secure_temp_file(suffix: str = ".json") -> str:
+    """Create a secure temporary file with restricted permissions."""
+    # Create temp directory if it doesn't exist
+    temp_dir = "/tmp/ocean"
+    os.makedirs(temp_dir, exist_ok=True)
+
+    # Create temporary file with secure permissions
+    fd, temp_path = tempfile.mkstemp(suffix=suffix, dir=temp_dir)
+    try:
+        # Set restrictive permissions (owner read/write only)
+        os.chmod(temp_path, stat.S_IRUSR | stat.S_IWUSR)
+        return temp_path
+    finally:
+        os.close(fd)
+
+async def get_items_to_parse_bulks(raw_data: dict[Any, Any], data_path: str, items_to_parse: str, items_to_parse_name: str, base_jq: str) -> AsyncGenerator[list[dict[str, Any]], None]:
+    # Validate inputs to prevent command injection
+    _validate_jq_expression(items_to_parse)
+    items_to_parse = items_to_parse.replace(base_jq, ".") if data_path else items_to_parse
+
+    temp_data_path = None
+    temp_output_path = None
+
+    try:
+        # Create secure temporary files
+        if not data_path:
+            raw_data_serialized = json.dumps(raw_data)
+            temp_data_path = _create_secure_temp_file("_input.json")
+            with open(temp_data_path, "w") as f:
+                f.write(raw_data_serialized)
+            data_path = temp_data_path
+
+        temp_output_path = _create_secure_temp_file("_parsed.json")
+
+        delete_target = items_to_parse.split('|', 1)[0].strip() if not items_to_parse.startswith('map(') else '.'
+        base_jq_object_string = await _build_base_jq_object_string(raw_data, base_jq, delete_target)
+
+        # Build jq expression safely
+        jq_expression = f""". as $all
+      | ($all | {items_to_parse}) as $items
+      | $items
+      | map({{{items_to_parse_name}: ., {base_jq_object_string}}})"""
+
+        # Use subprocess with list arguments instead of shell=True
+
+        jq_path = shutil.which("jq") or "/bin/jq"
+        jq_args = [jq_path, jq_expression, data_path]
+
+        with open(temp_output_path, "w") as output_file:
+            result = subprocess.run(
+                jq_args,
+                stdout=output_file,
+                stderr=subprocess.PIPE,
+                text=True,
+                check=False  # Don't raise exception, handle errors manually
+            )
+
+        if result.returncode != 0:
+            logger.error(f"Failed to parse items for JQ expression {items_to_parse}, error: {result.stderr}")
+            yield []
+        else:
+            with open(temp_output_path, "r") as f:
+                events_stream = get_events_as_a_stream(f, 'item', ocean.config.yield_items_to_parse_batch_size)
+                for items_bulk in events_stream:
+                    yield items_bulk
+
+    except ValueError as e:
+        logger.error(f"Invalid jq expression: {e}")
+        yield []
+    except Exception as e:
+        logger.error(f"Failed to parse items for JQ expression {items_to_parse}, error: {e}")
+        yield []
+    finally:
+        # Cleanup temporary files
+        for temp_path in [temp_data_path, temp_output_path]:
+            if temp_path and os.path.exists(temp_path):
+                try:
+                    os.remove(temp_path)
+                except OSError as e:
+                    logger.warning(f"Failed to cleanup temporary file {temp_path}: {e}")
 
 def unsupported_kind_response(
     kind: str, available_resync_kinds: list[str]
@@ -108,6 +260,44 @@ def unsupported_kind_response(
     logger.error(f"Kind {kind} is not supported in this integration")
     return [], [KindNotImplementedException(kind, available_resync_kinds)]
 
+async def _build_base_jq_object_string(raw_data: dict[Any, Any], base_jq: str, delete_target: str) -> str:
+    base_jq_object_before_parsing = await cast(JQEntityProcessor, ocean.app.integration.entity_processor)._search(raw_data, f"{base_jq} = {json.dumps("__all")}")
+    base_jq_object_before_parsing_serialized = json.dumps(base_jq_object_before_parsing)
+    base_jq_object_before_parsing_serialized = base_jq_object_before_parsing_serialized[1:-1] if len(base_jq_object_before_parsing_serialized) >= 2 else base_jq_object_before_parsing_serialized
+    base_jq_object_before_parsing_serialized = base_jq_object_before_parsing_serialized.replace("\"__all\"", f"(($all | del({delete_target})) // {{}})")
+    return base_jq_object_before_parsing_serialized
+
+
+def get_events_as_a_stream(
+        stream: Any,
+        target_items: str = "item",
+        max_buffer_size_mb: int = 1
+    ) -> Generator[list[dict[str, Any]], None, None]:
+        events = ijson.sendable_list()
+        coro = ijson.items_coro(events, target_items, use_float=True)
+
+        # Convert MB to bytes for the buffer size
+        buffer_size = max_buffer_size_mb * 1024 * 1024
+
+        # Read chunks from the stream until exhausted
+        while True:
+            chunk = stream.read(buffer_size)
+            if not chunk:  # End of stream
+                break
+
+            # Convert string to bytes if necessary (for text mode files)
+            if isinstance(chunk, str):
+                chunk = chunk.encode('utf-8')
+
+            coro.send(chunk)
+            yield events
+            del events[:]
+        try:
+            coro.close()
+        finally:
+            if events:
+                yield events
+                events[:] = []
 
 class ProcessWrapper(multiprocessing.Process):
     def __init__(self, *args, **kwargs):
@@ -134,3 +324,34 @@ def clear_http_client_context() -> None:
             _port_http_client.pop()
     except (RuntimeError, AttributeError):
         pass
+
+class _AiterReader:
+    """
+    Wraps an iterable of byte chunks (e.g., response.iter_bytes())
+    and exposes a .read(n) method that ijson expects.
+    """
+    def __init__(self, iterable):
+        self._iter = iter(iterable)
+        self._buf = bytearray()
+        self._eof = False
+
+    def read(self, n=-1):
+        # If n < 0, return everything until EOF
+        if n is None or n < 0:
+            chunks = [bytes(self._buf)]
+            self._buf.clear()
+            chunks.extend(self._iter)  # drain the iterator
+            return b"".join(chunks)
+
+        # Fill buffer until we have n bytes or hit EOF
+        while len(self._buf) < n and not self._eof:
+            try:
+                self._buf.extend(next(self._iter))
+            except StopIteration:
+                self._eof = True
+                break
+
+        # Serve up to n bytes
+        out = bytes(self._buf[:n])
+        del self._buf[:n]
+        return out