port-ocean 0.28.2__py3-none-any.whl → 0.29.0__py3-none-any.whl

port_ocean/core/integrations/mixins/utils.py

@@ -1,13 +1,21 @@
+import asyncio
+import json
+import multiprocessing
+import os
+import re
+import shutil
+import stat
+import subprocess
+import tempfile
 from contextlib import contextmanager
-from typing import Awaitable, Generator, Callable, cast
+from typing import Any, AsyncGenerator, Awaitable, Callable, Generator, cast
 
+import ijson
 from loguru import logger
 
-import asyncio
-import multiprocessing
-
+from port_ocean.clients.port.utils import _http_client as _port_http_client
+from port_ocean.context.ocean import ocean
 from port_ocean.core.handlers.entity_processor.jq_entity_processor import JQEntityProcessor
-from port_ocean.core.handlers.port_app_config.models import ResourceConfig
 from port_ocean.core.ocean_types import (
     ASYNC_GENERATOR_RESYNC_TYPE,
     RAW_RESULT,
@@ -20,11 +28,58 @@ from port_ocean.exceptions.core import (
     OceanAbortException,
     KindNotImplementedException,
 )
-
-from port_ocean.utils.async_http import _http_client
-from port_ocean.clients.port.utils import _http_client as _port_http_client
 from port_ocean.helpers.metric.metric import MetricType, MetricPhase
-from port_ocean.context.ocean import ocean
+from port_ocean.utils.async_http import _http_client
+
+def _process_path_type_items(
+    result: RAW_RESULT, items_to_parse: str | None = None
+) -> RAW_RESULT:
+    """
+    Process items in the result array to check for "__type": "path" fields.
+    If found, read the file contents and load them into a "content" field.
+    Skip processing if we're on the items_to_parse branch.
+    """
+    if not isinstance(result, list):
+        return result
+
+    # Skip processing if we're on the items_to_parse branch
+    if items_to_parse:
+        return result
+
+    processed_result = []
+    for item in result:
+        if isinstance(item, dict) and item.get("__type") == "path":
+            try:
+                # Read the file content and parse as JSON
+                file_path = item.get("file", {}).get("content", {}).get("path")
+                if file_path and os.path.exists(file_path):
+                    with open(file_path, "r") as f:
+                        content = json.loads(f.read())
+                    # Create a copy of the item with the content field
+                    processed_item = item.copy()
+                    processed_item["file"]["content"] = content
+                    processed_result.append(processed_item)
+                else:
+                    # If file doesn't exist, keep the original item
+                    processed_result.append(item)
+            except (json.JSONDecodeError, IOError, OSError) as e:
+                if isinstance(item, dict) and item.get("file") is not None:
+                    content = item["file"].get("content") if isinstance(item["file"].get("content"), dict) else {}
+                    data_path = content.get("path", None)
+                    logger.warning(
+                        f"Failed to read or parse file content for path {data_path}: {e}"
+                    )
+                else:
+                    logger.warning(
+                        f"Failed to read or parse file content for unknown path: {e}. item: {json.dumps(item)}"
+                    )
+                # Keep the original item if there's an error
+                processed_result.append(item)
+        else:
+            # Keep non-path type items as is
+            processed_result.append(item)
+
+    return processed_result
 
 @contextmanager
 def resync_error_handling() -> Generator[None, None, None]:
@@ -43,15 +98,16 @@ def resync_error_handling() -> Generator[None, None, None]:
 
 
 async def resync_function_wrapper(
-    fn: Callable[[str], Awaitable[RAW_RESULT]], kind: str
+    fn: Callable[[str], Awaitable[RAW_RESULT]], kind: str, items_to_parse: str | None = None
 ) -> RAW_RESULT:
     with resync_error_handling():
         results = await fn(kind)
-        return validate_result(results)
+        validated_results = validate_result(results)
+        return _process_path_type_items(validated_results, items_to_parse)
 
 
 async def resync_generator_wrapper(
-    fn: Callable[[str], ASYNC_GENERATOR_RESYNC_TYPE], kind: str, items_to_parse: str | None = None
+    fn: Callable[[str], ASYNC_GENERATOR_RESYNC_TYPE], kind: str, items_to_parse_name: str, items_to_parse: str | None = None
 ) -> ASYNC_GENERATOR_RESYNC_TYPE:
     generator = fn(kind)
     errors = []
@@ -61,27 +117,23 @@ async def resync_generator_wrapper(
                 with resync_error_handling():
                     result = await anext(generator)
                     if not ocean.config.yield_items_to_parse:
-                        yield validate_result(result)
+                        validated_result = validate_result(result)
+                        processed_result = _process_path_type_items(validated_result,items_to_parse)
+                        yield processed_result
                     else:
-                        batch_size = ocean.config.yield_items_to_parse_batch_size
                         if items_to_parse:
                             for data in result:
-                                items = await cast(JQEntityProcessor, ocean.app.integration.entity_processor)._search(data, items_to_parse)
-                                if not isinstance(items, list):
-                                    logger.warning(
-                                        f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items)}."
-                                        f" Skipping..."
-                                    )
-                                    yield []
-                                raw_data = [{"item": item, **data} for item in items]
-                                while True:
-                                    raw_data_batch = raw_data[:batch_size]
-                                    yield raw_data_batch
-                                    raw_data = raw_data[batch_size:]
-                                    if len(raw_data) == 0:
-                                        break
+                                data_path: str | None = None
+                                if isinstance(data, dict) and data.get("file") is not None:
+                                    content = data["file"].get("content") if isinstance(data["file"].get("content"), dict) else {}
+                                    data_path = content.get("path", None)
+                                bulks = get_items_to_parse_bulks(data, data_path, items_to_parse, items_to_parse_name, data.get("__base_jq", ".file.content"))
+                                async for bulk in bulks:
+                                    yield bulk
                         else:
-                            yield validate_result(result)
+                            validated_result = validate_result(result)
+                            processed_result = _process_path_type_items(validated_result, items_to_parse)
+                            yield processed_result
             except OceanAbortException as error:
                 errors.append(error)
                 ocean.metrics.inc_metric(
@@ -101,6 +153,106 @@ def is_resource_supported(
 ) -> bool:
     return bool(resync_event_mapping[kind] or resync_event_mapping[None])
 
+def _validate_jq_expression(expression: str) -> None:
+    """Validate jq expression to prevent command injection."""
+    try:
+        _ = cast(JQEntityProcessor, ocean.app.integration.entity_processor)._compile(expression)
+    except Exception as e:
+        raise ValueError(f"Invalid jq expression: {e}") from e
+    # Basic validation - reject expressions that could be dangerous
+    # Check for dangerous patterns (include, import, module)
+    dangerous_patterns = ['include', 'import', 'module', 'env', 'debug']
+    for pattern in dangerous_patterns:
+        # Use word boundary regex to match only complete words, not substrings
+        if re.search(rf'\b{re.escape(pattern)}\b', expression):
+            raise ValueError(f"Potentially dangerous pattern '{pattern}' found in jq expression")
+
+    # Special handling for 'env' - block environment variable access
+    if re.search(r'(?<!\w)\$ENV(?:\.)?', expression):
+        raise ValueError("Environment variable access '$ENV.' found in jq expression")
+    if re.search(r'\benv\.', expression):
+        raise ValueError("Environment variable access 'env.' found in jq expression")
+
+def _create_secure_temp_file(suffix: str = ".json") -> str:
+    """Create a secure temporary file with restricted permissions."""
+    # Create temp directory if it doesn't exist
+    temp_dir = "/tmp/ocean"
+    os.makedirs(temp_dir, exist_ok=True)
+
+    # Create temporary file with secure permissions
+    fd, temp_path = tempfile.mkstemp(suffix=suffix, dir=temp_dir)
+    try:
+        # Set restrictive permissions (owner read/write only)
+        os.chmod(temp_path, stat.S_IRUSR | stat.S_IWUSR)
+        return temp_path
+    finally:
+        os.close(fd)
+
+async def get_items_to_parse_bulks(raw_data: dict[Any, Any], data_path: str, items_to_parse: str, items_to_parse_name: str, base_jq: str) -> AsyncGenerator[list[dict[str, Any]], None]:
+    # Validate inputs to prevent command injection
+    _validate_jq_expression(items_to_parse)
+    items_to_parse = items_to_parse.replace(base_jq, ".") if data_path else items_to_parse
+
+    temp_data_path = None
+    temp_output_path = None
+
+    try:
+        # Create secure temporary files
+        if not data_path:
+            raw_data_serialized = json.dumps(raw_data)
+            temp_data_path = _create_secure_temp_file("_input.json")
+            with open(temp_data_path, "w") as f:
+                f.write(raw_data_serialized)
+            data_path = temp_data_path
+
+        temp_output_path = _create_secure_temp_file("_parsed.json")
+
+        delete_target = items_to_parse.split('|', 1)[0].strip() if not items_to_parse.startswith('map(') else '.'
+        base_jq_object_string = await _build_base_jq_object_string(raw_data, base_jq, delete_target)
+
+        # Build jq expression safely
+        jq_expression = f""". as $all
+      | ($all | {items_to_parse}) as $items
+      | $items
+      | map({{{items_to_parse_name}: ., {base_jq_object_string}}})"""
+
+        # Use subprocess with list arguments instead of shell=True
+
+        jq_path = shutil.which("jq") or "/bin/jq"
+        jq_args = [jq_path, jq_expression, data_path]
+
+        with open(temp_output_path, "w") as output_file:
+            result = subprocess.run(
+                jq_args,
+                stdout=output_file,
+                stderr=subprocess.PIPE,
+                text=True,
+                check=False  # Don't raise exception, handle errors manually
+            )
+
+        if result.returncode != 0:
+            logger.error(f"Failed to parse items for JQ expression {items_to_parse}, error: {result.stderr}")
+            yield []
+        else:
+            with open(temp_output_path, "r") as f:
+                events_stream = get_events_as_a_stream(f, 'item', ocean.config.yield_items_to_parse_batch_size)
+                for items_bulk in events_stream:
+                    yield items_bulk
+
+    except ValueError as e:
+        logger.error(f"Invalid jq expression: {e}")
+        yield []
+    except Exception as e:
+        logger.error(f"Failed to parse items for JQ expression {items_to_parse}, error: {e}")
+        yield []
+    finally:
+        # Cleanup temporary files
+        for temp_path in [temp_data_path, temp_output_path]:
+            if temp_path and os.path.exists(temp_path):
+                try:
+                    os.remove(temp_path)
+                except OSError as e:
+                    logger.warning(f"Failed to cleanup temporary file {temp_path}: {e}")
 
 def unsupported_kind_response(
     kind: str, available_resync_kinds: list[str]
@@ -108,6 +260,44 @@ def unsupported_kind_response(
     logger.error(f"Kind {kind} is not supported in this integration")
     return [], [KindNotImplementedException(kind, available_resync_kinds)]
 
+async def _build_base_jq_object_string(raw_data: dict[Any, Any], base_jq: str, delete_target: str) -> str:
+    base_jq_object_before_parsing = await cast(JQEntityProcessor, ocean.app.integration.entity_processor)._search(raw_data, f"{base_jq} = {json.dumps("__all")}")
+    base_jq_object_before_parsing_serialized = json.dumps(base_jq_object_before_parsing)
+    base_jq_object_before_parsing_serialized = base_jq_object_before_parsing_serialized[1:-1] if len(base_jq_object_before_parsing_serialized) >= 2 else base_jq_object_before_parsing_serialized
+    base_jq_object_before_parsing_serialized = base_jq_object_before_parsing_serialized.replace("\"__all\"", f"(($all | del({delete_target})) // {{}})")
+    return base_jq_object_before_parsing_serialized
+
+
+def get_events_as_a_stream(
+        stream: Any,
+        target_items: str = "item",
+        max_buffer_size_mb: int = 1
+    ) -> Generator[list[dict[str, Any]], None, None]:
+        events = ijson.sendable_list()
+        coro = ijson.items_coro(events, target_items, use_float=True)
+
+        # Convert MB to bytes for the buffer size
+        buffer_size = max_buffer_size_mb * 1024 * 1024
+
+        # Read chunks from the stream until exhausted
+        while True:
+            chunk = stream.read(buffer_size)
+            if not chunk:  # End of stream
+                break
+
+            # Convert string to bytes if necessary (for text mode files)
+            if isinstance(chunk, str):
+                chunk = chunk.encode('utf-8')
+
+            coro.send(chunk)
+            yield events
+            del events[:]
+        try:
+            coro.close()
+        finally:
+            if events:
+                yield events
+                events[:] = []
 
 class ProcessWrapper(multiprocessing.Process):
     def __init__(self, *args, **kwargs):
@@ -134,3 +324,34 @@ def clear_http_client_context() -> None:
             _port_http_client.pop()
     except (RuntimeError, AttributeError):
         pass
+
+class _AiterReader:
+    """
+    Wraps an iterable of byte chunks (e.g., response.iter_bytes())
+    and exposes a .read(n) method that ijson expects.
+    """
+    def __init__(self, iterable):
+        self._iter = iter(iterable)
+        self._buf = bytearray()
+        self._eof = False
+
+    def read(self, n=-1):
+        # If n < 0, return everything until EOF
+        if n is None or n < 0:
+            chunks = [bytes(self._buf)]
+            self._buf.clear()
+            chunks.extend(self._iter)  # drain the iterator
+            return b"".join(chunks)
+
+        # Fill buffer until we have n bytes or hit EOF
+        while len(self._buf) < n and not self._eof:
+            try:
+                self._buf.extend(next(self._iter))
+            except StopIteration:
+                self._eof = True
+                break
+
+        # Serve up to n bytes
+        out = bytes(self._buf[:n])
+        del self._buf[:n]
+        return out

port_ocean/core/models.py

@@ -1,11 +1,19 @@
 from dataclasses import dataclass, field
 from enum import Enum, StrEnum
-from typing import Any, TypedDict
-
+from typing import Any, Literal, TypedDict
 from pydantic import BaseModel
 from pydantic.fields import Field
 
 
+class EventListenerType(StrEnum):
+    WEBHOOK = "WEBHOOK"
+    KAFKA = "KAFKA"
+    POLLING = "POLLING"
+    ONCE = "ONCE"
+    WEBHOOKS_ONLY = "WEBHOOKS_ONLY"
+    ACTIONS_ONLY = "ACTIONS_ONLY"
+
+
 class CreatePortResourcesOrigin(StrEnum):
     Ocean = "Ocean"
     Port = "Port"
@@ -121,3 +129,28 @@ class EntityPortDiff:
     deleted: list[Entity] = field(default_factory=list)
     modified: list[Entity] = field(default_factory=list)
     created: list[Entity] = field(default_factory=list)
+
+
+class IntegrationFeatureFlag(StrEnum):
+    USE_PROVISIONED_DEFAULTS = "USE_PROVISIONED_DEFAULTS"
+    LAKEHOUSE_ELIGIBLE = "LAKEHOUSE_ELIGIBLE"
+    OCEAN_ACTIONS_PROCESSING_ENABLED = "OCEAN_ACTIONS_PROCESSING_ENABLED"
+
+
+class RunStatus(StrEnum):
+    IN_PROGRESS = "IN_PROGRESS"
+    SUCCESS = "SUCCESS"
+    FAILURE = "FAILURE"
+
+
+class IntegrationActionInvocationPayload(BaseModel):
+    type: Literal["INTEGRATION_ACTION"]
+    installationId: str
+    integrationActionType: str
+    integrationActionExecutionProperties: dict[str, Any] = Field(default_factory=dict)
+
+
+class ActionRun(BaseModel):
+    id: str
+    status: RunStatus
+    payload: IntegrationActionInvocationPayload

port_ocean/core/utils/utils.py

@@ -4,7 +4,7 @@ import json
 from typing import Iterable, Any, TypeVar, Callable, Awaitable
 
 from loguru import logger
-from pydantic import parse_obj_as, ValidationError
+from pydantic import BaseModel, parse_obj_as, ValidationError
 
 
 from port_ocean.clients.port.client import PortClient
@@ -79,6 +79,19 @@ async def gather_and_split_errors_from_results(
     return valid_items, errors
 
 
+def _get_entity_key(entity: Entity) -> tuple[str, str]:
+    identifier = entity.identifier
+    if isinstance(identifier, BaseModel):
+        identifier = identifier.dict()
+
+    key_part = (
+        json.dumps(identifier, sort_keys=True)
+        if isinstance(identifier, dict)
+        else str(identifier)
+    )
+    return key_part, entity.blueprint
+
+
 def get_port_diff(before: Iterable[Entity], after: Iterable[Entity]) -> EntityPortDiff:
     before_dict = {}
     after_dict = {}
@@ -88,12 +101,10 @@ def get_port_diff(before: Iterable[Entity], after: Iterable[Entity]) -> EntityPo
 
     # Create dictionaries for before and after lists
     for entity in before:
-        key = (entity.identifier, entity.blueprint)
-        before_dict[key] = entity
+        before_dict[_get_entity_key(entity)] = entity
 
     for entity in after:
-        key = (entity.identifier, entity.blueprint)
-        after_dict[key] = entity
+        after_dict[_get_entity_key(entity)] = entity
 
     # Find created, modified, and deleted objects
     for key, obj in after_dict.items():

port_ocean/exceptions/execution_manager.py

@@ -0,0 +1,22 @@
+class DuplicateActionExecutorError(Exception):
+    """
+    Raised when attempting to register an executor for an action that already has an existing executor.
+    """
+
+    pass
+
+
+class RunAlreadyAcknowledgedError(Exception):
+    """
+    Raised when attempting to acknowledge a run that has already been acknowledged.
+    """
+
+    pass
+
+
+class PartitionKeyNotFoundError(Exception):
+    """
+    Raised when attempting to extract a partition key that is not found in the invocation payload.
+    """
+
+    pass

port_ocean/helpers/metric/metric.py

@@ -131,7 +131,7 @@ class Metrics:
         self.load_metrics()
         self._integration_version: Optional[str] = None
         self._ocean_version: Optional[str] = None
-        self.event_id = ""
+        self._event_id = ""
         self.sync_state = SyncState.PENDING
 
     @property

port_ocean/helpers/retry.py

@@ -257,7 +257,7 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
             else:
                 response = await transport.handle_async_request(request)
 
-            await self._log_response_size_async(request, response)
+            self._log_response_size(request, response)
 
             return response
         except Exception as e:
@@ -345,32 +345,6 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
             return int(content_length)
         return None
 
-    async def _log_response_size_async(
-        self, request: httpx.Request, response: httpx.Response
-    ) -> None:
-        """Log the size of the response."""
-        if not self._should_log_response_size(request):
-            return
-
-        # Try to get content length from headers first
-        content_length = self._get_content_length(response)
-        if content_length is not None:
-            size_info = content_length
-        else:
-            # If no Content-Length header, try to get actual content size
-            try:
-                actual_size = len(await response.aread())
-                size_info = actual_size
-            except Exception as e:
-                cast(logging.Logger, self._logger).error(
-                    f"Error getting response size: {e}"
-                )
-                return
-
-        cast(logging.Logger, self._logger).info(
-            f"Response for {request.method} {request.url} - Size: {size_info} bytes"
-        )
-
     def _log_response_size(
         self, request: httpx.Request, response: httpx.Response
     ) -> None:
@@ -378,21 +352,11 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
             return
 
         content_length = self._get_content_length(response)
-        if content_length is not None:
-            size_info = content_length
-        else:
-            # If no Content-Length header, try to get actual content size
-            try:
-                actual_size = len(response.read())
-                size_info = actual_size
-            except Exception as e:
-                cast(logging.Logger, self._logger).error(
-                    f"Error getting response size: {e}"
-                )
-                return
+        if content_length is None:
+            return
 
         cast(logging.Logger, self._logger).info(
-            f"Response for {request.method} {request.url} - Size: {size_info} bytes"
+            f"Response for {request.method} {request.url} - Size: {content_length} bytes"
         )
 
     async def _should_retry_async(self, response: httpx.Response) -> bool:

port_ocean/log/logger_setup.py

@@ -61,9 +61,9 @@ def _http_loguru_handler(level: LogLevelType) -> None:
 
     http_memory_handler = HTTPMemoryHandler()
     signal_handler.register(
-        http_memory_handler.wait_for_lingering_threads, priority=-200
+        http_memory_handler.wait_for_lingering_threads, priority=-900
     )
-    signal_handler.register(http_memory_handler.flush, priority=-200)
+    signal_handler.register(http_memory_handler.flush, priority=-899)
 
     queue_listener = QueueListener(queue, http_memory_handler)
     queue_listener.start()

port_ocean/ocean.py

@@ -26,6 +26,7 @@ from port_ocean.core.handlers.resync_state_updater import ResyncStateUpdater
 from port_ocean.core.handlers.webhook.processor_manager import (
     LiveEventsProcessorManager,
 )
+from port_ocean.core.handlers.actions.execution_manager import ExecutionManager
 from port_ocean.core.integrations.base import BaseIntegration
 from port_ocean.core.models import ProcessExecutionMode
 from port_ocean.log.sensetive import sensitive_log_filter
@@ -88,6 +89,16 @@ class Ocean:
             max_wait_seconds_before_shutdown=self.config.max_wait_seconds_before_shutdown,
         )
 
+        self.execution_manager = ExecutionManager(
+            webhook_manager=self.webhook_manager,
+            signal_handler=signal_handler,
+            workers_count=self.config.actions_processor.workers_count,
+            runs_buffer_high_watermark=self.config.actions_processor.runs_buffer_high_watermark,
+            poll_check_interval_seconds=self.config.actions_processor.poll_check_interval_seconds,
+            visibility_timeout_ms=self.config.actions_processor.visibility_timeout_ms,
+            max_wait_seconds_before_shutdown=self.config.max_wait_seconds_before_shutdown,
+        )
+
         self.integration = (
             integration_class(ocean) if integration_class else BaseIntegration(ocean)
         )
@@ -105,7 +116,7 @@ class Ocean:
         This ensures Port is notified that the integration was interrupted.
         """
         try:
-            if self.metrics.event_id != "":
+            if self.metrics.event_id.find("-done") == -1:
                 await self.resync_state_updater.update_after_resync(
                     IntegrationStateStatus.Aborted
                 )
@@ -200,6 +211,24 @@ class Ocean:
                 )
         return None
 
+    async def _register_addons(self) -> None:
+        if self.base_url and self.config.event_listener.should_process_webhooks:
+            await self.webhook_manager.start_processing_event_messages()
+        else:
+            logger.warning(
+                "No base URL provided, or webhook processing is disabled is this event listener, skipping webhook processing"
+            )
+
+        if (
+            self.config.actions_processor.enabled
+            and self.config.event_listener.should_run_actions
+        ):
+            await self.execution_manager.start_processing_action_runs()
+        else:
+            logger.warning(
+                "Execution agent is not enabled, or actions processing is disabled in this event listener, skipping execution agent setup"
+            )
+
     def initialize_app(self) -> None:
         self.fast_api_app.include_router(self.integration_router, prefix="/integration")
         self.fast_api_app.include_router(
@@ -210,10 +239,7 @@ class Ocean:
         async def lifecycle(_: FastAPI) -> AsyncIterator[None]:
             try:
                 await self.integration.start()
-                if self.base_url:
-                    await self.webhook_manager.start_processing_event_messages()
-                else:
-                    logger.warning("No base URL provided, skipping webhook processing")
+                await self._register_addons()
                 await self._setup_scheduled_resync()
                 yield None
             except Exception: