port-ocean 0.28.5__py3-none-any.whl → 0.29.0__py3-none-any.whl

port_ocean/core/models.py

@@ -1,11 +1,19 @@
 from dataclasses import dataclass, field
 from enum import Enum, StrEnum
-from typing import Any, TypedDict
-
+from typing import Any, Literal, TypedDict
 from pydantic import BaseModel
 from pydantic.fields import Field
 
 
+class EventListenerType(StrEnum):
+    WEBHOOK = "WEBHOOK"
+    KAFKA = "KAFKA"
+    POLLING = "POLLING"
+    ONCE = "ONCE"
+    WEBHOOKS_ONLY = "WEBHOOKS_ONLY"
+    ACTIONS_ONLY = "ACTIONS_ONLY"
+
+
 class CreatePortResourcesOrigin(StrEnum):
     Ocean = "Ocean"
     Port = "Port"
@@ -121,3 +129,28 @@ class EntityPortDiff:
     deleted: list[Entity] = field(default_factory=list)
     modified: list[Entity] = field(default_factory=list)
     created: list[Entity] = field(default_factory=list)
+
+
+class IntegrationFeatureFlag(StrEnum):
+    USE_PROVISIONED_DEFAULTS = "USE_PROVISIONED_DEFAULTS"
+    LAKEHOUSE_ELIGIBLE = "LAKEHOUSE_ELIGIBLE"
+    OCEAN_ACTIONS_PROCESSING_ENABLED = "OCEAN_ACTIONS_PROCESSING_ENABLED"
+
+
+class RunStatus(StrEnum):
+    IN_PROGRESS = "IN_PROGRESS"
+    SUCCESS = "SUCCESS"
+    FAILURE = "FAILURE"
+
+
+class IntegrationActionInvocationPayload(BaseModel):
+    type: Literal["INTEGRATION_ACTION"]
+    installationId: str
+    integrationActionType: str
+    integrationActionExecutionProperties: dict[str, Any] = Field(default_factory=dict)
+
+
+class ActionRun(BaseModel):
+    id: str
+    status: RunStatus
+    payload: IntegrationActionInvocationPayload

port_ocean/core/utils/utils.py

@@ -4,7 +4,7 @@ import json
 from typing import Iterable, Any, TypeVar, Callable, Awaitable
 
 from loguru import logger
-from pydantic import parse_obj_as, ValidationError
+from pydantic import BaseModel, parse_obj_as, ValidationError
 
 
 from port_ocean.clients.port.client import PortClient
@@ -79,6 +79,19 @@ async def gather_and_split_errors_from_results(
     return valid_items, errors
 
 
+def _get_entity_key(entity: Entity) -> tuple[str, str]:
+    identifier = entity.identifier
+    if isinstance(identifier, BaseModel):
+        identifier = identifier.dict()
+
+    key_part = (
+        json.dumps(identifier, sort_keys=True)
+        if isinstance(identifier, dict)
+        else str(identifier)
+    )
+    return key_part, entity.blueprint
+
+
 def get_port_diff(before: Iterable[Entity], after: Iterable[Entity]) -> EntityPortDiff:
     before_dict = {}
     after_dict = {}
@@ -88,12 +101,10 @@ def get_port_diff(before: Iterable[Entity], after: Iterable[Entity]) -> EntityPo
 
     # Create dictionaries for before and after lists
     for entity in before:
-        key = (entity.identifier, entity.blueprint)
-        before_dict[key] = entity
+        before_dict[_get_entity_key(entity)] = entity
 
     for entity in after:
-        key = (entity.identifier, entity.blueprint)
-        after_dict[key] = entity
+        after_dict[_get_entity_key(entity)] = entity
 
     # Find created, modified, and deleted objects
     for key, obj in after_dict.items():

port_ocean/exceptions/execution_manager.py

@@ -0,0 +1,22 @@
+class DuplicateActionExecutorError(Exception):
+    """
+    Raised when attempting to register an executor for an action that already has an existing executor.
+    """
+
+    pass
+
+
+class RunAlreadyAcknowledgedError(Exception):
+    """
+    Raised when attempting to acknowledge a run that has already been acknowledged.
+    """
+
+    pass
+
+
+class PartitionKeyNotFoundError(Exception):
+    """
+    Raised when attempting to extract a partition key that is not found in the invocation payload.
+    """
+
+    pass

port_ocean/helpers/retry.py

@@ -257,7 +257,7 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
             else:
                 response = await transport.handle_async_request(request)
 
-            await self._log_response_size_async(request, response)
+            self._log_response_size(request, response)
 
             return response
         except Exception as e:
@@ -345,32 +345,6 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
             return int(content_length)
         return None
 
-    async def _log_response_size_async(
-        self, request: httpx.Request, response: httpx.Response
-    ) -> None:
-        """Log the size of the response."""
-        if not self._should_log_response_size(request):
-            return
-
-        # Try to get content length from headers first
-        content_length = self._get_content_length(response)
-        if content_length is not None:
-            size_info = content_length
-        else:
-            # If no Content-Length header, try to get actual content size
-            try:
-                actual_size = len(await response.aread())
-                size_info = actual_size
-            except Exception as e:
-                cast(logging.Logger, self._logger).error(
-                    f"Error getting response size: {e}"
-                )
-                return
-
-        cast(logging.Logger, self._logger).info(
-            f"Response for {request.method} {request.url} - Size: {size_info} bytes"
-        )
-
     def _log_response_size(
         self, request: httpx.Request, response: httpx.Response
     ) -> None:
@@ -378,21 +352,11 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
             return
 
         content_length = self._get_content_length(response)
-        if content_length is not None:
-            size_info = content_length
-        else:
-            # If no Content-Length header, try to get actual content size
-            try:
-                actual_size = len(response.read())
-                size_info = actual_size
-            except Exception as e:
-                cast(logging.Logger, self._logger).error(
-                    f"Error getting response size: {e}"
-                )
-                return
+        if content_length is None:
+            return
 
         cast(logging.Logger, self._logger).info(
-            f"Response for {request.method} {request.url} - Size: {size_info} bytes"
+            f"Response for {request.method} {request.url} - Size: {content_length} bytes"
         )
 
     async def _should_retry_async(self, response: httpx.Response) -> bool:

port_ocean/log/logger_setup.py

@@ -61,9 +61,9 @@ def _http_loguru_handler(level: LogLevelType) -> None:
 
     http_memory_handler = HTTPMemoryHandler()
     signal_handler.register(
-        http_memory_handler.wait_for_lingering_threads, priority=-200
+        http_memory_handler.wait_for_lingering_threads, priority=-900
     )
-    signal_handler.register(http_memory_handler.flush, priority=-200)
+    signal_handler.register(http_memory_handler.flush, priority=-899)
 
     queue_listener = QueueListener(queue, http_memory_handler)
     queue_listener.start()

port_ocean/ocean.py

@@ -26,6 +26,7 @@ from port_ocean.core.handlers.resync_state_updater import ResyncStateUpdater
 from port_ocean.core.handlers.webhook.processor_manager import (
     LiveEventsProcessorManager,
 )
+from port_ocean.core.handlers.actions.execution_manager import ExecutionManager
 from port_ocean.core.integrations.base import BaseIntegration
 from port_ocean.core.models import ProcessExecutionMode
 from port_ocean.log.sensetive import sensitive_log_filter
@@ -88,6 +89,16 @@ class Ocean:
             max_wait_seconds_before_shutdown=self.config.max_wait_seconds_before_shutdown,
         )
 
+        self.execution_manager = ExecutionManager(
+            webhook_manager=self.webhook_manager,
+            signal_handler=signal_handler,
+            workers_count=self.config.actions_processor.workers_count,
+            runs_buffer_high_watermark=self.config.actions_processor.runs_buffer_high_watermark,
+            poll_check_interval_seconds=self.config.actions_processor.poll_check_interval_seconds,
+            visibility_timeout_ms=self.config.actions_processor.visibility_timeout_ms,
+            max_wait_seconds_before_shutdown=self.config.max_wait_seconds_before_shutdown,
+        )
+
         self.integration = (
             integration_class(ocean) if integration_class else BaseIntegration(ocean)
         )
@@ -200,6 +211,24 @@ class Ocean:
                 )
         return None
 
+    async def _register_addons(self) -> None:
+        if self.base_url and self.config.event_listener.should_process_webhooks:
+            await self.webhook_manager.start_processing_event_messages()
+        else:
+            logger.warning(
+                "No base URL provided, or webhook processing is disabled is this event listener, skipping webhook processing"
+            )
+
+        if (
+            self.config.actions_processor.enabled
+            and self.config.event_listener.should_run_actions
+        ):
+            await self.execution_manager.start_processing_action_runs()
+        else:
+            logger.warning(
+                "Execution agent is not enabled, or actions processing is disabled in this event listener, skipping execution agent setup"
+            )
+
     def initialize_app(self) -> None:
         self.fast_api_app.include_router(self.integration_router, prefix="/integration")
         self.fast_api_app.include_router(
@@ -210,10 +239,7 @@ class Ocean:
         async def lifecycle(_: FastAPI) -> AsyncIterator[None]:
             try:
                 await self.integration.start()
-                if self.base_url:
-                    await self.webhook_manager.start_processing_event_messages()
-                else:
-                    logger.warning("No base URL provided, skipping webhook processing")
+                await self._register_addons()
                 await self._setup_scheduled_resync()
                 yield None
             except Exception:

port_ocean/tests/clients/port/mixins/test_entities.py

@@ -179,7 +179,7 @@ async def test_search_entities_uses_datasource_route_when_query_is_none(
 ) -> None:
     """Test that search_entities uses datasource route when query is None"""
     mock_response = MagicMock()
-    mock_response.json.return_value = {"entities": []}
+    mock_response.json.return_value = {"entities": [], "next": None}
     mock_response.is_error = False
     mock_response.status_code = 200
     mock_response.headers = {}
@@ -208,8 +208,74 @@ async def test_search_entities_uses_datasource_route_when_query_is_none(
         == "https://api.getport.io/v1/blueprints/entities/datasource-entities"
     )
 
-    expected_json = {
-        "datasource_prefix": "port-ocean/test-integration/",
-        "datasource_suffix": "/test-identifier/sync",
+    sent_json = call_args[1]["json"]
+    assert sent_json["datasource_prefix"] == "port-ocean/test-integration/"
+    assert sent_json["datasource_suffix"] == "/test-identifier/sync"
+
+
+async def test_search_entities_uses_datasource_route_when_query_is_none_two_pages(
+    entity_client: EntityClientMixin,
+) -> None:
+    """Test that search_entities uses datasource route when query is None"""
+    # First response with pagination token
+    mock_response_first = MagicMock()
+    mock_response_first.json.return_value = {
+        "entities": [
+            Entity(identifier="entity_1", blueprint="entity_1"),
+            Entity(identifier="entity_2", blueprint="entity_2"),
+        ],
+        "next": "next_page_token",
+    }
+    mock_response_first.is_error = False
+    mock_response_first.status_code = 200
+    mock_response_first.headers = {}
+
+    # Second response without pagination token (end of pagination)
+    mock_response_second = MagicMock()
+    mock_response_second.json.return_value = {
+        "entities": [Entity(identifier="entity_3", blueprint="entity_3")],
+        "next": None,
     }
-    assert call_args[1]["json"] == expected_json
+    mock_response_second.is_error = False
+    mock_response_second.status_code = 200
+    mock_response_second.headers = {}
+
+    # Mock the client to return different responses for each call
+    entity_client.client.post = AsyncMock(side_effect=[mock_response_first, mock_response_second])  # type: ignore
+    entity_client.auth.headers = AsyncMock(return_value={"Authorization": "Bearer test"})  # type: ignore
+
+    entity_client.auth.integration_type = "test-integration"
+    entity_client.auth.integration_identifier = "test-identifier"
+    entity_client.auth.api_url = "https://api.getport.io/v1"
+
+    mock_user_agent_type = MagicMock()
+    mock_user_agent_type.value = "sync"
+
+    entities = await entity_client.search_entities(
+        user_agent_type=mock_user_agent_type,
+        query=None,
+    )
+
+    # Should call the datasource-entities endpoint exactly twice for pagination
+    assert entity_client.client.post.await_count == 2
+    assert len(entities) == 3
+
+    # Check first call
+    first_call_args = entity_client.client.post.call_args_list[0]
+    assert (
+        first_call_args[0][0]
+        == "https://api.getport.io/v1/blueprints/entities/datasource-entities"
+    )
+    first_sent_json = first_call_args[1]["json"]
+    assert first_sent_json["datasource_prefix"] == "port-ocean/test-integration/"
+    assert first_sent_json["datasource_suffix"] == "/test-identifier/sync"
+
+    # Check second call
+    second_call_args = entity_client.client.post.call_args_list[1]
+    assert (
+        second_call_args[0][0]
+        == "https://api.getport.io/v1/blueprints/entities/datasource-entities"
+    )
+    second_sent_json = second_call_args[1]["json"]
+    assert second_sent_json["datasource_prefix"] == "port-ocean/test-integration/"
+    assert second_sent_json["datasource_suffix"] == "/test-identifier/sync"

port_ocean/tests/core/event_listener/test_kafka.py

@@ -1,11 +1,12 @@
 from port_ocean.core.event_listener.kafka import KafkaEventListenerSettings
+from port_ocean.core.models import EventListenerType
 import pytest
 from pydantic import ValidationError
 
 
 def test_default_kafka_settings() -> None:
     """Test default values are properly set"""
-    config = KafkaEventListenerSettings(type="KAFKA")
+    config = KafkaEventListenerSettings(type=EventListenerType.KAFKA)
     assert config.type == "KAFKA"
     assert config.security_protocol == "SASL_SSL"
     assert config.authentication_mechanism == "SCRAM-SHA-512"
@@ -17,28 +18,32 @@ def test_default_kafka_settings() -> None:
 def test_brokers_json_array_parsing() -> None:
     """Test that JSON array strings get converted to comma-separated"""
     json_brokers = '["broker1:9092", "broker2:9092", "broker3:9092"]'
-    config = KafkaEventListenerSettings(type="KAFKA", brokers=json_brokers)
+    config = KafkaEventListenerSettings(
+        type=EventListenerType.KAFKA, brokers=json_brokers
+    )
     assert config.brokers == "broker1:9092,broker2:9092,broker3:9092"
 
 
 def test_brokers_regular_string_unchanged() -> None:
     """Test that regular comma-separated strings pass through unchanged"""
     regular_brokers = "broker1:9092,broker2:9092"
-    config = KafkaEventListenerSettings(type="KAFKA", brokers=regular_brokers)
+    config = KafkaEventListenerSettings(
+        type=EventListenerType.KAFKA, brokers=regular_brokers
+    )
     assert config.brokers == regular_brokers
 
 
 def test_brokers_malformed_json_unchanged() -> None:
     """Test that malformed JSON strings don't break validation"""
     bad_json = "[broker1:9092, broker2:9092"
-    config = KafkaEventListenerSettings(type="KAFKA", brokers=bad_json)
+    config = KafkaEventListenerSettings(type=EventListenerType.KAFKA, brokers=bad_json)
     assert config.brokers == bad_json
 
 
 def test_custom_values() -> None:
     """Test overriding default values"""
     config = KafkaEventListenerSettings(
-        type="KAFKA",
+        type=EventListenerType.KAFKA,
         brokers="custom:9092",
         security_protocol="PLAINTEXT",
         authentication_mechanism="PLAIN",
@@ -60,11 +65,13 @@ def test_type_literal_validation() -> None:
 
 def test_empty_brokers_array() -> None:
     """Test empty JSON array becomes empty string"""
-    config = KafkaEventListenerSettings(type="KAFKA", brokers="[]")
+    config = KafkaEventListenerSettings(type=EventListenerType.KAFKA, brokers="[]")
     assert config.brokers == ""
 
 
 def test_single_broker_array() -> None:
     """Test single broker in JSON array"""
-    config = KafkaEventListenerSettings(type="KAFKA", brokers='["single:9092"]')
+    config = KafkaEventListenerSettings(
+        type=EventListenerType.KAFKA, brokers='["single:9092"]'
+    )
     assert config.brokers == "single:9092"