port-ocean 0.26.3__tar.gz → 0.27.0__tar.gz

{port_ocean-0.26.3 → port_ocean-0.27.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.26.3
+Version: 0.27.0
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port

{port_ocean-0.26.3 → port_ocean-0.27.0}/port_ocean/config/settings.py

@@ -9,6 +9,7 @@ from pydantic.main import BaseModel
 
 from port_ocean.config.base import BaseOceanModel, BaseOceanSettings
 from port_ocean.core.event_listener import EventListenerSettingsType
+
 from port_ocean.core.models import (
     CachingStorageMode,
     CreatePortResourcesOrigin,
@@ -88,6 +89,7 @@ class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
     event_listener: EventListenerSettingsType = Field(
         default=cast(EventListenerSettingsType, {"type": "POLLING"})
     )
+    event_workers_count: int = 1
     # If an identifier or type is not provided, it will be generated based on the integration name
     integration: IntegrationSettings = Field(
         default_factory=lambda: IntegrationSettings(type="", identifier="")

port_ocean-0.27.0/port_ocean/core/handlers/queue/__init__.py

@@ -0,0 +1,5 @@
+from .abstract_queue import AbstractQueue
+from .local_queue import LocalQueue
+from .group_queue import GroupQueue
+
+__all__ = ["AbstractQueue", "LocalQueue", "GroupQueue"]

{port_ocean-0.26.3 → port_ocean-0.27.0}/port_ocean/core/handlers/queue/abstract_queue.py

@@ -7,6 +7,9 @@ T = TypeVar("T")
 class AbstractQueue(ABC, Generic[T]):
     """Abstract interface for queues"""
 
+    def __init__(self, name: str | None = None):
+        pass
+
     @abstractmethod
     async def put(self, item: T) -> None:
         """Put an item into the queue"""
@@ -22,6 +25,11 @@ class AbstractQueue(ABC, Generic[T]):
         """Wait for all items to be processed"""
         pass
 
+    @abstractmethod
+    async def size(self) -> int:
+        """Size of the queue"""
+        pass
+
     @abstractmethod
     async def commit(self) -> None:
         """Mark item as processed"""

port_ocean-0.27.0/port_ocean/core/handlers/queue/group_queue.py

@@ -0,0 +1,138 @@
+import asyncio
+from collections import defaultdict, deque
+import time
+from typing import Deque, Dict, Optional, Set, TypeVar, Any
+from contextvars import ContextVar
+
+from loguru import logger
+
+from .abstract_queue import AbstractQueue
+
+T = TypeVar("T")
+MaybeStr = str | None
+
+_NO_GROUP = object()
+_current_group: ContextVar[Any] = ContextVar("current_group", default=_NO_GROUP)
+
+
+class GroupQueue(AbstractQueue[T]):
+    """Queue with exclusive processing per group."""
+
+    def __init__(
+        self,
+        group_key: MaybeStr = None,
+        name: MaybeStr = None,
+        lock_timeout: float = 300,
+    ):
+        super().__init__(name)
+        self.group_key = group_key
+        self._queues: Dict[MaybeStr, Deque[T]] = defaultdict(deque)
+        self._locked: Set[MaybeStr] = set()
+        self._queue_not_empty = asyncio.Condition()
+        self.lock_timeout = lock_timeout
+        self._lock_timestamps: Dict[MaybeStr, float] = {}
+        self._timeout_task: Optional[asyncio.Task[None]] = None
+
+    async def _background_timeout_check(self) -> None:
+        """Periodically release locks that have timed out."""
+        while True:
+            try:
+                await asyncio.sleep(self.lock_timeout / 4)
+                async with self._queue_not_empty:
+                    await self._release_expired_locks()
+            except asyncio.CancelledError:
+                break
+
+    def _extract_group_key(self, item: T) -> MaybeStr:
+        """Extract the group key from an item."""
+        if self.group_key is None:
+            return None
+        if not hasattr(item, self.group_key):
+            raise ValueError(
+                f"Item {item!r} lacks attribute '{self.group_key}' required for grouping"
+            )
+        return getattr(item, self.group_key)
+
+    async def put(self, item: T) -> None:
+        """Add item to its group's queue."""
+        group_key = self._extract_group_key(item)
+        async with self._queue_not_empty:
+            self._queues[group_key].append(item)
+            self._queue_not_empty.notify_all()
+
+    async def _release_expired_locks(self) -> None:
+        """Release locks that have exceeded the timeout."""
+        now = time.time()
+        expired_groups = []
+
+        for group, timestamp in list(self._lock_timestamps.items()):
+            if now - timestamp > self.lock_timeout:
+                expired_groups.append(group)
+                logger.warning(f"Releasing expired lock for group {group}")
+                self._locked.discard(group)
+                del self._lock_timestamps[group]
+
+        if expired_groups:
+            self._queue_not_empty.notify_all()
+
+    async def get(self) -> T:
+        """Get the next item from an unlocked group, locking that group."""
+        if self._timeout_task is None or self._timeout_task.done():
+            self._timeout_task = asyncio.create_task(self._background_timeout_check())
+
+        async with self._queue_not_empty:
+            while True:
+                await self._release_expired_locks()
+
+                for group, queue in self._queues.items():
+                    if queue and group not in self._locked:
+                        self._locked.add(group)
+                        self._lock_timestamps[group] = time.time()
+                        _current_group.set(group)
+                        return queue[0]
+
+                await self._queue_not_empty.wait()
+
+    async def commit(self) -> None:
+        """Remove the current item and unlock its group."""
+        group = _current_group.get()
+        if group is _NO_GROUP:
+            logger.warning("commit() called without active get()")
+            return
+
+        async with self._queue_not_empty:
+            queue = self._queues.get(group)
+            if queue:
+                queue.popleft()
+                if not queue:
+                    del self._queues[group]
+
+            self._locked.discard(group)
+            self._lock_timestamps.pop(group, None)
+            _current_group.set(_NO_GROUP)
+            self._queue_not_empty.notify_all()
+
+    async def teardown(self) -> None:
+        """Wait until all queues are empty and no groups are locked."""
+        async with self._queue_not_empty:
+            while any(self._queues.values()) or self._locked:
+                await self._queue_not_empty.wait()
+
+        if self._timeout_task and not self._timeout_task.done():
+            self._timeout_task.cancel()
+            try:
+                await self._timeout_task
+            except asyncio.CancelledError:
+                pass
+
+    async def size(self) -> int:
+        """Return total number of items across all groups."""
+        async with self._queue_not_empty:
+            return sum(len(queue) for queue in self._queues.values())
+
+    async def force_unlock_all(self) -> None:
+        """Force unlock all groups."""
+        async with self._queue_not_empty:
+            self._locked.clear()
+            self._lock_timestamps.clear()
+            self._queue_not_empty.notify_all()

{port_ocean-0.26.3 → port_ocean-0.27.0}/port_ocean/core/handlers/queue/local_queue.py

@@ -23,3 +23,6 @@ class LocalQueue(AbstractQueue[T]):
 
     async def commit(self) -> None:
         self._queue.task_done()
+
+    async def size(self) -> int:
+        return self._queue.qsize()

{port_ocean-0.26.3 → port_ocean-0.27.0}/port_ocean/core/handlers/webhook/processor_manager.py

@@ -1,4 +1,5 @@
-from typing import Dict, Type, Set
+from typing import Dict, Tuple, Type, Set, List
+
 from fastapi import APIRouter, Request
 from loguru import logger
 import asyncio
@@ -6,6 +7,7 @@ import asyncio
 from port_ocean.context.ocean import ocean
 from port_ocean.context.event import EventType, event_context
 from port_ocean.core.handlers.port_app_config.models import ResourceConfig
+from port_ocean.core.handlers.queue.abstract_queue import AbstractQueue
 from port_ocean.core.integrations.mixins.events import EventsMixin
 from port_ocean.core.integrations.mixins.live_events import LiveEventsMixin
 from port_ocean.exceptions.webhook_processor import WebhookEventNotSupportedError
@@ -15,7 +17,7 @@ from port_ocean.context.event import event
 
 from .abstract_webhook_processor import AbstractWebhookProcessor
 from port_ocean.utils.signal import SignalHandler
-from port_ocean.core.handlers.queue import AbstractQueue, LocalQueue
+from port_ocean.core.handlers.queue import LocalQueue
 
 
 class LiveEventsProcessorManager(LiveEventsMixin, EventsMixin):
@@ -31,22 +33,103 @@ class LiveEventsProcessorManager(LiveEventsMixin, EventsMixin):
         self._router = router
         self._processors_classes: Dict[str, list[Type[AbstractWebhookProcessor]]] = {}
         self._event_queues: Dict[str, AbstractQueue[WebhookEvent]] = {}
-        self._webhook_processor_tasks: Set[asyncio.Task[None]] = set()
+        self._event_processor_tasks: Set[asyncio.Task[None]] = set()
         self._max_event_processing_seconds = max_event_processing_seconds
         self._max_wait_seconds_before_shutdown = max_wait_seconds_before_shutdown
         signal_handler.register(self.shutdown)
 
     async def start_processing_event_messages(self) -> None:
-        """Start processing events for all registered paths"""
+        """Start processing events for all registered paths with N workers each."""
         await self.initialize_handlers()
         loop = asyncio.get_event_loop()
+        config = ocean.integration.context.config
+
         for path in self._event_queues.keys():
+            for worker_id in range(0, config.event_workers_count):
+                task = loop.create_task(self._process_webhook_events(path, worker_id))
+                self._event_processor_tasks.add(task)
+                task.add_done_callback(self._event_processor_tasks.discard)
+
+    async def _process_webhook_events(self, path: str, worker_id: int) -> None:
+        """Process webhook events from the queue for a given path."""
+        queue = self._event_queues[path]
+        while True:
+            event = None
+            matching_processors: List[
+                Tuple[ResourceConfig, AbstractWebhookProcessor]
+            ] = []
             try:
-                task = loop.create_task(self.process_queue(path))
-                self._webhook_processor_tasks.add(task)
-                task.add_done_callback(self._webhook_processor_tasks.discard)
+                event = await queue.get()
+                with logger.contextualize(
+                    worker=worker_id,
+                    webhook_path=path,
+                    trace_id=event.trace_id,
+                ):
+                    async with event_context(
+                        EventType.HTTP_REQUEST,
+                        trigger_type="machine",
+                    ):
+
+                        await ocean.integration.port_app_config_handler.get_port_app_config(
+                            use_cache=False
+                        )
+                        matching_processors = await self._extract_matching_processors(
+                            event, path
+                        )
+
+                        processing_results = await asyncio.gather(
+                            *(
+                                self._process_single_event(proc, path, res)
+                                for res, proc in matching_processors
+                            ),
+                            return_exceptions=True,
+                        )
+
+                        successful_results: List[WebhookEventRawResults] = []
+                        failed_exceptions: List[Exception] = []
+
+                        for result in processing_results:
+                            if isinstance(result, WebhookEventRawResults):
+                                successful_results.append(result)
+                            elif isinstance(result, Exception):
+                                failed_exceptions.append(result)
+
+                        if successful_results:
+                            logger.info(
+                                "Successfully processed webhook events",
+                                success_count=len(successful_results),
+                                failure_count=len(failed_exceptions),
+                            )
+
+                        if failed_exceptions:
+                            logger.warning(
+                                "Some webhook events failed processing",
+                                failures=[str(e) for e in failed_exceptions],
+                            )
+
+                        await self.sync_raw_results(successful_results)
+
+            except asyncio.CancelledError:
+                logger.info(f"Worker {worker_id} for {path} shutting down")
+                for _, proc in matching_processors:
+                    await proc.cancel()
+                    self._timestamp_event_error(proc.event)
+                break
             except Exception as e:
-                logger.exception(f"Error starting queue processor for {path}: {str(e)}")
+                logger.exception(
+                    f"Unexpected error in worker {worker_id} for {path}: {e}"
+                )
+                for _, proc in matching_processors:
+                    self._timestamp_event_error(proc.event)
+            finally:
+                try:
+                    if event is not None:
+                        await queue.commit()
+
+                except Exception as e:
+                    logger.exception(
+                        f"Unexpected error in queue commit in worker {worker_id} for {path}: {e}"
+                    )
 
     async def _extract_matching_processors(
         self, webhook_event: WebhookEvent, path: str
@@ -91,70 +174,6 @@ class LiveEventsProcessorManager(LiveEventsMixin, EventsMixin):
         )
         return created_processors
 
-    async def process_queue(self, path: str) -> None:
-        """Process events for a specific path in order"""
-        while True:
-            matching_processors_with_resource: list[
-                tuple[ResourceConfig, AbstractWebhookProcessor]
-            ] = []
-            webhook_event: WebhookEvent | None = None
-            try:
-                queue = self._event_queues[path]
-                webhook_event = await queue.get()
-                with logger.contextualize(
-                    webhook_path=path, trace_id=webhook_event.trace_id
-                ):
-                    async with event_context(
-                        EventType.HTTP_REQUEST,
-                        trigger_type="machine",
-                    ):
-                        # This forces the Processor manager to fetch the latest port app config for each event
-                        await ocean.integration.port_app_config_handler.get_port_app_config(
-                            use_cache=False
-                        )
-                        matching_processors_with_resource = (
-                            await self._extract_matching_processors(webhook_event, path)
-                        )
-                        webhook_event_raw_results_for_all_resources = await asyncio.gather(
-                            *(
-                                self._process_single_event(processor, path, resource)
-                                for resource, processor in matching_processors_with_resource
-                            ),
-                            return_exceptions=True,
-                        )
-
-                        successful_raw_results: list[WebhookEventRawResults] = [
-                            result
-                            for result in webhook_event_raw_results_for_all_resources
-                            if isinstance(result, WebhookEventRawResults)
-                        ]
-
-                        if successful_raw_results:
-                            logger.info(
-                                "Exporting raw event results to entities",
-                                webhook_event_raw_results_for_all_resources_length=len(
-                                    successful_raw_results
-                                ),
-                            )
-                            await self.sync_raw_results(successful_raw_results)
-            except asyncio.CancelledError:
-                logger.info(f"Queue processor for {path} is shutting down")
-                for _, processor in matching_processors_with_resource:
-                    await processor.cancel()
-                    self._timestamp_event_error(processor.event)
-                break
-            except Exception as e:
-                logger.exception(
-                    f"Unexpected error in queue processor for {path}: {str(e)}"
-                )
-                for _, processor in matching_processors_with_resource:
-                    self._timestamp_event_error(processor.event)
-            finally:
-                if webhook_event:
-                    await self._event_queues[path].commit()
-                    # Prevents committing empty events for cases where we shutdown while processing
-                    webhook_event = None
-
     def _timestamp_event_error(self, event: WebhookEvent) -> None:
         """Timestamp an event as having an error"""
         event.set_timestamp(LiveEventTimestamp.FinishedProcessingWithError)
@@ -279,12 +298,14 @@ class LiveEventsProcessorManager(LiveEventsMixin, EventsMixin):
             methods=["POST"],
         )
 
-    async def _cancel_all_tasks(self) -> None:
-        """Cancel all webhook processor tasks"""
-        for task in self._webhook_processor_tasks:
+    async def _cancel_all_event_processors(
+        self,
+    ) -> None:
+        """Cancel all event processor tasks"""
+        for task in self._event_processor_tasks:
             task.cancel()
 
-        await asyncio.gather(*self._webhook_processor_tasks, return_exceptions=True)
+        await asyncio.gather(*self._event_processor_tasks, return_exceptions=True)
 
     async def shutdown(self) -> None:
         """Gracefully shutdown all queue processors"""
@@ -299,5 +320,3 @@ class LiveEventsProcessorManager(LiveEventsMixin, EventsMixin):
             )
         except asyncio.TimeoutError:
             logger.warning("Shutdown timed out waiting for queues to empty")
-
-        await self._cancel_all_tasks()

{port_ocean-0.26.3 → port_ocean-0.27.0}/port_ocean/core/handlers/webhook/webhook_event.py

@@ -51,11 +51,13 @@ class WebhookEvent(LiveEvent):
         payload: EventPayload,
         headers: EventHeaders,
         original_request: Request | None = None,
+        group_id: str | None = None,
     ) -> None:
         self.trace_id = trace_id
         self.payload = payload
         self.headers = headers
         self._original_request = original_request
+        self.group_id = group_id
 
     @classmethod
     async def from_request(