port-ocean 0.18.5__py3-none-any.whl → 0.18.7__py3-none-any.whl

port_ocean/context/ocean.py

@@ -5,6 +5,7 @@ from pydantic.main import BaseModel
 from werkzeug.local import LocalProxy
 
 from port_ocean.clients.port.types import UserAgentType
+
 from port_ocean.core.models import Entity
 from port_ocean.core.ocean_types import (
     RESYNC_EVENT_LISTENER,
@@ -145,6 +146,37 @@ class PortOceanContext:
     async def sync_raw_all(self) -> None:
         await self.integration.sync_raw_all(trigger_type="manual")
 
+    def add_webhook_processor(
+        self,
+        path: str,
+        processor: type,
+        events_filter: Callable[[Any], bool] = lambda _: True,
+    ) -> None:
+        """
+        Registers a webhook processor for a specific path.
+
+        Args:
+            path: The path to register the webhook processor for.
+            processor: The processor to register.
+        Examples:
+            >>> from port_ocean.context.ocean import ocean
+            >>> from port_ocean.core.handlers.webhook import AbstractWebhookProcessor
+            >>> from port_ocean.core.handlers.webhook import WebhookEvent
+            >>> class MyWebhookProcessor(AbstractWebhookProcessor):
+            ...     async def authenticate(self, payload: EventPayload, headers: EventHeaders) -> bool:
+            ...         return True
+            ...     async def validate_payload(self, payload: EventPayload) -> bool:
+            ...         return True
+            ...     async def handle_event(self, payload: EventPayload) -> None:
+            ...         pass
+            >>> def events_filter(event: WebhookEvent) -> bool:
+            ...     return True
+            >>> ocean.add_webhook_processor('/my-webhook', MyWebhookProcessor, events_filter)
+        Raises:
+            ValueError: If the processor does not extend AbstractWebhookProcessor.
+        """
+        self.app.webhook_manager.register_processor(path, processor, events_filter)
+
 
 _port_ocean: PortOceanContext = PortOceanContext(None)
 

port_ocean/core/handlers/entities_state_applier/port/applier.py

@@ -78,6 +78,7 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
         self,
         entities: EntityDiff,
         user_agent_type: UserAgentType,
+        entity_deletion_threshold: float | None = None,
     ) -> None:
         diff = get_port_diff(entities["before"], entities["after"])
 
@@ -87,10 +88,25 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
         kept_entities = diff.created + diff.modified
 
         logger.info(
-            f"Determining entities to delete ({len(diff.deleted)}/{len(kept_entities)})"
+            f"Determining entities to delete ({len(diff.deleted)}/{len(kept_entities)})",
+            deleting_entities=len(diff.deleted),
+            keeping_entities=len(kept_entities),
+            entity_deletion_threshold=entity_deletion_threshold,
         )
 
-        await self._safe_delete(diff.deleted, kept_entities, user_agent_type)
+        deletion_rate = len(diff.deleted) / len(entities["before"])
+        if (
+            entity_deletion_threshold is not None
+            and deletion_rate <= entity_deletion_threshold
+        ):
+            await self._safe_delete(diff.deleted, kept_entities, user_agent_type)
+        else:
+            logger.info(
+                f"Skipping deletion of entities with delition rate {deletion_rate}",
+                deletion_rate=deletion_rate,
+                deleting_entities=len(diff.deleted),
+                total_entities=len(entities),
+            )
 
     async def upsert(
         self, entities: list[Entity], user_agent_type: UserAgentType

port_ocean/core/handlers/port_app_config/models.py

@@ -58,6 +58,9 @@ class PortAppConfig(BaseModel):
     create_missing_related_entities: bool = Field(
         alias="createMissingRelatedEntities", default=True
     )
+    entity_deletion_threshold: float = Field(
+        alias="entityDeletionThreshold", default=0.9
+    )
     resources: list[ResourceConfig] = Field(default_factory=list)
 
     def get_port_request_options(self) -> RequestOptions:

port_ocean/core/handlers/queue/__init__.py

@@ -0,0 +1,4 @@
+from .abstract_queue import AbstractQueue
+from .local_queue import LocalQueue
+
+__all__ = ["AbstractQueue", "LocalQueue"]

port_ocean/core/handlers/queue/abstract_queue.py

@@ -0,0 +1,28 @@
+from abc import ABC, abstractmethod
+from typing import Generic, TypeVar
+
+T = TypeVar("T")
+
+
+class AbstractQueue(ABC, Generic[T]):
+    """Abstract interface for queues"""
+
+    @abstractmethod
+    async def put(self, item: T) -> None:
+        """Put an item into the queue"""
+        pass
+
+    @abstractmethod
+    async def get(self) -> T:
+        """Get an item from the queue"""
+        pass
+
+    @abstractmethod
+    async def teardown(self) -> None:
+        """Wait for all items to be processed"""
+        pass
+
+    @abstractmethod
+    async def commit(self) -> None:
+        """Mark item as processed"""
+        pass

port_ocean/core/handlers/queue/local_queue.py

@@ -0,0 +1,25 @@
+import asyncio
+from typing import TypeVar
+
+from .abstract_queue import AbstractQueue
+
+T = TypeVar("T")
+
+
+class LocalQueue(AbstractQueue[T]):
+    """Implementation of Queue using asyncio.Queue"""
+
+    def __init__(self) -> None:
+        self._queue: asyncio.Queue[T] = asyncio.Queue()
+
+    async def put(self, item: T) -> None:
+        await self._queue.put(item)
+
+    async def get(self) -> T:
+        return await self._queue.get()
+
+    async def teardown(self) -> None:
+        await self._queue.join()
+
+    async def commit(self) -> None:
+        self._queue.task_done()

port_ocean/core/handlers/webhook/abstract_webhook_processor.py

@@ -0,0 +1,101 @@
+from abc import ABC, abstractmethod
+from loguru import logger
+
+from port_ocean.exceptions.webhook_processor import RetryableError
+
+from .webhook_event import WebhookEvent, EventPayload, EventHeaders
+
+
+class AbstractWebhookProcessor(ABC):
+    """
+    Abstract base class for webhook processors
+    Extend this class to implement custom webhook processing logic
+
+    Attributes:
+        max_retries: The maximum number of retries before giving up
+        initial_retry_delay_seconds: The initial delay before the first retry
+        max_retry_delay_seconds: The maximum delay between retries
+        exponential_base_seconds: The base for exponential backoff calculations
+
+    Args:
+        event: The webhook event to process
+
+    Examples:
+        >>> from port_ocean.core.handlers.webhook import AbstractWebhookProcessor
+        >>> from port_ocean.core.handlers.webhook import WebhookEvent
+        >>> class MyWebhookProcessor(AbstractWebhookProcessor):
+        ...     async def authenticate(self, payload: EventPayload, headers: EventHeaders) -> bool:
+        ...         return True
+        ...     async def validate_payload(self, payload: EventPayload) -> bool:
+        ...         return True
+        ...     async def handle_event(self, payload: EventPayload) -> None:
+        ...         pass
+    """
+
+    max_retries: int = 3
+    initial_retry_delay_seconds: float = 1.0
+    max_retry_delay_seconds: float = 30.0
+    exponential_base_seconds: float = 2.0
+
+    def __init__(self, event: WebhookEvent) -> None:
+        self.event = event
+        self.retry_count = 0
+
+    async def on_error(self, error: Exception) -> None:
+        """Hook to handle errors during processing. Override if needed"""
+        delay = self.calculate_retry_delay()
+
+        logger.error(
+            f"Attempt {self.retry_count}/{self.max_retries} failed. "
+            f"Retrying in {delay:.2f} seconds. Error: {str(error)}"
+        )
+
+    async def cancel(self) -> None:
+        """Handle cancellation of the request. Override if needed"""
+        pass
+
+    def validate_webhook_setup(self) -> bool:
+        """Validate webhook configuration. Override if needed"""
+        return True
+
+    def should_retry(self, error: Exception) -> bool:
+        """
+        Determine if the operation should be retried based on the error
+        Override to customize retry behavior
+        """
+        return isinstance(error, RetryableError)
+
+    def calculate_retry_delay(self) -> float:
+        """
+        Calculate the delay before the next retry using exponential backoff
+        Override to customize backoff strategy
+        """
+        delay = min(
+            self.initial_retry_delay_seconds
+            * (self.exponential_base_seconds**self.retry_count),
+            self.max_retry_delay_seconds,
+        )
+        return delay
+
+    async def before_processing(self) -> None:
+        """Hook to run before processing the event"""
+        pass
+
+    async def after_processing(self) -> None:
+        """Hook to run after processing the event"""
+        pass
+
+    @abstractmethod
+    async def authenticate(self, payload: EventPayload, headers: EventHeaders) -> bool:
+        """Authenticate the request."""
+        pass
+
+    @abstractmethod
+    async def validate_payload(self, payload: EventPayload) -> bool:
+        """Validate the payload structure and content."""
+        pass
+
+    @abstractmethod
+    async def handle_event(self, payload: EventPayload) -> None:
+        """Process the event."""
+        pass

port_ocean/core/handlers/webhook/processor_manager.py

@@ -0,0 +1,237 @@
+from typing import Dict, Type, Set, Callable
+from fastapi import APIRouter, Request
+from loguru import logger
+import asyncio
+from dataclasses import dataclass
+
+from .webhook_event import WebhookEvent, WebhookEventTimestamp
+
+
+from .abstract_webhook_processor import AbstractWebhookProcessor
+from port_ocean.utils.signal import SignalHandler
+from port_ocean.core.handlers.queue import AbstractQueue, LocalQueue
+
+
+@dataclass
+class ProcessorRegistration:
+    """Represents a registered processor with its filter"""
+
+    processor: Type[AbstractWebhookProcessor]
+    filter: Callable[[WebhookEvent], bool]
+
+
+class WebhookProcessorManager:
+    """Manages webhook processors and their routes"""
+
+    def __init__(
+        self,
+        router: APIRouter,
+        signal_handler: SignalHandler,
+        max_event_processing_seconds: float = 90.0,
+        max_wait_seconds_before_shutdown: float = 5.0,
+    ) -> None:
+        self._router = router
+        self._processors: Dict[str, list[ProcessorRegistration]] = {}
+        self._event_queues: Dict[str, AbstractQueue[WebhookEvent]] = {}
+        self._webhook_processor_tasks: Set[asyncio.Task[None]] = set()
+        self._max_event_processing_seconds = max_event_processing_seconds
+        self._max_wait_seconds_before_shutdown = max_wait_seconds_before_shutdown
+        signal_handler.register(self.shutdown)
+
+    async def start_processing_event_messages(self) -> None:
+        """Start processing events for all registered paths"""
+        loop = asyncio.get_event_loop()
+        for path in self._event_queues.keys():
+            try:
+                task = loop.create_task(self.process_queue(path))
+                self._webhook_processor_tasks.add(task)
+                task.add_done_callback(self._webhook_processor_tasks.discard)
+            except Exception as e:
+                logger.exception(f"Error starting queue processor for {path}: {str(e)}")
+
+    def _extract_matching_processors(
+        self, event: WebhookEvent, path: str
+    ) -> list[AbstractWebhookProcessor]:
+        """Find and extract the matching processor for an event"""
+        matching_processors = [
+            registration.processor
+            for registration in self._processors[path]
+            if registration.filter(event)
+        ]
+
+        if not matching_processors:
+            raise ValueError("No matching processors found")
+
+        created_processors: list[AbstractWebhookProcessor] = []
+        for processor_class in matching_processors:
+            processor = processor_class(event.clone())
+            created_processors.append(processor)
+        return created_processors
+
+    async def process_queue(self, path: str) -> None:
+        """Process events for a specific path in order"""
+        while True:
+            matching_processors: list[AbstractWebhookProcessor] = []
+            event: WebhookEvent | None = None
+            try:
+                event = await self._event_queues[path].get()
+                with logger.contextualize(webhook_path=path, trace_id=event.trace_id):
+                    matching_processors = self._extract_matching_processors(event, path)
+                    await asyncio.gather(
+                        *(
+                            self._process_single_event(processor, path)
+                            for processor in matching_processors
+                        )
+                    )
+            except asyncio.CancelledError:
+                logger.info(f"Queue processor for {path} is shutting down")
+                for processor in matching_processors:
+                    await processor.cancel()
+                    self._timestamp_event_error(processor.event)
+                break
+            except Exception as e:
+                logger.exception(
+                    f"Unexpected error in queue processor for {path}: {str(e)}"
+                )
+                for processor in matching_processors:
+                    self._timestamp_event_error(processor.event)
+            finally:
+                if event:
+                    await self._event_queues[path].commit()
+                    # Prevents committing empty events for cases where we shutdown while processing
+                    event = None
+
+    def _timestamp_event_error(self, event: WebhookEvent) -> None:
+        """Timestamp an event as having an error"""
+        event.set_timestamp(WebhookEventTimestamp.FinishedProcessingWithError)
+
+    async def _process_single_event(
+        self, processor: AbstractWebhookProcessor, path: str
+    ) -> None:
+        """Process a single event with a specific processor"""
+        try:
+            logger.debug("Start processing queued webhook")
+            processor.event.set_timestamp(WebhookEventTimestamp.StartedProcessing)
+
+            await self._execute_processor(processor)
+            processor.event.set_timestamp(
+                WebhookEventTimestamp.FinishedProcessingSuccessfully
+            )
+        except Exception as e:
+            logger.exception(f"Error processing queued webhook for {path}: {str(e)}")
+            self._timestamp_event_error(processor.event)
+
+    async def _execute_processor(self, processor: AbstractWebhookProcessor) -> None:
+        """Execute a single processor within a max processing time"""
+        try:
+            await asyncio.wait_for(
+                self._process_webhook_request(processor),
+                timeout=self._max_event_processing_seconds,
+            )
+        except asyncio.TimeoutError:
+            raise TimeoutError(
+                f"Processor processing timed out after {self._max_event_processing_seconds} seconds"
+            )
+
+    async def _process_webhook_request(
+        self, processor: AbstractWebhookProcessor
+    ) -> None:
+        """Process a webhook request with retry logic
+
+        Args:
+            processor: The webhook processor to use
+        """
+        await processor.before_processing()
+
+        payload = processor.event.payload
+        headers = processor.event.headers
+
+        if not await processor.authenticate(payload, headers):
+            raise ValueError("Authentication failed")
+
+        if not await processor.validate_payload(payload):
+            raise ValueError("Invalid payload")
+
+        while True:
+            try:
+                await processor.handle_event(payload)
+                break
+
+            except Exception as e:
+                await processor.on_error(e)
+
+                if (
+                    processor.should_retry(e)
+                    and processor.retry_count < processor.max_retries
+                ):
+                    processor.retry_count += 1
+                    delay = processor.calculate_retry_delay()
+                    await asyncio.sleep(delay)
+                    continue
+
+                raise
+
+        await processor.after_processing()
+
+    def register_processor(
+        self,
+        path: str,
+        processor: Type[AbstractWebhookProcessor],
+        event_filter: Callable[[WebhookEvent], bool] = lambda _: True,
+    ) -> None:
+        """Register a webhook processor for a specific path with optional filter"""
+
+        if not issubclass(processor, AbstractWebhookProcessor):
+            raise ValueError("Processor must extend AbstractWebhookProcessor")
+
+        if path not in self._processors:
+            self._processors[path] = []
+            self._event_queues[path] = LocalQueue()
+            self._register_route(path)
+
+        self._processors[path].append(
+            ProcessorRegistration(processor=processor, filter=event_filter)
+        )
+
+    def _register_route(self, path: str) -> None:
+        """Register a route for a specific path"""
+
+        async def handle_webhook(request: Request) -> Dict[str, str]:
+            """Handle incoming webhook requests for a specific path."""
+            try:
+                event = await WebhookEvent.from_request(request)
+                event.set_timestamp(WebhookEventTimestamp.AddedToQueue)
+                await self._event_queues[path].put(event)
+                return {"status": "ok"}
+            except Exception as e:
+                logger.exception(f"Error processing webhook: {str(e)}")
+                return {"status": "error", "message": str(e)}
+
+        self._router.add_api_route(
+            path,
+            handle_webhook,
+            methods=["POST"],
+        )
+
+    async def _cancel_all_tasks(self) -> None:
+        """Cancel all webhook processor tasks"""
+        for task in self._webhook_processor_tasks:
+            task.cancel()
+
+        await asyncio.gather(*self._webhook_processor_tasks, return_exceptions=True)
+
+    async def shutdown(self) -> None:
+        """Gracefully shutdown all queue processors"""
+        logger.warning("Shutting down webhook processor manager")
+
+        try:
+            await asyncio.wait_for(
+                asyncio.gather(
+                    *(queue.teardown() for queue in self._event_queues.values())
+                ),
+                timeout=self._max_wait_seconds_before_shutdown,
+            )
+        except asyncio.TimeoutError:
+            logger.warning("Shutdown timed out waiting for queues to empty")
+
+        await self._cancel_all_tasks()

port_ocean/core/handlers/webhook/webhook_event.py

@@ -0,0 +1,77 @@
+from enum import StrEnum
+from typing import Any, Dict, Type, TypeAlias
+from uuid import uuid4
+from fastapi import Request
+from loguru import logger
+
+
+EventPayload: TypeAlias = Dict[str, Any]
+EventHeaders: TypeAlias = Dict[str, str]
+
+
+class WebhookEventTimestamp(StrEnum):
+    """Enum for timestamp keys"""
+
+    AddedToQueue = "Added To Queue"
+    StartedProcessing = "Started Processing"
+    FinishedProcessingSuccessfully = "Finished Processing Successfully"
+    FinishedProcessingWithError = "Finished Processing With Error"
+
+
+class WebhookEvent:
+    """Represents a webhook event"""
+
+    def __init__(
+        self,
+        trace_id: str,
+        payload: EventPayload,
+        headers: EventHeaders,
+        original_request: Request | None = None,
+    ) -> None:
+        self.trace_id = trace_id
+        self.payload = payload
+        self.headers = headers
+        self._original_request = original_request
+
+    @classmethod
+    async def from_request(
+        cls: Type["WebhookEvent"], request: Request
+    ) -> "WebhookEvent":
+        trace_id = str(uuid4())
+        payload = await request.json()
+
+        return cls(
+            trace_id=trace_id,
+            payload=payload,
+            headers=dict(request.headers),
+            original_request=request,
+        )
+
+    @classmethod
+    def from_dict(cls: Type["WebhookEvent"], data: Dict[str, Any]) -> "WebhookEvent":
+        return cls(
+            trace_id=data["trace_id"],
+            payload=data["payload"],
+            headers=data["headers"],
+            original_request=None,
+        )
+
+    def clone(self) -> "WebhookEvent":
+        return WebhookEvent(
+            trace_id=self.trace_id,
+            payload=self.payload,
+            headers=self.headers,
+            original_request=self._original_request,
+        )
+
+    def set_timestamp(self, timestamp: WebhookEventTimestamp) -> None:
+        """Set a timestamp for a specific event"""
+        logger.info(
+            f"Webhook Event {timestamp.value}",
+            extra={
+                "trace_id": self.trace_id,
+                "payload": self.payload,
+                "headers": self.headers,
+                "timestamp_type": timestamp.value,
+            },
+        )

port_ocean/core/integrations/mixins/sync.py

@@ -96,12 +96,13 @@ class SyncMixin(HandlerMixin):
             IntegrationNotStartedException: If EntitiesStateApplier class is not initialized.
         """
         entities_at_port = await ocean.port_client.search_entities(user_agent_type)
+        app_config = await self.port_app_config_handler.get_port_app_config()
 
         modified_entities = await self.entities_state_applier.upsert(
             entities, user_agent_type
         )
         await self.entities_state_applier.delete_diff(
-            {"before": entities_at_port, "after": modified_entities}, user_agent_type
+            {"before": entities_at_port, "after": modified_entities}, user_agent_type, app_config.entity_deletion_threshold
         )
 
         logger.info("Finished syncing change")

port_ocean/core/integrations/mixins/sync_raw.py

@@ -220,26 +220,32 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
         )
         modified_objects = []
 
-        try:
-            changed_entities = await self._map_entities_compared_with_port(
-                objects_diff[0].entity_selector_diff.passed,
-                resource,
-                user_agent_type
-            )
-            if changed_entities:
-                logger.info("Upserting changed entities", changed_entities=len(changed_entities),
-                    total_entities=len(objects_diff[0].entity_selector_diff.passed))
-                await self.entities_state_applier.upsert(
-                    changed_entities, user_agent_type
-                )
-            else:
-                logger.info("Entities in batch didn't changed since last sync, skipping", total_entities=len(objects_diff[0].entity_selector_diff.passed))
-            modified_objects = [ocean.port_client._reduce_entity(entity) for entity in objects_diff[0].entity_selector_diff.passed]
-        except Exception as e:
-            logger.warning(f"Failed to resolve batch entities with Port, falling back to upserting all entities: {str(e)}")
-            modified_objects = await self.entities_state_applier.upsert(
-                objects_diff[0].entity_selector_diff.passed, user_agent_type
+        if event.event_type == EventType.RESYNC:
+            try:
+                changed_entities = await self._map_entities_compared_with_port(
+                    objects_diff[0].entity_selector_diff.passed,
+                    resource,
+                    user_agent_type
                 )
+                if changed_entities:
+                    logger.info("Upserting changed entities", changed_entities=len(changed_entities),
+                        total_entities=len(objects_diff[0].entity_selector_diff.passed))
+                    await self.entities_state_applier.upsert(
+                        changed_entities, user_agent_type
+                    )
+                else:
+                    logger.info("Entities in batch didn't changed since last sync, skipping", total_entities=len(objects_diff[0].entity_selector_diff.passed))
+                modified_objects = [ocean.port_client._reduce_entity(entity) for entity in objects_diff[0].entity_selector_diff.passed]
+            except Exception as e:
+                logger.warning(f"Failed to resolve batch entities with Port, falling back to upserting all entities: {str(e)}")
+                modified_objects = await self.entities_state_applier.upsert(
+                    objects_diff[0].entity_selector_diff.passed, user_agent_type
+                    )
+        else:
+           modified_objects = await self.entities_state_applier.upsert(
+                    objects_diff[0].entity_selector_diff.passed, user_agent_type
+                    )
+
 
         return CalculationResult(
             objects_diff[0].entity_selector_diff._replace(passed=modified_objects),
@@ -608,7 +614,7 @@ class SyncRawMixin(HandlerMixin, EventsMixin):
                     )
                     await self.entities_state_applier.delete_diff(
                         {"before": entities_at_port, "after": generated_entities},
-                        user_agent_type,
+                        user_agent_type, app_config.entity_deletion_threshold
                     )
 
                     logger.info("Resync finished successfully")

port_ocean/exceptions/webhook_processor.py

@@ -0,0 +1,4 @@
+class RetryableError(Exception):
+    """Base exception class for errors that should trigger a retry."""
+
+    pass