port-ocean 0.19.2__py3-none-any.whl → 0.20.0__py3-none-any.whl

port_ocean/config/settings.py

@@ -72,6 +72,7 @@ class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
     create_port_resources_origin: CreatePortResourcesOrigin | None = None
     send_raw_data_examples: bool = True
     oauth_access_token_file_path: str | None = None
+    base_url: str | None = None
     port: PortSettings
     event_listener: EventListenerSettingsType = Field(
         default=cast(EventListenerSettingsType, {"type": "POLLING"})

port_ocean/context/ocean.py

@@ -146,12 +146,7 @@ class PortOceanContext:
     async def sync_raw_all(self) -> None:
         await self.integration.sync_raw_all(trigger_type="manual")
 
-    def add_webhook_processor(
-        self,
-        path: str,
-        processor: type,
-        events_filter: Callable[[Any], bool] = lambda _: True,
-    ) -> None:
+    def add_webhook_processor(self, path: str, processor: type) -> None:
         """
         Registers a webhook processor for a specific path.
 
@@ -175,7 +170,7 @@ class PortOceanContext:
         Raises:
             ValueError: If the processor does not extend AbstractWebhookProcessor.
         """
-        self.app.webhook_manager.register_processor(path, processor, events_filter)
+        self.app.webhook_manager.register_processor(path, processor)
 
 
 _port_ocean: PortOceanContext = PortOceanContext(None)

port_ocean/core/handlers/webhook/abstract_webhook_processor.py

@@ -1,9 +1,15 @@
 from abc import ABC, abstractmethod
 from loguru import logger
 
+from port_ocean.core.handlers.port_app_config.models import ResourceConfig
 from port_ocean.exceptions.webhook_processor import RetryableError
 
-from .webhook_event import WebhookEvent, EventPayload, EventHeaders
+from .webhook_event import (
+    WebhookEvent,
+    EventPayload,
+    EventHeaders,
+    WebhookEventRawResults,
+)
 
 
 class AbstractWebhookProcessor(ABC):
@@ -96,6 +102,16 @@ class AbstractWebhookProcessor(ABC):
         pass
 
     @abstractmethod
-    async def handle_event(self, payload: EventPayload) -> None:
+    async def handle_event(
+        self, payload: EventPayload, resource: ResourceConfig
+    ) -> WebhookEventRawResults:
         """Process the event."""
         pass
+
+    @abstractmethod
+    def should_process_event(self, event: WebhookEvent) -> bool:
+        pass
+
+    @abstractmethod
+    def get_matching_kinds(self, event: WebhookEvent) -> list[str]:
+        pass

port_ocean/core/handlers/webhook/processor_manager.py

@@ -1,10 +1,15 @@
-from typing import Dict, Type, Set, Callable
+from copy import deepcopy
+from typing import Dict, Type, Set
 from fastapi import APIRouter, Request
 from loguru import logger
 import asyncio
-from dataclasses import dataclass
 
-from .webhook_event import WebhookEvent, WebhookEventTimestamp
+from port_ocean.context.event import EventType, event_context
+from port_ocean.core.handlers.port_app_config.models import ResourceConfig
+from port_ocean.core.integrations.mixins.events import EventsMixin
+from port_ocean.core.integrations.mixins.live_events import LiveEventsMixin
+from .webhook_event import WebhookEvent, WebhookEventRawResults, LiveEventTimestamp
+from port_ocean.context.event import event
 
 
 from .abstract_webhook_processor import AbstractWebhookProcessor
@@ -12,15 +17,7 @@ from port_ocean.utils.signal import SignalHandler
 from port_ocean.core.handlers.queue import AbstractQueue, LocalQueue
 
 
-@dataclass
-class ProcessorRegistration:
-    """Represents a registered processor with its filter"""
-
-    processor: Type[AbstractWebhookProcessor]
-    filter: Callable[[WebhookEvent], bool]
-
-
-class WebhookProcessorManager:
+class LiveEventsProcessorManager(LiveEventsMixin, EventsMixin):
     """Manages webhook processors and their routes"""
 
     def __init__(
@@ -31,7 +28,7 @@ class WebhookProcessorManager:
         max_wait_seconds_before_shutdown: float = 5.0,
     ) -> None:
         self._router = router
-        self._processors: Dict[str, list[ProcessorRegistration]] = {}
+        self._processors_classes: Dict[str, list[Type[AbstractWebhookProcessor]]] = {}
         self._event_queues: Dict[str, AbstractQueue[WebhookEvent]] = {}
         self._webhook_processor_tasks: Set[asyncio.Task[None]] = set()
         self._max_event_processing_seconds = max_event_processing_seconds
@@ -40,6 +37,7 @@ class WebhookProcessorManager:
 
     async def start_processing_event_messages(self) -> None:
         """Start processing events for all registered paths"""
+        await self.initialize_handlers()
         loop = asyncio.get_event_loop()
         for path in self._event_queues.keys():
             try:
@@ -50,42 +48,73 @@ class WebhookProcessorManager:
                 logger.exception(f"Error starting queue processor for {path}: {str(e)}")
 
     def _extract_matching_processors(
-        self, event: WebhookEvent, path: str
-    ) -> list[AbstractWebhookProcessor]:
+        self, webhook_event: WebhookEvent, path: str
+    ) -> list[tuple[ResourceConfig, AbstractWebhookProcessor]]:
         """Find and extract the matching processor for an event"""
-        matching_processors = [
-            registration.processor
-            for registration in self._processors[path]
-            if registration.filter(event)
-        ]
 
-        if not matching_processors:
+        created_processors: list[tuple[ResourceConfig, AbstractWebhookProcessor]] = []
+
+        for processor_class in self._processors_classes[path]:
+            processor = processor_class(webhook_event.clone())
+            if processor.should_process_event(webhook_event):
+                kinds = processor.get_matching_kinds(webhook_event)
+                for kind in kinds:
+                    for resource in event.port_app_config.resources:
+                        if resource.kind == kind:
+                            created_processors.append((resource, processor))
+
+        if not created_processors:
             raise ValueError("No matching processors found")
 
-        created_processors: list[AbstractWebhookProcessor] = []
-        for processor_class in matching_processors:
-            processor = processor_class(event.clone())
-            created_processors.append(processor)
+        logger.info(
+            "Found matching processors for webhook event",
+            processors_count=len(created_processors),
+            webhook_path=path,
+        )
         return created_processors
 
     async def process_queue(self, path: str) -> None:
         """Process events for a specific path in order"""
         while True:
-            matching_processors: list[AbstractWebhookProcessor] = []
-            event: WebhookEvent | None = None
+            matching_processors_with_resource: list[
+                tuple[ResourceConfig, AbstractWebhookProcessor]
+            ] = []
+            webhook_event: WebhookEvent | None = None
             try:
-                event = await self._event_queues[path].get()
-                with logger.contextualize(webhook_path=path, trace_id=event.trace_id):
-                    matching_processors = self._extract_matching_processors(event, path)
-                    await asyncio.gather(
-                        *(
-                            self._process_single_event(processor, path)
-                            for processor in matching_processors
+                queue = self._event_queues[path]
+                webhook_event = await queue.get()
+                with logger.contextualize(
+                    webhook_path=path, trace_id=webhook_event.trace_id
+                ):
+                    async with event_context(
+                        EventType.HTTP_REQUEST,
+                        trigger_type="machine",
+                        parent_override=webhook_event.event_context,
+                    ):
+                        matching_processors_with_resource = (
+                            self._extract_matching_processors(webhook_event, path)
                         )
-                    )
+                        webhook_event_raw_results_for_all_resources = await asyncio.gather(
+                            *(
+                                self._process_single_event(processor, path, resource)
+                                for resource, processor in matching_processors_with_resource
+                            )
+                        )
+                        if webhook_event_raw_results_for_all_resources and all(
+                            webhook_event_raw_results_for_all_resources
+                        ):
+                            logger.info(
+                                "Exporting raw event results to entities",
+                                webhook_event_raw_results_for_all_resources_length=len(
+                                    webhook_event_raw_results_for_all_resources
+                                ),
+                            )
+                            await self.sync_raw_results(
+                                webhook_event_raw_results_for_all_resources
+                            )
             except asyncio.CancelledError:
                 logger.info(f"Queue processor for {path} is shutting down")
-                for processor in matching_processors:
+                for _, processor in matching_processors_with_resource:
                     await processor.cancel()
                     self._timestamp_event_error(processor.event)
                 break
@@ -93,49 +122,55 @@ class WebhookProcessorManager:
                 logger.exception(
                     f"Unexpected error in queue processor for {path}: {str(e)}"
                 )
-                for processor in matching_processors:
+                for _, processor in matching_processors_with_resource:
                     self._timestamp_event_error(processor.event)
             finally:
-                if event:
+                if webhook_event:
                     await self._event_queues[path].commit()
                     # Prevents committing empty events for cases where we shutdown while processing
-                    event = None
+                    webhook_event = None
 
     def _timestamp_event_error(self, event: WebhookEvent) -> None:
         """Timestamp an event as having an error"""
-        event.set_timestamp(WebhookEventTimestamp.FinishedProcessingWithError)
+        event.set_timestamp(LiveEventTimestamp.FinishedProcessingWithError)
 
     async def _process_single_event(
-        self, processor: AbstractWebhookProcessor, path: str
-    ) -> None:
+        self, processor: AbstractWebhookProcessor, path: str, resource: ResourceConfig
+    ) -> WebhookEventRawResults:
         """Process a single event with a specific processor"""
         try:
             logger.debug("Start processing queued webhook")
-            processor.event.set_timestamp(WebhookEventTimestamp.StartedProcessing)
+            processor.event.set_timestamp(LiveEventTimestamp.StartedProcessing)
 
-            await self._execute_processor(processor)
+            webhook_event_raw_results = await self._execute_processor(
+                processor, resource
+            )
             processor.event.set_timestamp(
-                WebhookEventTimestamp.FinishedProcessingSuccessfully
+                LiveEventTimestamp.FinishedProcessingSuccessfully
             )
+            return webhook_event_raw_results
         except Exception as e:
             logger.exception(f"Error processing queued webhook for {path}: {str(e)}")
             self._timestamp_event_error(processor.event)
+            raise
 
-    async def _execute_processor(self, processor: AbstractWebhookProcessor) -> None:
+    async def _execute_processor(
+        self, processor: AbstractWebhookProcessor, resource: ResourceConfig
+    ) -> WebhookEventRawResults:
         """Execute a single processor within a max processing time"""
         try:
-            await asyncio.wait_for(
-                self._process_webhook_request(processor),
+            return await asyncio.wait_for(
+                self._process_webhook_request(processor, resource),
                 timeout=self._max_event_processing_seconds,
             )
         except asyncio.TimeoutError:
-            raise TimeoutError(
+            raise asyncio.TimeoutError(
                 f"Processor processing timed out after {self._max_event_processing_seconds} seconds"
             )
 
     async def _process_webhook_request(
-        self, processor: AbstractWebhookProcessor
-    ) -> None:
+        self, processor: AbstractWebhookProcessor, resource: ResourceConfig
+    ) -> WebhookEventRawResults:
         """Process a webhook request with retry logic
 
         Args:
@@ -152,9 +187,13 @@ class WebhookProcessorManager:
         if not await processor.validate_payload(payload):
             raise ValueError("Invalid payload")
 
+        webhook_event_raw_results = None
         while True:
             try:
-                await processor.handle_event(payload)
+                webhook_event_raw_results = await processor.handle_event(
+                    payload, resource
+                )
+                webhook_event_raw_results.resource = resource
                 break
 
             except Exception as e:
@@ -172,26 +211,28 @@ class WebhookProcessorManager:
                 raise
 
         await processor.after_processing()
+        return webhook_event_raw_results
 
     def register_processor(
-        self,
-        path: str,
-        processor: Type[AbstractWebhookProcessor],
-        event_filter: Callable[[WebhookEvent], bool] = lambda _: True,
+        self, path: str, processor: Type[AbstractWebhookProcessor]
     ) -> None:
-        """Register a webhook processor for a specific path with optional filter"""
+        """Register a webhook processor for a specific path with optional filter
+
+        Args:
+            path: The webhook path to register
+            processor: The processor class to register
+            kind: The resource kind to associate with this processor, or None to match any kind
+        """
 
         if not issubclass(processor, AbstractWebhookProcessor):
             raise ValueError("Processor must extend AbstractWebhookProcessor")
 
-        if path not in self._processors:
-            self._processors[path] = []
+        if path not in self._processors_classes:
+            self._processors_classes[path] = []
             self._event_queues[path] = LocalQueue()
             self._register_route(path)
 
-        self._processors[path].append(
-            ProcessorRegistration(processor=processor, filter=event_filter)
-        )
+        self._processors_classes[path].append(processor)
 
     def _register_route(self, path: str) -> None:
         """Register a route for a specific path"""
@@ -199,9 +240,10 @@ class WebhookProcessorManager:
         async def handle_webhook(request: Request) -> Dict[str, str]:
             """Handle incoming webhook requests for a specific path."""
             try:
-                event = await WebhookEvent.from_request(request)
-                event.set_timestamp(WebhookEventTimestamp.AddedToQueue)
-                await self._event_queues[path].put(event)
+                webhook_event = await WebhookEvent.from_request(request)
+                webhook_event.set_timestamp(LiveEventTimestamp.AddedToQueue)
+                webhook_event.set_event_context(deepcopy(event))
+                await self._event_queues[path].put(webhook_event)
                 return {"status": "ok"}
             except Exception as e:
                 logger.exception(f"Error processing webhook: {str(e)}")

port_ocean/core/handlers/webhook/webhook_event.py

@@ -1,15 +1,20 @@
+from abc import ABC
 from enum import StrEnum
-from typing import Any, Dict, Type, TypeAlias
+from typing import Any, Dict, Type, TypeAlias, Optional
 from uuid import uuid4
 from fastapi import Request
 from loguru import logger
 
+from port_ocean.context.event import EventContext
+from port_ocean.core.handlers.port_app_config.models import ResourceConfig
+from port_ocean.core.ocean_types import RAW_ITEM
+
 
 EventPayload: TypeAlias = Dict[str, Any]
 EventHeaders: TypeAlias = Dict[str, str]
 
 
-class WebhookEventTimestamp(StrEnum):
+class LiveEventTimestamp(StrEnum):
     """Enum for timestamp keys"""
 
     AddedToQueue = "Added To Queue"
@@ -18,7 +23,27 @@ class WebhookEventTimestamp(StrEnum):
     FinishedProcessingWithError = "Finished Processing With Error"
 
 
-class WebhookEvent:
+class LiveEvent(ABC):
+    """Represents a live event marker class"""
+
+    def set_timestamp(
+        self, timestamp: LiveEventTimestamp, params: Optional[Dict[str, Any]] = None
+    ) -> None:
+        """Set a timestamp for a specific event
+
+        Args:
+            timestamp: The timestamp type to set
+            params: Additional parameters to log with the event
+        """
+        log_params = params or {}
+        logger.info(
+            f"Event {timestamp.value}",
+            extra=log_params | {"timestamp_type": timestamp.value},
+        )
+        self._timestamp = timestamp
+
+
+class WebhookEvent(LiveEvent):
     """Represents a webhook event"""
 
     def __init__(
@@ -32,6 +57,7 @@ class WebhookEvent:
         self.payload = payload
         self.headers = headers
         self._original_request = original_request
+        self.event_context: EventContext | None = None
 
     @classmethod
     async def from_request(
@@ -64,14 +90,51 @@ class WebhookEvent:
             original_request=self._original_request,
         )
 
-    def set_timestamp(self, timestamp: WebhookEventTimestamp) -> None:
+    def set_timestamp(
+        self, timestamp: LiveEventTimestamp, params: Optional[Dict[str, Any]] = None
+    ) -> None:
         """Set a timestamp for a specific event"""
-        logger.info(
-            f"Webhook Event {timestamp.value}",
-            extra={
+        super().set_timestamp(
+            timestamp,
+            params={
                 "trace_id": self.trace_id,
                 "payload": self.payload,
                 "headers": self.headers,
-                "timestamp_type": timestamp.value,
             },
         )
+
+    def set_event_context(self, event_context: EventContext) -> None:
+        self.event_context = event_context
+
+
+class WebhookEventRawResults:
+    """
+    Class for webhook event to store the updated data for the event
+    """
+
+    def __init__(
+        self,
+        updated_raw_results: list[RAW_ITEM],
+        deleted_raw_results: list[RAW_ITEM],
+    ) -> None:
+        self._resource: ResourceConfig | None = None
+        self._updated_raw_results = updated_raw_results
+        self._deleted_raw_results = deleted_raw_results
+
+    @property
+    def resource(self) -> ResourceConfig:
+        if self._resource is None:
+            raise ValueError("Resource has not been set")
+        return self._resource
+
+    @resource.setter
+    def resource(self, value: ResourceConfig) -> None:
+        self._resource = value
+
+    @property
+    def updated_raw_results(self) -> list[RAW_ITEM]:
+        return self._updated_raw_results
+
+    @property
+    def deleted_raw_results(self) -> list[RAW_ITEM]:
+        return self._deleted_raw_results

port_ocean/core/integrations/mixins/live_events.py

@@ -0,0 +1,88 @@
+from loguru import logger
+from port_ocean.clients.port.types import UserAgentType
+from port_ocean.core.handlers.webhook.webhook_event import WebhookEventRawResults
+from port_ocean.core.integrations.mixins.handler import HandlerMixin
+from port_ocean.core.models import Entity
+from port_ocean.context.ocean import ocean
+
+
+
+class LiveEventsMixin(HandlerMixin):
+
+    async def sync_raw_results(self, webhook_events_raw_result: list[WebhookEventRawResults]) -> None:
+        """Process the webhook event raw results collected from multiple processors and export it.
+
+        Args:
+            webhook_events_raw_result: List of WebhookEventRawResults objects to process
+        """
+        entities_to_create, entities_to_delete = await self._parse_raw_event_results_to_entities(webhook_events_raw_result)
+        await self.entities_state_applier.upsert(entities_to_create, UserAgentType.exporter)
+        await self._delete_entities(entities_to_delete)
+
+
+    async def _parse_raw_event_results_to_entities(self, webhook_events_raw_result: list[WebhookEventRawResults]) -> tuple[list[Entity], list[Entity]]:
+        """Parse the webhook event raw results and return a list of entities.
+
+        Args:
+            webhook_events_raw_result: List of WebhookEventRawResults objects to process
+        """
+        entities: list[Entity] = []
+        entities_not_passed: list[Entity] = []
+        entities_to_delete: list[Entity] = []
+        for webhook_event_raw_result in webhook_events_raw_result:
+            for raw_item in webhook_event_raw_result.updated_raw_results:
+                calaculation_results = await self.entity_processor.parse_items(
+                    webhook_event_raw_result.resource, [raw_item], parse_all=True, send_raw_data_examples_amount=0
+                )
+                entities.extend(calaculation_results.entity_selector_diff.passed)
+                entities_not_passed.extend(calaculation_results.entity_selector_diff.failed)
+
+            for raw_item in webhook_event_raw_result.deleted_raw_results:
+                deletion_results = await self.entity_processor.parse_items(
+                    webhook_event_raw_result.resource, [raw_item], parse_all=True, send_raw_data_examples_amount=0
+                )
+                entities_to_delete.extend(deletion_results.entity_selector_diff.passed)
+
+        entities_to_remove = []
+        for entity in entities_to_delete + entities_not_passed:
+            if (entity.blueprint, entity.identifier) not in [(entity.blueprint, entity.identifier) for entity in entities]:
+                entities_to_remove.append(entity)
+
+        logger.info(f"Found {len(entities_to_remove)} entities to remove {', '.join(f'{entity.blueprint}/{entity.identifier}' for entity in entities_to_remove)}")
+        logger.info(f"Found {len(entities)} entities to upsert {', '.join(f'{entity.blueprint}/{entity.identifier}' for entity in entities)}")
+        return entities, entities_to_remove
+
+    async def _does_entity_exists(self, entity: Entity) -> bool:
+        """Check if this integration is the owner of the given entity.
+
+        Args:
+            entity: The entity to check ownership for
+
+        Returns:
+            bool: True if this integration is the owner of the entity, False otherwise
+        """
+        query = {
+            "combinator": "and",
+            "rules": [
+                {
+                    "property": "$identifier",
+                    "operator": "=",
+                    "value": entity.identifier
+                },
+                {
+                    "property": "$blueprint",
+                    "operator": "=",
+                    "value": entity.blueprint
+                }
+            ]
+        }
+        entities_at_port = await ocean.port_client.search_entities(
+            UserAgentType.exporter,
+            query
+        )
+        return len(entities_at_port) > 0
+
+    async def _delete_entities(self, entities: list[Entity]) -> None:
+        for entity in entities:
+            if await self._does_entity_exists(entity):
+                await self.entities_state_applier.delete([entity], UserAgentType.exporter)

port_ocean/ocean.py

@@ -26,7 +26,9 @@ from port_ocean.utils.misc import IntegrationStateStatus
 from port_ocean.utils.repeat import repeat_every
 from port_ocean.utils.signal import signal_handler
 from port_ocean.version import __integration_version__
-from port_ocean.core.handlers.webhook.processor_manager import WebhookProcessorManager
+from port_ocean.core.handlers.webhook.processor_manager import (
+    LiveEventsProcessorManager,
+)
 
 
 class Ocean:
@@ -54,7 +56,7 @@ class Ocean:
         )
         self.integration_router = integration_router or APIRouter()
 
-        self.webhook_manager = WebhookProcessorManager(
+        self.webhook_manager = LiveEventsProcessorManager(
             self.integration_router, signal_handler
         )
 
@@ -118,6 +120,17 @@ class Ocean:
             )
             await repeated_function()
 
+    @property
+    def base_url(self) -> str:
+        integration_config = self.config.integration.config
+        if isinstance(integration_config, BaseModel):
+            integration_config = integration_config.dict()
+        if integration_config.get("app_host"):
+            logger.warning(
+                "The OCEAN__INTEGRATION__CONFIG__APP_HOST field is deprecated. Please use the OCEAN__BASE_URL field instead."
+            )
+        return self.config.base_url or integration_config.get("app_host")
+
     def load_external_oauth_access_token(self) -> str | None:
         if self.config.oauth_access_token_file_path is not None:
             try:
@@ -137,7 +150,10 @@ class Ocean:
         async def lifecycle(_: FastAPI) -> AsyncIterator[None]:
             try:
                 await self.integration.start()
-                await self.webhook_manager.start_processing_event_messages()
+                if self.base_url:
+                    await self.webhook_manager.start_processing_event_messages()
+                else:
+                    logger.warning("No base URL provided, skipping webhook processing")
                 await self._setup_scheduled_resync()
                 yield None
             except Exception: