port-ocean 0.26.3__tar.gz → 0.27.10__tar.gz

{port_ocean-0.26.3 → port_ocean-0.27.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: port-ocean
-Version: 0.26.3
+Version: 0.27.10
 Summary: Port Ocean is a CLI tool for managing your Port projects.
 Home-page: https://app.getport.io
 Keywords: ocean,port-ocean,port
@@ -22,12 +22,15 @@ Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: Utilities
 Provides-Extra: cli
+Requires-Dist: aiofiles (>=24.1.0,<25.0.0)
 Requires-Dist: aiostream (>=0.5.2,<0.7.0)
 Requires-Dist: click (>=8.1.3,<9.0.0) ; extra == "cli"
 Requires-Dist: confluent-kafka (>=2.10.1,<3.0.0)
 Requires-Dist: cookiecutter (>=2.1.1,<3.0.0) ; extra == "cli"
-Requires-Dist: fastapi (>=0.115.3,<0.116.0)
+Requires-Dist: cryptography (>=44.0.1,<45.0.0)
+Requires-Dist: fastapi (>=0.116.0,<0.117.0)
 Requires-Dist: httpx (>=0.28.1,<0.29.0)
+Requires-Dist: ijson (>=3.4.0,<4.0.0)
 Requires-Dist: jinja2 (>=3.1.6)
 Requires-Dist: jinja2-time (>=0.2.0,<0.3.0) ; extra == "cli"
 Requires-Dist: jq (>=1.8.0,<2.0.0)

{port_ocean-0.26.3 → port_ocean-0.27.10}/integrations/_infra/Dockerfile.Deb

@@ -28,11 +28,14 @@ ARG INTEGRATION_VERSION
 ARG BUILD_CONTEXT
 ARG PROMETHEUS_MULTIPROC_DIR=/tmp/ocean/prometheus/metrics
 ARG OAUTH_CONFIG_DIR=/app/.config
+ARG STREAMING_LOCATION=/tmp/ocean/streaming
 
 ENV LIBRDKAFKA_VERSION=2.8.2 \
-    PROMETHEUS_MULTIPROC_DIR=${PROMETHEUS_MULTIPROC_DIR}
+    PROMETHEUS_MULTIPROC_DIR=${PROMETHEUS_MULTIPROC_DIR} \
+    STREAMING_LOCATION=${STREAMING_LOCATION}
 
 RUN mkdir -p ${PROMETHEUS_MULTIPROC_DIR}
+RUN mkdir -p ${STREAMING_LOCATION}
 RUN chown -R ocean:appgroup /tmp/ocean && chmod -R 755 /tmp/ocean
 
 RUN mkdir -p ${OAUTH_CONFIG_DIR}

{port_ocean-0.26.3 → port_ocean-0.27.10}/integrations/_infra/Dockerfile.local

@@ -33,13 +33,15 @@ RUN apt-get update \
 
 ARG BUILD_CONTEXT
 ARG PROMETHEUS_MULTIPROC_DIR=/tmp/ocean/prometheus/metrics
+ARG STREAMING_LOCATION=/tmp/ocean/streaming
 
 ENV PROMETHEUS_MULTIPROC_DIR=${PROMETHEUS_MULTIPROC_DIR}
-
+ENV STREAMING_LOCATION=${STREAMING_LOCATION}
 # Create /tmp/ocean directory and set permissions
 
 
 RUN mkdir -p ${PROMETHEUS_MULTIPROC_DIR}
+RUN mkdir -p ${STREAMING_LOCATION}
 
 WORKDIR /app
 

{port_ocean-0.26.3 → port_ocean-0.27.10}/port_ocean/clients/port/authentication.py

@@ -35,6 +35,7 @@ class PortAuthentication:
         integration_identifier: str,
         integration_type: str,
         integration_version: str,
+        ingest_url: str,
     ):
         self.client = client
         self.api_url = api_url
@@ -43,6 +44,7 @@ class PortAuthentication:
         self.integration_identifier = integration_identifier
         self.integration_type = integration_type
         self.integration_version = integration_version
+        self.ingest_url = ingest_url
         self.last_token_object: TokenResponse | None = None
 
     async def _get_token(self, client_id: str, client_secret: str) -> TokenResponse:

{port_ocean-0.26.3 → port_ocean-0.27.10}/port_ocean/clients/port/client.py

@@ -1,3 +1,5 @@
+from typing import Any
+
 from loguru import logger
 
 from port_ocean.clients.port.authentication import PortAuthentication
@@ -10,11 +12,10 @@ from port_ocean.clients.port.types import (
     KafkaCreds,
 )
 from port_ocean.clients.port.utils import (
-    handle_port_status_code,
     get_internal_http_client,
+    handle_port_status_code,
 )
 from port_ocean.exceptions.clients import KafkaCredentialsNotFound
-from typing import Any
 
 
 class PortClient(
@@ -32,6 +33,7 @@ class PortClient(
         integration_identifier: str,
         integration_type: str,
         integration_version: str,
+        ingest_url: str,
     ):
         self.api_url = f"{base_url}/v1"
         self.client = get_internal_http_client(self)
@@ -43,6 +45,7 @@ class PortClient(
             integration_identifier,
             integration_type,
             integration_version,
+            ingest_url,
         )
         EntityClientMixin.__init__(self, self.auth, self.client)
         IntegrationClientMixin.__init__(

{port_ocean-0.26.3 → port_ocean-0.27.10}/port_ocean/clients/port/mixins/integrations.py

@@ -296,7 +296,7 @@ class IntegrationClientMixin:
         logger.debug("starting POST raw data request", raw_data=raw_data)
         headers = await self.auth.headers()
         response = await self.client.post(
-            f"{self.auth.api_url}/lakehouse/integration/{self.integration_identifier}/sync/{sync_id}/kind/{kind}/items",
+            f"{self.auth.ingest_url}/lakehouse/integration-type/{self.auth.integration_type}/integration/{self.integration_identifier}/sync/{sync_id}/kind/{kind}/items",
             headers=headers,
             json={
                 "items": raw_data,

{port_ocean-0.26.3 → port_ocean-0.27.10}/port_ocean/config/settings.py

@@ -46,6 +46,7 @@ class PortSettings(BaseOceanModel, extra=Extra.allow):
     client_secret: str = Field(..., sensitive=True)
     base_url: AnyHttpUrl = parse_obj_as(AnyHttpUrl, "https://api.getport.io")
     port_app_config_cache_ttl: int = 60
+    ingest_url: AnyHttpUrl = parse_obj_as(AnyHttpUrl, "https://ingest.getport.io")
 
 
 class IntegrationSettings(BaseOceanModel, extra=Extra.allow):
@@ -72,6 +73,13 @@ class MetricsSettings(BaseOceanModel, extra=Extra.allow):
     webhook_url: str | None = Field(default=None)
 
 
+class StreamingSettings(BaseOceanModel, extra=Extra.allow):
+    enabled: bool = Field(default=False)
+    max_buffer_size_mb: int = Field(default=1024 * 1024 * 20)  # 20 mb
+    chunk_size: int = Field(default=1024 * 64)  # 64 kb
+    location: str = Field(default="/tmp/ocean/streaming")
+
+
 class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
     _integration_config_model: BaseModel | None = None
 
@@ -88,6 +96,7 @@ class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
     event_listener: EventListenerSettingsType = Field(
         default=cast(EventListenerSettingsType, {"type": "POLLING"})
     )
+    event_workers_count: int = 1
     # If an identifier or type is not provided, it will be generated based on the integration name
     integration: IntegrationSettings = Field(
         default_factory=lambda: IntegrationSettings(type="", identifier="")
@@ -109,6 +118,10 @@ class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
     upsert_entities_batch_max_length: int = 20
     upsert_entities_batch_max_size_in_bytes: int = 1024 * 1024
     lakehouse_enabled: bool = False
+    yield_items_to_parse: bool = False
+    yield_items_to_parse_batch_size: int = 10
+
+    streaming: StreamingSettings = Field(default_factory=lambda: StreamingSettings())
 
     @validator("process_execution_mode")
     def validate_process_execution_mode(

{port_ocean-0.26.3 → port_ocean-0.27.10}/port_ocean/core/event_listener/kafka.py

@@ -16,6 +16,7 @@ from port_ocean.core.event_listener.base import (
     EventListenerEvents,
     EventListenerSettings,
 )
+from pydantic import validator
 
 
 class KafkaEventListenerSettings(EventListenerSettings):
@@ -46,6 +47,19 @@ class KafkaEventListenerSettings(EventListenerSettings):
     kafka_security_enabled: bool = True
     consumer_poll_timeout: int = 1
 
+    @validator("brokers")
+    @classmethod
+    def parse_brokers(cls, v: str) -> str:
+        # If it's a JSON array string, parse and join
+        if v.strip().startswith("[") and v.strip().endswith("]"):
+            try:
+                parsed = json.loads(v)
+                if isinstance(parsed, list):
+                    return ",".join(parsed)
+            except json.JSONDecodeError:
+                pass
+        return v
+
     def get_changelog_destination_details(self) -> dict[str, Any]:
         """
         Returns the changelog destination configuration for the Kafka event listener.

{port_ocean-0.26.3 → port_ocean-0.27.10}/port_ocean/core/handlers/entities_state_applier/port/applier.py

@@ -87,6 +87,15 @@ class HttpEntitiesStateApplier(BaseEntitiesStateApplier):
         diff = get_port_diff(entities["before"], entities["after"])
 
         if not diff.deleted:
+            ocean.metrics.inc_metric(
+                name=MetricType.OBJECT_COUNT_NAME,
+                labels=[
+                    ocean.metrics.current_resource_kind(),
+                    MetricPhase.DELETE,
+                    MetricPhase.DeletionResult.DELETED,
+                ],
+                value=0,
+            )
             return
 
         kept_entities = diff.created + diff.modified

{port_ocean-0.26.3 → port_ocean-0.27.10}/port_ocean/core/handlers/entity_processor/jq_entity_processor.py

@@ -242,19 +242,21 @@ class JQEntityProcessor(BaseEntityProcessor):
         data: dict[str, Any],
         raw_entity_mappings: dict[str, Any],
         items_to_parse: str | None,
+        items_to_parse_name: str,
         selector_query: str,
         parse_all: bool = False,
     ) -> tuple[list[MappedEntity], list[Exception]]:
         raw_data = [data.copy()]
-        if items_to_parse:
-            items = await self._search(data, items_to_parse)
-            if not isinstance(items, list):
-                logger.warning(
-                    f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items)}."
-                    f" Skipping..."
-                )
-                return [], []
-            raw_data = [{"item": item, **data} for item in items]
+        if not ocean.config.yield_items_to_parse:
+            if items_to_parse:
+                items = await self._search(data, items_to_parse)
+                if not isinstance(items, list):
+                    logger.warning(
+                        f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items)}."
+                        f" Skipping..."
+                    )
+                    return [], []
+                raw_data = [{items_to_parse_name: item, **data} for item in items]
 
         entities, errors = await gather_and_split_errors_from_results(
             [
@@ -303,6 +305,7 @@ class JQEntityProcessor(BaseEntityProcessor):
                 self._calculate_entity,
                 raw_entity_mappings,
                 mapping.port.items_to_parse,
+                mapping.port.items_to_parse_name,
                 mapping.selector.query,
                 parse_all,
             )

{port_ocean-0.26.3 → port_ocean-0.27.10}/port_ocean/core/handlers/port_app_config/models.py

@@ -39,6 +39,7 @@ class MappingsConfig(BaseModel):
 class PortResourceConfig(BaseModel):
     entity: MappingsConfig
     items_to_parse: str | None = Field(alias="itemsToParse")
+    items_to_parse_name: str | None = Field(alias="itemsToParseName", default="item")
 
 
 class Selector(BaseModel):

port_ocean-0.27.10/port_ocean/core/handlers/queue/__init__.py

@@ -0,0 +1,5 @@
+from .abstract_queue import AbstractQueue
+from .local_queue import LocalQueue
+from .group_queue import GroupQueue
+
+__all__ = ["AbstractQueue", "LocalQueue", "GroupQueue"]

{port_ocean-0.26.3 → port_ocean-0.27.10}/port_ocean/core/handlers/queue/abstract_queue.py

@@ -7,6 +7,9 @@ T = TypeVar("T")
 class AbstractQueue(ABC, Generic[T]):
     """Abstract interface for queues"""
 
+    def __init__(self, name: str | None = None):
+        pass
+
     @abstractmethod
     async def put(self, item: T) -> None:
         """Put an item into the queue"""
@@ -22,6 +25,11 @@ class AbstractQueue(ABC, Generic[T]):
         """Wait for all items to be processed"""
         pass
 
+    @abstractmethod
+    async def size(self) -> int:
+        """Size of the queue"""
+        pass
+
     @abstractmethod
     async def commit(self) -> None:
         """Mark item as processed"""

port_ocean-0.27.10/port_ocean/core/handlers/queue/group_queue.py

@@ -0,0 +1,138 @@
+import asyncio
+from collections import defaultdict, deque
+import time
+from typing import Deque, Dict, Optional, Set, TypeVar, Any
+from contextvars import ContextVar
+
+from loguru import logger
+
+from .abstract_queue import AbstractQueue
+
+T = TypeVar("T")
+MaybeStr = str | None
+
+_NO_GROUP = object()
+_current_group: ContextVar[Any] = ContextVar("current_group", default=_NO_GROUP)
+
+
+class GroupQueue(AbstractQueue[T]):
+    """Queue with exclusive processing per group."""
+
+    def __init__(
+        self,
+        group_key: MaybeStr = None,
+        name: MaybeStr = None,
+        lock_timeout: float = 300,
+    ):
+        super().__init__(name)
+        self.group_key = group_key
+        self._queues: Dict[MaybeStr, Deque[T]] = defaultdict(deque)
+        self._locked: Set[MaybeStr] = set()
+        self._queue_not_empty = asyncio.Condition()
+        self.lock_timeout = lock_timeout
+        self._lock_timestamps: Dict[MaybeStr, float] = {}
+        self._timeout_task: Optional[asyncio.Task[None]] = None
+
+    async def _background_timeout_check(self) -> None:
+        """Periodically release locks that have timed out."""
+        while True:
+            try:
+                await asyncio.sleep(self.lock_timeout / 4)
+                async with self._queue_not_empty:
+                    await self._release_expired_locks()
+            except asyncio.CancelledError:
+                break
+
+    def _extract_group_key(self, item: T) -> MaybeStr:
+        """Extract the group key from an item."""
+        if self.group_key is None:
+            return None
+        if not hasattr(item, self.group_key):
+            raise ValueError(
+                f"Item {item!r} lacks attribute '{self.group_key}' required for grouping"
+            )
+        return getattr(item, self.group_key)
+
+    async def put(self, item: T) -> None:
+        """Add item to its group's queue."""
+        group_key = self._extract_group_key(item)
+        async with self._queue_not_empty:
+            self._queues[group_key].append(item)
+            self._queue_not_empty.notify_all()
+
+    async def _release_expired_locks(self) -> None:
+        """Release locks that have exceeded the timeout."""
+        now = time.time()
+        expired_groups = []
+
+        for group, timestamp in list(self._lock_timestamps.items()):
+            if now - timestamp > self.lock_timeout:
+                expired_groups.append(group)
+                logger.warning(f"Releasing expired lock for group {group}")
+                self._locked.discard(group)
+                del self._lock_timestamps[group]
+
+        if expired_groups:
+            self._queue_not_empty.notify_all()
+
+    async def get(self) -> T:
+        """Get the next item from an unlocked group, locking that group."""
+        if self._timeout_task is None or self._timeout_task.done():
+            self._timeout_task = asyncio.create_task(self._background_timeout_check())
+
+        async with self._queue_not_empty:
+            while True:
+                await self._release_expired_locks()
+
+                for group, queue in self._queues.items():
+                    if queue and group not in self._locked:
+                        self._locked.add(group)
+                        self._lock_timestamps[group] = time.time()
+                        _current_group.set(group)
+                        return queue[0]
+
+                await self._queue_not_empty.wait()
+
+    async def commit(self) -> None:
+        """Remove the current item and unlock its group."""
+        group = _current_group.get()
+        if group is _NO_GROUP:
+            logger.warning("commit() called without active get()")
+            return
+
+        async with self._queue_not_empty:
+            queue = self._queues.get(group)
+            if queue:
+                queue.popleft()
+                if not queue:
+                    del self._queues[group]
+
+            self._locked.discard(group)
+            self._lock_timestamps.pop(group, None)
+            _current_group.set(_NO_GROUP)
+            self._queue_not_empty.notify_all()
+
+    async def teardown(self) -> None:
+        """Wait until all queues are empty and no groups are locked."""
+        async with self._queue_not_empty:
+            while any(self._queues.values()) or self._locked:
+                await self._queue_not_empty.wait()
+
+        if self._timeout_task and not self._timeout_task.done():
+            self._timeout_task.cancel()
+            try:
+                await self._timeout_task
+            except asyncio.CancelledError:
+                pass
+
+    async def size(self) -> int:
+        """Return total number of items across all groups."""
+        async with self._queue_not_empty:
+            return sum(len(queue) for queue in self._queues.values())
+
+    async def force_unlock_all(self) -> None:
+        """Force unlock all groups."""
+        async with self._queue_not_empty:
+            self._locked.clear()
+            self._lock_timestamps.clear()
+            self._queue_not_empty.notify_all()

{port_ocean-0.26.3 → port_ocean-0.27.10}/port_ocean/core/handlers/queue/local_queue.py

@@ -23,3 +23,6 @@ class LocalQueue(AbstractQueue[T]):
 
     async def commit(self) -> None:
         self._queue.task_done()
+
+    async def size(self) -> int:
+        return self._queue.qsize()

{port_ocean-0.26.3 → port_ocean-0.27.10}/port_ocean/core/handlers/resync_state_updater/updater.py

@@ -91,6 +91,8 @@ class ResyncStateUpdater:
             value=int(status == IntegrationStateStatus.Completed),
         )
 
+        ocean.metrics.sync_state = status.value
+
         await ocean.metrics.send_metrics_to_webhook(
             kind=ocean.metrics.current_resource_kind()
         )

{port_ocean-0.26.3 → port_ocean-0.27.10}/port_ocean/core/handlers/webhook/processor_manager.py

@@ -1,4 +1,5 @@
-from typing import Dict, Type, Set
+from typing import Dict, Tuple, Type, Set, List
+
 from fastapi import APIRouter, Request
 from loguru import logger
 import asyncio
@@ -6,6 +7,7 @@ import asyncio
 from port_ocean.context.ocean import ocean
 from port_ocean.context.event import EventType, event_context
 from port_ocean.core.handlers.port_app_config.models import ResourceConfig
+from port_ocean.core.handlers.queue.abstract_queue import AbstractQueue
 from port_ocean.core.integrations.mixins.events import EventsMixin
 from port_ocean.core.integrations.mixins.live_events import LiveEventsMixin
 from port_ocean.exceptions.webhook_processor import WebhookEventNotSupportedError
@@ -15,7 +17,7 @@ from port_ocean.context.event import event
 
 from .abstract_webhook_processor import AbstractWebhookProcessor
 from port_ocean.utils.signal import SignalHandler
-from port_ocean.core.handlers.queue import AbstractQueue, LocalQueue
+from port_ocean.core.handlers.queue import LocalQueue
 
 
 class LiveEventsProcessorManager(LiveEventsMixin, EventsMixin):
@@ -31,22 +33,103 @@ class LiveEventsProcessorManager(LiveEventsMixin, EventsMixin):
         self._router = router
         self._processors_classes: Dict[str, list[Type[AbstractWebhookProcessor]]] = {}
         self._event_queues: Dict[str, AbstractQueue[WebhookEvent]] = {}
-        self._webhook_processor_tasks: Set[asyncio.Task[None]] = set()
+        self._event_processor_tasks: Set[asyncio.Task[None]] = set()
         self._max_event_processing_seconds = max_event_processing_seconds
         self._max_wait_seconds_before_shutdown = max_wait_seconds_before_shutdown
         signal_handler.register(self.shutdown)
 
     async def start_processing_event_messages(self) -> None:
-        """Start processing events for all registered paths"""
+        """Start processing events for all registered paths with N workers each."""
         await self.initialize_handlers()
         loop = asyncio.get_event_loop()
+        config = ocean.integration.context.config
+
         for path in self._event_queues.keys():
+            for worker_id in range(0, config.event_workers_count):
+                task = loop.create_task(self._process_webhook_events(path, worker_id))
+                self._event_processor_tasks.add(task)
+                task.add_done_callback(self._event_processor_tasks.discard)
+
+    async def _process_webhook_events(self, path: str, worker_id: int) -> None:
+        """Process webhook events from the queue for a given path."""
+        queue = self._event_queues[path]
+        while True:
+            event = None
+            matching_processors: List[
+                Tuple[ResourceConfig, AbstractWebhookProcessor]
+            ] = []
             try:
-                task = loop.create_task(self.process_queue(path))
-                self._webhook_processor_tasks.add(task)
-                task.add_done_callback(self._webhook_processor_tasks.discard)
+                event = await queue.get()
+                with logger.contextualize(
+                    worker=worker_id,
+                    webhook_path=path,
+                    trace_id=event.trace_id,
+                ):
+                    async with event_context(
+                        EventType.HTTP_REQUEST,
+                        trigger_type="machine",
+                    ):
+
+                        await ocean.integration.port_app_config_handler.get_port_app_config(
+                            use_cache=False
+                        )
+                        matching_processors = await self._extract_matching_processors(
+                            event, path
+                        )
+
+                        processing_results = await asyncio.gather(
+                            *(
+                                self._process_single_event(proc, path, res)
+                                for res, proc in matching_processors
+                            ),
+                            return_exceptions=True,
+                        )
+
+                        successful_results: List[WebhookEventRawResults] = []
+                        failed_exceptions: List[Exception] = []
+
+                        for result in processing_results:
+                            if isinstance(result, WebhookEventRawResults):
+                                successful_results.append(result)
+                            elif isinstance(result, Exception):
+                                failed_exceptions.append(result)
+
+                        if successful_results:
+                            logger.info(
+                                "Successfully processed webhook events",
+                                success_count=len(successful_results),
+                                failure_count=len(failed_exceptions),
+                            )
+
+                        if failed_exceptions:
+                            logger.warning(
+                                "Some webhook events failed processing",
+                                failures=[str(e) for e in failed_exceptions],
+                            )
+
+                        await self.sync_raw_results(successful_results)
+
+            except asyncio.CancelledError:
+                logger.info(f"Worker {worker_id} for {path} shutting down")
+                for _, proc in matching_processors:
+                    await proc.cancel()
+                    self._timestamp_event_error(proc.event)
+                break
             except Exception as e:
-                logger.exception(f"Error starting queue processor for {path}: {str(e)}")
+                logger.exception(
+                    f"Unexpected error in worker {worker_id} for {path}: {e}"
+                )
+                for _, proc in matching_processors:
+                    self._timestamp_event_error(proc.event)
+            finally:
+                try:
+                    if event is not None:
+                        await queue.commit()
+
+                except Exception as e:
+                    logger.exception(
+                        f"Unexpected error in queue commit in worker {worker_id} for {path}: {e}"
+                    )
 
     async def _extract_matching_processors(
         self, webhook_event: WebhookEvent, path: str
@@ -91,70 +174,6 @@ class LiveEventsProcessorManager(LiveEventsMixin, EventsMixin):
         )
         return created_processors
 
-    async def process_queue(self, path: str) -> None:
-        """Process events for a specific path in order"""
-        while True:
-            matching_processors_with_resource: list[
-                tuple[ResourceConfig, AbstractWebhookProcessor]
-            ] = []
-            webhook_event: WebhookEvent | None = None
-            try:
-                queue = self._event_queues[path]
-                webhook_event = await queue.get()
-                with logger.contextualize(
-                    webhook_path=path, trace_id=webhook_event.trace_id
-                ):
-                    async with event_context(
-                        EventType.HTTP_REQUEST,
-                        trigger_type="machine",
-                    ):
-                        # This forces the Processor manager to fetch the latest port app config for each event
-                        await ocean.integration.port_app_config_handler.get_port_app_config(
-                            use_cache=False
-                        )
-                        matching_processors_with_resource = (
-                            await self._extract_matching_processors(webhook_event, path)
-                        )
-                        webhook_event_raw_results_for_all_resources = await asyncio.gather(
-                            *(
-                                self._process_single_event(processor, path, resource)
-                                for resource, processor in matching_processors_with_resource
-                            ),
-                            return_exceptions=True,
-                        )
-
-                        successful_raw_results: list[WebhookEventRawResults] = [
-                            result
-                            for result in webhook_event_raw_results_for_all_resources
-                            if isinstance(result, WebhookEventRawResults)
-                        ]
-
-                        if successful_raw_results:
-                            logger.info(
-                                "Exporting raw event results to entities",
-                                webhook_event_raw_results_for_all_resources_length=len(
-                                    successful_raw_results
-                                ),
-                            )
-                            await self.sync_raw_results(successful_raw_results)
-            except asyncio.CancelledError:
-                logger.info(f"Queue processor for {path} is shutting down")
-                for _, processor in matching_processors_with_resource:
-                    await processor.cancel()
-                    self._timestamp_event_error(processor.event)
-                break
-            except Exception as e:
-                logger.exception(
-                    f"Unexpected error in queue processor for {path}: {str(e)}"
-                )
-                for _, processor in matching_processors_with_resource:
-                    self._timestamp_event_error(processor.event)
-            finally:
-                if webhook_event:
-                    await self._event_queues[path].commit()
-                    # Prevents committing empty events for cases where we shutdown while processing
-                    webhook_event = None
-
     def _timestamp_event_error(self, event: WebhookEvent) -> None:
         """Timestamp an event as having an error"""
         event.set_timestamp(LiveEventTimestamp.FinishedProcessingWithError)
@@ -279,12 +298,14 @@ class LiveEventsProcessorManager(LiveEventsMixin, EventsMixin):
             methods=["POST"],
         )
 
-    async def _cancel_all_tasks(self) -> None:
-        """Cancel all webhook processor tasks"""
-        for task in self._webhook_processor_tasks:
+    async def _cancel_all_event_processors(
+        self,
+    ) -> None:
+        """Cancel all event processor tasks"""
+        for task in self._event_processor_tasks:
             task.cancel()
 
-        await asyncio.gather(*self._webhook_processor_tasks, return_exceptions=True)
+        await asyncio.gather(*self._event_processor_tasks, return_exceptions=True)
 
     async def shutdown(self) -> None:
         """Gracefully shutdown all queue processors"""
@@ -299,5 +320,3 @@ class LiveEventsProcessorManager(LiveEventsMixin, EventsMixin):
             )
         except asyncio.TimeoutError:
             logger.warning("Shutdown timed out waiting for queues to empty")
-
-        await self._cancel_all_tasks()