port-ocean 0.19.3__py3-none-any.whl → 0.20.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -146,12 +146,7 @@ class PortOceanContext:
146
146
  async def sync_raw_all(self) -> None:
147
147
  await self.integration.sync_raw_all(trigger_type="manual")
148
148
 
149
- def add_webhook_processor(
150
- self,
151
- path: str,
152
- processor: type,
153
- events_filter: Callable[[Any], bool] = lambda _: True,
154
- ) -> None:
149
+ def add_webhook_processor(self, path: str, processor: type) -> None:
155
150
  """
156
151
  Registers a webhook processor for a specific path.
157
152
 
@@ -175,7 +170,7 @@ class PortOceanContext:
175
170
  Raises:
176
171
  ValueError: If the processor does not extend AbstractWebhookProcessor.
177
172
  """
178
- self.app.webhook_manager.register_processor(path, processor, events_filter)
173
+ self.app.webhook_manager.register_processor(path, processor)
179
174
 
180
175
 
181
176
  _port_ocean: PortOceanContext = PortOceanContext(None)
@@ -1,9 +1,15 @@
1
1
  from abc import ABC, abstractmethod
2
2
  from loguru import logger
3
3
 
4
+ from port_ocean.core.handlers.port_app_config.models import ResourceConfig
4
5
  from port_ocean.exceptions.webhook_processor import RetryableError
5
6
 
6
- from .webhook_event import WebhookEvent, EventPayload, EventHeaders
7
+ from .webhook_event import (
8
+ WebhookEvent,
9
+ EventPayload,
10
+ EventHeaders,
11
+ WebhookEventRawResults,
12
+ )
7
13
 
8
14
 
9
15
  class AbstractWebhookProcessor(ABC):
@@ -96,6 +102,16 @@ class AbstractWebhookProcessor(ABC):
96
102
  pass
97
103
 
98
104
  @abstractmethod
99
- async def handle_event(self, payload: EventPayload) -> None:
105
+ async def handle_event(
106
+ self, payload: EventPayload, resource: ResourceConfig
107
+ ) -> WebhookEventRawResults:
100
108
  """Process the event."""
101
109
  pass
110
+
111
+ @abstractmethod
112
+ def should_process_event(self, event: WebhookEvent) -> bool:
113
+ pass
114
+
115
+ @abstractmethod
116
+ def get_matching_kinds(self, event: WebhookEvent) -> list[str]:
117
+ pass
@@ -1,10 +1,15 @@
1
- from typing import Dict, Type, Set, Callable
1
+ from copy import deepcopy
2
+ from typing import Dict, Type, Set
2
3
  from fastapi import APIRouter, Request
3
4
  from loguru import logger
4
5
  import asyncio
5
- from dataclasses import dataclass
6
6
 
7
- from .webhook_event import WebhookEvent, WebhookEventTimestamp
7
+ from port_ocean.context.event import EventType, event_context
8
+ from port_ocean.core.handlers.port_app_config.models import ResourceConfig
9
+ from port_ocean.core.integrations.mixins.events import EventsMixin
10
+ from port_ocean.core.integrations.mixins.live_events import LiveEventsMixin
11
+ from .webhook_event import WebhookEvent, WebhookEventRawResults, LiveEventTimestamp
12
+ from port_ocean.context.event import event
8
13
 
9
14
 
10
15
  from .abstract_webhook_processor import AbstractWebhookProcessor
@@ -12,15 +17,7 @@ from port_ocean.utils.signal import SignalHandler
12
17
  from port_ocean.core.handlers.queue import AbstractQueue, LocalQueue
13
18
 
14
19
 
15
- @dataclass
16
- class ProcessorRegistration:
17
- """Represents a registered processor with its filter"""
18
-
19
- processor: Type[AbstractWebhookProcessor]
20
- filter: Callable[[WebhookEvent], bool]
21
-
22
-
23
- class WebhookProcessorManager:
20
+ class LiveEventsProcessorManager(LiveEventsMixin, EventsMixin):
24
21
  """Manages webhook processors and their routes"""
25
22
 
26
23
  def __init__(
@@ -31,7 +28,7 @@ class WebhookProcessorManager:
31
28
  max_wait_seconds_before_shutdown: float = 5.0,
32
29
  ) -> None:
33
30
  self._router = router
34
- self._processors: Dict[str, list[ProcessorRegistration]] = {}
31
+ self._processors_classes: Dict[str, list[Type[AbstractWebhookProcessor]]] = {}
35
32
  self._event_queues: Dict[str, AbstractQueue[WebhookEvent]] = {}
36
33
  self._webhook_processor_tasks: Set[asyncio.Task[None]] = set()
37
34
  self._max_event_processing_seconds = max_event_processing_seconds
@@ -40,6 +37,7 @@ class WebhookProcessorManager:
40
37
 
41
38
  async def start_processing_event_messages(self) -> None:
42
39
  """Start processing events for all registered paths"""
40
+ await self.initialize_handlers()
43
41
  loop = asyncio.get_event_loop()
44
42
  for path in self._event_queues.keys():
45
43
  try:
@@ -50,42 +48,73 @@ class WebhookProcessorManager:
50
48
  logger.exception(f"Error starting queue processor for {path}: {str(e)}")
51
49
 
52
50
  def _extract_matching_processors(
53
- self, event: WebhookEvent, path: str
54
- ) -> list[AbstractWebhookProcessor]:
51
+ self, webhook_event: WebhookEvent, path: str
52
+ ) -> list[tuple[ResourceConfig, AbstractWebhookProcessor]]:
55
53
  """Find and extract the matching processor for an event"""
56
- matching_processors = [
57
- registration.processor
58
- for registration in self._processors[path]
59
- if registration.filter(event)
60
- ]
61
54
 
62
- if not matching_processors:
55
+ created_processors: list[tuple[ResourceConfig, AbstractWebhookProcessor]] = []
56
+
57
+ for processor_class in self._processors_classes[path]:
58
+ processor = processor_class(webhook_event.clone())
59
+ if processor.should_process_event(webhook_event):
60
+ kinds = processor.get_matching_kinds(webhook_event)
61
+ for kind in kinds:
62
+ for resource in event.port_app_config.resources:
63
+ if resource.kind == kind:
64
+ created_processors.append((resource, processor))
65
+
66
+ if not created_processors:
63
67
  raise ValueError("No matching processors found")
64
68
 
65
- created_processors: list[AbstractWebhookProcessor] = []
66
- for processor_class in matching_processors:
67
- processor = processor_class(event.clone())
68
- created_processors.append(processor)
69
+ logger.info(
70
+ "Found matching processors for webhook event",
71
+ processors_count=len(created_processors),
72
+ webhook_path=path,
73
+ )
69
74
  return created_processors
70
75
 
71
76
  async def process_queue(self, path: str) -> None:
72
77
  """Process events for a specific path in order"""
73
78
  while True:
74
- matching_processors: list[AbstractWebhookProcessor] = []
75
- event: WebhookEvent | None = None
79
+ matching_processors_with_resource: list[
80
+ tuple[ResourceConfig, AbstractWebhookProcessor]
81
+ ] = []
82
+ webhook_event: WebhookEvent | None = None
76
83
  try:
77
- event = await self._event_queues[path].get()
78
- with logger.contextualize(webhook_path=path, trace_id=event.trace_id):
79
- matching_processors = self._extract_matching_processors(event, path)
80
- await asyncio.gather(
81
- *(
82
- self._process_single_event(processor, path)
83
- for processor in matching_processors
84
+ queue = self._event_queues[path]
85
+ webhook_event = await queue.get()
86
+ with logger.contextualize(
87
+ webhook_path=path, trace_id=webhook_event.trace_id
88
+ ):
89
+ async with event_context(
90
+ EventType.HTTP_REQUEST,
91
+ trigger_type="machine",
92
+ parent_override=webhook_event.event_context,
93
+ ):
94
+ matching_processors_with_resource = (
95
+ self._extract_matching_processors(webhook_event, path)
84
96
  )
85
- )
97
+ webhook_event_raw_results_for_all_resources = await asyncio.gather(
98
+ *(
99
+ self._process_single_event(processor, path, resource)
100
+ for resource, processor in matching_processors_with_resource
101
+ )
102
+ )
103
+ if webhook_event_raw_results_for_all_resources and all(
104
+ webhook_event_raw_results_for_all_resources
105
+ ):
106
+ logger.info(
107
+ "Exporting raw event results to entities",
108
+ webhook_event_raw_results_for_all_resources_length=len(
109
+ webhook_event_raw_results_for_all_resources
110
+ ),
111
+ )
112
+ await self.sync_raw_results(
113
+ webhook_event_raw_results_for_all_resources
114
+ )
86
115
  except asyncio.CancelledError:
87
116
  logger.info(f"Queue processor for {path} is shutting down")
88
- for processor in matching_processors:
117
+ for _, processor in matching_processors_with_resource:
89
118
  await processor.cancel()
90
119
  self._timestamp_event_error(processor.event)
91
120
  break
@@ -93,49 +122,55 @@ class WebhookProcessorManager:
93
122
  logger.exception(
94
123
  f"Unexpected error in queue processor for {path}: {str(e)}"
95
124
  )
96
- for processor in matching_processors:
125
+ for _, processor in matching_processors_with_resource:
97
126
  self._timestamp_event_error(processor.event)
98
127
  finally:
99
- if event:
128
+ if webhook_event:
100
129
  await self._event_queues[path].commit()
101
130
  # Prevents committing empty events for cases where we shutdown while processing
102
- event = None
131
+ webhook_event = None
103
132
 
104
133
  def _timestamp_event_error(self, event: WebhookEvent) -> None:
105
134
  """Timestamp an event as having an error"""
106
- event.set_timestamp(WebhookEventTimestamp.FinishedProcessingWithError)
135
+ event.set_timestamp(LiveEventTimestamp.FinishedProcessingWithError)
107
136
 
108
137
  async def _process_single_event(
109
- self, processor: AbstractWebhookProcessor, path: str
110
- ) -> None:
138
+ self, processor: AbstractWebhookProcessor, path: str, resource: ResourceConfig
139
+ ) -> WebhookEventRawResults:
111
140
  """Process a single event with a specific processor"""
112
141
  try:
113
142
  logger.debug("Start processing queued webhook")
114
- processor.event.set_timestamp(WebhookEventTimestamp.StartedProcessing)
143
+ processor.event.set_timestamp(LiveEventTimestamp.StartedProcessing)
115
144
 
116
- await self._execute_processor(processor)
145
+ webhook_event_raw_results = await self._execute_processor(
146
+ processor, resource
147
+ )
117
148
  processor.event.set_timestamp(
118
- WebhookEventTimestamp.FinishedProcessingSuccessfully
149
+ LiveEventTimestamp.FinishedProcessingSuccessfully
119
150
  )
151
+ return webhook_event_raw_results
120
152
  except Exception as e:
121
153
  logger.exception(f"Error processing queued webhook for {path}: {str(e)}")
122
154
  self._timestamp_event_error(processor.event)
155
+ raise
123
156
 
124
- async def _execute_processor(self, processor: AbstractWebhookProcessor) -> None:
157
+ async def _execute_processor(
158
+ self, processor: AbstractWebhookProcessor, resource: ResourceConfig
159
+ ) -> WebhookEventRawResults:
125
160
  """Execute a single processor within a max processing time"""
126
161
  try:
127
- await asyncio.wait_for(
128
- self._process_webhook_request(processor),
162
+ return await asyncio.wait_for(
163
+ self._process_webhook_request(processor, resource),
129
164
  timeout=self._max_event_processing_seconds,
130
165
  )
131
166
  except asyncio.TimeoutError:
132
- raise TimeoutError(
167
+ raise asyncio.TimeoutError(
133
168
  f"Processor processing timed out after {self._max_event_processing_seconds} seconds"
134
169
  )
135
170
 
136
171
  async def _process_webhook_request(
137
- self, processor: AbstractWebhookProcessor
138
- ) -> None:
172
+ self, processor: AbstractWebhookProcessor, resource: ResourceConfig
173
+ ) -> WebhookEventRawResults:
139
174
  """Process a webhook request with retry logic
140
175
 
141
176
  Args:
@@ -152,9 +187,13 @@ class WebhookProcessorManager:
152
187
  if not await processor.validate_payload(payload):
153
188
  raise ValueError("Invalid payload")
154
189
 
190
+ webhook_event_raw_results = None
155
191
  while True:
156
192
  try:
157
- await processor.handle_event(payload)
193
+ webhook_event_raw_results = await processor.handle_event(
194
+ payload, resource
195
+ )
196
+ webhook_event_raw_results.resource = resource
158
197
  break
159
198
 
160
199
  except Exception as e:
@@ -172,26 +211,28 @@ class WebhookProcessorManager:
172
211
  raise
173
212
 
174
213
  await processor.after_processing()
214
+ return webhook_event_raw_results
175
215
 
176
216
  def register_processor(
177
- self,
178
- path: str,
179
- processor: Type[AbstractWebhookProcessor],
180
- event_filter: Callable[[WebhookEvent], bool] = lambda _: True,
217
+ self, path: str, processor: Type[AbstractWebhookProcessor]
181
218
  ) -> None:
182
- """Register a webhook processor for a specific path with optional filter"""
219
+ """Register a webhook processor for a specific path with optional filter
220
+
221
+ Args:
222
+ path: The webhook path to register
223
+ processor: The processor class to register
224
+ kind: The resource kind to associate with this processor, or None to match any kind
225
+ """
183
226
 
184
227
  if not issubclass(processor, AbstractWebhookProcessor):
185
228
  raise ValueError("Processor must extend AbstractWebhookProcessor")
186
229
 
187
- if path not in self._processors:
188
- self._processors[path] = []
230
+ if path not in self._processors_classes:
231
+ self._processors_classes[path] = []
189
232
  self._event_queues[path] = LocalQueue()
190
233
  self._register_route(path)
191
234
 
192
- self._processors[path].append(
193
- ProcessorRegistration(processor=processor, filter=event_filter)
194
- )
235
+ self._processors_classes[path].append(processor)
195
236
 
196
237
  def _register_route(self, path: str) -> None:
197
238
  """Register a route for a specific path"""
@@ -199,9 +240,10 @@ class WebhookProcessorManager:
199
240
  async def handle_webhook(request: Request) -> Dict[str, str]:
200
241
  """Handle incoming webhook requests for a specific path."""
201
242
  try:
202
- event = await WebhookEvent.from_request(request)
203
- event.set_timestamp(WebhookEventTimestamp.AddedToQueue)
204
- await self._event_queues[path].put(event)
243
+ webhook_event = await WebhookEvent.from_request(request)
244
+ webhook_event.set_timestamp(LiveEventTimestamp.AddedToQueue)
245
+ webhook_event.set_event_context(deepcopy(event))
246
+ await self._event_queues[path].put(webhook_event)
205
247
  return {"status": "ok"}
206
248
  except Exception as e:
207
249
  logger.exception(f"Error processing webhook: {str(e)}")
@@ -1,15 +1,20 @@
1
+ from abc import ABC
1
2
  from enum import StrEnum
2
- from typing import Any, Dict, Type, TypeAlias
3
+ from typing import Any, Dict, Type, TypeAlias, Optional
3
4
  from uuid import uuid4
4
5
  from fastapi import Request
5
6
  from loguru import logger
6
7
 
8
+ from port_ocean.context.event import EventContext
9
+ from port_ocean.core.handlers.port_app_config.models import ResourceConfig
10
+ from port_ocean.core.ocean_types import RAW_ITEM
11
+
7
12
 
8
13
  EventPayload: TypeAlias = Dict[str, Any]
9
14
  EventHeaders: TypeAlias = Dict[str, str]
10
15
 
11
16
 
12
- class WebhookEventTimestamp(StrEnum):
17
+ class LiveEventTimestamp(StrEnum):
13
18
  """Enum for timestamp keys"""
14
19
 
15
20
  AddedToQueue = "Added To Queue"
@@ -18,7 +23,27 @@ class WebhookEventTimestamp(StrEnum):
18
23
  FinishedProcessingWithError = "Finished Processing With Error"
19
24
 
20
25
 
21
- class WebhookEvent:
26
+ class LiveEvent(ABC):
27
+ """Represents a live event marker class"""
28
+
29
+ def set_timestamp(
30
+ self, timestamp: LiveEventTimestamp, params: Optional[Dict[str, Any]] = None
31
+ ) -> None:
32
+ """Set a timestamp for a specific event
33
+
34
+ Args:
35
+ timestamp: The timestamp type to set
36
+ params: Additional parameters to log with the event
37
+ """
38
+ log_params = params or {}
39
+ logger.info(
40
+ f"Event {timestamp.value}",
41
+ extra=log_params | {"timestamp_type": timestamp.value},
42
+ )
43
+ self._timestamp = timestamp
44
+
45
+
46
+ class WebhookEvent(LiveEvent):
22
47
  """Represents a webhook event"""
23
48
 
24
49
  def __init__(
@@ -32,6 +57,7 @@ class WebhookEvent:
32
57
  self.payload = payload
33
58
  self.headers = headers
34
59
  self._original_request = original_request
60
+ self.event_context: EventContext | None = None
35
61
 
36
62
  @classmethod
37
63
  async def from_request(
@@ -64,14 +90,51 @@ class WebhookEvent:
64
90
  original_request=self._original_request,
65
91
  )
66
92
 
67
- def set_timestamp(self, timestamp: WebhookEventTimestamp) -> None:
93
+ def set_timestamp(
94
+ self, timestamp: LiveEventTimestamp, params: Optional[Dict[str, Any]] = None
95
+ ) -> None:
68
96
  """Set a timestamp for a specific event"""
69
- logger.info(
70
- f"Webhook Event {timestamp.value}",
71
- extra={
97
+ super().set_timestamp(
98
+ timestamp,
99
+ params={
72
100
  "trace_id": self.trace_id,
73
101
  "payload": self.payload,
74
102
  "headers": self.headers,
75
- "timestamp_type": timestamp.value,
76
103
  },
77
104
  )
105
+
106
+ def set_event_context(self, event_context: EventContext) -> None:
107
+ self.event_context = event_context
108
+
109
+
110
+ class WebhookEventRawResults:
111
+ """
112
+ Class for webhook event to store the updated data for the event
113
+ """
114
+
115
+ def __init__(
116
+ self,
117
+ updated_raw_results: list[RAW_ITEM],
118
+ deleted_raw_results: list[RAW_ITEM],
119
+ ) -> None:
120
+ self._resource: ResourceConfig | None = None
121
+ self._updated_raw_results = updated_raw_results
122
+ self._deleted_raw_results = deleted_raw_results
123
+
124
+ @property
125
+ def resource(self) -> ResourceConfig:
126
+ if self._resource is None:
127
+ raise ValueError("Resource has not been set")
128
+ return self._resource
129
+
130
+ @resource.setter
131
+ def resource(self, value: ResourceConfig) -> None:
132
+ self._resource = value
133
+
134
+ @property
135
+ def updated_raw_results(self) -> list[RAW_ITEM]:
136
+ return self._updated_raw_results
137
+
138
+ @property
139
+ def deleted_raw_results(self) -> list[RAW_ITEM]:
140
+ return self._deleted_raw_results
@@ -0,0 +1,88 @@
1
+ from loguru import logger
2
+ from port_ocean.clients.port.types import UserAgentType
3
+ from port_ocean.core.handlers.webhook.webhook_event import WebhookEventRawResults
4
+ from port_ocean.core.integrations.mixins.handler import HandlerMixin
5
+ from port_ocean.core.models import Entity
6
+ from port_ocean.context.ocean import ocean
7
+
8
+
9
+
10
+ class LiveEventsMixin(HandlerMixin):
11
+
12
+ async def sync_raw_results(self, webhook_events_raw_result: list[WebhookEventRawResults]) -> None:
13
+ """Process the webhook event raw results collected from multiple processors and export it.
14
+
15
+ Args:
16
+ webhook_events_raw_result: List of WebhookEventRawResults objects to process
17
+ """
18
+ entities_to_create, entities_to_delete = await self._parse_raw_event_results_to_entities(webhook_events_raw_result)
19
+ await self.entities_state_applier.upsert(entities_to_create, UserAgentType.exporter)
20
+ await self._delete_entities(entities_to_delete)
21
+
22
+
23
+ async def _parse_raw_event_results_to_entities(self, webhook_events_raw_result: list[WebhookEventRawResults]) -> tuple[list[Entity], list[Entity]]:
24
+ """Parse the webhook event raw results and return a list of entities.
25
+
26
+ Args:
27
+ webhook_events_raw_result: List of WebhookEventRawResults objects to process
28
+ """
29
+ entities: list[Entity] = []
30
+ entities_not_passed: list[Entity] = []
31
+ entities_to_delete: list[Entity] = []
32
+ for webhook_event_raw_result in webhook_events_raw_result:
33
+ for raw_item in webhook_event_raw_result.updated_raw_results:
34
+ calaculation_results = await self.entity_processor.parse_items(
35
+ webhook_event_raw_result.resource, [raw_item], parse_all=True, send_raw_data_examples_amount=0
36
+ )
37
+ entities.extend(calaculation_results.entity_selector_diff.passed)
38
+ entities_not_passed.extend(calaculation_results.entity_selector_diff.failed)
39
+
40
+ for raw_item in webhook_event_raw_result.deleted_raw_results:
41
+ deletion_results = await self.entity_processor.parse_items(
42
+ webhook_event_raw_result.resource, [raw_item], parse_all=True, send_raw_data_examples_amount=0
43
+ )
44
+ entities_to_delete.extend(deletion_results.entity_selector_diff.passed)
45
+
46
+ entities_to_remove = []
47
+ for entity in entities_to_delete + entities_not_passed:
48
+ if (entity.blueprint, entity.identifier) not in [(entity.blueprint, entity.identifier) for entity in entities]:
49
+ entities_to_remove.append(entity)
50
+
51
+ logger.info(f"Found {len(entities_to_remove)} entities to remove {', '.join(f'{entity.blueprint}/{entity.identifier}' for entity in entities_to_remove)}")
52
+ logger.info(f"Found {len(entities)} entities to upsert {', '.join(f'{entity.blueprint}/{entity.identifier}' for entity in entities)}")
53
+ return entities, entities_to_remove
54
+
55
+ async def _does_entity_exists(self, entity: Entity) -> bool:
56
+ """Check if this integration is the owner of the given entity.
57
+
58
+ Args:
59
+ entity: The entity to check ownership for
60
+
61
+ Returns:
62
+ bool: True if this integration is the owner of the entity, False otherwise
63
+ """
64
+ query = {
65
+ "combinator": "and",
66
+ "rules": [
67
+ {
68
+ "property": "$identifier",
69
+ "operator": "=",
70
+ "value": entity.identifier
71
+ },
72
+ {
73
+ "property": "$blueprint",
74
+ "operator": "=",
75
+ "value": entity.blueprint
76
+ }
77
+ ]
78
+ }
79
+ entities_at_port = await ocean.port_client.search_entities(
80
+ UserAgentType.exporter,
81
+ query
82
+ )
83
+ return len(entities_at_port) > 0
84
+
85
+ async def _delete_entities(self, entities: list[Entity]) -> None:
86
+ for entity in entities:
87
+ if await self._does_entity_exists(entity):
88
+ await self.entities_state_applier.delete([entity], UserAgentType.exporter)
port_ocean/ocean.py CHANGED
@@ -26,7 +26,9 @@ from port_ocean.utils.misc import IntegrationStateStatus
26
26
  from port_ocean.utils.repeat import repeat_every
27
27
  from port_ocean.utils.signal import signal_handler
28
28
  from port_ocean.version import __integration_version__
29
- from port_ocean.core.handlers.webhook.processor_manager import WebhookProcessorManager
29
+ from port_ocean.core.handlers.webhook.processor_manager import (
30
+ LiveEventsProcessorManager,
31
+ )
30
32
 
31
33
 
32
34
  class Ocean:
@@ -54,7 +56,7 @@ class Ocean:
54
56
  )
55
57
  self.integration_router = integration_router or APIRouter()
56
58
 
57
- self.webhook_manager = WebhookProcessorManager(
59
+ self.webhook_manager = LiveEventsProcessorManager(
58
60
  self.integration_router, signal_handler
59
61
  )
60
62