port-ocean 0.19.3__py3-none-any.whl → 0.20.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- port_ocean/context/ocean.py +2 -7
- port_ocean/core/handlers/webhook/abstract_webhook_processor.py +18 -2
- port_ocean/core/handlers/webhook/processor_manager.py +107 -65
- port_ocean/core/handlers/webhook/webhook_event.py +71 -8
- port_ocean/core/integrations/mixins/live_events.py +88 -0
- port_ocean/ocean.py +4 -2
- port_ocean/tests/core/handlers/mixins/test_live_events.py +404 -0
- port_ocean/tests/core/handlers/webhook/test_abstract_webhook_processor.py +88 -97
- port_ocean/tests/core/handlers/webhook/test_processor_manager.py +1161 -288
- port_ocean/tests/core/handlers/webhook/test_webhook_event.py +97 -56
- {port_ocean-0.19.3.dist-info → port_ocean-0.20.0.dist-info}/METADATA +1 -1
- {port_ocean-0.19.3.dist-info → port_ocean-0.20.0.dist-info}/RECORD +15 -13
- {port_ocean-0.19.3.dist-info → port_ocean-0.20.0.dist-info}/LICENSE.md +0 -0
- {port_ocean-0.19.3.dist-info → port_ocean-0.20.0.dist-info}/WHEEL +0 -0
- {port_ocean-0.19.3.dist-info → port_ocean-0.20.0.dist-info}/entry_points.txt +0 -0
port_ocean/context/ocean.py
CHANGED
@@ -146,12 +146,7 @@ class PortOceanContext:
|
|
146
146
|
async def sync_raw_all(self) -> None:
|
147
147
|
await self.integration.sync_raw_all(trigger_type="manual")
|
148
148
|
|
149
|
-
def add_webhook_processor(
|
150
|
-
self,
|
151
|
-
path: str,
|
152
|
-
processor: type,
|
153
|
-
events_filter: Callable[[Any], bool] = lambda _: True,
|
154
|
-
) -> None:
|
149
|
+
def add_webhook_processor(self, path: str, processor: type) -> None:
|
155
150
|
"""
|
156
151
|
Registers a webhook processor for a specific path.
|
157
152
|
|
@@ -175,7 +170,7 @@ class PortOceanContext:
|
|
175
170
|
Raises:
|
176
171
|
ValueError: If the processor does not extend AbstractWebhookProcessor.
|
177
172
|
"""
|
178
|
-
self.app.webhook_manager.register_processor(path, processor
|
173
|
+
self.app.webhook_manager.register_processor(path, processor)
|
179
174
|
|
180
175
|
|
181
176
|
_port_ocean: PortOceanContext = PortOceanContext(None)
|
@@ -1,9 +1,15 @@
|
|
1
1
|
from abc import ABC, abstractmethod
|
2
2
|
from loguru import logger
|
3
3
|
|
4
|
+
from port_ocean.core.handlers.port_app_config.models import ResourceConfig
|
4
5
|
from port_ocean.exceptions.webhook_processor import RetryableError
|
5
6
|
|
6
|
-
from .webhook_event import
|
7
|
+
from .webhook_event import (
|
8
|
+
WebhookEvent,
|
9
|
+
EventPayload,
|
10
|
+
EventHeaders,
|
11
|
+
WebhookEventRawResults,
|
12
|
+
)
|
7
13
|
|
8
14
|
|
9
15
|
class AbstractWebhookProcessor(ABC):
|
@@ -96,6 +102,16 @@ class AbstractWebhookProcessor(ABC):
|
|
96
102
|
pass
|
97
103
|
|
98
104
|
@abstractmethod
|
99
|
-
async def handle_event(
|
105
|
+
async def handle_event(
|
106
|
+
self, payload: EventPayload, resource: ResourceConfig
|
107
|
+
) -> WebhookEventRawResults:
|
100
108
|
"""Process the event."""
|
101
109
|
pass
|
110
|
+
|
111
|
+
@abstractmethod
|
112
|
+
def should_process_event(self, event: WebhookEvent) -> bool:
|
113
|
+
pass
|
114
|
+
|
115
|
+
@abstractmethod
|
116
|
+
def get_matching_kinds(self, event: WebhookEvent) -> list[str]:
|
117
|
+
pass
|
@@ -1,10 +1,15 @@
|
|
1
|
-
from
|
1
|
+
from copy import deepcopy
|
2
|
+
from typing import Dict, Type, Set
|
2
3
|
from fastapi import APIRouter, Request
|
3
4
|
from loguru import logger
|
4
5
|
import asyncio
|
5
|
-
from dataclasses import dataclass
|
6
6
|
|
7
|
-
from .
|
7
|
+
from port_ocean.context.event import EventType, event_context
|
8
|
+
from port_ocean.core.handlers.port_app_config.models import ResourceConfig
|
9
|
+
from port_ocean.core.integrations.mixins.events import EventsMixin
|
10
|
+
from port_ocean.core.integrations.mixins.live_events import LiveEventsMixin
|
11
|
+
from .webhook_event import WebhookEvent, WebhookEventRawResults, LiveEventTimestamp
|
12
|
+
from port_ocean.context.event import event
|
8
13
|
|
9
14
|
|
10
15
|
from .abstract_webhook_processor import AbstractWebhookProcessor
|
@@ -12,15 +17,7 @@ from port_ocean.utils.signal import SignalHandler
|
|
12
17
|
from port_ocean.core.handlers.queue import AbstractQueue, LocalQueue
|
13
18
|
|
14
19
|
|
15
|
-
|
16
|
-
class ProcessorRegistration:
|
17
|
-
"""Represents a registered processor with its filter"""
|
18
|
-
|
19
|
-
processor: Type[AbstractWebhookProcessor]
|
20
|
-
filter: Callable[[WebhookEvent], bool]
|
21
|
-
|
22
|
-
|
23
|
-
class WebhookProcessorManager:
|
20
|
+
class LiveEventsProcessorManager(LiveEventsMixin, EventsMixin):
|
24
21
|
"""Manages webhook processors and their routes"""
|
25
22
|
|
26
23
|
def __init__(
|
@@ -31,7 +28,7 @@ class WebhookProcessorManager:
|
|
31
28
|
max_wait_seconds_before_shutdown: float = 5.0,
|
32
29
|
) -> None:
|
33
30
|
self._router = router
|
34
|
-
self.
|
31
|
+
self._processors_classes: Dict[str, list[Type[AbstractWebhookProcessor]]] = {}
|
35
32
|
self._event_queues: Dict[str, AbstractQueue[WebhookEvent]] = {}
|
36
33
|
self._webhook_processor_tasks: Set[asyncio.Task[None]] = set()
|
37
34
|
self._max_event_processing_seconds = max_event_processing_seconds
|
@@ -40,6 +37,7 @@ class WebhookProcessorManager:
|
|
40
37
|
|
41
38
|
async def start_processing_event_messages(self) -> None:
|
42
39
|
"""Start processing events for all registered paths"""
|
40
|
+
await self.initialize_handlers()
|
43
41
|
loop = asyncio.get_event_loop()
|
44
42
|
for path in self._event_queues.keys():
|
45
43
|
try:
|
@@ -50,42 +48,73 @@ class WebhookProcessorManager:
|
|
50
48
|
logger.exception(f"Error starting queue processor for {path}: {str(e)}")
|
51
49
|
|
52
50
|
def _extract_matching_processors(
|
53
|
-
self,
|
54
|
-
) -> list[AbstractWebhookProcessor]:
|
51
|
+
self, webhook_event: WebhookEvent, path: str
|
52
|
+
) -> list[tuple[ResourceConfig, AbstractWebhookProcessor]]:
|
55
53
|
"""Find and extract the matching processor for an event"""
|
56
|
-
matching_processors = [
|
57
|
-
registration.processor
|
58
|
-
for registration in self._processors[path]
|
59
|
-
if registration.filter(event)
|
60
|
-
]
|
61
54
|
|
62
|
-
|
55
|
+
created_processors: list[tuple[ResourceConfig, AbstractWebhookProcessor]] = []
|
56
|
+
|
57
|
+
for processor_class in self._processors_classes[path]:
|
58
|
+
processor = processor_class(webhook_event.clone())
|
59
|
+
if processor.should_process_event(webhook_event):
|
60
|
+
kinds = processor.get_matching_kinds(webhook_event)
|
61
|
+
for kind in kinds:
|
62
|
+
for resource in event.port_app_config.resources:
|
63
|
+
if resource.kind == kind:
|
64
|
+
created_processors.append((resource, processor))
|
65
|
+
|
66
|
+
if not created_processors:
|
63
67
|
raise ValueError("No matching processors found")
|
64
68
|
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
+
logger.info(
|
70
|
+
"Found matching processors for webhook event",
|
71
|
+
processors_count=len(created_processors),
|
72
|
+
webhook_path=path,
|
73
|
+
)
|
69
74
|
return created_processors
|
70
75
|
|
71
76
|
async def process_queue(self, path: str) -> None:
|
72
77
|
"""Process events for a specific path in order"""
|
73
78
|
while True:
|
74
|
-
|
75
|
-
|
79
|
+
matching_processors_with_resource: list[
|
80
|
+
tuple[ResourceConfig, AbstractWebhookProcessor]
|
81
|
+
] = []
|
82
|
+
webhook_event: WebhookEvent | None = None
|
76
83
|
try:
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
+
queue = self._event_queues[path]
|
85
|
+
webhook_event = await queue.get()
|
86
|
+
with logger.contextualize(
|
87
|
+
webhook_path=path, trace_id=webhook_event.trace_id
|
88
|
+
):
|
89
|
+
async with event_context(
|
90
|
+
EventType.HTTP_REQUEST,
|
91
|
+
trigger_type="machine",
|
92
|
+
parent_override=webhook_event.event_context,
|
93
|
+
):
|
94
|
+
matching_processors_with_resource = (
|
95
|
+
self._extract_matching_processors(webhook_event, path)
|
84
96
|
)
|
85
|
-
|
97
|
+
webhook_event_raw_results_for_all_resources = await asyncio.gather(
|
98
|
+
*(
|
99
|
+
self._process_single_event(processor, path, resource)
|
100
|
+
for resource, processor in matching_processors_with_resource
|
101
|
+
)
|
102
|
+
)
|
103
|
+
if webhook_event_raw_results_for_all_resources and all(
|
104
|
+
webhook_event_raw_results_for_all_resources
|
105
|
+
):
|
106
|
+
logger.info(
|
107
|
+
"Exporting raw event results to entities",
|
108
|
+
webhook_event_raw_results_for_all_resources_length=len(
|
109
|
+
webhook_event_raw_results_for_all_resources
|
110
|
+
),
|
111
|
+
)
|
112
|
+
await self.sync_raw_results(
|
113
|
+
webhook_event_raw_results_for_all_resources
|
114
|
+
)
|
86
115
|
except asyncio.CancelledError:
|
87
116
|
logger.info(f"Queue processor for {path} is shutting down")
|
88
|
-
for processor in
|
117
|
+
for _, processor in matching_processors_with_resource:
|
89
118
|
await processor.cancel()
|
90
119
|
self._timestamp_event_error(processor.event)
|
91
120
|
break
|
@@ -93,49 +122,55 @@ class WebhookProcessorManager:
|
|
93
122
|
logger.exception(
|
94
123
|
f"Unexpected error in queue processor for {path}: {str(e)}"
|
95
124
|
)
|
96
|
-
for processor in
|
125
|
+
for _, processor in matching_processors_with_resource:
|
97
126
|
self._timestamp_event_error(processor.event)
|
98
127
|
finally:
|
99
|
-
if
|
128
|
+
if webhook_event:
|
100
129
|
await self._event_queues[path].commit()
|
101
130
|
# Prevents committing empty events for cases where we shutdown while processing
|
102
|
-
|
131
|
+
webhook_event = None
|
103
132
|
|
104
133
|
def _timestamp_event_error(self, event: WebhookEvent) -> None:
|
105
134
|
"""Timestamp an event as having an error"""
|
106
|
-
event.set_timestamp(
|
135
|
+
event.set_timestamp(LiveEventTimestamp.FinishedProcessingWithError)
|
107
136
|
|
108
137
|
async def _process_single_event(
|
109
|
-
self, processor: AbstractWebhookProcessor, path: str
|
110
|
-
) ->
|
138
|
+
self, processor: AbstractWebhookProcessor, path: str, resource: ResourceConfig
|
139
|
+
) -> WebhookEventRawResults:
|
111
140
|
"""Process a single event with a specific processor"""
|
112
141
|
try:
|
113
142
|
logger.debug("Start processing queued webhook")
|
114
|
-
processor.event.set_timestamp(
|
143
|
+
processor.event.set_timestamp(LiveEventTimestamp.StartedProcessing)
|
115
144
|
|
116
|
-
await self._execute_processor(
|
145
|
+
webhook_event_raw_results = await self._execute_processor(
|
146
|
+
processor, resource
|
147
|
+
)
|
117
148
|
processor.event.set_timestamp(
|
118
|
-
|
149
|
+
LiveEventTimestamp.FinishedProcessingSuccessfully
|
119
150
|
)
|
151
|
+
return webhook_event_raw_results
|
120
152
|
except Exception as e:
|
121
153
|
logger.exception(f"Error processing queued webhook for {path}: {str(e)}")
|
122
154
|
self._timestamp_event_error(processor.event)
|
155
|
+
raise
|
123
156
|
|
124
|
-
async def _execute_processor(
|
157
|
+
async def _execute_processor(
|
158
|
+
self, processor: AbstractWebhookProcessor, resource: ResourceConfig
|
159
|
+
) -> WebhookEventRawResults:
|
125
160
|
"""Execute a single processor within a max processing time"""
|
126
161
|
try:
|
127
|
-
await asyncio.wait_for(
|
128
|
-
self._process_webhook_request(processor),
|
162
|
+
return await asyncio.wait_for(
|
163
|
+
self._process_webhook_request(processor, resource),
|
129
164
|
timeout=self._max_event_processing_seconds,
|
130
165
|
)
|
131
166
|
except asyncio.TimeoutError:
|
132
|
-
raise TimeoutError(
|
167
|
+
raise asyncio.TimeoutError(
|
133
168
|
f"Processor processing timed out after {self._max_event_processing_seconds} seconds"
|
134
169
|
)
|
135
170
|
|
136
171
|
async def _process_webhook_request(
|
137
|
-
self, processor: AbstractWebhookProcessor
|
138
|
-
) ->
|
172
|
+
self, processor: AbstractWebhookProcessor, resource: ResourceConfig
|
173
|
+
) -> WebhookEventRawResults:
|
139
174
|
"""Process a webhook request with retry logic
|
140
175
|
|
141
176
|
Args:
|
@@ -152,9 +187,13 @@ class WebhookProcessorManager:
|
|
152
187
|
if not await processor.validate_payload(payload):
|
153
188
|
raise ValueError("Invalid payload")
|
154
189
|
|
190
|
+
webhook_event_raw_results = None
|
155
191
|
while True:
|
156
192
|
try:
|
157
|
-
await processor.handle_event(
|
193
|
+
webhook_event_raw_results = await processor.handle_event(
|
194
|
+
payload, resource
|
195
|
+
)
|
196
|
+
webhook_event_raw_results.resource = resource
|
158
197
|
break
|
159
198
|
|
160
199
|
except Exception as e:
|
@@ -172,26 +211,28 @@ class WebhookProcessorManager:
|
|
172
211
|
raise
|
173
212
|
|
174
213
|
await processor.after_processing()
|
214
|
+
return webhook_event_raw_results
|
175
215
|
|
176
216
|
def register_processor(
|
177
|
-
self,
|
178
|
-
path: str,
|
179
|
-
processor: Type[AbstractWebhookProcessor],
|
180
|
-
event_filter: Callable[[WebhookEvent], bool] = lambda _: True,
|
217
|
+
self, path: str, processor: Type[AbstractWebhookProcessor]
|
181
218
|
) -> None:
|
182
|
-
"""Register a webhook processor for a specific path with optional filter
|
219
|
+
"""Register a webhook processor for a specific path with optional filter
|
220
|
+
|
221
|
+
Args:
|
222
|
+
path: The webhook path to register
|
223
|
+
processor: The processor class to register
|
224
|
+
kind: The resource kind to associate with this processor, or None to match any kind
|
225
|
+
"""
|
183
226
|
|
184
227
|
if not issubclass(processor, AbstractWebhookProcessor):
|
185
228
|
raise ValueError("Processor must extend AbstractWebhookProcessor")
|
186
229
|
|
187
|
-
if path not in self.
|
188
|
-
self.
|
230
|
+
if path not in self._processors_classes:
|
231
|
+
self._processors_classes[path] = []
|
189
232
|
self._event_queues[path] = LocalQueue()
|
190
233
|
self._register_route(path)
|
191
234
|
|
192
|
-
self.
|
193
|
-
ProcessorRegistration(processor=processor, filter=event_filter)
|
194
|
-
)
|
235
|
+
self._processors_classes[path].append(processor)
|
195
236
|
|
196
237
|
def _register_route(self, path: str) -> None:
|
197
238
|
"""Register a route for a specific path"""
|
@@ -199,9 +240,10 @@ class WebhookProcessorManager:
|
|
199
240
|
async def handle_webhook(request: Request) -> Dict[str, str]:
|
200
241
|
"""Handle incoming webhook requests for a specific path."""
|
201
242
|
try:
|
202
|
-
|
203
|
-
|
204
|
-
|
243
|
+
webhook_event = await WebhookEvent.from_request(request)
|
244
|
+
webhook_event.set_timestamp(LiveEventTimestamp.AddedToQueue)
|
245
|
+
webhook_event.set_event_context(deepcopy(event))
|
246
|
+
await self._event_queues[path].put(webhook_event)
|
205
247
|
return {"status": "ok"}
|
206
248
|
except Exception as e:
|
207
249
|
logger.exception(f"Error processing webhook: {str(e)}")
|
@@ -1,15 +1,20 @@
|
|
1
|
+
from abc import ABC
|
1
2
|
from enum import StrEnum
|
2
|
-
from typing import Any, Dict, Type, TypeAlias
|
3
|
+
from typing import Any, Dict, Type, TypeAlias, Optional
|
3
4
|
from uuid import uuid4
|
4
5
|
from fastapi import Request
|
5
6
|
from loguru import logger
|
6
7
|
|
8
|
+
from port_ocean.context.event import EventContext
|
9
|
+
from port_ocean.core.handlers.port_app_config.models import ResourceConfig
|
10
|
+
from port_ocean.core.ocean_types import RAW_ITEM
|
11
|
+
|
7
12
|
|
8
13
|
EventPayload: TypeAlias = Dict[str, Any]
|
9
14
|
EventHeaders: TypeAlias = Dict[str, str]
|
10
15
|
|
11
16
|
|
12
|
-
class
|
17
|
+
class LiveEventTimestamp(StrEnum):
|
13
18
|
"""Enum for timestamp keys"""
|
14
19
|
|
15
20
|
AddedToQueue = "Added To Queue"
|
@@ -18,7 +23,27 @@ class WebhookEventTimestamp(StrEnum):
|
|
18
23
|
FinishedProcessingWithError = "Finished Processing With Error"
|
19
24
|
|
20
25
|
|
21
|
-
class
|
26
|
+
class LiveEvent(ABC):
|
27
|
+
"""Represents a live event marker class"""
|
28
|
+
|
29
|
+
def set_timestamp(
|
30
|
+
self, timestamp: LiveEventTimestamp, params: Optional[Dict[str, Any]] = None
|
31
|
+
) -> None:
|
32
|
+
"""Set a timestamp for a specific event
|
33
|
+
|
34
|
+
Args:
|
35
|
+
timestamp: The timestamp type to set
|
36
|
+
params: Additional parameters to log with the event
|
37
|
+
"""
|
38
|
+
log_params = params or {}
|
39
|
+
logger.info(
|
40
|
+
f"Event {timestamp.value}",
|
41
|
+
extra=log_params | {"timestamp_type": timestamp.value},
|
42
|
+
)
|
43
|
+
self._timestamp = timestamp
|
44
|
+
|
45
|
+
|
46
|
+
class WebhookEvent(LiveEvent):
|
22
47
|
"""Represents a webhook event"""
|
23
48
|
|
24
49
|
def __init__(
|
@@ -32,6 +57,7 @@ class WebhookEvent:
|
|
32
57
|
self.payload = payload
|
33
58
|
self.headers = headers
|
34
59
|
self._original_request = original_request
|
60
|
+
self.event_context: EventContext | None = None
|
35
61
|
|
36
62
|
@classmethod
|
37
63
|
async def from_request(
|
@@ -64,14 +90,51 @@ class WebhookEvent:
|
|
64
90
|
original_request=self._original_request,
|
65
91
|
)
|
66
92
|
|
67
|
-
def set_timestamp(
|
93
|
+
def set_timestamp(
|
94
|
+
self, timestamp: LiveEventTimestamp, params: Optional[Dict[str, Any]] = None
|
95
|
+
) -> None:
|
68
96
|
"""Set a timestamp for a specific event"""
|
69
|
-
|
70
|
-
|
71
|
-
|
97
|
+
super().set_timestamp(
|
98
|
+
timestamp,
|
99
|
+
params={
|
72
100
|
"trace_id": self.trace_id,
|
73
101
|
"payload": self.payload,
|
74
102
|
"headers": self.headers,
|
75
|
-
"timestamp_type": timestamp.value,
|
76
103
|
},
|
77
104
|
)
|
105
|
+
|
106
|
+
def set_event_context(self, event_context: EventContext) -> None:
|
107
|
+
self.event_context = event_context
|
108
|
+
|
109
|
+
|
110
|
+
class WebhookEventRawResults:
|
111
|
+
"""
|
112
|
+
Class for webhook event to store the updated data for the event
|
113
|
+
"""
|
114
|
+
|
115
|
+
def __init__(
|
116
|
+
self,
|
117
|
+
updated_raw_results: list[RAW_ITEM],
|
118
|
+
deleted_raw_results: list[RAW_ITEM],
|
119
|
+
) -> None:
|
120
|
+
self._resource: ResourceConfig | None = None
|
121
|
+
self._updated_raw_results = updated_raw_results
|
122
|
+
self._deleted_raw_results = deleted_raw_results
|
123
|
+
|
124
|
+
@property
|
125
|
+
def resource(self) -> ResourceConfig:
|
126
|
+
if self._resource is None:
|
127
|
+
raise ValueError("Resource has not been set")
|
128
|
+
return self._resource
|
129
|
+
|
130
|
+
@resource.setter
|
131
|
+
def resource(self, value: ResourceConfig) -> None:
|
132
|
+
self._resource = value
|
133
|
+
|
134
|
+
@property
|
135
|
+
def updated_raw_results(self) -> list[RAW_ITEM]:
|
136
|
+
return self._updated_raw_results
|
137
|
+
|
138
|
+
@property
|
139
|
+
def deleted_raw_results(self) -> list[RAW_ITEM]:
|
140
|
+
return self._deleted_raw_results
|
@@ -0,0 +1,88 @@
|
|
1
|
+
from loguru import logger
|
2
|
+
from port_ocean.clients.port.types import UserAgentType
|
3
|
+
from port_ocean.core.handlers.webhook.webhook_event import WebhookEventRawResults
|
4
|
+
from port_ocean.core.integrations.mixins.handler import HandlerMixin
|
5
|
+
from port_ocean.core.models import Entity
|
6
|
+
from port_ocean.context.ocean import ocean
|
7
|
+
|
8
|
+
|
9
|
+
|
10
|
+
class LiveEventsMixin(HandlerMixin):
|
11
|
+
|
12
|
+
async def sync_raw_results(self, webhook_events_raw_result: list[WebhookEventRawResults]) -> None:
|
13
|
+
"""Process the webhook event raw results collected from multiple processors and export it.
|
14
|
+
|
15
|
+
Args:
|
16
|
+
webhook_events_raw_result: List of WebhookEventRawResults objects to process
|
17
|
+
"""
|
18
|
+
entities_to_create, entities_to_delete = await self._parse_raw_event_results_to_entities(webhook_events_raw_result)
|
19
|
+
await self.entities_state_applier.upsert(entities_to_create, UserAgentType.exporter)
|
20
|
+
await self._delete_entities(entities_to_delete)
|
21
|
+
|
22
|
+
|
23
|
+
async def _parse_raw_event_results_to_entities(self, webhook_events_raw_result: list[WebhookEventRawResults]) -> tuple[list[Entity], list[Entity]]:
|
24
|
+
"""Parse the webhook event raw results and return a list of entities.
|
25
|
+
|
26
|
+
Args:
|
27
|
+
webhook_events_raw_result: List of WebhookEventRawResults objects to process
|
28
|
+
"""
|
29
|
+
entities: list[Entity] = []
|
30
|
+
entities_not_passed: list[Entity] = []
|
31
|
+
entities_to_delete: list[Entity] = []
|
32
|
+
for webhook_event_raw_result in webhook_events_raw_result:
|
33
|
+
for raw_item in webhook_event_raw_result.updated_raw_results:
|
34
|
+
calaculation_results = await self.entity_processor.parse_items(
|
35
|
+
webhook_event_raw_result.resource, [raw_item], parse_all=True, send_raw_data_examples_amount=0
|
36
|
+
)
|
37
|
+
entities.extend(calaculation_results.entity_selector_diff.passed)
|
38
|
+
entities_not_passed.extend(calaculation_results.entity_selector_diff.failed)
|
39
|
+
|
40
|
+
for raw_item in webhook_event_raw_result.deleted_raw_results:
|
41
|
+
deletion_results = await self.entity_processor.parse_items(
|
42
|
+
webhook_event_raw_result.resource, [raw_item], parse_all=True, send_raw_data_examples_amount=0
|
43
|
+
)
|
44
|
+
entities_to_delete.extend(deletion_results.entity_selector_diff.passed)
|
45
|
+
|
46
|
+
entities_to_remove = []
|
47
|
+
for entity in entities_to_delete + entities_not_passed:
|
48
|
+
if (entity.blueprint, entity.identifier) not in [(entity.blueprint, entity.identifier) for entity in entities]:
|
49
|
+
entities_to_remove.append(entity)
|
50
|
+
|
51
|
+
logger.info(f"Found {len(entities_to_remove)} entities to remove {', '.join(f'{entity.blueprint}/{entity.identifier}' for entity in entities_to_remove)}")
|
52
|
+
logger.info(f"Found {len(entities)} entities to upsert {', '.join(f'{entity.blueprint}/{entity.identifier}' for entity in entities)}")
|
53
|
+
return entities, entities_to_remove
|
54
|
+
|
55
|
+
async def _does_entity_exists(self, entity: Entity) -> bool:
|
56
|
+
"""Check if this integration is the owner of the given entity.
|
57
|
+
|
58
|
+
Args:
|
59
|
+
entity: The entity to check ownership for
|
60
|
+
|
61
|
+
Returns:
|
62
|
+
bool: True if this integration is the owner of the entity, False otherwise
|
63
|
+
"""
|
64
|
+
query = {
|
65
|
+
"combinator": "and",
|
66
|
+
"rules": [
|
67
|
+
{
|
68
|
+
"property": "$identifier",
|
69
|
+
"operator": "=",
|
70
|
+
"value": entity.identifier
|
71
|
+
},
|
72
|
+
{
|
73
|
+
"property": "$blueprint",
|
74
|
+
"operator": "=",
|
75
|
+
"value": entity.blueprint
|
76
|
+
}
|
77
|
+
]
|
78
|
+
}
|
79
|
+
entities_at_port = await ocean.port_client.search_entities(
|
80
|
+
UserAgentType.exporter,
|
81
|
+
query
|
82
|
+
)
|
83
|
+
return len(entities_at_port) > 0
|
84
|
+
|
85
|
+
async def _delete_entities(self, entities: list[Entity]) -> None:
|
86
|
+
for entity in entities:
|
87
|
+
if await self._does_entity_exists(entity):
|
88
|
+
await self.entities_state_applier.delete([entity], UserAgentType.exporter)
|
port_ocean/ocean.py
CHANGED
@@ -26,7 +26,9 @@ from port_ocean.utils.misc import IntegrationStateStatus
|
|
26
26
|
from port_ocean.utils.repeat import repeat_every
|
27
27
|
from port_ocean.utils.signal import signal_handler
|
28
28
|
from port_ocean.version import __integration_version__
|
29
|
-
from port_ocean.core.handlers.webhook.processor_manager import
|
29
|
+
from port_ocean.core.handlers.webhook.processor_manager import (
|
30
|
+
LiveEventsProcessorManager,
|
31
|
+
)
|
30
32
|
|
31
33
|
|
32
34
|
class Ocean:
|
@@ -54,7 +56,7 @@ class Ocean:
|
|
54
56
|
)
|
55
57
|
self.integration_router = integration_router or APIRouter()
|
56
58
|
|
57
|
-
self.webhook_manager =
|
59
|
+
self.webhook_manager = LiveEventsProcessorManager(
|
58
60
|
self.integration_router, signal_handler
|
59
61
|
)
|
60
62
|
|